test: update retrieval accuracy benchmarks

2026-01-29 15:24:10 +08:00 · 2025-10-27 13:45:48 +01:00
parent b2c58d2b97
commit 1a5e6199ac
10 changed files with 5686 additions and 5709 deletions
--- a/README.md
+++ b/README.md
@@ -55,7 +55,7 @@ users[2]{id,name,role}:
 🛒 E-commerce Order          ███████████████░░░░░░░░░░     203 tokens  (JSON:    338)  💰 39.9% saved
 ```

-**Total:** 15,172 tokens (TOON) vs 29,096 tokens (JSON) → 47.9% savings
+**Total:** 15,172 tokens (TOON) vs 29,096 tokens (JSON) → **47.9% savings**

 <details>
 <summary><strong>View detailed examples</strong></summary>
@@ -200,19 +200,22 @@ metrics[5]{date,views,clicks,conversions,revenue}:
 Tested across **2 LLMs** with data retrieval tasks:

 ```
-gpt-4o-mini          ██████████████░░░░░░ 72.3% accuracy
-claude-haiku-4-5     ███████████████░░░░░ 76.7% accuracy
+gpt-5-nano
+  toon         ███████████████████░  97.5% (155/159)
+  markdown-kv  ███████████████████░  95.6% (152/159)
+  yaml         ███████████████████░  94.3% (150/159)
+  json         ███████████████████░  93.7% (149/159)
+  csv          ███████████████████░  93.7% (149/159)
+
+claude-haiku-4-5
+  markdown-kv  ███████████████░░░░░  76.7% (122/159)
+  toon         ███████████████░░░░░  75.5% (120/159)
+  json         ███████████████░░░░░  75.5% (120/159)
+  csv          ███████████████░░░░░  75.5% (120/159)
+  yaml         ███████████████░░░░░  74.8% (119/159)
 ```

-**TOON achieves 73.9% accuracy (vs JSON's 73.6%) while using 46.3% fewer tokens.**
-
-| Format | Accuracy | Average Tokens |
-| ------ | -------- | -------------- |
-| `toon` | 73.9% | 4.678 |
-| `json` | 73.6% | 8.713 |
-| `markdown-kv` | 73.6% | 8.649 |
-| `csv` | 72.3% | 4.745 |
-| `yaml` | 71.7% | 7.091 |
+**Tradeoff:** TOON achieves 86.5% accuracy (vs JSON's 84.6%) while using 46.3% fewer tokens.

 <details>
 <summary><strong>View detailed breakdown by dataset and model</strong></summary>
@@ -223,53 +226,53 @@ claude-haiku-4-5     ███████████████░░░░

 | Format | Accuracy | Tokens | Correct/Total |
 |--------|----------|--------|---------------|
-| `toon` | 72.4% | 2.483 | 84/116 |
-| `csv` | 69.0% | 2.337 | 80/116 |
-| `yaml` | 68.1% | 4.969 | 79/116 |
-| `markdown-kv` | 68.1% | 6.270 | 79/116 |
-| `json` | 68.1% | 6.347 | 79/116 |
+| `toon` | 86.2% | 2.483 | 100/116 |
+| `csv` | 80.2% | 2.337 | 93/116 |
+| `yaml` | 82.8% | 4.969 | 96/116 |
+| `markdown-kv` | 84.5% | 6.270 | 98/116 |
+| `json` | 84.5% | 6.347 | 98/116 |

 ##### E-commerce orders with nested structures

 | Format | Accuracy | Tokens | Correct/Total |
 |--------|----------|--------|---------------|
-| `toon` | 84.1% | 5.967 | 74/88 |
-| `csv` | 83.0% | 6.735 | 73/88 |
-| `yaml` | 81.8% | 7.328 | 72/88 |
-| `markdown-kv` | 86.4% | 9.110 | 76/88 |
-| `json` | 84.1% | 9.694 | 74/88 |
+| `toon` | 90.9% | 5.967 | 80/88 |
+| `csv` | 90.9% | 6.735 | 80/88 |
+| `yaml` | 89.8% | 7.328 | 79/88 |
+| `markdown-kv` | 90.9% | 9.110 | 80/88 |
+| `json` | 89.8% | 9.694 | 79/88 |

 ##### Time-series analytics data

 | Format | Accuracy | Tokens | Correct/Total |
 |--------|----------|--------|---------------|
-| `csv` | 72.4% | 1.393 | 42/58 |
-| `toon` | 70.7% | 1.515 | 41/58 |
-| `yaml` | 72.4% | 2.938 | 42/58 |
-| `json` | 74.1% | 3.665 | 43/58 |
-| `markdown-kv` | 70.7% | 3.779 | 41/58 |
+| `csv` | 87.9% | 1.393 | 51/58 |
+| `toon` | 86.2% | 1.515 | 50/58 |
+| `yaml` | 86.2% | 2.938 | 50/58 |
+| `json` | 87.9% | 3.665 | 51/58 |
+| `markdown-kv` | 86.2% | 3.779 | 50/58 |

 ##### Popular GitHub repositories

 | Format | Accuracy | Tokens | Correct/Total |
 |--------|----------|--------|---------------|
-| `toon` | 64.3% | 8.745 | 36/56 |
-| `csv` | 62.5% | 8.513 | 35/56 |
-| `json` | 67.9% | 15.145 | 38/56 |
-| `markdown-kv` | 67.9% | 15.436 | 38/56 |
-| `yaml` | 62.5% | 13.129 | 35/56 |
+| `csv` | 80.4% | 8.513 | 45/56 |
+| `toon` | 80.4% | 8.745 | 45/56 |
+| `yaml` | 78.6% | 13.129 | 44/56 |
+| `markdown-kv` | 82.1% | 15.436 | 46/56 |
+| `json` | 73.2% | 15.145 | 41/56 |

 #### Performance by Model

-##### gpt-4o-mini
+##### gpt-5-nano

 | Format | Accuracy | Correct/Total |
 |--------|----------|---------------|
-| `toon` | 72.3% | 115/159 |
-| `json` | 71.7% | 114/159 |
-| `markdown-kv` | 70.4% | 112/159 |
-| `csv` | 69.2% | 110/159 |
-| `yaml` | 68.6% | 109/159 |
+| `toon` | 97.5% | 155/159 |
+| `markdown-kv` | 95.6% | 152/159 |
+| `yaml` | 94.3% | 150/159 |
+| `json` | 93.7% | 149/159 |
+| `csv` | 93.7% | 149/159 |

 ##### claude-haiku-4-5

--- a/benchmarks/results/accuracy/raw-results.json
+++ b/benchmarks/results/accuracy/raw-results.json
--- a/benchmarks/results/accuracy/report.md
+++ b/benchmarks/results/accuracy/report.md
@@ -3,19 +3,22 @@
 Tested across **2 LLMs** with data retrieval tasks:

 ```
-gpt-4o-mini          ██████████████░░░░░░ 72.3% accuracy
-claude-haiku-4-5     ███████████████░░░░░ 76.7% accuracy
+gpt-5-nano
+  toon         ███████████████████░  97.5% (155/159)
+  markdown-kv  ███████████████████░  95.6% (152/159)
+  yaml         ███████████████████░  94.3% (150/159)
+  json         ███████████████████░  93.7% (149/159)
+  csv          ███████████████████░  93.7% (149/159)
+
+claude-haiku-4-5
+  markdown-kv  ███████████████░░░░░  76.7% (122/159)
+  toon         ███████████████░░░░░  75.5% (120/159)
+  json         ███████████████░░░░░  75.5% (120/159)
+  csv          ███████████████░░░░░  75.5% (120/159)
+  yaml         ███████████████░░░░░  74.8% (119/159)
 ```

-**TOON achieves 73.9% accuracy (vs JSON's 73.6%) while using 46.3% fewer tokens.**
-
-| Format | Accuracy | Average Tokens |
-| ------ | -------- | -------------- |
-| `toon` | 73.9% | 4.678 |
-| `json` | 73.6% | 8.713 |
-| `markdown-kv` | 73.6% | 8.649 |
-| `csv` | 72.3% | 4.745 |
-| `yaml` | 71.7% | 7.091 |
+**Tradeoff:** TOON achieves 86.5% accuracy (vs JSON's 84.6%) while using 46.3% fewer tokens.

 <details>
 <summary><strong>View detailed breakdown by dataset and model</strong></summary>
@@ -26,53 +29,53 @@ claude-haiku-4-5     ███████████████░░░░

 | Format | Accuracy | Tokens | Correct/Total |
 |--------|----------|--------|---------------|
-| `toon` | 72.4% | 2.483 | 84/116 |
-| `csv` | 69.0% | 2.337 | 80/116 |
-| `yaml` | 68.1% | 4.969 | 79/116 |
-| `markdown-kv` | 68.1% | 6.270 | 79/116 |
-| `json` | 68.1% | 6.347 | 79/116 |
+| `toon` | 86.2% | 2.483 | 100/116 |
+| `csv` | 80.2% | 2.337 | 93/116 |
+| `yaml` | 82.8% | 4.969 | 96/116 |
+| `markdown-kv` | 84.5% | 6.270 | 98/116 |
+| `json` | 84.5% | 6.347 | 98/116 |

 ##### E-commerce orders with nested structures

 | Format | Accuracy | Tokens | Correct/Total |
 |--------|----------|--------|---------------|
-| `toon` | 84.1% | 5.967 | 74/88 |
-| `csv` | 83.0% | 6.735 | 73/88 |
-| `yaml` | 81.8% | 7.328 | 72/88 |
-| `markdown-kv` | 86.4% | 9.110 | 76/88 |
-| `json` | 84.1% | 9.694 | 74/88 |
+| `toon` | 90.9% | 5.967 | 80/88 |
+| `csv` | 90.9% | 6.735 | 80/88 |
+| `yaml` | 89.8% | 7.328 | 79/88 |
+| `markdown-kv` | 90.9% | 9.110 | 80/88 |
+| `json` | 89.8% | 9.694 | 79/88 |

 ##### Time-series analytics data

 | Format | Accuracy | Tokens | Correct/Total |
 |--------|----------|--------|---------------|
-| `csv` | 72.4% | 1.393 | 42/58 |
-| `toon` | 70.7% | 1.515 | 41/58 |
-| `yaml` | 72.4% | 2.938 | 42/58 |
-| `json` | 74.1% | 3.665 | 43/58 |
-| `markdown-kv` | 70.7% | 3.779 | 41/58 |
+| `csv` | 87.9% | 1.393 | 51/58 |
+| `toon` | 86.2% | 1.515 | 50/58 |
+| `yaml` | 86.2% | 2.938 | 50/58 |
+| `json` | 87.9% | 3.665 | 51/58 |
+| `markdown-kv` | 86.2% | 3.779 | 50/58 |

 ##### Popular GitHub repositories

 | Format | Accuracy | Tokens | Correct/Total |
 |--------|----------|--------|---------------|
-| `toon` | 64.3% | 8.745 | 36/56 |
-| `csv` | 62.5% | 8.513 | 35/56 |
-| `json` | 67.9% | 15.145 | 38/56 |
-| `markdown-kv` | 67.9% | 15.436 | 38/56 |
-| `yaml` | 62.5% | 13.129 | 35/56 |
+| `csv` | 80.4% | 8.513 | 45/56 |
+| `toon` | 80.4% | 8.745 | 45/56 |
+| `yaml` | 78.6% | 13.129 | 44/56 |
+| `markdown-kv` | 82.1% | 15.436 | 46/56 |
+| `json` | 73.2% | 15.145 | 41/56 |

 #### Performance by Model

-##### gpt-4o-mini
+##### gpt-5-nano

 | Format | Accuracy | Correct/Total |
 |--------|----------|---------------|
-| `toon` | 72.3% | 115/159 |
-| `json` | 71.7% | 114/159 |
-| `markdown-kv` | 70.4% | 112/159 |
-| `csv` | 69.2% | 110/159 |
-| `yaml` | 68.6% | 109/159 |
+| `toon` | 97.5% | 155/159 |
+| `markdown-kv` | 95.6% | 152/159 |
+| `yaml` | 94.3% | 150/159 |
+| `json` | 93.7% | 149/159 |
+| `csv` | 93.7% | 149/159 |

 ##### claude-haiku-4-5

--- a/benchmarks/results/accuracy/summary.json
+++ b/benchmarks/results/accuracy/summary.json
@@ -2,53 +2,48 @@
  "formatResults": [
    {
      "format": "toon",
-      "accuracy": 0.7389937106918238,
+      "accuracy": 0.8647798742138365,
      "totalTokens": 4678,
-      "avgInputTokens": 4675,
-      "avgLatency": 1424,
-      "correctCount": 235,
-      "totalCount": 318
-    },
-    {
-      "format": "json",
-      "accuracy": 0.7358490566037735,
-      "totalTokens": 8713,
-      "avgInputTokens": 9177,
-      "avgLatency": 1678,
-      "correctCount": 234,
+      "averageLatency": 5016,
+      "correctCount": 275,
      "totalCount": 318
    },
    {
      "format": "markdown-kv",
-      "accuracy": 0.7358490566037735,
+      "accuracy": 0.8616352201257862,
      "totalTokens": 8649,
-      "avgInputTokens": 8242,
-      "avgLatency": 1724,
-      "correctCount": 234,
+      "averageLatency": 4628,
+      "correctCount": 274,
+      "totalCount": 318
+    },
+    {
+      "format": "json",
+      "accuracy": 0.8459119496855346,
+      "totalTokens": 8713,
+      "averageLatency": 5369,
+      "correctCount": 269,
      "totalCount": 318
    },
    {
      "format": "csv",
-      "accuracy": 0.7232704402515723,
+      "accuracy": 0.8459119496855346,
      "totalTokens": 4745,
-      "avgInputTokens": 4878,
-      "avgLatency": 1573,
-      "correctCount": 230,
+      "averageLatency": 5168,
+      "correctCount": 269,
      "totalCount": 318
    },
    {
      "format": "yaml",
-      "accuracy": 0.7169811320754716,
+      "accuracy": 0.8459119496855346,
      "totalTokens": 7091,
-      "avgInputTokens": 7136,
-      "avgLatency": 1602,
-      "correctCount": 228,
+      "averageLatency": 4299,
+      "correctCount": 269,
      "totalCount": 318
    }
  ],
  "questions": 159,
  "models": [
-    "gpt-4o-mini",
+    "gpt-5-nano",
    "claude-haiku-4-5"
  ],
  "datasets": [
@@ -91,5 +86,5 @@
    "yaml-analytics": 2938,
    "yaml-github": 13129
  },
-  "timestamp": "2025-10-27T10:46:35.127Z"
+  "timestamp": "2025-10-27T12:43:38.288Z"
 }
--- a/benchmarks/results/token-efficiency.md
+++ b/benchmarks/results/token-efficiency.md
@@ -7,7 +7,7 @@
 🛒 E-commerce Order          ███████████████░░░░░░░░░░     203 tokens  (JSON:    338)  💰 39.9% saved
 ```

-**Total:** 15,172 tokens (TOON) vs 29,096 tokens (JSON) → 47.9% savings
+**Total:** 15,172 tokens (TOON) vs 29,096 tokens (JSON) → **47.9% savings**

 <details>
 <summary><strong>View detailed examples</strong></summary>
--- a/benchmarks/scripts/accuracy-benchmark.ts
+++ b/benchmarks/scripts/accuracy-benchmark.ts
@@ -1,5 +1,5 @@
 /**
- * TOON LLM Accuracy Benchmark
+ * LLM Retrieval Accuracy Benchmark
 *
 * Main entry point that orchestrates the full benchmark:
 * 1. Generate questions from datasets
@@ -20,7 +20,7 @@ import { formatters } from '../src/formatters'
 import { generateQuestions } from '../src/questions'
 import { calculateFormatResults, calculateTokenCounts, saveResults } from '../src/report'

-consola.start('LLM Accuracy Benchmark for TOON')
+consola.start('Retrieval Accuracy Benchmark for TOON')

 // Check if results already exist
 const resultsDir = path.join(BENCHMARKS_DIR, 'results', 'accuracy')
@@ -82,10 +82,10 @@ else {
  // Format datasets once (reuse for all questions)
  const formattedDatasets: Record<string, Record<string, string>> = {}
  for (const [formatName, formatter] of Object.entries(formatters)) {
-    formattedDatasets[formatName] = {}
+    formattedDatasets[formatName] ??= {}
+
    for (const dataset of datasets) {
-      const formatted = formatter(dataset.data)
-      formattedDatasets[formatName]![dataset.name] = formatted
+      formattedDatasets[formatName]![dataset.name] = formatter(dataset.data)
    }
  }

@@ -108,7 +108,7 @@ else {
    tasks,
    async (task, index) => {
      const formattedData = formattedDatasets[task.formatName]![task.question.dataset]!
-      const model = activeModels[task.modelName as keyof typeof activeModels]
+      const model = activeModels[task.modelName as keyof typeof activeModels]!

      const result = await evaluateQuestion(
        task.question,
@@ -121,7 +121,7 @@ else {
      // Progress update
      if ((index + 1) % 10 === 0) {
        const percent = (((index + 1) / total) * 100).toFixed(1)
-        console.log(`⏳ Progress: ${index + 1}/${total} (${percent}%)`)
+        consola.start(`Progress: ${index + 1}/${total} (${percent}%)`)
      }

      return result
--- a/benchmarks/scripts/token-efficiency-benchmark.ts
+++ b/benchmarks/scripts/token-efficiency-benchmark.ts
@@ -59,7 +59,7 @@ let totalToonTokens = 0
 const results: BenchmarkResult[] = []

 for (const example of BENCHMARK_EXAMPLES) {
-  const data = await example.getData()
+  const data = example.getData()

  const jsonString = JSON.stringify(data, undefined, 2)
  const toonString = encode(data)
--- a/benchmarks/src/evaluate.ts
+++ b/benchmarks/src/evaluate.ts
@@ -20,18 +20,69 @@ import { RATE_LIMIT_DELAY_MS } from './constants'
 * Models used for evaluation
 */
 export const models: Record<string, LanguageModelV2> = {
-  'gpt-4o-mini': openai('gpt-4o-mini'),
+  'gpt-5-nano': openai('gpt-5-nano'),
  'claude-haiku-4-5': anthropic('claude-haiku-4-5-20251001'),
 }

 /**
- * Validate an answer using LLM-as-judge approach
- * More robust than string matching for LLM outputs
+ * Evaluate a single question with a specific format and model
 */
-export async function validateAnswer(
-  actual: string,
-  expected: string,
-  question: string,
+export async function evaluateQuestion(
+  question: Question,
+  formatName: string,
+  formattedData: string,
+  model: LanguageModelV2,
+  modelName: string,
+): Promise<EvaluationResult> {
+  const prompt = `Given the following data in ${formatName} format:
+
+\`\`\`
+${formattedData}
+\`\`\`
+
+Question: ${question.prompt}
+
+Provide only the direct answer, without any additional explanation or formatting.`
+
+  const startTime = performance.now()
+  const { text, usage } = await generateText({
+    model,
+    prompt,
+    temperature: model.modelId.startsWith('gpt-') ? undefined : 0,
+  })
+
+  await setTimeout(RATE_LIMIT_DELAY_MS)
+
+  const latencyMs = performance.now() - startTime
+  const correct = await validateAnswer({
+    actual: text.trim(),
+    expected: question.groundTruth,
+    question: question.prompt,
+  })
+
+  return {
+    questionId: question.id,
+    format: formatName,
+    model: modelName,
+    expected: question.groundTruth,
+    actual: text.trim(),
+    correct,
+    inputTokens: usage.inputTokens,
+    outputTokens: usage.outputTokens,
+    latencyMs,
+  }
+}
+
+/**
+ * Validate an answer using LLM-as-judge approach
+ */
+async function validateAnswer(
+  {
+    actual,
+    expected,
+    question,
+  }:
+  { actual: string, expected: string, question: string },
 ): Promise<boolean> {
  const prompt = `You are validating answers to questions about structured data.

@@ -49,10 +100,9 @@ Respond with only "YES" or "NO".`

  try {
    const { text } = await generateText({
-      model: models['gpt-4o-mini']!,
+      model: models['claude-haiku-4-5']!,
      prompt,
      temperature: 0,
-      maxOutputTokens: 16,
    })

    await setTimeout(RATE_LIMIT_DELAY_MS)
@@ -65,69 +115,3 @@ Respond with only "YES" or "NO".`
    return actual.toLowerCase().trim() === expected.toLowerCase().trim()
  }
 }
-
-/**
- * Evaluate a single question with a specific format and model
- */
-export async function evaluateQuestion(
-  question: Question,
-  formatName: string,
-  formattedData: string,
-  model: any,
-  modelName: string,
-): Promise<EvaluationResult> {
-  const prompt = `Given the following data in ${formatName} format:
-
-\`\`\`
-${formattedData}
-\`\`\`
-
-Question: ${question.prompt}
-
-Provide only the direct answer, without any additional explanation or formatting.`
-
-  const startTime = Date.now()
-
-  try {
-    const { text, usage } = await generateText({
-      model,
-      prompt,
-      temperature: 0,
-      maxOutputTokens: 50,
-    })
-
-    await setTimeout(RATE_LIMIT_DELAY_MS)
-
-    const latencyMs = Date.now() - startTime
-    const correct = await validateAnswer(text.trim(), question.groundTruth, question.prompt)
-
-    return {
-      questionId: question.id,
-      format: formatName,
-      model: modelName,
-      expected: question.groundTruth,
-      actual: text.trim(),
-      correct,
-      inputTokens: usage.inputTokens ?? 0,
-      outputTokens: usage.outputTokens ?? 0,
-      latencyMs,
-    }
-  }
-  catch (error) {
-    consola.error(`Error evaluating ${question.id} with ${formatName}/${modelName}:`, error)
-
-    await setTimeout(RATE_LIMIT_DELAY_MS)
-
-    return {
-      questionId: question.id,
-      format: formatName,
-      model: modelName,
-      expected: question.groundTruth,
-      actual: '',
-      correct: false,
-      inputTokens: 0,
-      outputTokens: 0,
-      latencyMs: Date.now() - startTime,
-    }
-  }
-}
--- a/benchmarks/src/report.ts
+++ b/benchmarks/src/report.ts
@@ -37,15 +37,13 @@ export function calculateFormatResults(
      .filter(([key]) => key.startsWith(`${formatName}-`))
      .reduce((sum, [, tokens]) => sum + tokens, 0) / datasets.length

-    const avgInputTokens = formatResults.reduce((sum, r) => sum + r.inputTokens, 0) / totalCount
-    const avgLatency = formatResults.reduce((sum, r) => sum + r.latencyMs, 0) / totalCount
+    const averageLatency = formatResults.reduce((sum, r) => sum + r.latencyMs, 0) / totalCount

    return {
      format: formatName,
      accuracy,
      totalTokens: Math.round(avgTokens),
-      avgInputTokens: Math.round(avgInputTokens),
-      avgLatency: Math.round(avgLatency),
+      averageLatency: Math.round(averageLatency),
      correctCount,
      totalCount,
    }
@@ -69,11 +67,13 @@ export function generateMarkdownReport(
  const toon = formatResults.find(r => r.format === 'toon')
  const json = formatResults.find(r => r.format === 'json')

-  // Model-by-model breakdown (most interesting result)
+  // Model-by-model breakdown with ASCII bars
  const modelCount = Object.keys(models).length
  lines.push(`Tested across **${modelCount} ${modelCount === 1 ? 'LLM' : 'LLMs'}** with data retrieval tasks:`, '', '```')

-  for (const modelName of Object.keys(models)) {
+  const modelNames = Object.keys(models)
+  for (let i = 0; i < modelNames.length; i++) {
+    const modelName = modelNames[i]!
    const modelResults = formatResults.map((fr) => {
      const modelFormatResults = results.filter(r => r.model === modelName && r.format === fr.format)
      const correctCount = modelFormatResults.filter(r => r.correct).length
@@ -88,10 +88,16 @@ export function generateMarkdownReport(
      }
    }).sort((a, b) => b.accuracy - a.accuracy)

-    const bestResult = modelResults[0]!
-    const bar = createTokenBar(bestResult.accuracy, 1, 20)
-
-    lines.push(`${modelName.padEnd(20)} ${bar} ${(bestResult.accuracy * 100).toFixed(1)}% accuracy`)
+    // Add blank line before model name, except for first model
+    if (i > 0)
+      lines.push('')
+    lines.push(modelName)
+    for (const result of modelResults) {
+      const bar = createProgressBar(result.accuracy, 1, 20)
+      const accuracyStr = `${(result.accuracy * 100).toFixed(1)}%`.padStart(6)
+      const countStr = `(${result.correctCount}/${result.totalCount})`
+      lines.push(`  ${result.format.padEnd(12)} ${bar} ${accuracyStr} ${countStr}`)
+    }
  }

  lines.push('```', '')
@@ -100,24 +106,12 @@ export function generateMarkdownReport(
  if (toon && json) {
    const tokenSavings = ((1 - toon.totalTokens / json.totalTokens) * 100).toFixed(1)
    lines.push(
-      `**TOON achieves ${(toon.accuracy * 100).toFixed(1)}% accuracy (vs JSON's ${(json.accuracy * 100).toFixed(1)}%) while using ${tokenSavings}% fewer tokens.**`,
+      `**Tradeoff:** TOON achieves ${(toon.accuracy * 100).toFixed(1)}% accuracy (vs JSON's ${(json.accuracy * 100).toFixed(1)}%) while using ${tokenSavings}% fewer tokens.`,
      '',
    )
  }

-  // Simple format comparison table
-  lines.push(
-    '| Format | Accuracy | Average Tokens |',
-    '| ------ | -------- | -------------- |',
-  )
-
-  for (const result of formatResults) {
-    lines.push(
-      `| \`${result.format}\` | ${(result.accuracy * 100).toFixed(1)}% | ${result.totalTokens.toLocaleString()} |`,
-    )
-  }
-
-  lines.push('', '<details>', '<summary><strong>View detailed breakdown by dataset and model</strong></summary>', '', '#### Performance by Dataset', '')
+  lines.push('<details>', '<summary><strong>View detailed breakdown by dataset and model</strong></summary>', '', '#### Performance by Dataset', '')

  for (const dataset of datasets) {
    lines.push(`##### ${dataset.description}`, '')
@@ -173,7 +167,7 @@ export function generateMarkdownReport(
  }

  // Model breakdown
-  lines.push('', '#### Performance by Model', '')
+  lines.push('#### Performance by Model', '')

  for (const modelName of Object.keys(models)) {
    lines.push(`##### ${modelName}`, '')
@@ -203,7 +197,6 @@ export function generateMarkdownReport(

  // Methodology
  lines.push(
-    '',
    '#### Methodology',
    '',
    '- **Semantic validation**: LLM-as-judge validates responses semantically (not exact string matching).',
@@ -252,20 +245,20 @@ export async function saveResults(
  // Save raw results
  await fsp.writeFile(
    path.join(resultsDir, 'raw-results.json'),
-    JSON.stringify(results, undefined, 2),
+    `${JSON.stringify(results, undefined, 2)}\n`,
  )

  // Save summary
  await fsp.writeFile(
    path.join(resultsDir, 'summary.json'),
-    JSON.stringify({
+    `${JSON.stringify({
      formatResults,
      questions: questions.length,
      models: Object.keys(models),
      datasets: datasets.map(d => ({ name: d.name, description: d.description })),
      tokenCounts,
      timestamp: new Date().toISOString(),
-    }, undefined, 2),
+    }, undefined, 2)}\n`,
  )

  // Generate markdown report
@@ -279,9 +272,9 @@ export async function saveResults(
 }

 /**
- * Generate visual bar chart for token counts
+ * Generate visual progress bar using ASCII characters (█ for filled, ░ for empty)
 */
-function createTokenBar(tokens: number, maxTokens: number, width = 30): string {
+function createProgressBar(tokens: number, maxTokens: number, width = 30): string {
  const filled = Math.round((tokens / maxTokens) * width)
  const empty = width - filled
  return '█'.repeat(filled) + '░'.repeat(empty)
--- a/benchmarks/src/types.ts
+++ b/benchmarks/src/types.ts
@@ -19,8 +19,8 @@ export interface EvaluationResult {
  expected: string
  actual: string
  correct: boolean
-  inputTokens: number
-  outputTokens: number
+  inputTokens?: number
+  outputTokens?: number
  latencyMs: number
 }

@@ -28,8 +28,7 @@ export interface FormatResult {
  format: string
  accuracy: number
  totalTokens: number
-  avgInputTokens: number
-  avgLatency: number
+  averageLatency: number
  correctCount: number
  totalCount: number
 }