From b9f54ba585078e31696845d0449db598e8c12553 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Mon, 27 Oct 2025 14:18:37 +0100 Subject: [PATCH] docs: update benchmark reports' readability --- README.md | 71 +++++++++---------- benchmarks/results/accuracy/report.md | 2 +- benchmarks/results/accuracy/summary.json | 2 +- benchmarks/results/token-efficiency.md | 64 ++++++++--------- .../scripts/token-efficiency-benchmark.ts | 6 +- benchmarks/src/questions.ts | 17 ++--- benchmarks/src/report.ts | 2 +- 7 files changed, 82 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 00393ef..d48d69b 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,8 @@ users[2]{id,name,role}: 2,Bob,user ``` +I built TOON to save tokens when sending large datasets to LLMs at work, where I tend to have uniform arrays of objects that benefit from the tabular format. +
Another reason @@ -38,9 +40,6 @@ users[2]{id,name,role}:
-> [!NOTE] -> I built TOON to save tokens when sending large datasets to LLMs at work, where I tend to have uniform arrays of objects that benefit from the tabular format. - ## Key Features - 💸 **Token-efficient:** typically 30–60% fewer tokens than JSON @@ -57,12 +56,12 @@ users[2]{id,name,role}: ``` ⭐ GitHub Repositories ██████████████░░░░░░░░░░░ 8,745 tokens (JSON: 15,145) 💰 42.3% saved -📈 Analytics Time Series ██████████░░░░░░░░░░░░░░░ 3,631 tokens (JSON: 9,024) 💰 59.8% saved -👥 API Response ██████████████░░░░░░░░░░░ 2,593 tokens (JSON: 4,589) 💰 43.5% saved -🛒 E-commerce Order ███████████████░░░░░░░░░░ 203 tokens (JSON: 338) 💰 39.9% saved +📈 Daily Analytics ██████████░░░░░░░░░░░░░░░ 3,630 tokens (JSON: 9,023) 💰 59.8% saved +👥 API Response ██████████████░░░░░░░░░░░ 2,597 tokens (JSON: 4,589) 💰 43.4% saved +🛒 E-Commerce Order ████████████████░░░░░░░░░ 164 tokens (JSON: 256) 💰 35.9% saved ``` -**Total:** 15,172 tokens (TOON) vs 29,096 tokens (JSON) → **47.9% savings** +**Total:** 15,136 tokens (TOON) vs 29,013 tokens (JSON) → 47.8% savings
View detailed examples @@ -132,65 +131,65 @@ repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watc --- -#### 📈 Analytics Time Series +#### 📈 Daily Analytics **Configuration:** 180 days of web metrics (views, clicks, conversions, revenue) **Savings:** 5,393 tokens (59.8% reduction) -**JSON** (9,024 tokens): +**JSON** (9,023 tokens): ```json { "metrics": [ { "date": "2024-12-31", - "views": 3769, - "clicks": 400, - "conversions": 59, - "revenue": 198.98 + "views": 1953, + "clicks": 224, + "conversions": 60, + "revenue": 409.79 }, { "date": "2025-01-01", - "views": 5742, - "clicks": 463, - "conversions": 28, - "revenue": 295.77 + "views": 2981, + "clicks": 242, + "conversions": 109, + "revenue": 467.73 }, { "date": "2025-01-02", - "views": 3669, - "clicks": 336, - "conversions": 102, - "revenue": 624.23 + "views": 3842, + "clicks": 100, + "conversions": 15, + "revenue": 569.44 }, { "date": "2025-01-03", - "views": 1332, - "clicks": 304, - "conversions": 99, - "revenue": 113.06 + "views": 4083, + "clicks": 161, + "conversions": 73, + "revenue": 444.75 }, { "date": "2025-01-04", - "views": 1444, - "clicks": 222, - "conversions": 88, - "revenue": 986.69 + "views": 5382, + "clicks": 257, + "conversions": 63, + "revenue": 457.28 } ] } ``` -**TOON** (3,631 tokens): +**TOON** (3,630 tokens): ``` metrics[5]{date,views,clicks,conversions,revenue}: - 2024-12-31,3769,400,59,198.98 - 2025-01-01,5742,463,28,295.77 - 2025-01-02,3669,336,102,624.23 - 2025-01-03,1332,304,99,113.06 - 2025-01-04,1444,222,88,986.69 + 2024-12-31,1953,224,60,409.79 + 2025-01-01,2981,242,109,467.73 + 2025-01-02,3842,100,15,569.44 + 2025-01-03,4083,161,73,444.75 + 2025-01-04,5382,257,63,457.28 ```
@@ -222,7 +221,7 @@ claude-haiku-4-5 yaml ███████████████░░░░░ 74.8% (119/159) ``` -**Tradeoff:** TOON achieves 86.5% accuracy (vs JSON's 84.6%) while using 46.3% fewer tokens. +**Advantage:** TOON achieves **86.5% accuracy** (vs JSON's 84.6%) while using **46.3% fewer tokens**.
View detailed breakdown by dataset and model diff --git a/benchmarks/results/accuracy/report.md b/benchmarks/results/accuracy/report.md index b44276b..eccae66 100644 --- a/benchmarks/results/accuracy/report.md +++ b/benchmarks/results/accuracy/report.md @@ -18,7 +18,7 @@ claude-haiku-4-5 yaml ███████████████░░░░░ 74.8% (119/159) ``` -**Tradeoff:** TOON achieves 86.5% accuracy (vs JSON's 84.6%) while using 46.3% fewer tokens. +**Advantage:** TOON achieves **86.5% accuracy** (vs JSON's 84.6%) while using **46.3% fewer tokens**.
View detailed breakdown by dataset and model diff --git a/benchmarks/results/accuracy/summary.json b/benchmarks/results/accuracy/summary.json index dbbd353..abbcc06 100644 --- a/benchmarks/results/accuracy/summary.json +++ b/benchmarks/results/accuracy/summary.json @@ -86,5 +86,5 @@ "yaml-analytics": 2938, "yaml-github": 13129 }, - "timestamp": "2025-10-27T13:04:50.634Z" + "timestamp": "2025-10-27T13:17:28.071Z" } diff --git a/benchmarks/results/token-efficiency.md b/benchmarks/results/token-efficiency.md index 619904c..85fb74d 100644 --- a/benchmarks/results/token-efficiency.md +++ b/benchmarks/results/token-efficiency.md @@ -2,12 +2,12 @@ ``` ⭐ GitHub Repositories ██████████████░░░░░░░░░░░ 8,745 tokens (JSON: 15,145) 💰 42.3% saved -📈 Analytics Time Series ██████████░░░░░░░░░░░░░░░ 3,631 tokens (JSON: 9,024) 💰 59.8% saved -👥 API Response ██████████████░░░░░░░░░░░ 2,593 tokens (JSON: 4,589) 💰 43.5% saved -🛒 E-commerce Order ███████████████░░░░░░░░░░ 203 tokens (JSON: 338) 💰 39.9% saved +📈 Daily Analytics ██████████░░░░░░░░░░░░░░░ 3,630 tokens (JSON: 9,023) 💰 59.8% saved +👥 API Response ██████████████░░░░░░░░░░░ 2,597 tokens (JSON: 4,589) 💰 43.4% saved +🛒 E-Commerce Order ████████████████░░░░░░░░░ 164 tokens (JSON: 256) 💰 35.9% saved ``` -**Total:** 15,172 tokens (TOON) vs 29,096 tokens (JSON) → **47.9% savings** +**Total:** 15,136 tokens (TOON) vs 29,013 tokens (JSON) → 47.8% savings
View detailed examples @@ -77,65 +77,65 @@ repositories[3]{id,name,repo,description,createdAt,updatedAt,pushedAt,stars,watc --- -#### 📈 Analytics Time Series +#### 📈 Daily Analytics **Configuration:** 180 days of web metrics (views, clicks, conversions, revenue) **Savings:** 5,393 tokens (59.8% reduction) -**JSON** (9,024 tokens): +**JSON** (9,023 tokens): ```json { "metrics": [ { "date": "2024-12-31", - "views": 3769, - "clicks": 400, - "conversions": 59, - "revenue": 198.98 + "views": 1953, + "clicks": 224, + "conversions": 60, + "revenue": 409.79 }, { "date": "2025-01-01", - "views": 5742, - "clicks": 463, - "conversions": 28, - "revenue": 295.77 + "views": 2981, + "clicks": 242, + "conversions": 109, + "revenue": 467.73 }, { "date": "2025-01-02", - "views": 3669, - "clicks": 336, - "conversions": 102, - "revenue": 624.23 + "views": 3842, + "clicks": 100, + "conversions": 15, + "revenue": 569.44 }, { "date": "2025-01-03", - "views": 1332, - "clicks": 304, - "conversions": 99, - "revenue": 113.06 + "views": 4083, + "clicks": 161, + "conversions": 73, + "revenue": 444.75 }, { "date": "2025-01-04", - "views": 1444, - "clicks": 222, - "conversions": 88, - "revenue": 986.69 + "views": 5382, + "clicks": 257, + "conversions": 63, + "revenue": 457.28 } ] } ``` -**TOON** (3,631 tokens): +**TOON** (3,630 tokens): ``` metrics[5]{date,views,clicks,conversions,revenue}: - 2024-12-31,3769,400,59,198.98 - 2025-01-01,5742,463,28,295.77 - 2025-01-02,3669,336,102,624.23 - 2025-01-03,1332,304,99,113.06 - 2025-01-04,1444,222,88,986.69 + 2024-12-31,1953,224,60,409.79 + 2025-01-01,2981,242,109,467.73 + 2025-01-02,3842,100,15,569.44 + 2025-01-03,4083,161,73,444.75 + 2025-01-04,5382,257,63,457.28 ```
diff --git a/benchmarks/scripts/token-efficiency-benchmark.ts b/benchmarks/scripts/token-efficiency-benchmark.ts index f498f3e..3b07473 100644 --- a/benchmarks/scripts/token-efficiency-benchmark.ts +++ b/benchmarks/scripts/token-efficiency-benchmark.ts @@ -30,7 +30,7 @@ const BENCHMARK_EXAMPLES = [ showDetailed: true, }, { - name: 'Analytics Time Series', + name: 'Daily Analytics', emoji: '📈', description: '180 days of web metrics (views, clicks, conversions, revenue)', getData: () => generateAnalytics(180), @@ -44,7 +44,7 @@ const BENCHMARK_EXAMPLES = [ showDetailed: false, }, { - name: 'E-commerce Order', + name: 'E-Commerce Order', emoji: '🛒', description: 'Single nested order with customer and items', getData: generateOrder, @@ -113,7 +113,7 @@ const detailedExamples = results })), } } - else if (result.name === 'Analytics Time Series') { + else if (result.name === 'Daily Analytics') { displayData = { metrics: result.data.metrics.slice(0, 5) } } diff --git a/benchmarks/src/questions.ts b/benchmarks/src/questions.ts index e211dce..46220dd 100644 --- a/benchmarks/src/questions.ts +++ b/benchmarks/src/questions.ts @@ -1,5 +1,3 @@ -/* eslint-disable no-console */ - /** * Question generation for TOON benchmarks * @@ -12,6 +10,7 @@ */ import type { Question } from './types' +import { consola } from 'consola' import { datasets } from './datasets' /** @@ -387,12 +386,14 @@ export function generateQuestions(): Question[] { } } - console.log(`📊 Question breakdown:`) - console.log(` Tabular: ${questions.filter(q => q.dataset === 'tabular').length}`) - console.log(` Nested: ${questions.filter(q => q.dataset === 'nested').length}`) - console.log(` Analytics: ${questions.filter(q => q.dataset === 'analytics').length}`) - console.log(` GitHub: ${questions.filter(q => q.dataset === 'github').length}`) - console.log(` Total: ${questions.length}`) + consola.info(`Question breakdown:`) + consola.box(` +Tabular: ${questions.filter(q => q.dataset === 'tabular').length} +Nested: ${questions.filter(q => q.dataset === 'nested').length} +Analytics: ${questions.filter(q => q.dataset === 'analytics').length} +GitHub: ${questions.filter(q => q.dataset === 'github').length} +Total: ${questions.length} +`.trim()) return questions } diff --git a/benchmarks/src/report.ts b/benchmarks/src/report.ts index 35891af..8da12b8 100644 --- a/benchmarks/src/report.ts +++ b/benchmarks/src/report.ts @@ -94,7 +94,7 @@ export function generateMarkdownReport( // Build summary comparison const summaryComparison = toon && json - ? `**Tradeoff:** TOON achieves ${(toon.accuracy * 100).toFixed(1)}% accuracy (vs JSON's ${(json.accuracy * 100).toFixed(1)}%) while using ${((1 - toon.totalTokens / json.totalTokens) * 100).toFixed(1)}% fewer tokens.` + ? `**Advantage:** TOON achieves **${(toon.accuracy * 100).toFixed(1)}% accuracy** (vs JSON's ${(json.accuracy * 100).toFixed(1)}%) while using **${((1 - toon.totalTokens / json.totalTokens) * 100).toFixed(1)}% fewer tokens**.` : '' // Build performance by dataset