import type { Dataset } from '../src/types' import * as fsp from 'node:fs/promises' import * as path from 'node:path' import * as prompts from '@clack/prompts' import { encode } from '../../packages/toon/src' import { BENCHMARKS_DIR, FORMATTER_DISPLAY_NAMES, ROOT_DIR } from '../src/constants' import { TOKEN_EFFICIENCY_DATASETS } from '../src/datasets' import { formatters, supportsCSV } from '../src/formatters' import { createProgressBar, ensureDir, tokenize } from '../src/utils' interface FormatMetrics { name: string tokens: number savings: number savingsPercent: number } interface BenchmarkResult { dataset: Dataset formats: FormatMetrics[] } // Constants const DATASET_ICONS: Record = { 'tabular': '๐Ÿ‘ฅ', 'nested': '๐Ÿ›’', 'analytics': '๐Ÿ“ˆ', 'github': 'โญ', 'event-logs': '๐Ÿงพ', 'nested-config': '๐Ÿงฉ', } const COMPARISON_FORMAT_ORDER = ['json-pretty', 'json-compact', 'yaml', 'xml'] as const const PROGRESS_BAR_CONFIG = { filled: 'โ–“', empty: 'โ–‘' } as const const PROGRESS_BAR_WIDTH = 20 const TOKEN_PADDING = 7 const LABEL_PADDING = 60 const COMPARISON_LABEL_PADDING = 30 const SEPARATOR = 'โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€' const DEFAULT_DATASET_ICON = '๐Ÿ“Š' const DETAILED_EXAMPLE_DATASETS = ['github', 'analytics'] as const const GITHUB_REPO_LIMIT = 3 const GITHUB_DESC_LIMIT = 80 const ANALYTICS_METRICS_LIMIT = 5 prompts.intro('Token Efficiency Benchmark') /** * Format a comparison line showing savings vs TOON */ function formatComparisonLine(format: FormatMetrics): string { const label = FORMATTER_DISPLAY_NAMES[format.name] || format.name.toUpperCase() const signedPercent = format.savingsPercent >= 0 ? `โˆ’${format.savingsPercent.toFixed(1)}%` : `+${Math.abs(format.savingsPercent).toFixed(1)}%` const labelWithSavings = `vs ${label} (${signedPercent})`.padEnd(COMPARISON_LABEL_PADDING) const tokenStr = format.tokens.toLocaleString('en-US').padStart(TOKEN_PADDING) return ` ${labelWithSavings}${tokenStr}` } /** * Calculate total tokens and savings for a set of datasets */ function calculateTotalMetrics(datasets: BenchmarkResult[], formatNames: readonly string[]) { const totalToonTokens = datasets.reduce((sum, r) => { const toon = r.formats.find(f => f.name === 'toon')! return sum + toon.tokens }, 0) const totals = formatNames.map((formatName) => { const totalTokens = datasets.reduce((sum, r) => { const format = r.formats.find(f => f.name === formatName) return sum + (format?.tokens || 0) }, 0) const savings = totalTokens - totalToonTokens const savingsPercent = (savings / totalTokens) * 100 return { name: formatName, tokens: totalTokens, savingsPercent } }) return { totalToonTokens, totals } } /** * Generate total lines for a track */ function generateTotalLines( totalToonTokens: number, totals: { name: string, tokens: number, savingsPercent: number }[], baselineFormat?: { name: string, tokens: number }, ) { const lines: string[] = ['Total '] if (baselineFormat) { // Flat-only track with CSV baseline const csvPercentage = Math.min(100, (baselineFormat.tokens / totalToonTokens) * 100) const csvBar = createProgressBar(csvPercentage, 100, PROGRESS_BAR_WIDTH, PROGRESS_BAR_CONFIG) const csvStr = baselineFormat.tokens.toLocaleString('en-US').padStart(TOKEN_PADDING) lines.push(`csv ${csvBar} ${csvStr} tokens`) const overheadPercent = ((totalToonTokens - baselineFormat.tokens) / totalToonTokens) * 100 const toonBar = createProgressBar(100, 100, PROGRESS_BAR_WIDTH, PROGRESS_BAR_CONFIG) const toonStr = totalToonTokens.toLocaleString('en-US').padStart(TOKEN_PADDING) lines.push(`toon ${toonBar} ${toonStr} tokens (+${overheadPercent.toFixed(1)}% vs CSV)`) } else { // Mixed-structure track const totalPercentage = Math.min(100, (totalToonTokens / totals[0]!.tokens) * 100) const totalBar = createProgressBar(totalPercentage, 100, PROGRESS_BAR_WIDTH, PROGRESS_BAR_CONFIG) const toonStr = totalToonTokens.toLocaleString('en-US').padStart(TOKEN_PADDING) lines.push(`toon ${totalBar} ${toonStr} tokens`) } // Add comparison lines for (const format of totals) { lines.push(formatComparisonLine({ name: format.name, tokens: format.tokens, savings: 0, // Not used in this context savingsPercent: format.savingsPercent, })) } return lines.join('\n') } /** * Generate bar chart for a dataset */ function generateDatasetChart(result: BenchmarkResult): string { const { dataset, formats } = result const toon = formats.find(f => f.name === 'toon')! const jsonPretty = formats.find(f => f.name === 'json-pretty')! const emoji = DATASET_ICONS[dataset.name] || DEFAULT_DATASET_ICON const eligibility = dataset.metadata.tabularEligibility const name = `${dataset.description} [eligibility: ${eligibility}%]` const percentage = Math.min(100, 100 - jsonPretty.savingsPercent) const bar = createProgressBar(percentage, 100, PROGRESS_BAR_WIDTH, PROGRESS_BAR_CONFIG) const toonStr = toon.tokens.toLocaleString('en-US') const line1 = `${emoji} ${name.padEnd(LABEL_PADDING)}\ntoon ${bar} ${toonStr.padStart(TOKEN_PADDING)} tokens` const comparisonLines = COMPARISON_FORMAT_ORDER.map((formatName) => { const format = formats.find(f => f.name === formatName) if (!format) return null return formatComparisonLine(format) }).filter(Boolean) return [line1, ...comparisonLines].join('\n') } const results: BenchmarkResult[] = [] // Calculate token counts for all datasets for (const dataset of TOKEN_EFFICIENCY_DATASETS) { const formatMetrics: FormatMetrics[] = [] const tokensByFormat: Record = {} // Calculate tokens for each format for (const [formatName, formatter] of Object.entries(formatters)) { // Skip CSV for datasets that don't support it if (formatName === 'csv' && !supportsCSV(dataset)) continue const formattedString = formatter(dataset.data) const tokens = tokenize(formattedString) tokensByFormat[formatName] = tokens } // Calculate savings vs TOON const toonTokens = tokensByFormat.toon! for (const [formatName, tokens] of Object.entries(tokensByFormat)) { const savings = tokens - toonTokens formatMetrics.push({ name: formatName, tokens, savings, savingsPercent: formatName === 'toon' ? 0 : (savings / tokens) * 100, }) } results.push({ dataset, formats: formatMetrics, }) } // Separate datasets by CSV support const mixedStructureDatasets = results.filter(r => !supportsCSV(r.dataset)) const flatOnlyDatasets = results.filter(r => supportsCSV(r.dataset)) // Mixed-Structure Track (no CSV) const mixedCharts = mixedStructureDatasets .map(result => generateDatasetChart(result)) .join('\n\n') // Flat-Only Track (with CSV) const flatCharts = flatOnlyDatasets .map((result) => { const csv = result.formats.find(f => f.name === 'csv') const toon = result.formats.find(f => f.name === 'toon')! if (!csv) return generateDatasetChart(result) // Special handling to show CSV first with TOON overhead const { dataset } = result const emoji = DATASET_ICONS[dataset.name] || DEFAULT_DATASET_ICON const eligibility = dataset.metadata.tabularEligibility const name = `${dataset.description} [eligibility: ${eligibility}%]` // CSV line const csvPercentage = Math.min(100, (csv.tokens / toon.tokens) * 100) const csvBar = createProgressBar(csvPercentage, 100, PROGRESS_BAR_WIDTH, PROGRESS_BAR_CONFIG) const csvStr = csv.tokens.toLocaleString('en-US') const line1 = `${emoji} ${name.padEnd(LABEL_PADDING)}\ncsv ${csvBar} ${csvStr.padStart(TOKEN_PADDING)} tokens` // TOON line with overhead vs CSV const toonOverhead = toon.tokens - csv.tokens const toonOverheadPercent = (toonOverhead / toon.tokens) * 100 const toonBar = createProgressBar(100, 100, PROGRESS_BAR_WIDTH, PROGRESS_BAR_CONFIG) const toonStr = toon.tokens.toLocaleString('en-US') const toonVsCSV = toonOverheadPercent >= 0 ? `(+${toonOverheadPercent.toFixed(1)}% vs CSV)` : `(${toonOverheadPercent.toFixed(1)}% vs CSV)` const toonLine = `toon ${toonBar} ${toonStr.padStart(TOKEN_PADDING)} tokens ${toonVsCSV}` // Other format comparisons (vs TOON) const comparisonLines = COMPARISON_FORMAT_ORDER.map((formatName) => { const format = result.formats.find(f => f.name === formatName) if (!format) return null return formatComparisonLine(format) }).filter(Boolean) return [line1, toonLine, ...comparisonLines].join('\n') }) .join('\n\n') // Calculate totals for mixed structure const { totalToonTokens: totalToonTokensMixed, totals: mixedTotals } = calculateTotalMetrics(mixedStructureDatasets, COMPARISON_FORMAT_ORDER) const mixedTotalLines = generateTotalLines(totalToonTokensMixed, mixedTotals) // Calculate totals for flat-only const { totalToonTokens: totalToonTokensFlat, totals: flatTotals } = calculateTotalMetrics(flatOnlyDatasets, COMPARISON_FORMAT_ORDER) const totalCSVTokensFlat = flatOnlyDatasets.reduce((sum, r) => { const csv = r.formats.find(f => f.name === 'csv') return sum + (csv?.tokens || 0) }, 0) const flatTotalLines = generateTotalLines(totalToonTokensFlat, flatTotals, { name: 'csv', tokens: totalCSVTokensFlat }) const barChartSection = ` ## Mixed-Structure Track Datasets with nested or semi-uniform structures. CSV excluded as it cannot properly represent these structures. \`\`\` ${mixedCharts} ${SEPARATOR} ${mixedTotalLines} \`\`\` ## Flat-Only Track Datasets with flat tabular structures where CSV is applicable. \`\`\` ${flatCharts} ${SEPARATOR} ${flatTotalLines} \`\`\` `.trim() // Generate detailed examples (optional: show a few examples) const detailedExamples = results .filter(r => DETAILED_EXAMPLE_DATASETS.includes(r.dataset.name as any)) .map((result, i, filtered) => { let displayData = result.dataset.data // Truncate for display if (result.dataset.name === 'github') { displayData = { repositories: displayData.repositories.slice(0, GITHUB_REPO_LIMIT).map((repo: Record) => ({ ...repo, description: repo.description?.slice(0, GITHUB_DESC_LIMIT) + (repo.description?.length > GITHUB_DESC_LIMIT ? 'โ€ฆ' : ''), })), } } else if (result.dataset.name === 'analytics') { displayData = { metrics: displayData.metrics.slice(0, ANALYTICS_METRICS_LIMIT) } } const separator = i < filtered.length - 1 ? '\n\n---' : '' const emoji = DATASET_ICONS[result.dataset.name] || DEFAULT_DATASET_ICON const json = result.formats.find(f => f.name === 'json-pretty')! const toon = result.formats.find(f => f.name === 'toon')! return `#### ${emoji} ${result.dataset.description} **Savings:** ${json.savings.toLocaleString('en-US')} tokens (${json.savingsPercent.toFixed(1)}% reduction vs JSON) **JSON** (${json.tokens.toLocaleString('en-US')} tokens): \`\`\`json ${JSON.stringify(displayData, undefined, 2)} \`\`\` **TOON** (${toon.tokens.toLocaleString('en-US')} tokens): \`\`\` ${encode(displayData)} \`\`\`${separator}` }) .join('\n\n') const markdown = ` ${barChartSection}
View detailed examples ${detailedExamples}
`.trimStart() prompts.log.message(barChartSection) const resultsDir = path.join(BENCHMARKS_DIR, 'results') await ensureDir(resultsDir) const outputFilePath = path.join(resultsDir, 'token-efficiency.md') await fsp.writeFile(outputFilePath, markdown, 'utf-8') prompts.log.success(`Report saved to \`${path.relative(ROOT_DIR, outputFilePath)}\``)