mirror of
https://github.com/voson-wang/toon.git
synced 2026-01-29 23:34:10 +08:00
345 lines
12 KiB
TypeScript
345 lines
12 KiB
TypeScript
import type { Dataset } from '../src/types'
|
||
import * as fsp from 'node:fs/promises'
|
||
import * as path from 'node:path'
|
||
import * as prompts from '@clack/prompts'
|
||
import { encode } from '../../packages/toon/src'
|
||
import { BENCHMARKS_DIR, FORMATTER_DISPLAY_NAMES, ROOT_DIR } from '../src/constants'
|
||
import { TOKEN_EFFICIENCY_DATASETS } from '../src/datasets'
|
||
import { formatters, supportsCSV } from '../src/formatters'
|
||
import { createProgressBar, ensureDir, tokenize } from '../src/utils'
|
||
|
||
interface FormatMetrics {
|
||
name: string
|
||
tokens: number
|
||
savings: number
|
||
savingsPercent: number
|
||
}
|
||
|
||
interface BenchmarkResult {
|
||
dataset: Dataset
|
||
formats: FormatMetrics[]
|
||
}
|
||
|
||
// Constants
|
||
const DATASET_ICONS: Record<string, string> = {
|
||
'tabular': '👥',
|
||
'nested': '🛒',
|
||
'analytics': '📈',
|
||
'github': '⭐',
|
||
'event-logs': '🧾',
|
||
'nested-config': '🧩',
|
||
}
|
||
|
||
const COMPARISON_FORMAT_ORDER = ['json-pretty', 'json-compact', 'yaml', 'xml'] as const
|
||
|
||
const PROGRESS_BAR_CONFIG = { filled: '▓', empty: '░' } as const
|
||
const PROGRESS_BAR_WIDTH = 20
|
||
const TOKEN_PADDING = 7
|
||
const LABEL_PADDING = 60
|
||
const COMPARISON_LABEL_PADDING = 30
|
||
|
||
const SEPARATOR = '─────────────────────────────────────────────────────────────────────────────────'
|
||
const DEFAULT_DATASET_ICON = '📊'
|
||
|
||
const DETAILED_EXAMPLE_DATASETS = ['github', 'analytics'] as const
|
||
const GITHUB_REPO_LIMIT = 3
|
||
const GITHUB_DESC_LIMIT = 80
|
||
const ANALYTICS_METRICS_LIMIT = 5
|
||
|
||
prompts.intro('Token Efficiency Benchmark')
|
||
|
||
/**
|
||
* Format a comparison line showing savings vs TOON
|
||
*/
|
||
function formatComparisonLine(format: FormatMetrics): string {
|
||
const label = FORMATTER_DISPLAY_NAMES[format.name] || format.name.toUpperCase()
|
||
const signedPercent = format.savingsPercent >= 0
|
||
? `−${format.savingsPercent.toFixed(1)}%`
|
||
: `+${Math.abs(format.savingsPercent).toFixed(1)}%`
|
||
const labelWithSavings = `vs ${label} (${signedPercent})`.padEnd(COMPARISON_LABEL_PADDING)
|
||
const tokenStr = format.tokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
|
||
return ` ${labelWithSavings}${tokenStr}`
|
||
}
|
||
|
||
/**
|
||
* Calculate total tokens and savings for a set of datasets
|
||
*/
|
||
function calculateTotalMetrics(datasets: BenchmarkResult[], formatNames: readonly string[]) {
|
||
const totalToonTokens = datasets.reduce((sum, r) => {
|
||
const toon = r.formats.find(f => f.name === 'toon')!
|
||
return sum + toon.tokens
|
||
}, 0)
|
||
|
||
const totals = formatNames.map((formatName) => {
|
||
const totalTokens = datasets.reduce((sum, r) => {
|
||
const format = r.formats.find(f => f.name === formatName)
|
||
return sum + (format?.tokens || 0)
|
||
}, 0)
|
||
const savings = totalTokens - totalToonTokens
|
||
const savingsPercent = (savings / totalTokens) * 100
|
||
return { name: formatName, tokens: totalTokens, savingsPercent }
|
||
})
|
||
|
||
return { totalToonTokens, totals }
|
||
}
|
||
|
||
/**
|
||
* Generate total lines for a track
|
||
*/
|
||
function generateTotalLines(
|
||
totalToonTokens: number,
|
||
totals: { name: string, tokens: number, savingsPercent: number }[],
|
||
baselineFormat?: { name: string, tokens: number },
|
||
) {
|
||
const lines: string[] = ['Total ']
|
||
|
||
if (baselineFormat) {
|
||
// Flat-only track with CSV baseline
|
||
const csvPercentage = Math.min(100, (baselineFormat.tokens / totalToonTokens) * 100)
|
||
const csvBar = createProgressBar(csvPercentage, 100, PROGRESS_BAR_WIDTH, PROGRESS_BAR_CONFIG)
|
||
const csvStr = baselineFormat.tokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
|
||
lines.push(`csv ${csvBar} ${csvStr} tokens`)
|
||
|
||
const overheadPercent = ((totalToonTokens - baselineFormat.tokens) / totalToonTokens) * 100
|
||
const toonBar = createProgressBar(100, 100, PROGRESS_BAR_WIDTH, PROGRESS_BAR_CONFIG)
|
||
const toonStr = totalToonTokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
|
||
lines.push(`toon ${toonBar} ${toonStr} tokens (+${overheadPercent.toFixed(1)}% vs CSV)`)
|
||
}
|
||
else {
|
||
// Mixed-structure track
|
||
const totalPercentage = Math.min(100, (totalToonTokens / totals[0]!.tokens) * 100)
|
||
const totalBar = createProgressBar(totalPercentage, 100, PROGRESS_BAR_WIDTH, PROGRESS_BAR_CONFIG)
|
||
const toonStr = totalToonTokens.toLocaleString('en-US').padStart(TOKEN_PADDING)
|
||
lines.push(`toon ${totalBar} ${toonStr} tokens`)
|
||
}
|
||
|
||
// Add comparison lines
|
||
for (const format of totals) {
|
||
lines.push(formatComparisonLine({
|
||
name: format.name,
|
||
tokens: format.tokens,
|
||
savings: 0, // Not used in this context
|
||
savingsPercent: format.savingsPercent,
|
||
}))
|
||
}
|
||
|
||
return lines.join('\n')
|
||
}
|
||
|
||
/**
|
||
* Generate bar chart for a dataset
|
||
*/
|
||
function generateDatasetChart(result: BenchmarkResult): string {
|
||
const { dataset, formats } = result
|
||
const toon = formats.find(f => f.name === 'toon')!
|
||
const jsonPretty = formats.find(f => f.name === 'json-pretty')!
|
||
|
||
const emoji = DATASET_ICONS[dataset.name] || DEFAULT_DATASET_ICON
|
||
const eligibility = dataset.metadata.tabularEligibility
|
||
const name = `${dataset.description} [eligibility: ${eligibility}%]`
|
||
const percentage = Math.min(100, 100 - jsonPretty.savingsPercent)
|
||
const bar = createProgressBar(percentage, 100, PROGRESS_BAR_WIDTH, PROGRESS_BAR_CONFIG)
|
||
const toonStr = toon.tokens.toLocaleString('en-US')
|
||
|
||
const line1 = `${emoji} ${name.padEnd(LABEL_PADDING)}\ntoon ${bar} ${toonStr.padStart(TOKEN_PADDING)} tokens`
|
||
|
||
const comparisonLines = COMPARISON_FORMAT_ORDER.map((formatName) => {
|
||
const format = formats.find(f => f.name === formatName)
|
||
if (!format)
|
||
return null
|
||
|
||
return formatComparisonLine(format)
|
||
}).filter(Boolean)
|
||
|
||
return [line1, ...comparisonLines].join('\n')
|
||
}
|
||
|
||
const results: BenchmarkResult[] = []
|
||
|
||
// Calculate token counts for all datasets
|
||
for (const dataset of TOKEN_EFFICIENCY_DATASETS) {
|
||
const formatMetrics: FormatMetrics[] = []
|
||
const tokensByFormat: Record<string, number> = {}
|
||
|
||
// Calculate tokens for each format
|
||
for (const [formatName, formatter] of Object.entries(formatters)) {
|
||
// Skip CSV for datasets that don't support it
|
||
if (formatName === 'csv' && !supportsCSV(dataset))
|
||
continue
|
||
|
||
const formattedString = formatter(dataset.data)
|
||
const tokens = tokenize(formattedString)
|
||
tokensByFormat[formatName] = tokens
|
||
}
|
||
|
||
// Calculate savings vs TOON
|
||
const toonTokens = tokensByFormat.toon!
|
||
for (const [formatName, tokens] of Object.entries(tokensByFormat)) {
|
||
const savings = tokens - toonTokens
|
||
formatMetrics.push({
|
||
name: formatName,
|
||
tokens,
|
||
savings,
|
||
savingsPercent: formatName === 'toon' ? 0 : (savings / tokens) * 100,
|
||
})
|
||
}
|
||
|
||
results.push({
|
||
dataset,
|
||
formats: formatMetrics,
|
||
})
|
||
}
|
||
|
||
// Separate datasets by CSV support
|
||
const mixedStructureDatasets = results.filter(r => !supportsCSV(r.dataset))
|
||
const flatOnlyDatasets = results.filter(r => supportsCSV(r.dataset))
|
||
|
||
// Mixed-Structure Track (no CSV)
|
||
const mixedCharts = mixedStructureDatasets
|
||
.map(result => generateDatasetChart(result))
|
||
.join('\n\n')
|
||
|
||
// Flat-Only Track (with CSV)
|
||
const flatCharts = flatOnlyDatasets
|
||
.map((result) => {
|
||
const csv = result.formats.find(f => f.name === 'csv')
|
||
const toon = result.formats.find(f => f.name === 'toon')!
|
||
|
||
if (!csv)
|
||
return generateDatasetChart(result)
|
||
|
||
// Special handling to show CSV first with TOON overhead
|
||
const { dataset } = result
|
||
const emoji = DATASET_ICONS[dataset.name] || DEFAULT_DATASET_ICON
|
||
const eligibility = dataset.metadata.tabularEligibility
|
||
const name = `${dataset.description} [eligibility: ${eligibility}%]`
|
||
|
||
// CSV line
|
||
const csvPercentage = Math.min(100, (csv.tokens / toon.tokens) * 100)
|
||
const csvBar = createProgressBar(csvPercentage, 100, PROGRESS_BAR_WIDTH, PROGRESS_BAR_CONFIG)
|
||
const csvStr = csv.tokens.toLocaleString('en-US')
|
||
|
||
const line1 = `${emoji} ${name.padEnd(LABEL_PADDING)}\ncsv ${csvBar} ${csvStr.padStart(TOKEN_PADDING)} tokens`
|
||
|
||
// TOON line with overhead vs CSV
|
||
const toonOverhead = toon.tokens - csv.tokens
|
||
const toonOverheadPercent = (toonOverhead / toon.tokens) * 100
|
||
const toonBar = createProgressBar(100, 100, PROGRESS_BAR_WIDTH, PROGRESS_BAR_CONFIG)
|
||
const toonStr = toon.tokens.toLocaleString('en-US')
|
||
const toonVsCSV = toonOverheadPercent >= 0
|
||
? `(+${toonOverheadPercent.toFixed(1)}% vs CSV)`
|
||
: `(${toonOverheadPercent.toFixed(1)}% vs CSV)`
|
||
const toonLine = `toon ${toonBar} ${toonStr.padStart(TOKEN_PADDING)} tokens ${toonVsCSV}`
|
||
|
||
// Other format comparisons (vs TOON)
|
||
const comparisonLines = COMPARISON_FORMAT_ORDER.map((formatName) => {
|
||
const format = result.formats.find(f => f.name === formatName)
|
||
if (!format)
|
||
return null
|
||
|
||
return formatComparisonLine(format)
|
||
}).filter(Boolean)
|
||
|
||
return [line1, toonLine, ...comparisonLines].join('\n')
|
||
})
|
||
.join('\n\n')
|
||
|
||
// Calculate totals for mixed structure
|
||
const { totalToonTokens: totalToonTokensMixed, totals: mixedTotals } = calculateTotalMetrics(mixedStructureDatasets, COMPARISON_FORMAT_ORDER)
|
||
const mixedTotalLines = generateTotalLines(totalToonTokensMixed, mixedTotals)
|
||
|
||
// Calculate totals for flat-only
|
||
const { totalToonTokens: totalToonTokensFlat, totals: flatTotals } = calculateTotalMetrics(flatOnlyDatasets, COMPARISON_FORMAT_ORDER)
|
||
const totalCSVTokensFlat = flatOnlyDatasets.reduce((sum, r) => {
|
||
const csv = r.formats.find(f => f.name === 'csv')
|
||
return sum + (csv?.tokens || 0)
|
||
}, 0)
|
||
const flatTotalLines = generateTotalLines(totalToonTokensFlat, flatTotals, { name: 'csv', tokens: totalCSVTokensFlat })
|
||
|
||
const barChartSection = `
|
||
## Mixed-Structure Track
|
||
|
||
Datasets with nested or semi-uniform structures. CSV excluded as it cannot properly represent these structures.
|
||
|
||
\`\`\`
|
||
${mixedCharts}
|
||
|
||
${SEPARATOR}
|
||
${mixedTotalLines}
|
||
\`\`\`
|
||
|
||
## Flat-Only Track
|
||
|
||
Datasets with flat tabular structures where CSV is applicable.
|
||
|
||
\`\`\`
|
||
${flatCharts}
|
||
|
||
${SEPARATOR}
|
||
${flatTotalLines}
|
||
\`\`\`
|
||
`.trim()
|
||
|
||
// Generate detailed examples (optional: show a few examples)
|
||
const detailedExamples = results
|
||
.filter(r => DETAILED_EXAMPLE_DATASETS.includes(r.dataset.name as any))
|
||
.map((result, i, filtered) => {
|
||
let displayData = result.dataset.data
|
||
|
||
// Truncate for display
|
||
if (result.dataset.name === 'github') {
|
||
displayData = {
|
||
repositories: displayData.repositories.slice(0, GITHUB_REPO_LIMIT).map((repo: Record<string, any>) => ({
|
||
...repo,
|
||
description: repo.description?.slice(0, GITHUB_DESC_LIMIT) + (repo.description?.length > GITHUB_DESC_LIMIT ? '…' : ''),
|
||
})),
|
||
}
|
||
}
|
||
else if (result.dataset.name === 'analytics') {
|
||
displayData = { metrics: displayData.metrics.slice(0, ANALYTICS_METRICS_LIMIT) }
|
||
}
|
||
|
||
const separator = i < filtered.length - 1 ? '\n\n---' : ''
|
||
const emoji = DATASET_ICONS[result.dataset.name] || DEFAULT_DATASET_ICON
|
||
const json = result.formats.find(f => f.name === 'json-pretty')!
|
||
const toon = result.formats.find(f => f.name === 'toon')!
|
||
|
||
return `#### ${emoji} ${result.dataset.description}
|
||
|
||
**Savings:** ${json.savings.toLocaleString('en-US')} tokens (${json.savingsPercent.toFixed(1)}% reduction vs JSON)
|
||
|
||
**JSON** (${json.tokens.toLocaleString('en-US')} tokens):
|
||
|
||
\`\`\`json
|
||
${JSON.stringify(displayData, undefined, 2)}
|
||
\`\`\`
|
||
|
||
**TOON** (${toon.tokens.toLocaleString('en-US')} tokens):
|
||
|
||
\`\`\`
|
||
${encode(displayData)}
|
||
\`\`\`${separator}`
|
||
})
|
||
.join('\n\n')
|
||
|
||
const markdown = `
|
||
${barChartSection}
|
||
|
||
<details>
|
||
<summary><strong>View detailed examples</strong></summary>
|
||
|
||
${detailedExamples}
|
||
|
||
</details>
|
||
`.trimStart()
|
||
|
||
prompts.log.message(barChartSection)
|
||
|
||
const resultsDir = path.join(BENCHMARKS_DIR, 'results')
|
||
await ensureDir(resultsDir)
|
||
|
||
const outputFilePath = path.join(resultsDir, 'token-efficiency.md')
|
||
await fsp.writeFile(outputFilePath, markdown, 'utf-8')
|
||
|
||
prompts.log.success(`Report saved to \`${path.relative(ROOT_DIR, outputFilePath)}\``)
|