github/toon
1
0
Fork 0
mirror of https://github.com/voson-wang/toon.git synced 2026-01-29 23:34:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: encodeLines for streaming encoding to TOON

Browse Source
This commit is contained in:
Johann Schopplich
2025-11-20 18:37:46 +01:00
parent 660ed21e01
commit 2e992aed74
7 changed files with 269 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
packages/cli/src/conversion.ts
View File

@@ -5,7 +5,7 @@ import * as path from 'node:path'
import process from 'node:process'
import { consola } from 'consola'
import { estimateTokenCount } from 'tokenx'
import { decode, encode } from '../../toon/src'
import { decode, encode, encodeLines } from '../../toon/src'
import { formatInputLabel, readInput } from './utils'
export async function encodeToToon(config: {
@@ -34,7 +34,17 @@ export async function encodeToToon(config: {
flattenDepth: config.flattenDepth,
}
const toonOutput = encode(data, encodeOptions)
let toonOutput: string
// When printing stats, we need the full string for token counting
if (config.printStats) {
toonOutput = encode(data, encodeOptions)
}
else {
// Use streaming encoder for non-stats path
const lines = Array.from(encodeLines(data, encodeOptions))
toonOutput = lines.join('\n')
}
if (config.output) {
await fsp.writeFile(config.output, toonOutput, 'utf-8')