feat(cli): stream output for both encoding and decoding

This commit is contained in:
Johann Schopplich
2025-11-21 16:52:34 +01:00
parent cfbbb09358
commit 9ebad53ea3
6 changed files with 486 additions and 24 deletions

View File

@@ -7,6 +7,7 @@ import process from 'node:process'
import { consola } from 'consola'
import { estimateTokenCount } from 'tokenx'
import { decode, encode, encodeLines } from '../../toon/src'
import { jsonStringifyLines } from './json-stringify-stream'
import { formatInputLabel, readInput } from './utils'
export async function encodeToToon(config: {
@@ -62,7 +63,6 @@ export async function encodeToToon(config: {
consola.success(`Saved ~${diff} tokens (-${percent}%)`)
}
else {
// Use streaming encoder for memory-efficient output
await writeStreamingToon(encodeLines(data, encodeOptions), config.output)
if (config.output) {
@@ -95,25 +95,52 @@ export async function decodeToJson(config: {
throw new Error(`Failed to decode TOON: ${error instanceof Error ? error.message : String(error)}`)
}
const jsonOutput = JSON.stringify(data, undefined, config.indent)
await writeStreamingJson(jsonStringifyLines(data, config.indent), config.output)
if (config.output) {
await fsp.writeFile(config.output, jsonOutput, 'utf-8')
const relativeInputPath = formatInputLabel(config.input)
const relativeOutputPath = path.relative(process.cwd(), config.output)
consola.success(`Decoded \`${relativeInputPath}\`\`${relativeOutputPath}\``)
}
}
/**
* Writes JSON chunks to a file or stdout using streaming approach.
* Chunks are written one at a time without building the full string in memory.
*/
async function writeStreamingJson(
chunks: Iterable<string>,
outputPath?: string,
): Promise<void> {
// Stream to file using fs/promises API
if (outputPath) {
let fileHandle: FileHandle | undefined
try {
fileHandle = await fsp.open(outputPath, 'w')
for (const chunk of chunks) {
await fileHandle.write(chunk)
}
}
finally {
await fileHandle?.close()
}
}
// Stream to stdout
else {
console.log(jsonOutput)
for (const chunk of chunks) {
process.stdout.write(chunk)
}
// Add final newline for stdout
process.stdout.write('\n')
}
}
/**
* Writes TOON lines to a file or stdout using streaming approach.
* Lines are written one at a time without building the full string in memory.
*
* @param lines - Iterable of TOON lines (without trailing newlines)
* @param outputPath - File path to write to, or undefined for stdout
*/
async function writeStreamingToon(
lines: Iterable<string>,