feat(cli): stream output for both encoding and decoding

This commit is contained in:
Johann Schopplich
2025-11-21 16:52:34 +01:00
parent cfbbb09358
commit 9ebad53ea3
6 changed files with 486 additions and 24 deletions

View File

@@ -7,6 +7,7 @@ import process from 'node:process'
import { consola } from 'consola'
import { estimateTokenCount } from 'tokenx'
import { decode, encode, encodeLines } from '../../toon/src'
import { jsonStringifyLines } from './json-stringify-stream'
import { formatInputLabel, readInput } from './utils'
export async function encodeToToon(config: {
@@ -62,7 +63,6 @@ export async function encodeToToon(config: {
consola.success(`Saved ~${diff} tokens (-${percent}%)`)
}
else {
// Use streaming encoder for memory-efficient output
await writeStreamingToon(encodeLines(data, encodeOptions), config.output)
if (config.output) {
@@ -95,25 +95,52 @@ export async function decodeToJson(config: {
throw new Error(`Failed to decode TOON: ${error instanceof Error ? error.message : String(error)}`)
}
const jsonOutput = JSON.stringify(data, undefined, config.indent)
await writeStreamingJson(jsonStringifyLines(data, config.indent), config.output)
if (config.output) {
await fsp.writeFile(config.output, jsonOutput, 'utf-8')
const relativeInputPath = formatInputLabel(config.input)
const relativeOutputPath = path.relative(process.cwd(), config.output)
consola.success(`Decoded \`${relativeInputPath}\`\`${relativeOutputPath}\``)
}
}
/**
* Writes JSON chunks to a file or stdout using streaming approach.
* Chunks are written one at a time without building the full string in memory.
*/
async function writeStreamingJson(
chunks: Iterable<string>,
outputPath?: string,
): Promise<void> {
// Stream to file using fs/promises API
if (outputPath) {
let fileHandle: FileHandle | undefined
try {
fileHandle = await fsp.open(outputPath, 'w')
for (const chunk of chunks) {
await fileHandle.write(chunk)
}
}
finally {
await fileHandle?.close()
}
}
// Stream to stdout
else {
console.log(jsonOutput)
for (const chunk of chunks) {
process.stdout.write(chunk)
}
// Add final newline for stdout
process.stdout.write('\n')
}
}
/**
* Writes TOON lines to a file or stdout using streaming approach.
* Lines are written one at a time without building the full string in memory.
*
* @param lines - Iterable of TOON lines (without trailing newlines)
* @param outputPath - File path to write to, or undefined for stdout
*/
async function writeStreamingToon(
lines: Iterable<string>,

View File

@@ -0,0 +1,161 @@
/**
* Streaming JSON stringifier.
*
* Yields JSON tokens one at a time, allowing streaming output without holding
* the entire JSON string in memory.
*
* @param value - The value to stringify (must be JSON-serializable)
* @param indent - Number of spaces for indentation (0 = compact, >0 = pretty)
* @returns Generator that yields JSON string chunks
*
* @example
* ```ts
* const data = { name: "Alice", scores: [95, 87, 92] }
* for (const chunk of jsonStringifyLines(data, 2)) {
* process.stdout.write(chunk)
* }
* ```
*/
export function* jsonStringifyLines(
value: unknown,
indent: number = 2,
): Iterable<string> {
yield* stringifyValue(value, 0, indent)
}
/**
* Internal generator for recursive stringification.
*/
function* stringifyValue(
value: unknown,
depth: number,
indent: number,
): Iterable<string> {
// Handle null
if (value === null) {
yield 'null'
return
}
const type = typeof value
// Handle primitives
if (type === 'boolean' || type === 'number') {
yield JSON.stringify(value)
return
}
if (type === 'string') {
yield JSON.stringify(value)
return
}
// Handle arrays
if (Array.isArray(value)) {
yield* stringifyArray(value, depth, indent)
return
}
// Handle objects
if (type === 'object') {
yield* stringifyObject(value as Record<string, unknown>, depth, indent)
return
}
// Undefined, functions, symbols become null in JSON
yield 'null'
}
/**
* Stringify an array with proper formatting.
*/
function* stringifyArray(
arr: unknown[],
depth: number,
indent: number,
): Iterable<string> {
if (arr.length === 0) {
yield '[]'
return
}
yield '['
if (indent > 0) {
// Pretty-printed format
for (let i = 0; i < arr.length; i++) {
yield '\n'
yield ' '.repeat((depth + 1) * indent)
yield* stringifyValue(arr[i], depth + 1, indent)
if (i < arr.length - 1) {
yield ','
}
}
yield '\n'
yield ' '.repeat(depth * indent)
yield ']'
}
else {
// Compact format
for (let i = 0; i < arr.length; i++) {
yield* stringifyValue(arr[i], depth + 1, indent)
if (i < arr.length - 1) {
yield ','
}
}
yield ']'
}
}
/**
* Stringify an object with proper formatting.
*/
function* stringifyObject(
obj: Record<string, unknown>,
depth: number,
indent: number,
): Iterable<string> {
const keys = Object.keys(obj)
if (keys.length === 0) {
yield '{}'
return
}
yield '{'
if (indent > 0) {
// Pretty-printed format
for (let i = 0; i < keys.length; i++) {
const key = keys[i]!
const value = obj[key]
yield '\n'
yield ' '.repeat((depth + 1) * indent)
yield JSON.stringify(key)
yield ': '
yield* stringifyValue(value, depth + 1, indent)
if (i < keys.length - 1) {
yield ','
}
}
yield '\n'
yield ' '.repeat(depth * indent)
yield '}'
}
else {
// Compact format
for (let i = 0; i < keys.length; i++) {
const key = keys[i]!
const value = obj[key]
yield JSON.stringify(key)
yield ':'
yield* stringifyValue(value, depth + 1, indent)
if (i < keys.length - 1) {
yield ','
}
}
yield '}'
}
}