mirror of
https://github.com/voson-wang/toon.git
synced 2026-01-29 23:34:10 +08:00
feat(cli): stream output for both encoding and decoding
This commit is contained in:
@@ -7,6 +7,7 @@ import process from 'node:process'
|
||||
import { consola } from 'consola'
|
||||
import { estimateTokenCount } from 'tokenx'
|
||||
import { decode, encode, encodeLines } from '../../toon/src'
|
||||
import { jsonStringifyLines } from './json-stringify-stream'
|
||||
import { formatInputLabel, readInput } from './utils'
|
||||
|
||||
export async function encodeToToon(config: {
|
||||
@@ -62,7 +63,6 @@ export async function encodeToToon(config: {
|
||||
consola.success(`Saved ~${diff} tokens (-${percent}%)`)
|
||||
}
|
||||
else {
|
||||
// Use streaming encoder for memory-efficient output
|
||||
await writeStreamingToon(encodeLines(data, encodeOptions), config.output)
|
||||
|
||||
if (config.output) {
|
||||
@@ -95,25 +95,52 @@ export async function decodeToJson(config: {
|
||||
throw new Error(`Failed to decode TOON: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
|
||||
const jsonOutput = JSON.stringify(data, undefined, config.indent)
|
||||
await writeStreamingJson(jsonStringifyLines(data, config.indent), config.output)
|
||||
|
||||
if (config.output) {
|
||||
await fsp.writeFile(config.output, jsonOutput, 'utf-8')
|
||||
const relativeInputPath = formatInputLabel(config.input)
|
||||
const relativeOutputPath = path.relative(process.cwd(), config.output)
|
||||
consola.success(`Decoded \`${relativeInputPath}\` → \`${relativeOutputPath}\``)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes JSON chunks to a file or stdout using streaming approach.
|
||||
* Chunks are written one at a time without building the full string in memory.
|
||||
*/
|
||||
async function writeStreamingJson(
|
||||
chunks: Iterable<string>,
|
||||
outputPath?: string,
|
||||
): Promise<void> {
|
||||
// Stream to file using fs/promises API
|
||||
if (outputPath) {
|
||||
let fileHandle: FileHandle | undefined
|
||||
|
||||
try {
|
||||
fileHandle = await fsp.open(outputPath, 'w')
|
||||
|
||||
for (const chunk of chunks) {
|
||||
await fileHandle.write(chunk)
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await fileHandle?.close()
|
||||
}
|
||||
}
|
||||
// Stream to stdout
|
||||
else {
|
||||
console.log(jsonOutput)
|
||||
for (const chunk of chunks) {
|
||||
process.stdout.write(chunk)
|
||||
}
|
||||
|
||||
// Add final newline for stdout
|
||||
process.stdout.write('\n')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes TOON lines to a file or stdout using streaming approach.
|
||||
* Lines are written one at a time without building the full string in memory.
|
||||
*
|
||||
* @param lines - Iterable of TOON lines (without trailing newlines)
|
||||
* @param outputPath - File path to write to, or undefined for stdout
|
||||
*/
|
||||
async function writeStreamingToon(
|
||||
lines: Iterable<string>,
|
||||
|
||||
Reference in New Issue
Block a user