mirror of
https://github.com/voson-wang/toon.git
synced 2026-01-29 23:34:10 +08:00
feat(cli): stream output for both encoding and decoding
This commit is contained in:
@@ -7,6 +7,7 @@ import process from 'node:process'
|
||||
import { consola } from 'consola'
|
||||
import { estimateTokenCount } from 'tokenx'
|
||||
import { decode, encode, encodeLines } from '../../toon/src'
|
||||
import { jsonStringifyLines } from './json-stringify-stream'
|
||||
import { formatInputLabel, readInput } from './utils'
|
||||
|
||||
export async function encodeToToon(config: {
|
||||
@@ -62,7 +63,6 @@ export async function encodeToToon(config: {
|
||||
consola.success(`Saved ~${diff} tokens (-${percent}%)`)
|
||||
}
|
||||
else {
|
||||
// Use streaming encoder for memory-efficient output
|
||||
await writeStreamingToon(encodeLines(data, encodeOptions), config.output)
|
||||
|
||||
if (config.output) {
|
||||
@@ -95,25 +95,52 @@ export async function decodeToJson(config: {
|
||||
throw new Error(`Failed to decode TOON: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
|
||||
const jsonOutput = JSON.stringify(data, undefined, config.indent)
|
||||
await writeStreamingJson(jsonStringifyLines(data, config.indent), config.output)
|
||||
|
||||
if (config.output) {
|
||||
await fsp.writeFile(config.output, jsonOutput, 'utf-8')
|
||||
const relativeInputPath = formatInputLabel(config.input)
|
||||
const relativeOutputPath = path.relative(process.cwd(), config.output)
|
||||
consola.success(`Decoded \`${relativeInputPath}\` → \`${relativeOutputPath}\``)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes JSON chunks to a file or stdout using streaming approach.
|
||||
* Chunks are written one at a time without building the full string in memory.
|
||||
*/
|
||||
async function writeStreamingJson(
|
||||
chunks: Iterable<string>,
|
||||
outputPath?: string,
|
||||
): Promise<void> {
|
||||
// Stream to file using fs/promises API
|
||||
if (outputPath) {
|
||||
let fileHandle: FileHandle | undefined
|
||||
|
||||
try {
|
||||
fileHandle = await fsp.open(outputPath, 'w')
|
||||
|
||||
for (const chunk of chunks) {
|
||||
await fileHandle.write(chunk)
|
||||
}
|
||||
}
|
||||
finally {
|
||||
await fileHandle?.close()
|
||||
}
|
||||
}
|
||||
// Stream to stdout
|
||||
else {
|
||||
console.log(jsonOutput)
|
||||
for (const chunk of chunks) {
|
||||
process.stdout.write(chunk)
|
||||
}
|
||||
|
||||
// Add final newline for stdout
|
||||
process.stdout.write('\n')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes TOON lines to a file or stdout using streaming approach.
|
||||
* Lines are written one at a time without building the full string in memory.
|
||||
*
|
||||
* @param lines - Iterable of TOON lines (without trailing newlines)
|
||||
* @param outputPath - File path to write to, or undefined for stdout
|
||||
*/
|
||||
async function writeStreamingToon(
|
||||
lines: Iterable<string>,
|
||||
|
||||
161
packages/cli/src/json-stringify-stream.ts
Normal file
161
packages/cli/src/json-stringify-stream.ts
Normal file
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* Streaming JSON stringifier.
|
||||
*
|
||||
* Yields JSON tokens one at a time, allowing streaming output without holding
|
||||
* the entire JSON string in memory.
|
||||
*
|
||||
* @param value - The value to stringify (must be JSON-serializable)
|
||||
* @param indent - Number of spaces for indentation (0 = compact, >0 = pretty)
|
||||
* @returns Generator that yields JSON string chunks
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* const data = { name: "Alice", scores: [95, 87, 92] }
|
||||
* for (const chunk of jsonStringifyLines(data, 2)) {
|
||||
* process.stdout.write(chunk)
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
export function* jsonStringifyLines(
|
||||
value: unknown,
|
||||
indent: number = 2,
|
||||
): Iterable<string> {
|
||||
yield* stringifyValue(value, 0, indent)
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal generator for recursive stringification.
|
||||
*/
|
||||
function* stringifyValue(
|
||||
value: unknown,
|
||||
depth: number,
|
||||
indent: number,
|
||||
): Iterable<string> {
|
||||
// Handle null
|
||||
if (value === null) {
|
||||
yield 'null'
|
||||
return
|
||||
}
|
||||
|
||||
const type = typeof value
|
||||
|
||||
// Handle primitives
|
||||
if (type === 'boolean' || type === 'number') {
|
||||
yield JSON.stringify(value)
|
||||
return
|
||||
}
|
||||
|
||||
if (type === 'string') {
|
||||
yield JSON.stringify(value)
|
||||
return
|
||||
}
|
||||
|
||||
// Handle arrays
|
||||
if (Array.isArray(value)) {
|
||||
yield* stringifyArray(value, depth, indent)
|
||||
return
|
||||
}
|
||||
|
||||
// Handle objects
|
||||
if (type === 'object') {
|
||||
yield* stringifyObject(value as Record<string, unknown>, depth, indent)
|
||||
return
|
||||
}
|
||||
|
||||
// Undefined, functions, symbols become null in JSON
|
||||
yield 'null'
|
||||
}
|
||||
|
||||
/**
|
||||
* Stringify an array with proper formatting.
|
||||
*/
|
||||
function* stringifyArray(
|
||||
arr: unknown[],
|
||||
depth: number,
|
||||
indent: number,
|
||||
): Iterable<string> {
|
||||
if (arr.length === 0) {
|
||||
yield '[]'
|
||||
return
|
||||
}
|
||||
|
||||
yield '['
|
||||
|
||||
if (indent > 0) {
|
||||
// Pretty-printed format
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
yield '\n'
|
||||
yield ' '.repeat((depth + 1) * indent)
|
||||
yield* stringifyValue(arr[i], depth + 1, indent)
|
||||
if (i < arr.length - 1) {
|
||||
yield ','
|
||||
}
|
||||
}
|
||||
yield '\n'
|
||||
yield ' '.repeat(depth * indent)
|
||||
yield ']'
|
||||
}
|
||||
else {
|
||||
// Compact format
|
||||
for (let i = 0; i < arr.length; i++) {
|
||||
yield* stringifyValue(arr[i], depth + 1, indent)
|
||||
if (i < arr.length - 1) {
|
||||
yield ','
|
||||
}
|
||||
}
|
||||
yield ']'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stringify an object with proper formatting.
|
||||
*/
|
||||
function* stringifyObject(
|
||||
obj: Record<string, unknown>,
|
||||
depth: number,
|
||||
indent: number,
|
||||
): Iterable<string> {
|
||||
const keys = Object.keys(obj)
|
||||
|
||||
if (keys.length === 0) {
|
||||
yield '{}'
|
||||
return
|
||||
}
|
||||
|
||||
yield '{'
|
||||
|
||||
if (indent > 0) {
|
||||
// Pretty-printed format
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
const key = keys[i]!
|
||||
const value = obj[key]
|
||||
|
||||
yield '\n'
|
||||
yield ' '.repeat((depth + 1) * indent)
|
||||
yield JSON.stringify(key)
|
||||
yield ': '
|
||||
yield* stringifyValue(value, depth + 1, indent)
|
||||
if (i < keys.length - 1) {
|
||||
yield ','
|
||||
}
|
||||
}
|
||||
yield '\n'
|
||||
yield ' '.repeat(depth * indent)
|
||||
yield '}'
|
||||
}
|
||||
else {
|
||||
// Compact format
|
||||
for (let i = 0; i < keys.length; i++) {
|
||||
const key = keys[i]!
|
||||
const value = obj[key]
|
||||
|
||||
yield JSON.stringify(key)
|
||||
yield ':'
|
||||
yield* stringifyValue(value, depth + 1, indent)
|
||||
if (i < keys.length - 1) {
|
||||
yield ','
|
||||
}
|
||||
}
|
||||
yield '}'
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user