mirror of
https://github.com/voson-wang/toon.git
synced 2026-01-29 23:34:10 +08:00
feat: encodeLines for streaming encoding to TOON
This commit is contained in:
@@ -102,6 +102,16 @@ cat data.json | toon -
|
|||||||
cat data.toon | toon --decode
|
cat data.toon | toon --decode
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Performance
|
||||||
|
|
||||||
|
### Streaming Encoding
|
||||||
|
|
||||||
|
JSON→TOON conversions use line-by-line encoding internally, which avoids holding the entire TOON document in memory. This makes the CLI efficient for large datasets without requiring additional configuration.
|
||||||
|
|
||||||
|
::: info Token Statistics
|
||||||
|
When using the `--stats` flag, the CLI builds the full TOON string once to compute accurate token counts. For maximum memory efficiency on very large files, omit `--stats`.
|
||||||
|
:::
|
||||||
|
|
||||||
## Options
|
## Options
|
||||||
|
|
||||||
| Option | Description |
|
| Option | Description |
|
||||||
|
|||||||
@@ -127,6 +127,68 @@ encode(data, { delimiter: '\t', keyFolding: 'safe' })
|
|||||||
```
|
```
|
||||||
:::
|
:::
|
||||||
|
|
||||||
|
## `encodeLines(value, options?)`
|
||||||
|
|
||||||
|
Converts any JSON-serializable value to TOON format as a sequence of lines, without building the full string in memory. Suitable for streaming large outputs to files, HTTP responses, or process stdout.
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { encodeLines } from '@toon-format/toon'
|
||||||
|
|
||||||
|
// Stream to stdout
|
||||||
|
for (const line of encodeLines(data)) {
|
||||||
|
console.log(line)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write to file line-by-line
|
||||||
|
const lines = encodeLines(data, { indent: 2, delimiter: '\t' })
|
||||||
|
for (const line of lines) {
|
||||||
|
await writeToStream(`${line}\n`)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect to array
|
||||||
|
const lineArray = Array.from(encodeLines(data))
|
||||||
|
```
|
||||||
|
|
||||||
|
### Parameters
|
||||||
|
|
||||||
|
| Parameter | Type | Description |
|
||||||
|
|-----------|------|-------------|
|
||||||
|
| `value` | `unknown` | Any JSON-serializable value (object, array, primitive, or nested structure) |
|
||||||
|
| `options` | `EncodeOptions?` | Optional encoding options (same as `encode()`) |
|
||||||
|
|
||||||
|
### Return Value
|
||||||
|
|
||||||
|
Returns an `Iterable<string>` that yields TOON lines one at a time. Each yielded string is a single line without a trailing newline character.
|
||||||
|
|
||||||
|
::: info Relationship to `encode()`
|
||||||
|
`encode(value, options)` is equivalent to:
|
||||||
|
```ts
|
||||||
|
Array.from(encodeLines(value, options)).join('\n')
|
||||||
|
```
|
||||||
|
:::
|
||||||
|
|
||||||
|
### Example
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { createWriteStream } from 'node:fs'
|
||||||
|
import { encodeLines } from '@toon-format/toon'
|
||||||
|
|
||||||
|
const data = {
|
||||||
|
items: Array.from({ length: 100000 }, (_, i) => ({
|
||||||
|
id: i,
|
||||||
|
name: `Item ${i}`,
|
||||||
|
value: Math.random()
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stream large dataset to file
|
||||||
|
const stream = createWriteStream('output.toon')
|
||||||
|
for (const line of encodeLines(data, { delimiter: '\t' })) {
|
||||||
|
stream.write(`${line}\n`)
|
||||||
|
}
|
||||||
|
stream.end()
|
||||||
|
```
|
||||||
|
|
||||||
## `decode(input, options?)`
|
## `decode(input, options?)`
|
||||||
|
|
||||||
Converts a TOON-formatted string back to JavaScript values.
|
Converts a TOON-formatted string back to JavaScript values.
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import * as path from 'node:path'
|
|||||||
import process from 'node:process'
|
import process from 'node:process'
|
||||||
import { consola } from 'consola'
|
import { consola } from 'consola'
|
||||||
import { estimateTokenCount } from 'tokenx'
|
import { estimateTokenCount } from 'tokenx'
|
||||||
import { decode, encode } from '../../toon/src'
|
import { decode, encode, encodeLines } from '../../toon/src'
|
||||||
import { formatInputLabel, readInput } from './utils'
|
import { formatInputLabel, readInput } from './utils'
|
||||||
|
|
||||||
export async function encodeToToon(config: {
|
export async function encodeToToon(config: {
|
||||||
@@ -34,7 +34,17 @@ export async function encodeToToon(config: {
|
|||||||
flattenDepth: config.flattenDepth,
|
flattenDepth: config.flattenDepth,
|
||||||
}
|
}
|
||||||
|
|
||||||
const toonOutput = encode(data, encodeOptions)
|
let toonOutput: string
|
||||||
|
|
||||||
|
// When printing stats, we need the full string for token counting
|
||||||
|
if (config.printStats) {
|
||||||
|
toonOutput = encode(data, encodeOptions)
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
// Use streaming encoder for non-stats path
|
||||||
|
const lines = Array.from(encodeLines(data, encodeOptions))
|
||||||
|
toonOutput = lines.join('\n')
|
||||||
|
}
|
||||||
|
|
||||||
if (config.output) {
|
if (config.output) {
|
||||||
await fsp.writeFile(config.output, toonOutput, 'utf-8')
|
await fsp.writeFile(config.output, toonOutput, 'utf-8')
|
||||||
|
|||||||
@@ -1,34 +1,42 @@
|
|||||||
import type { Depth, JsonArray, JsonObject, JsonPrimitive, JsonValue, ResolvedEncodeOptions } from '../types'
|
import type { Depth, JsonArray, JsonObject, JsonPrimitive, JsonValue, ResolvedEncodeOptions } from '../types'
|
||||||
import { DOT, LIST_ITEM_MARKER } from '../constants'
|
import { DOT, LIST_ITEM_MARKER, LIST_ITEM_PREFIX } from '../constants'
|
||||||
import { tryFoldKeyChain } from './folding'
|
import { tryFoldKeyChain } from './folding'
|
||||||
import { isArrayOfArrays, isArrayOfObjects, isArrayOfPrimitives, isEmptyObject, isJsonArray, isJsonObject, isJsonPrimitive } from './normalize'
|
import { isArrayOfArrays, isArrayOfObjects, isArrayOfPrimitives, isEmptyObject, isJsonArray, isJsonObject, isJsonPrimitive } from './normalize'
|
||||||
import { encodeAndJoinPrimitives, encodeKey, encodePrimitive, formatHeader } from './primitives'
|
import { encodeAndJoinPrimitives, encodeKey, encodePrimitive, formatHeader } from './primitives'
|
||||||
import { LineWriter } from './writer'
|
|
||||||
|
|
||||||
// #region Encode normalized JsonValue
|
// #region Encode normalized JsonValue
|
||||||
|
|
||||||
export function encodeValue(value: JsonValue, options: ResolvedEncodeOptions): string {
|
export function* encodeJsonValue(value: JsonValue, options: ResolvedEncodeOptions, depth: Depth): Generator<string> {
|
||||||
if (isJsonPrimitive(value)) {
|
if (isJsonPrimitive(value)) {
|
||||||
return encodePrimitive(value, options.delimiter)
|
// Primitives at root level are returned as a single line
|
||||||
}
|
const encodedPrimitive = encodePrimitive(value, options.delimiter)
|
||||||
|
|
||||||
const writer = new LineWriter(options.indent)
|
if (encodedPrimitive !== '')
|
||||||
|
yield encodedPrimitive
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
if (isJsonArray(value)) {
|
if (isJsonArray(value)) {
|
||||||
encodeArray(undefined, value, writer, 0, options)
|
yield* encodeArrayLines(undefined, value, depth, options)
|
||||||
}
|
}
|
||||||
else if (isJsonObject(value)) {
|
else if (isJsonObject(value)) {
|
||||||
encodeObject(value, writer, 0, options)
|
yield* encodeObjectLines(value, depth, options)
|
||||||
}
|
}
|
||||||
|
|
||||||
return writer.toString()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// #endregion
|
// #endregion
|
||||||
|
|
||||||
// #region Object encoding
|
// #region Object encoding
|
||||||
|
|
||||||
export function encodeObject(value: JsonObject, writer: LineWriter, depth: Depth, options: ResolvedEncodeOptions, rootLiteralKeys?: Set<string>, pathPrefix?: string, remainingDepth?: number): void {
|
export function* encodeObjectLines(
|
||||||
|
value: JsonObject,
|
||||||
|
depth: Depth,
|
||||||
|
options: ResolvedEncodeOptions,
|
||||||
|
rootLiteralKeys?: Set<string>,
|
||||||
|
pathPrefix?: string,
|
||||||
|
remainingDepth?: number,
|
||||||
|
): Generator<string> {
|
||||||
const keys = Object.keys(value)
|
const keys = Object.keys(value)
|
||||||
|
|
||||||
// At root level (depth 0), collect all literal dotted keys for collision checking
|
// At root level (depth 0), collect all literal dotted keys for collision checking
|
||||||
@@ -39,11 +47,20 @@ export function encodeObject(value: JsonObject, writer: LineWriter, depth: Depth
|
|||||||
const effectiveFlattenDepth = remainingDepth ?? options.flattenDepth
|
const effectiveFlattenDepth = remainingDepth ?? options.flattenDepth
|
||||||
|
|
||||||
for (const [key, val] of Object.entries(value)) {
|
for (const [key, val] of Object.entries(value)) {
|
||||||
encodeKeyValuePair(key, val, writer, depth, options, keys, rootLiteralKeys, pathPrefix, effectiveFlattenDepth)
|
yield* encodeKeyValuePairLines(key, val, depth, options, keys, rootLiteralKeys, pathPrefix, effectiveFlattenDepth)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function encodeKeyValuePair(key: string, value: JsonValue, writer: LineWriter, depth: Depth, options: ResolvedEncodeOptions, siblings?: readonly string[], rootLiteralKeys?: Set<string>, pathPrefix?: string, flattenDepth?: number): void {
|
export function* encodeKeyValuePairLines(
|
||||||
|
key: string,
|
||||||
|
value: JsonValue,
|
||||||
|
depth: Depth,
|
||||||
|
options: ResolvedEncodeOptions,
|
||||||
|
siblings?: readonly string[],
|
||||||
|
rootLiteralKeys?: Set<string>,
|
||||||
|
pathPrefix?: string,
|
||||||
|
flattenDepth?: number,
|
||||||
|
): Generator<string> {
|
||||||
const currentPath = pathPrefix ? `${pathPrefix}${DOT}${key}` : key
|
const currentPath = pathPrefix ? `${pathPrefix}${DOT}${key}` : key
|
||||||
const effectiveFlattenDepth = flattenDepth ?? options.flattenDepth
|
const effectiveFlattenDepth = flattenDepth ?? options.flattenDepth
|
||||||
|
|
||||||
@@ -59,26 +76,26 @@ export function encodeKeyValuePair(key: string, value: JsonValue, writer: LineWr
|
|||||||
if (remainder === undefined) {
|
if (remainder === undefined) {
|
||||||
// The folded chain ended at a leaf (primitive, array, or empty object)
|
// The folded chain ended at a leaf (primitive, array, or empty object)
|
||||||
if (isJsonPrimitive(leafValue)) {
|
if (isJsonPrimitive(leafValue)) {
|
||||||
writer.push(depth, `${encodedFoldedKey}: ${encodePrimitive(leafValue, options.delimiter)}`)
|
yield indentedLine(depth, `${encodedFoldedKey}: ${encodePrimitive(leafValue, options.delimiter)}`, options.indent)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
else if (isJsonArray(leafValue)) {
|
else if (isJsonArray(leafValue)) {
|
||||||
encodeArray(foldedKey, leafValue, writer, depth, options)
|
yield* encodeArrayLines(foldedKey, leafValue, depth, options)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
else if (isJsonObject(leafValue) && isEmptyObject(leafValue)) {
|
else if (isJsonObject(leafValue) && isEmptyObject(leafValue)) {
|
||||||
writer.push(depth, `${encodedFoldedKey}:`)
|
yield indentedLine(depth, `${encodedFoldedKey}:`, options.indent)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Case 2: Partially folded with a tail object
|
// Case 2: Partially folded with a tail object
|
||||||
if (isJsonObject(remainder)) {
|
if (isJsonObject(remainder)) {
|
||||||
writer.push(depth, `${encodedFoldedKey}:`)
|
yield indentedLine(depth, `${encodedFoldedKey}:`, options.indent)
|
||||||
// Calculate remaining depth budget (subtract segments already folded)
|
// Calculate remaining depth budget (subtract segments already folded)
|
||||||
const remainingDepth = effectiveFlattenDepth - segmentCount
|
const remainingDepth = effectiveFlattenDepth - segmentCount
|
||||||
const foldedPath = pathPrefix ? `${pathPrefix}${DOT}${foldedKey}` : foldedKey
|
const foldedPath = pathPrefix ? `${pathPrefix}${DOT}${foldedKey}` : foldedKey
|
||||||
encodeObject(remainder, writer, depth + 1, options, rootLiteralKeys, foldedPath, remainingDepth)
|
yield* encodeObjectLines(remainder, depth + 1, options, rootLiteralKeys, foldedPath, remainingDepth)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -88,15 +105,15 @@ export function encodeKeyValuePair(key: string, value: JsonValue, writer: LineWr
|
|||||||
const encodedKey = encodeKey(key)
|
const encodedKey = encodeKey(key)
|
||||||
|
|
||||||
if (isJsonPrimitive(value)) {
|
if (isJsonPrimitive(value)) {
|
||||||
writer.push(depth, `${encodedKey}: ${encodePrimitive(value, options.delimiter)}`)
|
yield indentedLine(depth, `${encodedKey}: ${encodePrimitive(value, options.delimiter)}`, options.indent)
|
||||||
}
|
}
|
||||||
else if (isJsonArray(value)) {
|
else if (isJsonArray(value)) {
|
||||||
encodeArray(key, value, writer, depth, options)
|
yield* encodeArrayLines(key, value, depth, options)
|
||||||
}
|
}
|
||||||
else if (isJsonObject(value)) {
|
else if (isJsonObject(value)) {
|
||||||
writer.push(depth, `${encodedKey}:`)
|
yield indentedLine(depth, `${encodedKey}:`, options.indent)
|
||||||
if (!isEmptyObject(value)) {
|
if (!isEmptyObject(value)) {
|
||||||
encodeObject(value, writer, depth + 1, options, rootLiteralKeys, currentPath, effectiveFlattenDepth)
|
yield* encodeObjectLines(value, depth + 1, options, rootLiteralKeys, currentPath, effectiveFlattenDepth)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -105,23 +122,22 @@ export function encodeKeyValuePair(key: string, value: JsonValue, writer: LineWr
|
|||||||
|
|
||||||
// #region Array encoding
|
// #region Array encoding
|
||||||
|
|
||||||
export function encodeArray(
|
export function* encodeArrayLines(
|
||||||
key: string | undefined,
|
key: string | undefined,
|
||||||
value: JsonArray,
|
value: JsonArray,
|
||||||
writer: LineWriter,
|
|
||||||
depth: Depth,
|
depth: Depth,
|
||||||
options: ResolvedEncodeOptions,
|
options: ResolvedEncodeOptions,
|
||||||
): void {
|
): Generator<string> {
|
||||||
if (value.length === 0) {
|
if (value.length === 0) {
|
||||||
const header = formatHeader(0, { key, delimiter: options.delimiter })
|
const header = formatHeader(0, { key, delimiter: options.delimiter })
|
||||||
writer.push(depth, header)
|
yield indentedLine(depth, header, options.indent)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Primitive array
|
// Primitive array
|
||||||
if (isArrayOfPrimitives(value)) {
|
if (isArrayOfPrimitives(value)) {
|
||||||
const arrayLine = encodeInlineArrayLine(value, options.delimiter, key)
|
const arrayLine = encodeInlineArrayLine(value, options.delimiter, key)
|
||||||
writer.push(depth, arrayLine)
|
yield indentedLine(depth, arrayLine, options.indent)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -129,7 +145,7 @@ export function encodeArray(
|
|||||||
if (isArrayOfArrays(value)) {
|
if (isArrayOfArrays(value)) {
|
||||||
const allPrimitiveArrays = value.every(arr => isArrayOfPrimitives(arr))
|
const allPrimitiveArrays = value.every(arr => isArrayOfPrimitives(arr))
|
||||||
if (allPrimitiveArrays) {
|
if (allPrimitiveArrays) {
|
||||||
encodeArrayOfArraysAsListItems(key, value, writer, depth, options)
|
yield* encodeArrayOfArraysAsListItemsLines(key, value, depth, options)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -138,36 +154,35 @@ export function encodeArray(
|
|||||||
if (isArrayOfObjects(value)) {
|
if (isArrayOfObjects(value)) {
|
||||||
const header = extractTabularHeader(value)
|
const header = extractTabularHeader(value)
|
||||||
if (header) {
|
if (header) {
|
||||||
encodeArrayOfObjectsAsTabular(key, value, header, writer, depth, options)
|
yield* encodeArrayOfObjectsAsTabularLines(key, value, header, depth, options)
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
encodeMixedArrayAsListItems(key, value, writer, depth, options)
|
yield* encodeMixedArrayAsListItemsLines(key, value, depth, options)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mixed array: fallback to expanded format
|
// Mixed array: fallback to expanded format
|
||||||
encodeMixedArrayAsListItems(key, value, writer, depth, options)
|
yield* encodeMixedArrayAsListItemsLines(key, value, depth, options)
|
||||||
}
|
}
|
||||||
|
|
||||||
// #endregion
|
// #endregion
|
||||||
|
|
||||||
// #region Array of arrays (expanded format)
|
// #region Array of arrays (expanded format)
|
||||||
|
|
||||||
export function encodeArrayOfArraysAsListItems(
|
export function* encodeArrayOfArraysAsListItemsLines(
|
||||||
prefix: string | undefined,
|
prefix: string | undefined,
|
||||||
values: readonly JsonArray[],
|
values: readonly JsonArray[],
|
||||||
writer: LineWriter,
|
|
||||||
depth: Depth,
|
depth: Depth,
|
||||||
options: ResolvedEncodeOptions,
|
options: ResolvedEncodeOptions,
|
||||||
): void {
|
): Generator<string> {
|
||||||
const header = formatHeader(values.length, { key: prefix, delimiter: options.delimiter })
|
const header = formatHeader(values.length, { key: prefix, delimiter: options.delimiter })
|
||||||
writer.push(depth, header)
|
yield indentedLine(depth, header, options.indent)
|
||||||
|
|
||||||
for (const arr of values) {
|
for (const arr of values) {
|
||||||
if (isArrayOfPrimitives(arr)) {
|
if (isArrayOfPrimitives(arr)) {
|
||||||
const arrayLine = encodeInlineArrayLine(arr, options.delimiter)
|
const arrayLine = encodeInlineArrayLine(arr, options.delimiter)
|
||||||
writer.pushListItem(depth + 1, arrayLine)
|
yield indentedListItem(depth + 1, arrayLine, options.indent)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -186,18 +201,17 @@ export function encodeInlineArrayLine(values: readonly JsonPrimitive[], delimite
|
|||||||
|
|
||||||
// #region Array of objects (tabular format)
|
// #region Array of objects (tabular format)
|
||||||
|
|
||||||
export function encodeArrayOfObjectsAsTabular(
|
export function* encodeArrayOfObjectsAsTabularLines(
|
||||||
prefix: string | undefined,
|
prefix: string | undefined,
|
||||||
rows: readonly JsonObject[],
|
rows: readonly JsonObject[],
|
||||||
header: readonly string[],
|
header: readonly string[],
|
||||||
writer: LineWriter,
|
|
||||||
depth: Depth,
|
depth: Depth,
|
||||||
options: ResolvedEncodeOptions,
|
options: ResolvedEncodeOptions,
|
||||||
): void {
|
): Generator<string> {
|
||||||
const formattedHeader = formatHeader(rows.length, { key: prefix, fields: header, delimiter: options.delimiter })
|
const formattedHeader = formatHeader(rows.length, { key: prefix, fields: header, delimiter: options.delimiter })
|
||||||
writer.push(depth, `${formattedHeader}`)
|
yield indentedLine(depth, formattedHeader, options.indent)
|
||||||
|
|
||||||
writeTabularRows(rows, header, writer, depth + 1, options)
|
yield* writeTabularRowsLines(rows, header, depth + 1, options)
|
||||||
}
|
}
|
||||||
|
|
||||||
export function extractTabularHeader(rows: readonly JsonObject[]): string[] | undefined {
|
export function extractTabularHeader(rows: readonly JsonObject[]): string[] | undefined {
|
||||||
@@ -240,17 +254,16 @@ export function isTabularArray(
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
function writeTabularRows(
|
function* writeTabularRowsLines(
|
||||||
rows: readonly JsonObject[],
|
rows: readonly JsonObject[],
|
||||||
header: readonly string[],
|
header: readonly string[],
|
||||||
writer: LineWriter,
|
|
||||||
depth: Depth,
|
depth: Depth,
|
||||||
options: ResolvedEncodeOptions,
|
options: ResolvedEncodeOptions,
|
||||||
): void {
|
): Generator<string> {
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
const values = header.map(key => row[key])
|
const values = header.map(key => row[key])
|
||||||
const joinedValue = encodeAndJoinPrimitives(values as JsonPrimitive[], options.delimiter)
|
const joinedValue = encodeAndJoinPrimitives(values as JsonPrimitive[], options.delimiter)
|
||||||
writer.push(depth, joinedValue)
|
yield indentedLine(depth, joinedValue, options.indent)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -258,24 +271,27 @@ function writeTabularRows(
|
|||||||
|
|
||||||
// #region Array of objects (expanded format)
|
// #region Array of objects (expanded format)
|
||||||
|
|
||||||
export function encodeMixedArrayAsListItems(
|
export function* encodeMixedArrayAsListItemsLines(
|
||||||
prefix: string | undefined,
|
prefix: string | undefined,
|
||||||
items: readonly JsonValue[],
|
items: readonly JsonValue[],
|
||||||
writer: LineWriter,
|
|
||||||
depth: Depth,
|
depth: Depth,
|
||||||
options: ResolvedEncodeOptions,
|
options: ResolvedEncodeOptions,
|
||||||
): void {
|
): Generator<string> {
|
||||||
const header = formatHeader(items.length, { key: prefix, delimiter: options.delimiter })
|
const header = formatHeader(items.length, { key: prefix, delimiter: options.delimiter })
|
||||||
writer.push(depth, header)
|
yield indentedLine(depth, header, options.indent)
|
||||||
|
|
||||||
for (const item of items) {
|
for (const item of items) {
|
||||||
encodeListItemValue(item, writer, depth + 1, options)
|
yield* encodeListItemValueLines(item, depth + 1, options)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function encodeObjectAsListItem(obj: JsonObject, writer: LineWriter, depth: Depth, options: ResolvedEncodeOptions): void {
|
export function* encodeObjectAsListItemLines(
|
||||||
|
obj: JsonObject,
|
||||||
|
depth: Depth,
|
||||||
|
options: ResolvedEncodeOptions,
|
||||||
|
): Generator<string> {
|
||||||
if (isEmptyObject(obj)) {
|
if (isEmptyObject(obj)) {
|
||||||
writer.push(depth, LIST_ITEM_MARKER)
|
yield indentedLine(depth, LIST_ITEM_MARKER, options.indent)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -284,13 +300,13 @@ export function encodeObjectAsListItem(obj: JsonObject, writer: LineWriter, dept
|
|||||||
const encodedKey = encodeKey(firstKey)
|
const encodedKey = encodeKey(firstKey)
|
||||||
|
|
||||||
if (isJsonPrimitive(firstValue)) {
|
if (isJsonPrimitive(firstValue)) {
|
||||||
writer.pushListItem(depth, `${encodedKey}: ${encodePrimitive(firstValue, options.delimiter)}`)
|
yield indentedListItem(depth, `${encodedKey}: ${encodePrimitive(firstValue, options.delimiter)}`, options.indent)
|
||||||
}
|
}
|
||||||
else if (isJsonArray(firstValue)) {
|
else if (isJsonArray(firstValue)) {
|
||||||
if (isArrayOfPrimitives(firstValue)) {
|
if (isArrayOfPrimitives(firstValue)) {
|
||||||
// Inline format for primitive arrays
|
// Inline format for primitive arrays
|
||||||
const arrayPropertyLine = encodeInlineArrayLine(firstValue, options.delimiter, firstKey)
|
const arrayPropertyLine = encodeInlineArrayLine(firstValue, options.delimiter, firstKey)
|
||||||
writer.pushListItem(depth, arrayPropertyLine)
|
yield indentedListItem(depth, arrayPropertyLine, options.indent)
|
||||||
}
|
}
|
||||||
else if (isArrayOfObjects(firstValue)) {
|
else if (isArrayOfObjects(firstValue)) {
|
||||||
// Check if array of objects can use tabular format
|
// Check if array of objects can use tabular format
|
||||||
@@ -298,38 +314,38 @@ export function encodeObjectAsListItem(obj: JsonObject, writer: LineWriter, dept
|
|||||||
if (header) {
|
if (header) {
|
||||||
// Tabular format for uniform arrays of objects
|
// Tabular format for uniform arrays of objects
|
||||||
const formattedHeader = formatHeader(firstValue.length, { key: firstKey, fields: header, delimiter: options.delimiter })
|
const formattedHeader = formatHeader(firstValue.length, { key: firstKey, fields: header, delimiter: options.delimiter })
|
||||||
writer.pushListItem(depth, formattedHeader)
|
yield indentedListItem(depth, formattedHeader, options.indent)
|
||||||
writeTabularRows(firstValue, header, writer, depth + 1, options)
|
yield* writeTabularRowsLines(firstValue, header, depth + 1, options)
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
// Fall back to list format for non-uniform arrays of objects
|
// Fall back to list format for non-uniform arrays of objects
|
||||||
writer.pushListItem(depth, `${encodedKey}[${firstValue.length}]:`)
|
yield indentedListItem(depth, `${encodedKey}[${firstValue.length}]:`, options.indent)
|
||||||
for (const item of firstValue) {
|
for (const item of firstValue) {
|
||||||
encodeObjectAsListItem(item, writer, depth + 1, options)
|
yield* encodeObjectAsListItemLines(item, depth + 1, options)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
// Complex arrays on separate lines (array of arrays, etc.)
|
// Complex arrays on separate lines (array of arrays, etc.)
|
||||||
writer.pushListItem(depth, `${encodedKey}[${firstValue.length}]:`)
|
yield indentedListItem(depth, `${encodedKey}[${firstValue.length}]:`, options.indent)
|
||||||
|
|
||||||
// Encode array contents at depth + 1
|
// Encode array contents at depth + 1
|
||||||
for (const item of firstValue) {
|
for (const item of firstValue) {
|
||||||
encodeListItemValue(item, writer, depth + 1, options)
|
yield* encodeListItemValueLines(item, depth + 1, options)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if (isJsonObject(firstValue)) {
|
else if (isJsonObject(firstValue)) {
|
||||||
writer.pushListItem(depth, `${encodedKey}:`)
|
yield indentedListItem(depth, `${encodedKey}:`, options.indent)
|
||||||
if (!isEmptyObject(firstValue)) {
|
if (!isEmptyObject(firstValue)) {
|
||||||
encodeObject(firstValue, writer, depth + 2, options)
|
yield* encodeObjectLines(firstValue, depth + 2, options)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remaining entries on indented lines
|
// Remaining entries on indented lines
|
||||||
for (let i = 1; i < entries.length; i++) {
|
for (let i = 1; i < entries.length; i++) {
|
||||||
const [key, value] = entries[i]!
|
const [key, value] = entries[i]!
|
||||||
encodeKeyValuePair(key, value, writer, depth + 1, options)
|
yield* encodeKeyValuePairLines(key, value, depth + 1, options)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -337,22 +353,34 @@ export function encodeObjectAsListItem(obj: JsonObject, writer: LineWriter, dept
|
|||||||
|
|
||||||
// #region List item encoding helpers
|
// #region List item encoding helpers
|
||||||
|
|
||||||
function encodeListItemValue(
|
function* encodeListItemValueLines(
|
||||||
value: JsonValue,
|
value: JsonValue,
|
||||||
writer: LineWriter,
|
|
||||||
depth: Depth,
|
depth: Depth,
|
||||||
options: ResolvedEncodeOptions,
|
options: ResolvedEncodeOptions,
|
||||||
): void {
|
): Generator<string> {
|
||||||
if (isJsonPrimitive(value)) {
|
if (isJsonPrimitive(value)) {
|
||||||
writer.pushListItem(depth, encodePrimitive(value, options.delimiter))
|
yield indentedListItem(depth, encodePrimitive(value, options.delimiter), options.indent)
|
||||||
}
|
}
|
||||||
else if (isJsonArray(value) && isArrayOfPrimitives(value)) {
|
else if (isJsonArray(value) && isArrayOfPrimitives(value)) {
|
||||||
const arrayLine = encodeInlineArrayLine(value, options.delimiter)
|
const arrayLine = encodeInlineArrayLine(value, options.delimiter)
|
||||||
writer.pushListItem(depth, arrayLine)
|
yield indentedListItem(depth, arrayLine, options.indent)
|
||||||
}
|
}
|
||||||
else if (isJsonObject(value)) {
|
else if (isJsonObject(value)) {
|
||||||
encodeObjectAsListItem(value, writer, depth, options)
|
yield* encodeObjectAsListItemLines(value, depth, options)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// #endregion
|
// #endregion
|
||||||
|
|
||||||
|
// #region Indentation helpers
|
||||||
|
|
||||||
|
function indentedLine(depth: Depth, content: string, indentSize: number): string {
|
||||||
|
const indentation = ' '.repeat(indentSize * depth)
|
||||||
|
return indentation + content
|
||||||
|
}
|
||||||
|
|
||||||
|
function indentedListItem(depth: Depth, content: string, indentSize: number): string {
|
||||||
|
return indentedLine(depth, LIST_ITEM_PREFIX + content, indentSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
// #endregion
|
||||||
|
|||||||
@@ -1,24 +0,0 @@
|
|||||||
import type { Depth } from '../types'
|
|
||||||
import { LIST_ITEM_PREFIX } from '../constants'
|
|
||||||
|
|
||||||
export class LineWriter {
|
|
||||||
private readonly lines: string[] = []
|
|
||||||
private readonly indentationString: string
|
|
||||||
|
|
||||||
constructor(indentSize: number) {
|
|
||||||
this.indentationString = ' '.repeat(indentSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
push(depth: Depth, content: string): void {
|
|
||||||
const indent = this.indentationString.repeat(depth)
|
|
||||||
this.lines.push(indent + content)
|
|
||||||
}
|
|
||||||
|
|
||||||
pushListItem(depth: Depth, content: string): void {
|
|
||||||
this.push(depth, `${LIST_ITEM_PREFIX}${content}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
toString(): string {
|
|
||||||
return this.lines.join('\n')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -3,7 +3,7 @@ import { DEFAULT_DELIMITER } from './constants'
|
|||||||
import { decodeValueFromLines } from './decode/decoders'
|
import { decodeValueFromLines } from './decode/decoders'
|
||||||
import { expandPathsSafe } from './decode/expand'
|
import { expandPathsSafe } from './decode/expand'
|
||||||
import { LineCursor, toParsedLines } from './decode/scanner'
|
import { LineCursor, toParsedLines } from './decode/scanner'
|
||||||
import { encodeValue } from './encode/encoders'
|
import { encodeJsonValue } from './encode/encoders'
|
||||||
import { normalizeValue } from './encode/normalize'
|
import { normalizeValue } from './encode/normalize'
|
||||||
|
|
||||||
export { DEFAULT_DELIMITER, DELIMITERS } from './constants'
|
export { DEFAULT_DELIMITER, DELIMITERS } from './constants'
|
||||||
@@ -20,6 +20,36 @@ export type {
|
|||||||
ResolvedEncodeOptions,
|
ResolvedEncodeOptions,
|
||||||
} from './types'
|
} from './types'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Encodes a JavaScript value into TOON format as a sequence of lines.
|
||||||
|
*
|
||||||
|
* This function yields TOON lines one at a time without building the full string,
|
||||||
|
* making it suitable for streaming large outputs to files, HTTP responses, or process stdout.
|
||||||
|
*
|
||||||
|
* @param input - Any JavaScript value (objects, arrays, primitives)
|
||||||
|
* @param options - Optional encoding configuration
|
||||||
|
* @returns Iterable of TOON lines (without trailing newlines)
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* ```ts
|
||||||
|
* // Stream to stdout
|
||||||
|
* for (const line of encodeLines({ name: 'Alice', age: 30 })) {
|
||||||
|
* console.log(line)
|
||||||
|
* }
|
||||||
|
*
|
||||||
|
* // Collect to array
|
||||||
|
* const lines = Array.from(encodeLines(data))
|
||||||
|
*
|
||||||
|
* // Equivalent to encode()
|
||||||
|
* const toonString = Array.from(encodeLines(data, options)).join('\n')
|
||||||
|
* ```
|
||||||
|
*/
|
||||||
|
export function encodeLines(input: unknown, options?: EncodeOptions): Iterable<string> {
|
||||||
|
const normalizedValue = normalizeValue(input)
|
||||||
|
const resolvedOptions = resolveOptions(options)
|
||||||
|
return encodeJsonValue(normalizedValue, resolvedOptions, 0)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Encodes a JavaScript value into TOON format string.
|
* Encodes a JavaScript value into TOON format string.
|
||||||
*
|
*
|
||||||
@@ -42,9 +72,7 @@ export type {
|
|||||||
* ```
|
* ```
|
||||||
*/
|
*/
|
||||||
export function encode(input: unknown, options?: EncodeOptions): string {
|
export function encode(input: unknown, options?: EncodeOptions): string {
|
||||||
const normalizedValue = normalizeValue(input)
|
return Array.from(encodeLines(input, options)).join('\n')
|
||||||
const resolvedOptions = resolveOptions(options)
|
|
||||||
return encodeValue(normalizedValue, resolvedOptions)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
56
packages/toon/test/encodeLines.test.ts
Normal file
56
packages/toon/test/encodeLines.test.ts
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
import { describe, expect, it } from 'vitest'
|
||||||
|
import { encodeLines } from '../src/index'
|
||||||
|
|
||||||
|
describe('encodeLines', () => {
|
||||||
|
it('should yield lines without newline characters', () => {
|
||||||
|
const value = { name: 'Alice', age: 30, city: 'Paris' }
|
||||||
|
const lines = Array.from(encodeLines(value))
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
expect(line).not.toContain('\n')
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should yield zero lines for empty object', () => {
|
||||||
|
const lines = Array.from(encodeLines({}))
|
||||||
|
|
||||||
|
expect(lines.length).toBe(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should be iterable with for-of loop', () => {
|
||||||
|
const value = { x: 10, y: 20 }
|
||||||
|
const collectedLines: string[] = []
|
||||||
|
|
||||||
|
for (const line of encodeLines(value)) {
|
||||||
|
collectedLines.push(line)
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(collectedLines.length).toBe(2)
|
||||||
|
expect(collectedLines[0]).toBe('x: 10')
|
||||||
|
expect(collectedLines[1]).toBe('y: 20')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should not have trailing spaces in lines', () => {
|
||||||
|
const value = {
|
||||||
|
user: {
|
||||||
|
name: 'Alice',
|
||||||
|
tags: ['a', 'b'],
|
||||||
|
nested: {
|
||||||
|
deep: 'value',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
const lines = Array.from(encodeLines(value))
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
expect(line).not.toMatch(/\s$/)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should yield correct number of lines', () => {
|
||||||
|
const value = { a: 1, b: 2, c: 3 }
|
||||||
|
const lines = Array.from(encodeLines(value))
|
||||||
|
|
||||||
|
expect(lines.length).toBe(3)
|
||||||
|
})
|
||||||
|
})
|
||||||
Reference in New Issue
Block a user