mirror of
https://github.com/voson-wang/toon.git
synced 2026-01-29 15:24:10 +08:00
feat!: publish to @toon-format/toon and @toon-format/cli
This commit is contained in:
3
packages/cli/bin/toon.mjs
Executable file
3
packages/cli/bin/toon.mjs
Executable file
@@ -0,0 +1,3 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict'
|
||||
import('../dist/index.js')
|
||||
41
packages/cli/package.json
Normal file
41
packages/cli/package.json
Normal file
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"name": "@toon-format/cli",
|
||||
"type": "module",
|
||||
"version": "0.6.0",
|
||||
"packageManager": "pnpm@10.19.0",
|
||||
"description": "CLI for encoding and decoding Token-Oriented Object Notation (TOON)",
|
||||
"author": "Johann Schopplich <hello@johannschopplich.com>",
|
||||
"license": "MIT",
|
||||
"homepage": "https://toonformat.dev",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/johannschopplich/toon.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/johannschopplich/toon/issues"
|
||||
},
|
||||
"sideEffects": false,
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"types": "./dist/index.d.ts",
|
||||
"bin": {
|
||||
"toon": "bin/toon.mjs"
|
||||
},
|
||||
"files": [
|
||||
"bin",
|
||||
"dist"
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "tsx ./src/index.ts",
|
||||
"build": "tsdown"
|
||||
},
|
||||
"dependencies": {
|
||||
"citty": "^0.1.6",
|
||||
"consola": "^3.4.2",
|
||||
"tokenx": "^1.2.0"
|
||||
}
|
||||
}
|
||||
216
packages/cli/src/index.ts
Normal file
216
packages/cli/src/index.ts
Normal file
@@ -0,0 +1,216 @@
|
||||
import type { DecodeOptions, Delimiter, EncodeOptions } from '../../toon/src'
|
||||
import * as fsp from 'node:fs/promises'
|
||||
import * as path from 'node:path'
|
||||
import process from 'node:process'
|
||||
import { defineCommand, runMain } from 'citty'
|
||||
import { consola } from 'consola'
|
||||
import { estimateTokenCount } from 'tokenx'
|
||||
import { name, version } from '../../toon/package.json' with { type: 'json' }
|
||||
import { decode, DEFAULT_DELIMITER, DELIMITERS, encode } from '../../toon/src'
|
||||
|
||||
const main = defineCommand({
|
||||
meta: {
|
||||
name,
|
||||
description: 'TOON CLI — Convert between JSON and TOON formats',
|
||||
version,
|
||||
},
|
||||
args: {
|
||||
input: {
|
||||
type: 'positional',
|
||||
description: 'Input file path',
|
||||
required: true,
|
||||
},
|
||||
output: {
|
||||
type: 'string',
|
||||
description: 'Output file path',
|
||||
alias: 'o',
|
||||
},
|
||||
encode: {
|
||||
type: 'boolean',
|
||||
description: 'Encode JSON to TOON (auto-detected by default)',
|
||||
alias: 'e',
|
||||
},
|
||||
decode: {
|
||||
type: 'boolean',
|
||||
description: 'Decode TOON to JSON (auto-detected by default)',
|
||||
alias: 'd',
|
||||
},
|
||||
delimiter: {
|
||||
type: 'string',
|
||||
description: 'Delimiter for arrays: comma (,), tab (\\t), or pipe (|)',
|
||||
default: ',',
|
||||
},
|
||||
indent: {
|
||||
type: 'string',
|
||||
description: 'Indentation size',
|
||||
default: '2',
|
||||
},
|
||||
lengthMarker: {
|
||||
type: 'boolean',
|
||||
description: 'Use length marker (#) for arrays',
|
||||
default: false,
|
||||
},
|
||||
strict: {
|
||||
type: 'boolean',
|
||||
description: 'Enable strict mode for decoding',
|
||||
default: true,
|
||||
},
|
||||
stats: {
|
||||
type: 'boolean',
|
||||
description: 'Show token statistics',
|
||||
default: false,
|
||||
},
|
||||
},
|
||||
async run({ args }) {
|
||||
const input = args.input || args._[0]
|
||||
if (!input) {
|
||||
throw new Error('Input file path is required')
|
||||
}
|
||||
|
||||
const inputPath = path.resolve(input)
|
||||
const outputPath = args.output ? path.resolve(args.output) : undefined
|
||||
|
||||
// Parse and validate indent
|
||||
const indent = Number.parseInt(args.indent || '2', 10)
|
||||
if (Number.isNaN(indent) || indent < 0) {
|
||||
throw new Error(`Invalid indent value: ${args.indent}`)
|
||||
}
|
||||
|
||||
// Validate delimiter
|
||||
const delimiter = args.delimiter || DEFAULT_DELIMITER
|
||||
if (!(Object.values(DELIMITERS)).includes(delimiter as Delimiter)) {
|
||||
throw new Error(`Invalid delimiter "${delimiter}". Valid delimiters are: comma (,), tab (\\t), pipe (|)`)
|
||||
}
|
||||
|
||||
const mode = detectMode(inputPath, args.encode, args.decode)
|
||||
|
||||
try {
|
||||
if (mode === 'encode') {
|
||||
await encodeToToon({
|
||||
input: inputPath,
|
||||
output: outputPath,
|
||||
delimiter: delimiter as Delimiter,
|
||||
indent,
|
||||
lengthMarker: args.lengthMarker === true ? '#' : false,
|
||||
printStats: args.stats === true,
|
||||
})
|
||||
}
|
||||
else {
|
||||
await decodeToJson({
|
||||
input: inputPath,
|
||||
output: outputPath,
|
||||
indent,
|
||||
strict: args.strict !== false,
|
||||
})
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
consola.error(error)
|
||||
process.exit(1)
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
function detectMode(
|
||||
inputFile: string,
|
||||
encodeFlag?: boolean,
|
||||
decodeFlag?: boolean,
|
||||
): 'encode' | 'decode' {
|
||||
// Explicit flags take precedence
|
||||
if (encodeFlag)
|
||||
return 'encode'
|
||||
if (decodeFlag)
|
||||
return 'decode'
|
||||
|
||||
// Auto-detect based on file extension
|
||||
if (inputFile.endsWith('.json'))
|
||||
return 'encode'
|
||||
if (inputFile.endsWith('.toon'))
|
||||
return 'decode'
|
||||
|
||||
// Default to encode
|
||||
return 'encode'
|
||||
}
|
||||
|
||||
async function encodeToToon(config: {
|
||||
input: string
|
||||
output?: string
|
||||
delimiter: Delimiter
|
||||
indent: number
|
||||
lengthMarker: NonNullable<EncodeOptions['lengthMarker']>
|
||||
printStats: boolean
|
||||
}) {
|
||||
const jsonContent = await fsp.readFile(config.input, 'utf-8')
|
||||
|
||||
let data: unknown
|
||||
try {
|
||||
data = JSON.parse(jsonContent)
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Failed to parse JSON: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
|
||||
const encodeOptions: EncodeOptions = {
|
||||
delimiter: config.delimiter,
|
||||
indent: config.indent,
|
||||
lengthMarker: config.lengthMarker,
|
||||
}
|
||||
|
||||
const toonOutput = encode(data, encodeOptions)
|
||||
|
||||
if (config.output) {
|
||||
await fsp.writeFile(config.output, toonOutput, 'utf-8')
|
||||
const relativeInputPath = path.relative(process.cwd(), config.input)
|
||||
const relativeOutputPath = path.relative(process.cwd(), config.output)
|
||||
consola.success(`Encoded \`${relativeInputPath}\` → \`${relativeOutputPath}\``)
|
||||
}
|
||||
else {
|
||||
console.log(toonOutput)
|
||||
}
|
||||
|
||||
if (config.printStats) {
|
||||
const jsonTokens = estimateTokenCount(jsonContent)
|
||||
const toonTokens = estimateTokenCount(toonOutput)
|
||||
const diff = jsonTokens - toonTokens
|
||||
const percent = ((diff / jsonTokens) * 100).toFixed(1)
|
||||
|
||||
console.log()
|
||||
consola.info(`Token estimates: ~${jsonTokens} (JSON) → ~${toonTokens} (TOON)`)
|
||||
consola.success(`Saved ~${diff} tokens (-${percent}%)`)
|
||||
}
|
||||
}
|
||||
|
||||
async function decodeToJson(config: {
|
||||
input: string
|
||||
output?: string
|
||||
indent: number
|
||||
strict: boolean
|
||||
}) {
|
||||
const toonContent = await fsp.readFile(config.input, 'utf-8')
|
||||
|
||||
let data: unknown
|
||||
try {
|
||||
const decodeOptions: DecodeOptions = {
|
||||
indent: config.indent,
|
||||
strict: config.strict,
|
||||
}
|
||||
data = decode(toonContent, decodeOptions)
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error(`Failed to decode TOON: ${error instanceof Error ? error.message : String(error)}`)
|
||||
}
|
||||
|
||||
const jsonOutput = JSON.stringify(data, undefined, config.indent)
|
||||
|
||||
if (config.output) {
|
||||
await fsp.writeFile(config.output, jsonOutput, 'utf-8')
|
||||
const relativeInputPath = path.relative(process.cwd(), config.input)
|
||||
const relativeOutputPath = path.relative(process.cwd(), config.output)
|
||||
consola.success(`Decoded \`${relativeInputPath}\` → \`${relativeOutputPath}\``)
|
||||
}
|
||||
else {
|
||||
console.log(jsonOutput)
|
||||
}
|
||||
}
|
||||
|
||||
runMain(main)
|
||||
9
packages/cli/tsdown.config.ts
Normal file
9
packages/cli/tsdown.config.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
import type { UserConfig, UserConfigFn } from 'tsdown/config'
|
||||
import { defineConfig } from 'tsdown/config'
|
||||
|
||||
const config: UserConfig | UserConfigFn = defineConfig({
|
||||
entry: 'src/index.ts',
|
||||
dts: true,
|
||||
})
|
||||
|
||||
export default config
|
||||
32
packages/toon/package.json
Normal file
32
packages/toon/package.json
Normal file
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"name": "@toon-format/toon",
|
||||
"type": "module",
|
||||
"version": "0.6.0",
|
||||
"packageManager": "pnpm@10.19.0",
|
||||
"description": "Token-Oriented Object Notation (TOON) – a token-efficient JSON alternative for LLM prompts",
|
||||
"author": "Johann Schopplich <hello@johannschopplich.com>",
|
||||
"license": "MIT",
|
||||
"homepage": "https://toonformat.dev",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/johannschopplich/toon.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/johannschopplich/toon/issues"
|
||||
},
|
||||
"sideEffects": false,
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"types": "./dist/index.d.ts",
|
||||
"files": [
|
||||
"dist"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsdown",
|
||||
"test": "vitest"
|
||||
}
|
||||
}
|
||||
58
packages/toon/src/constants.ts
Normal file
58
packages/toon/src/constants.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
// #region List markers
|
||||
|
||||
export const LIST_ITEM_MARKER = '-'
|
||||
export const LIST_ITEM_PREFIX = '- '
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Structural characters
|
||||
|
||||
export const COMMA = ','
|
||||
export const COLON = ':'
|
||||
export const SPACE = ' '
|
||||
export const PIPE = '|'
|
||||
export const HASH = '#'
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Brackets and braces
|
||||
|
||||
export const OPEN_BRACKET = '['
|
||||
export const CLOSE_BRACKET = ']'
|
||||
export const OPEN_BRACE = '{'
|
||||
export const CLOSE_BRACE = '}'
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Literals
|
||||
|
||||
export const NULL_LITERAL = 'null'
|
||||
export const TRUE_LITERAL = 'true'
|
||||
export const FALSE_LITERAL = 'false'
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Escape characters
|
||||
|
||||
export const BACKSLASH = '\\'
|
||||
export const DOUBLE_QUOTE = '"'
|
||||
export const NEWLINE = '\n'
|
||||
export const CARRIAGE_RETURN = '\r'
|
||||
export const TAB = '\t'
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Delimiters
|
||||
|
||||
export const DELIMITERS = {
|
||||
comma: COMMA as ',',
|
||||
tab: TAB as '\t',
|
||||
pipe: PIPE as '|',
|
||||
} as const
|
||||
|
||||
export type DelimiterKey = keyof typeof DELIMITERS
|
||||
export type Delimiter = typeof DELIMITERS[DelimiterKey]
|
||||
|
||||
export const DEFAULT_DELIMITER: Delimiter = DELIMITERS.comma
|
||||
|
||||
// #endregion
|
||||
363
packages/toon/src/decode/decoders.ts
Normal file
363
packages/toon/src/decode/decoders.ts
Normal file
@@ -0,0 +1,363 @@
|
||||
import type { ArrayHeaderInfo, Delimiter, Depth, JsonArray, JsonObject, JsonPrimitive, JsonValue, ParsedLine, ResolvedDecodeOptions } from '../types'
|
||||
import type { LineCursor } from './scanner'
|
||||
import { COLON, DEFAULT_DELIMITER, LIST_ITEM_PREFIX } from '../constants'
|
||||
import { findClosingQuote } from '../shared/string-utils'
|
||||
import { isArrayHeaderAfterHyphen, isObjectFirstFieldAfterHyphen, mapRowValuesToPrimitives, parseArrayHeaderLine, parseDelimitedValues, parseKeyToken, parsePrimitiveToken } from './parser'
|
||||
import { assertExpectedCount, validateNoBlankLinesInRange, validateNoExtraListItems, validateNoExtraTabularRows } from './validation'
|
||||
|
||||
// #region Entry decoding
|
||||
|
||||
export function decodeValueFromLines(cursor: LineCursor, options: ResolvedDecodeOptions): JsonValue {
|
||||
const first = cursor.peek()
|
||||
if (!first) {
|
||||
throw new ReferenceError('No content to decode')
|
||||
}
|
||||
|
||||
// Check for root array
|
||||
if (isArrayHeaderAfterHyphen(first.content)) {
|
||||
const headerInfo = parseArrayHeaderLine(first.content, DEFAULT_DELIMITER)
|
||||
if (headerInfo) {
|
||||
cursor.advance() // Move past the header line
|
||||
return decodeArrayFromHeader(headerInfo.header, headerInfo.inlineValues, cursor, 0, options)
|
||||
}
|
||||
}
|
||||
|
||||
// Check for single primitive value
|
||||
if (cursor.length === 1 && !isKeyValueLine(first)) {
|
||||
return parsePrimitiveToken(first.content.trim())
|
||||
}
|
||||
|
||||
// Default to object
|
||||
return decodeObject(cursor, 0, options)
|
||||
}
|
||||
|
||||
function isKeyValueLine(line: ParsedLine): boolean {
|
||||
const content = line.content
|
||||
// Look for unquoted colon or quoted key followed by colon
|
||||
if (content.startsWith('"')) {
|
||||
// Quoted key - find the closing quote
|
||||
const closingQuoteIndex = findClosingQuote(content, 0)
|
||||
if (closingQuoteIndex === -1) {
|
||||
return false
|
||||
}
|
||||
// Check if there's a colon after the quoted key
|
||||
return closingQuoteIndex + 1 < content.length && content[closingQuoteIndex + 1] === COLON
|
||||
}
|
||||
else {
|
||||
// Unquoted key - look for first colon not inside quotes
|
||||
return content.includes(COLON)
|
||||
}
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Object decoding
|
||||
|
||||
function decodeObject(cursor: LineCursor, baseDepth: Depth, options: ResolvedDecodeOptions): JsonObject {
|
||||
const obj: JsonObject = {}
|
||||
|
||||
while (!cursor.atEnd()) {
|
||||
const line = cursor.peek()
|
||||
if (!line || line.depth < baseDepth) {
|
||||
break
|
||||
}
|
||||
|
||||
if (line.depth === baseDepth) {
|
||||
const [key, value] = decodeKeyValuePair(line, cursor, baseDepth, options)
|
||||
obj[key] = value
|
||||
}
|
||||
else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return obj
|
||||
}
|
||||
|
||||
function decodeKeyValue(
|
||||
content: string,
|
||||
cursor: LineCursor,
|
||||
baseDepth: Depth,
|
||||
options: ResolvedDecodeOptions,
|
||||
): { key: string, value: JsonValue, followDepth: Depth } {
|
||||
// Check for array header first (before parsing key)
|
||||
const arrayHeader = parseArrayHeaderLine(content, DEFAULT_DELIMITER)
|
||||
if (arrayHeader && arrayHeader.header.key) {
|
||||
const value = decodeArrayFromHeader(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options)
|
||||
// After an array, subsequent fields are at baseDepth + 1 (where array content is)
|
||||
return {
|
||||
key: arrayHeader.header.key,
|
||||
value,
|
||||
followDepth: baseDepth + 1,
|
||||
}
|
||||
}
|
||||
|
||||
// Regular key-value pair
|
||||
const { key, end } = parseKeyToken(content, 0)
|
||||
const rest = content.slice(end).trim()
|
||||
|
||||
// No value after colon - expect nested object or empty
|
||||
if (!rest) {
|
||||
const nextLine = cursor.peek()
|
||||
if (nextLine && nextLine.depth > baseDepth) {
|
||||
const nested = decodeObject(cursor, baseDepth + 1, options)
|
||||
return { key, value: nested, followDepth: baseDepth + 1 }
|
||||
}
|
||||
// Empty object
|
||||
return { key, value: {}, followDepth: baseDepth + 1 }
|
||||
}
|
||||
|
||||
// Inline primitive value
|
||||
const value = parsePrimitiveToken(rest)
|
||||
return { key, value, followDepth: baseDepth + 1 }
|
||||
}
|
||||
|
||||
function decodeKeyValuePair(
|
||||
line: ParsedLine,
|
||||
cursor: LineCursor,
|
||||
baseDepth: Depth,
|
||||
options: ResolvedDecodeOptions,
|
||||
): [key: string, value: JsonValue] {
|
||||
cursor.advance()
|
||||
const { key, value } = decodeKeyValue(line.content, cursor, baseDepth, options)
|
||||
return [key, value]
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Array decoding
|
||||
|
||||
function decodeArrayFromHeader(
|
||||
header: ArrayHeaderInfo,
|
||||
inlineValues: string | undefined,
|
||||
cursor: LineCursor,
|
||||
baseDepth: Depth,
|
||||
options: ResolvedDecodeOptions,
|
||||
): JsonArray {
|
||||
// Inline primitive array
|
||||
if (inlineValues) {
|
||||
// For inline arrays, cursor should already be advanced or will be by caller
|
||||
return decodeInlinePrimitiveArray(header, inlineValues, options)
|
||||
}
|
||||
|
||||
// For multi-line arrays (tabular or list), the cursor should already be positioned
|
||||
// at the array header line, but we haven't advanced past it yet
|
||||
|
||||
// Tabular array
|
||||
if (header.fields && header.fields.length > 0) {
|
||||
return decodeTabularArray(header, cursor, baseDepth, options)
|
||||
}
|
||||
|
||||
// List array
|
||||
return decodeListArray(header, cursor, baseDepth, options)
|
||||
}
|
||||
|
||||
function decodeInlinePrimitiveArray(
|
||||
header: ArrayHeaderInfo,
|
||||
inlineValues: string,
|
||||
options: ResolvedDecodeOptions,
|
||||
): JsonPrimitive[] {
|
||||
if (!inlineValues.trim()) {
|
||||
assertExpectedCount(0, header.length, 'inline array items', options)
|
||||
return []
|
||||
}
|
||||
|
||||
const values = parseDelimitedValues(inlineValues, header.delimiter)
|
||||
const primitives = mapRowValuesToPrimitives(values)
|
||||
|
||||
assertExpectedCount(primitives.length, header.length, 'inline array items', options)
|
||||
|
||||
return primitives
|
||||
}
|
||||
|
||||
function decodeListArray(
|
||||
header: ArrayHeaderInfo,
|
||||
cursor: LineCursor,
|
||||
baseDepth: Depth,
|
||||
options: ResolvedDecodeOptions,
|
||||
): JsonValue[] {
|
||||
const items: JsonValue[] = []
|
||||
const itemDepth = baseDepth + 1
|
||||
|
||||
// Track line range for blank line validation
|
||||
let startLine: number | undefined
|
||||
let endLine: number | undefined
|
||||
|
||||
while (!cursor.atEnd() && items.length < header.length) {
|
||||
const line = cursor.peek()
|
||||
if (!line || line.depth < itemDepth) {
|
||||
break
|
||||
}
|
||||
|
||||
if (line.depth === itemDepth && line.content.startsWith(LIST_ITEM_PREFIX)) {
|
||||
// Track first and last item line numbers
|
||||
if (startLine === undefined) {
|
||||
startLine = line.lineNumber
|
||||
}
|
||||
endLine = line.lineNumber
|
||||
|
||||
const item = decodeListItem(cursor, itemDepth, header.delimiter, options)
|
||||
items.push(item)
|
||||
|
||||
// Update endLine to the current cursor position (after item was decoded)
|
||||
const currentLine = cursor.current()
|
||||
if (currentLine) {
|
||||
endLine = currentLine.lineNumber
|
||||
}
|
||||
}
|
||||
else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
assertExpectedCount(items.length, header.length, 'list array items', options)
|
||||
|
||||
// In strict mode, check for blank lines inside the array
|
||||
if (options.strict && startLine !== undefined && endLine !== undefined) {
|
||||
validateNoBlankLinesInRange(
|
||||
startLine, // From first item line
|
||||
endLine, // To last item line
|
||||
cursor.getBlankLines(),
|
||||
options.strict,
|
||||
'list array',
|
||||
)
|
||||
}
|
||||
|
||||
// In strict mode, check for extra items
|
||||
if (options.strict) {
|
||||
validateNoExtraListItems(cursor, itemDepth, header.length)
|
||||
}
|
||||
|
||||
return items
|
||||
}
|
||||
|
||||
function decodeTabularArray(
|
||||
header: ArrayHeaderInfo,
|
||||
cursor: LineCursor,
|
||||
baseDepth: Depth,
|
||||
options: ResolvedDecodeOptions,
|
||||
): JsonObject[] {
|
||||
const objects: JsonObject[] = []
|
||||
const rowDepth = baseDepth + 1
|
||||
|
||||
// Track line range for blank line validation
|
||||
let startLine: number | undefined
|
||||
let endLine: number | undefined
|
||||
|
||||
while (!cursor.atEnd() && objects.length < header.length) {
|
||||
const line = cursor.peek()
|
||||
if (!line || line.depth < rowDepth) {
|
||||
break
|
||||
}
|
||||
|
||||
if (line.depth === rowDepth) {
|
||||
// Track first and last row line numbers
|
||||
if (startLine === undefined) {
|
||||
startLine = line.lineNumber
|
||||
}
|
||||
endLine = line.lineNumber
|
||||
|
||||
cursor.advance()
|
||||
const values = parseDelimitedValues(line.content, header.delimiter)
|
||||
assertExpectedCount(values.length, header.fields!.length, 'tabular row values', options)
|
||||
|
||||
const primitives = mapRowValuesToPrimitives(values)
|
||||
const obj: JsonObject = {}
|
||||
|
||||
for (let i = 0; i < header.fields!.length; i++) {
|
||||
obj[header.fields![i]!] = primitives[i]!
|
||||
}
|
||||
|
||||
objects.push(obj)
|
||||
}
|
||||
else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
assertExpectedCount(objects.length, header.length, 'tabular rows', options)
|
||||
|
||||
// In strict mode, check for blank lines inside the array
|
||||
if (options.strict && startLine !== undefined && endLine !== undefined) {
|
||||
validateNoBlankLinesInRange(
|
||||
startLine, // From first row line
|
||||
endLine, // To last row line
|
||||
cursor.getBlankLines(),
|
||||
options.strict,
|
||||
'tabular array',
|
||||
)
|
||||
}
|
||||
|
||||
// In strict mode, check for extra rows
|
||||
if (options.strict) {
|
||||
validateNoExtraTabularRows(cursor, rowDepth, header)
|
||||
}
|
||||
|
||||
return objects
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region List item decoding
|
||||
|
||||
function decodeListItem(
|
||||
cursor: LineCursor,
|
||||
baseDepth: Depth,
|
||||
activeDelimiter: Delimiter,
|
||||
options: ResolvedDecodeOptions,
|
||||
): JsonValue {
|
||||
const line = cursor.next()
|
||||
if (!line) {
|
||||
throw new ReferenceError('Expected list item')
|
||||
}
|
||||
|
||||
const afterHyphen = line.content.slice(LIST_ITEM_PREFIX.length)
|
||||
|
||||
// Check for array header after hyphen
|
||||
if (isArrayHeaderAfterHyphen(afterHyphen)) {
|
||||
const arrayHeader = parseArrayHeaderLine(afterHyphen, DEFAULT_DELIMITER)
|
||||
if (arrayHeader) {
|
||||
return decodeArrayFromHeader(arrayHeader.header, arrayHeader.inlineValues, cursor, baseDepth, options)
|
||||
}
|
||||
}
|
||||
|
||||
// Check for object first field after hyphen
|
||||
if (isObjectFirstFieldAfterHyphen(afterHyphen)) {
|
||||
return decodeObjectFromListItem(line, cursor, baseDepth, options)
|
||||
}
|
||||
|
||||
// Primitive value
|
||||
return parsePrimitiveToken(afterHyphen)
|
||||
}
|
||||
|
||||
function decodeObjectFromListItem(
|
||||
firstLine: ParsedLine,
|
||||
cursor: LineCursor,
|
||||
baseDepth: Depth,
|
||||
options: ResolvedDecodeOptions,
|
||||
): JsonObject {
|
||||
const afterHyphen = firstLine.content.slice(LIST_ITEM_PREFIX.length)
|
||||
const { key, value, followDepth } = decodeKeyValue(afterHyphen, cursor, baseDepth, options)
|
||||
|
||||
const obj: JsonObject = { [key]: value }
|
||||
|
||||
// Read subsequent fields
|
||||
while (!cursor.atEnd()) {
|
||||
const line = cursor.peek()
|
||||
if (!line || line.depth < followDepth) {
|
||||
break
|
||||
}
|
||||
|
||||
if (line.depth === followDepth && !line.content.startsWith(LIST_ITEM_PREFIX)) {
|
||||
const [k, v] = decodeKeyValuePair(line, cursor, followDepth, options)
|
||||
obj[k] = v
|
||||
}
|
||||
else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return obj
|
||||
}
|
||||
|
||||
// #endregion
|
||||
288
packages/toon/src/decode/parser.ts
Normal file
288
packages/toon/src/decode/parser.ts
Normal file
@@ -0,0 +1,288 @@
|
||||
import type { ArrayHeaderInfo, Delimiter, JsonPrimitive } from '../types'
|
||||
import { BACKSLASH, CLOSE_BRACE, CLOSE_BRACKET, COLON, DELIMITERS, DOUBLE_QUOTE, FALSE_LITERAL, HASH, NULL_LITERAL, OPEN_BRACE, OPEN_BRACKET, PIPE, TAB, TRUE_LITERAL } from '../constants'
|
||||
import { isBooleanOrNullLiteral, isNumericLiteral } from '../shared/literal-utils'
|
||||
import { findClosingQuote, findUnquotedChar, unescapeString } from '../shared/string-utils'
|
||||
|
||||
// #region Array header parsing
|
||||
|
||||
export function parseArrayHeaderLine(
|
||||
content: string,
|
||||
defaultDelimiter: Delimiter,
|
||||
): { header: ArrayHeaderInfo, inlineValues?: string } | undefined {
|
||||
// Don't match if the line starts with a quote (it's a quoted key, not an array)
|
||||
if (content.trimStart().startsWith(DOUBLE_QUOTE)) {
|
||||
return
|
||||
}
|
||||
|
||||
// Find the bracket segment first
|
||||
const bracketStart = content.indexOf(OPEN_BRACKET)
|
||||
if (bracketStart === -1) {
|
||||
return
|
||||
}
|
||||
|
||||
const bracketEnd = content.indexOf(CLOSE_BRACKET, bracketStart)
|
||||
if (bracketEnd === -1) {
|
||||
return
|
||||
}
|
||||
|
||||
// Find the colon that comes after all brackets and braces
|
||||
let colonIndex = bracketEnd + 1
|
||||
let braceEnd = colonIndex
|
||||
|
||||
// Check for fields segment (braces come after bracket)
|
||||
const braceStart = content.indexOf(OPEN_BRACE, bracketEnd)
|
||||
if (braceStart !== -1 && braceStart < content.indexOf(COLON, bracketEnd)) {
|
||||
const foundBraceEnd = content.indexOf(CLOSE_BRACE, braceStart)
|
||||
if (foundBraceEnd !== -1) {
|
||||
braceEnd = foundBraceEnd + 1
|
||||
}
|
||||
}
|
||||
|
||||
// Now find colon after brackets and braces
|
||||
colonIndex = content.indexOf(COLON, Math.max(bracketEnd, braceEnd))
|
||||
if (colonIndex === -1) {
|
||||
return
|
||||
}
|
||||
|
||||
const key = bracketStart > 0 ? content.slice(0, bracketStart) : undefined
|
||||
const afterColon = content.slice(colonIndex + 1).trim()
|
||||
|
||||
const bracketContent = content.slice(bracketStart + 1, bracketEnd)
|
||||
|
||||
// Try to parse bracket segment
|
||||
let parsedBracket: ReturnType<typeof parseBracketSegment>
|
||||
try {
|
||||
parsedBracket = parseBracketSegment(bracketContent, defaultDelimiter)
|
||||
}
|
||||
catch {
|
||||
return
|
||||
}
|
||||
|
||||
const { length, delimiter, hasLengthMarker } = parsedBracket
|
||||
|
||||
// Check for fields segment
|
||||
let fields: string[] | undefined
|
||||
if (braceStart !== -1 && braceStart < colonIndex) {
|
||||
const foundBraceEnd = content.indexOf(CLOSE_BRACE, braceStart)
|
||||
if (foundBraceEnd !== -1 && foundBraceEnd < colonIndex) {
|
||||
const fieldsContent = content.slice(braceStart + 1, foundBraceEnd)
|
||||
fields = parseDelimitedValues(fieldsContent, delimiter).map(field => parseStringLiteral(field.trim()))
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
header: {
|
||||
key,
|
||||
length,
|
||||
delimiter,
|
||||
fields,
|
||||
hasLengthMarker,
|
||||
},
|
||||
inlineValues: afterColon || undefined,
|
||||
}
|
||||
}
|
||||
|
||||
export function parseBracketSegment(
|
||||
seg: string,
|
||||
defaultDelimiter: Delimiter,
|
||||
): { length: number, delimiter: Delimiter, hasLengthMarker: boolean } {
|
||||
let hasLengthMarker = false
|
||||
let content = seg
|
||||
|
||||
// Check for length marker
|
||||
if (content.startsWith(HASH)) {
|
||||
hasLengthMarker = true
|
||||
content = content.slice(1)
|
||||
}
|
||||
|
||||
// Check for delimiter suffix
|
||||
let delimiter = defaultDelimiter
|
||||
if (content.endsWith(TAB)) {
|
||||
delimiter = DELIMITERS.tab
|
||||
content = content.slice(0, -1)
|
||||
}
|
||||
else if (content.endsWith(PIPE)) {
|
||||
delimiter = DELIMITERS.pipe
|
||||
content = content.slice(0, -1)
|
||||
}
|
||||
|
||||
const length = Number.parseInt(content, 10)
|
||||
if (Number.isNaN(length)) {
|
||||
throw new TypeError(`Invalid array length: ${seg}`)
|
||||
}
|
||||
|
||||
return { length, delimiter, hasLengthMarker }
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Delimited value parsing
|
||||
|
||||
export function parseDelimitedValues(input: string, delimiter: Delimiter): string[] {
|
||||
const values: string[] = []
|
||||
let current = ''
|
||||
let inQuotes = false
|
||||
let i = 0
|
||||
|
||||
while (i < input.length) {
|
||||
const char = input[i]
|
||||
|
||||
if (char === BACKSLASH && i + 1 < input.length && inQuotes) {
|
||||
// Escape sequence in quoted string
|
||||
current += char + input[i + 1]
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
|
||||
if (char === DOUBLE_QUOTE) {
|
||||
inQuotes = !inQuotes
|
||||
current += char
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if (char === delimiter && !inQuotes) {
|
||||
values.push(current.trim())
|
||||
current = ''
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
current += char
|
||||
i++
|
||||
}
|
||||
|
||||
// Add last value
|
||||
if (current || values.length > 0) {
|
||||
values.push(current.trim())
|
||||
}
|
||||
|
||||
return values
|
||||
}
|
||||
|
||||
export function mapRowValuesToPrimitives(values: string[]): JsonPrimitive[] {
|
||||
return values.map(v => parsePrimitiveToken(v))
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Primitive and key parsing
|
||||
|
||||
export function parsePrimitiveToken(token: string): JsonPrimitive {
|
||||
const trimmed = token.trim()
|
||||
|
||||
// Empty token
|
||||
if (!trimmed) {
|
||||
return ''
|
||||
}
|
||||
|
||||
// Quoted string (if starts with quote, it MUST be properly quoted)
|
||||
if (trimmed.startsWith(DOUBLE_QUOTE)) {
|
||||
return parseStringLiteral(trimmed)
|
||||
}
|
||||
|
||||
// Boolean or null literals
|
||||
if (isBooleanOrNullLiteral(trimmed)) {
|
||||
if (trimmed === TRUE_LITERAL)
|
||||
return true
|
||||
if (trimmed === FALSE_LITERAL)
|
||||
return false
|
||||
if (trimmed === NULL_LITERAL)
|
||||
return null
|
||||
}
|
||||
|
||||
// Numeric literal
|
||||
if (isNumericLiteral(trimmed)) {
|
||||
return Number.parseFloat(trimmed)
|
||||
}
|
||||
|
||||
// Unquoted string
|
||||
return trimmed
|
||||
}
|
||||
|
||||
export function parseStringLiteral(token: string): string {
|
||||
const trimmed = token.trim()
|
||||
|
||||
if (trimmed.startsWith(DOUBLE_QUOTE)) {
|
||||
// Find the closing quote, accounting for escaped quotes
|
||||
const closingQuoteIndex = findClosingQuote(trimmed, 0)
|
||||
|
||||
if (closingQuoteIndex === -1) {
|
||||
// No closing quote was found
|
||||
throw new SyntaxError('Unterminated string: missing closing quote')
|
||||
}
|
||||
|
||||
if (closingQuoteIndex !== trimmed.length - 1) {
|
||||
throw new SyntaxError('Unexpected characters after closing quote')
|
||||
}
|
||||
|
||||
const content = trimmed.slice(1, closingQuoteIndex)
|
||||
return unescapeString(content)
|
||||
}
|
||||
|
||||
return trimmed
|
||||
}
|
||||
|
||||
export function parseUnquotedKey(content: string, start: number): { key: string, end: number } {
|
||||
let end = start
|
||||
while (end < content.length && content[end] !== COLON) {
|
||||
end++
|
||||
}
|
||||
|
||||
// Validate that a colon was found
|
||||
if (end >= content.length || content[end] !== COLON) {
|
||||
throw new SyntaxError('Missing colon after key')
|
||||
}
|
||||
|
||||
const key = content.slice(start, end).trim()
|
||||
|
||||
// Skip the colon
|
||||
end++
|
||||
|
||||
return { key, end }
|
||||
}
|
||||
|
||||
export function parseQuotedKey(content: string, start: number): { key: string, end: number } {
|
||||
// Find the closing quote, accounting for escaped quotes
|
||||
const closingQuoteIndex = findClosingQuote(content, start)
|
||||
|
||||
if (closingQuoteIndex === -1) {
|
||||
throw new SyntaxError('Unterminated quoted key')
|
||||
}
|
||||
|
||||
// Extract and unescape the key content
|
||||
const keyContent = content.slice(start + 1, closingQuoteIndex)
|
||||
const key = unescapeString(keyContent)
|
||||
let end = closingQuoteIndex + 1
|
||||
|
||||
// Validate and skip colon after quoted key
|
||||
if (end >= content.length || content[end] !== COLON) {
|
||||
throw new SyntaxError('Missing colon after key')
|
||||
}
|
||||
end++
|
||||
|
||||
return { key, end }
|
||||
}
|
||||
|
||||
export function parseKeyToken(content: string, start: number): { key: string, end: number } {
|
||||
if (content[start] === DOUBLE_QUOTE) {
|
||||
return parseQuotedKey(content, start)
|
||||
}
|
||||
else {
|
||||
return parseUnquotedKey(content, start)
|
||||
}
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Array content detection helpers
|
||||
|
||||
export function isArrayHeaderAfterHyphen(content: string): boolean {
|
||||
return content.trim().startsWith(OPEN_BRACKET) && findUnquotedChar(content, COLON) !== -1
|
||||
}
|
||||
|
||||
export function isObjectFirstFieldAfterHyphen(content: string): boolean {
|
||||
return findUnquotedChar(content, COLON) !== -1
|
||||
}
|
||||
|
||||
// #endregion
|
||||
119
packages/toon/src/decode/scanner.ts
Normal file
119
packages/toon/src/decode/scanner.ts
Normal file
@@ -0,0 +1,119 @@
|
||||
import type { BlankLineInfo, Depth, ParsedLine } from '../types'
|
||||
import { SPACE, TAB } from '../constants'
|
||||
|
||||
export interface ScanResult {
|
||||
lines: ParsedLine[]
|
||||
blankLines: BlankLineInfo[]
|
||||
}
|
||||
|
||||
export class LineCursor {
|
||||
private lines: ParsedLine[]
|
||||
private index: number
|
||||
private blankLines: BlankLineInfo[]
|
||||
|
||||
constructor(lines: ParsedLine[], blankLines: BlankLineInfo[] = []) {
|
||||
this.lines = lines
|
||||
this.index = 0
|
||||
this.blankLines = blankLines
|
||||
}
|
||||
|
||||
getBlankLines(): BlankLineInfo[] {
|
||||
return this.blankLines
|
||||
}
|
||||
|
||||
peek(): ParsedLine | undefined {
|
||||
return this.lines[this.index]
|
||||
}
|
||||
|
||||
next(): ParsedLine | undefined {
|
||||
return this.lines[this.index++]
|
||||
}
|
||||
|
||||
current(): ParsedLine | undefined {
|
||||
return this.index > 0 ? this.lines[this.index - 1] : undefined
|
||||
}
|
||||
|
||||
advance(): void {
|
||||
this.index++
|
||||
}
|
||||
|
||||
atEnd(): boolean {
|
||||
return this.index >= this.lines.length
|
||||
}
|
||||
|
||||
get length(): number {
|
||||
return this.lines.length
|
||||
}
|
||||
|
||||
peekAtDepth(targetDepth: Depth): ParsedLine | undefined {
|
||||
const line = this.peek()
|
||||
if (!line || line.depth < targetDepth) {
|
||||
return undefined
|
||||
}
|
||||
if (line.depth === targetDepth) {
|
||||
return line
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
hasMoreAtDepth(targetDepth: Depth): boolean {
|
||||
return this.peekAtDepth(targetDepth) !== undefined
|
||||
}
|
||||
}
|
||||
|
||||
export function toParsedLines(source: string, indentSize: number, strict: boolean): ScanResult {
|
||||
if (!source.trim()) {
|
||||
return { lines: [], blankLines: [] }
|
||||
}
|
||||
|
||||
const lines = source.split('\n')
|
||||
const parsed: ParsedLine[] = []
|
||||
const blankLines: BlankLineInfo[] = []
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const raw = lines[i]!
|
||||
const lineNumber = i + 1
|
||||
let indent = 0
|
||||
while (indent < raw.length && raw[indent] === SPACE) {
|
||||
indent++
|
||||
}
|
||||
|
||||
const content = raw.slice(indent)
|
||||
|
||||
// Track blank lines
|
||||
if (!content.trim()) {
|
||||
const depth = computeDepthFromIndent(indent, indentSize)
|
||||
blankLines.push({ lineNumber, indent, depth })
|
||||
continue
|
||||
}
|
||||
|
||||
const depth = computeDepthFromIndent(indent, indentSize)
|
||||
|
||||
// Strict mode validation
|
||||
if (strict) {
|
||||
// Find the full leading whitespace region (spaces and tabs)
|
||||
let wsEnd = 0
|
||||
while (wsEnd < raw.length && (raw[wsEnd] === SPACE || raw[wsEnd] === TAB)) {
|
||||
wsEnd++
|
||||
}
|
||||
|
||||
// Check for tabs in leading whitespace (before actual content)
|
||||
if (raw.slice(0, wsEnd).includes(TAB)) {
|
||||
throw new SyntaxError(`Line ${lineNumber}: Tabs are not allowed in indentation in strict mode`)
|
||||
}
|
||||
|
||||
// Check for exact multiples of indentSize
|
||||
if (indent > 0 && indent % indentSize !== 0) {
|
||||
throw new SyntaxError(`Line ${lineNumber}: Indentation must be exact multiple of ${indentSize}, but found ${indent} spaces`)
|
||||
}
|
||||
}
|
||||
|
||||
parsed.push({ raw, indent, content, depth, lineNumber })
|
||||
}
|
||||
|
||||
return { lines: parsed, blankLines }
|
||||
}
|
||||
|
||||
function computeDepthFromIndent(indentSpaces: number, indentSize: number): Depth {
|
||||
return Math.floor(indentSpaces / indentSize)
|
||||
}
|
||||
135
packages/toon/src/decode/validation.ts
Normal file
135
packages/toon/src/decode/validation.ts
Normal file
@@ -0,0 +1,135 @@
|
||||
import type { ArrayHeaderInfo, BlankLineInfo, Delimiter, Depth, ResolvedDecodeOptions } from '../types'
|
||||
import type { LineCursor } from './scanner'
|
||||
import { COLON, LIST_ITEM_PREFIX } from '../constants'
|
||||
|
||||
/**
|
||||
* Asserts that the actual count matches the expected count in strict mode.
|
||||
*
|
||||
* @param actual The actual count
|
||||
* @param expected The expected count
|
||||
* @param itemType The type of items being counted (e.g., `list array items`, `tabular rows`)
|
||||
* @param options Decode options
|
||||
* @throws RangeError if counts don't match in strict mode
|
||||
*/
|
||||
export function assertExpectedCount(
|
||||
actual: number,
|
||||
expected: number,
|
||||
itemType: string,
|
||||
options: ResolvedDecodeOptions,
|
||||
): void {
|
||||
if (options.strict && actual !== expected) {
|
||||
throw new RangeError(`Expected ${expected} ${itemType}, but got ${actual}`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that there are no extra list items beyond the expected count.
|
||||
*
|
||||
* @param cursor The line cursor
|
||||
* @param itemDepth The expected depth of items
|
||||
* @param expectedCount The expected number of items
|
||||
* @throws RangeError if extra items are found
|
||||
*/
|
||||
export function validateNoExtraListItems(
|
||||
cursor: LineCursor,
|
||||
itemDepth: Depth,
|
||||
expectedCount: number,
|
||||
): void {
|
||||
if (cursor.atEnd())
|
||||
return
|
||||
|
||||
const nextLine = cursor.peek()
|
||||
if (nextLine && nextLine.depth === itemDepth && nextLine.content.startsWith(LIST_ITEM_PREFIX)) {
|
||||
throw new RangeError(`Expected ${expectedCount} list array items, but found more`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that there are no extra tabular rows beyond the expected count.
|
||||
*
|
||||
* @param cursor The line cursor
|
||||
* @param rowDepth The expected depth of rows
|
||||
* @param header The array header info containing length and delimiter
|
||||
* @throws RangeError if extra rows are found
|
||||
*/
|
||||
export function validateNoExtraTabularRows(
|
||||
cursor: LineCursor,
|
||||
rowDepth: Depth,
|
||||
header: ArrayHeaderInfo,
|
||||
): void {
|
||||
if (cursor.atEnd())
|
||||
return
|
||||
|
||||
const nextLine = cursor.peek()
|
||||
if (
|
||||
nextLine
|
||||
&& nextLine.depth === rowDepth
|
||||
&& !nextLine.content.startsWith(LIST_ITEM_PREFIX)
|
||||
&& isDataRow(nextLine.content, header.delimiter)
|
||||
) {
|
||||
throw new RangeError(`Expected ${header.length} tabular rows, but found more`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that there are no blank lines within a specific line range and depth.
|
||||
*
|
||||
* @remarks
|
||||
* In strict mode, blank lines inside arrays/tabular rows are not allowed.
|
||||
*
|
||||
* @param startLine The starting line number (inclusive)
|
||||
* @param endLine The ending line number (inclusive)
|
||||
* @param blankLines Array of blank line information
|
||||
* @param strict Whether strict mode is enabled
|
||||
* @param context Description of the context (e.g., "list array", "tabular array")
|
||||
* @throws SyntaxError if blank lines are found in strict mode
|
||||
*/
|
||||
export function validateNoBlankLinesInRange(
|
||||
startLine: number,
|
||||
endLine: number,
|
||||
blankLines: BlankLineInfo[],
|
||||
strict: boolean,
|
||||
context: string,
|
||||
): void {
|
||||
if (!strict)
|
||||
return
|
||||
|
||||
// Find blank lines within the range
|
||||
// Note: We don't filter by depth because ANY blank line between array items is an error,
|
||||
// regardless of its indentation level
|
||||
const blanksInRange = blankLines.filter(
|
||||
blank => blank.lineNumber > startLine
|
||||
&& blank.lineNumber < endLine,
|
||||
)
|
||||
|
||||
if (blanksInRange.length > 0) {
|
||||
throw new SyntaxError(
|
||||
`Line ${blanksInRange[0]!.lineNumber}: Blank lines inside ${context} are not allowed in strict mode`,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a line represents a data row (as opposed to a key-value pair) in a tabular array.
|
||||
*
|
||||
* @param content The line content
|
||||
* @param delimiter The delimiter used in the table
|
||||
* @returns true if the line is a data row, false if it's a key-value pair
|
||||
*/
|
||||
function isDataRow(content: string, delimiter: Delimiter): boolean {
|
||||
const colonPos = content.indexOf(COLON)
|
||||
const delimiterPos = content.indexOf(delimiter)
|
||||
|
||||
// No colon = definitely a data row
|
||||
if (colonPos === -1) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Has delimiter and it comes before colon = data row
|
||||
if (delimiterPos !== -1 && delimiterPos < colonPos) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Colon before delimiter or no delimiter = key-value pair
|
||||
return false
|
||||
}
|
||||
320
packages/toon/src/encode/encoders.ts
Normal file
320
packages/toon/src/encode/encoders.ts
Normal file
@@ -0,0 +1,320 @@
|
||||
import type { Depth, JsonArray, JsonObject, JsonPrimitive, JsonValue, ResolvedEncodeOptions } from '../types'
|
||||
import { LIST_ITEM_MARKER } from '../constants'
|
||||
import { isArrayOfArrays, isArrayOfObjects, isArrayOfPrimitives, isJsonArray, isJsonObject, isJsonPrimitive } from './normalize'
|
||||
import { encodeAndJoinPrimitives, encodeKey, encodePrimitive, formatHeader } from './primitives'
|
||||
import { LineWriter } from './writer'
|
||||
|
||||
// #region Encode normalized JsonValue
|
||||
|
||||
export function encodeValue(value: JsonValue, options: ResolvedEncodeOptions): string {
|
||||
if (isJsonPrimitive(value)) {
|
||||
return encodePrimitive(value, options.delimiter)
|
||||
}
|
||||
|
||||
const writer = new LineWriter(options.indent)
|
||||
|
||||
if (isJsonArray(value)) {
|
||||
encodeArray(undefined, value, writer, 0, options)
|
||||
}
|
||||
else if (isJsonObject(value)) {
|
||||
encodeObject(value, writer, 0, options)
|
||||
}
|
||||
|
||||
return writer.toString()
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Object encoding
|
||||
|
||||
export function encodeObject(value: JsonObject, writer: LineWriter, depth: Depth, options: ResolvedEncodeOptions): void {
|
||||
const keys = Object.keys(value)
|
||||
|
||||
for (const key of keys) {
|
||||
encodeKeyValuePair(key, value[key]!, writer, depth, options)
|
||||
}
|
||||
}
|
||||
|
||||
export function encodeKeyValuePair(key: string, value: JsonValue, writer: LineWriter, depth: Depth, options: ResolvedEncodeOptions): void {
|
||||
const encodedKey = encodeKey(key)
|
||||
|
||||
if (isJsonPrimitive(value)) {
|
||||
writer.push(depth, `${encodedKey}: ${encodePrimitive(value, options.delimiter)}`)
|
||||
}
|
||||
else if (isJsonArray(value)) {
|
||||
encodeArray(key, value, writer, depth, options)
|
||||
}
|
||||
else if (isJsonObject(value)) {
|
||||
const nestedKeys = Object.keys(value)
|
||||
if (nestedKeys.length === 0) {
|
||||
// Empty object
|
||||
writer.push(depth, `${encodedKey}:`)
|
||||
}
|
||||
else {
|
||||
writer.push(depth, `${encodedKey}:`)
|
||||
encodeObject(value, writer, depth + 1, options)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Array encoding
|
||||
|
||||
export function encodeArray(
|
||||
key: string | undefined,
|
||||
value: JsonArray,
|
||||
writer: LineWriter,
|
||||
depth: Depth,
|
||||
options: ResolvedEncodeOptions,
|
||||
): void {
|
||||
if (value.length === 0) {
|
||||
const header = formatHeader(0, { key, delimiter: options.delimiter, lengthMarker: options.lengthMarker })
|
||||
writer.push(depth, header)
|
||||
return
|
||||
}
|
||||
|
||||
// Primitive array
|
||||
if (isArrayOfPrimitives(value)) {
|
||||
const formatted = encodeInlineArrayLine(value, options.delimiter, key, options.lengthMarker)
|
||||
writer.push(depth, formatted)
|
||||
return
|
||||
}
|
||||
|
||||
// Array of arrays (all primitives)
|
||||
if (isArrayOfArrays(value)) {
|
||||
const allPrimitiveArrays = value.every(arr => isArrayOfPrimitives(arr))
|
||||
if (allPrimitiveArrays) {
|
||||
encodeArrayOfArraysAsListItems(key, value, writer, depth, options)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Array of objects
|
||||
if (isArrayOfObjects(value)) {
|
||||
const header = extractTabularHeader(value)
|
||||
if (header) {
|
||||
encodeArrayOfObjectsAsTabular(key, value, header, writer, depth, options)
|
||||
}
|
||||
else {
|
||||
encodeMixedArrayAsListItems(key, value, writer, depth, options)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Mixed array: fallback to expanded format
|
||||
encodeMixedArrayAsListItems(key, value, writer, depth, options)
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Array of arrays (expanded format)
|
||||
|
||||
export function encodeArrayOfArraysAsListItems(
|
||||
prefix: string | undefined,
|
||||
values: readonly JsonArray[],
|
||||
writer: LineWriter,
|
||||
depth: Depth,
|
||||
options: ResolvedEncodeOptions,
|
||||
): void {
|
||||
const header = formatHeader(values.length, { key: prefix, delimiter: options.delimiter, lengthMarker: options.lengthMarker })
|
||||
writer.push(depth, header)
|
||||
|
||||
for (const arr of values) {
|
||||
if (isArrayOfPrimitives(arr)) {
|
||||
const inline = encodeInlineArrayLine(arr, options.delimiter, undefined, options.lengthMarker)
|
||||
writer.pushListItem(depth + 1, inline)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function encodeInlineArrayLine(values: readonly JsonPrimitive[], delimiter: string, prefix?: string, lengthMarker?: '#' | false): string {
|
||||
const header = formatHeader(values.length, { key: prefix, delimiter, lengthMarker })
|
||||
const joinedValue = encodeAndJoinPrimitives(values, delimiter)
|
||||
// Only add space if there are values
|
||||
if (values.length === 0) {
|
||||
return header
|
||||
}
|
||||
return `${header} ${joinedValue}`
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Array of objects (tabular format)
|
||||
|
||||
export function encodeArrayOfObjectsAsTabular(
|
||||
prefix: string | undefined,
|
||||
rows: readonly JsonObject[],
|
||||
header: readonly string[],
|
||||
writer: LineWriter,
|
||||
depth: Depth,
|
||||
options: ResolvedEncodeOptions,
|
||||
): void {
|
||||
const formattedHeader = formatHeader(rows.length, { key: prefix, fields: header, delimiter: options.delimiter, lengthMarker: options.lengthMarker })
|
||||
writer.push(depth, `${formattedHeader}`)
|
||||
|
||||
writeTabularRows(rows, header, writer, depth + 1, options)
|
||||
}
|
||||
|
||||
export function extractTabularHeader(rows: readonly JsonObject[]): string[] | undefined {
|
||||
if (rows.length === 0)
|
||||
return
|
||||
|
||||
const firstRow = rows[0]!
|
||||
const firstKeys = Object.keys(firstRow)
|
||||
if (firstKeys.length === 0)
|
||||
return
|
||||
|
||||
if (isTabularArray(rows, firstKeys)) {
|
||||
return firstKeys
|
||||
}
|
||||
}
|
||||
|
||||
export function isTabularArray(
|
||||
rows: readonly JsonObject[],
|
||||
header: readonly string[],
|
||||
): boolean {
|
||||
for (const row of rows) {
|
||||
const keys = Object.keys(row)
|
||||
|
||||
// All objects must have the same keys (but order can differ)
|
||||
if (keys.length !== header.length) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check that all header keys exist in the row and all values are primitives
|
||||
for (const key of header) {
|
||||
if (!(key in row)) {
|
||||
return false
|
||||
}
|
||||
if (!isJsonPrimitive(row[key])) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
function writeTabularRows(
|
||||
rows: readonly JsonObject[],
|
||||
header: readonly string[],
|
||||
writer: LineWriter,
|
||||
depth: Depth,
|
||||
options: ResolvedEncodeOptions,
|
||||
): void {
|
||||
for (const row of rows) {
|
||||
const values = header.map(key => row[key])
|
||||
const joinedValue = encodeAndJoinPrimitives(values as JsonPrimitive[], options.delimiter)
|
||||
writer.push(depth, joinedValue)
|
||||
}
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Array of objects (expanded format)
|
||||
|
||||
export function encodeMixedArrayAsListItems(
|
||||
prefix: string | undefined,
|
||||
items: readonly JsonValue[],
|
||||
writer: LineWriter,
|
||||
depth: Depth,
|
||||
options: ResolvedEncodeOptions,
|
||||
): void {
|
||||
const header = formatHeader(items.length, { key: prefix, delimiter: options.delimiter, lengthMarker: options.lengthMarker })
|
||||
writer.push(depth, header)
|
||||
|
||||
for (const item of items) {
|
||||
encodeListItemValue(item, writer, depth + 1, options)
|
||||
}
|
||||
}
|
||||
|
||||
export function encodeObjectAsListItem(obj: JsonObject, writer: LineWriter, depth: Depth, options: ResolvedEncodeOptions): void {
|
||||
const keys = Object.keys(obj)
|
||||
if (keys.length === 0) {
|
||||
writer.push(depth, LIST_ITEM_MARKER)
|
||||
return
|
||||
}
|
||||
|
||||
// First key-value on the same line as "- "
|
||||
const firstKey = keys[0]!
|
||||
const encodedKey = encodeKey(firstKey)
|
||||
const firstValue = obj[firstKey]!
|
||||
|
||||
if (isJsonPrimitive(firstValue)) {
|
||||
writer.pushListItem(depth, `${encodedKey}: ${encodePrimitive(firstValue, options.delimiter)}`)
|
||||
}
|
||||
else if (isJsonArray(firstValue)) {
|
||||
if (isArrayOfPrimitives(firstValue)) {
|
||||
// Inline format for primitive arrays
|
||||
const formatted = encodeInlineArrayLine(firstValue, options.delimiter, firstKey, options.lengthMarker)
|
||||
writer.pushListItem(depth, formatted)
|
||||
}
|
||||
else if (isArrayOfObjects(firstValue)) {
|
||||
// Check if array of objects can use tabular format
|
||||
const header = extractTabularHeader(firstValue)
|
||||
if (header) {
|
||||
// Tabular format for uniform arrays of objects
|
||||
const formattedHeader = formatHeader(firstValue.length, { key: firstKey, fields: header, delimiter: options.delimiter, lengthMarker: options.lengthMarker })
|
||||
writer.pushListItem(depth, formattedHeader)
|
||||
writeTabularRows(firstValue, header, writer, depth + 1, options)
|
||||
}
|
||||
else {
|
||||
// Fall back to list format for non-uniform arrays of objects
|
||||
writer.pushListItem(depth, `${encodedKey}[${firstValue.length}]:`)
|
||||
for (const item of firstValue) {
|
||||
encodeObjectAsListItem(item, writer, depth + 1, options)
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Complex arrays on separate lines (array of arrays, etc.)
|
||||
writer.pushListItem(depth, `${encodedKey}[${firstValue.length}]:`)
|
||||
|
||||
// Encode array contents at depth + 1
|
||||
for (const item of firstValue) {
|
||||
encodeListItemValue(item, writer, depth + 1, options)
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (isJsonObject(firstValue)) {
|
||||
const nestedKeys = Object.keys(firstValue)
|
||||
if (nestedKeys.length === 0) {
|
||||
writer.pushListItem(depth, `${encodedKey}:`)
|
||||
}
|
||||
else {
|
||||
writer.pushListItem(depth, `${encodedKey}:`)
|
||||
encodeObject(firstValue, writer, depth + 2, options)
|
||||
}
|
||||
}
|
||||
|
||||
// Remaining keys on indented lines
|
||||
for (let i = 1; i < keys.length; i++) {
|
||||
const key = keys[i]!
|
||||
encodeKeyValuePair(key, obj[key]!, writer, depth + 1, options)
|
||||
}
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region List item encoding helpers
|
||||
|
||||
function encodeListItemValue(
|
||||
value: JsonValue,
|
||||
writer: LineWriter,
|
||||
depth: Depth,
|
||||
options: ResolvedEncodeOptions,
|
||||
): void {
|
||||
if (isJsonPrimitive(value)) {
|
||||
writer.pushListItem(depth, encodePrimitive(value, options.delimiter))
|
||||
}
|
||||
else if (isJsonArray(value) && isArrayOfPrimitives(value)) {
|
||||
const inline = encodeInlineArrayLine(value, options.delimiter, undefined, options.lengthMarker)
|
||||
writer.pushListItem(depth, inline)
|
||||
}
|
||||
else if (isJsonObject(value)) {
|
||||
encodeObjectAsListItem(value, writer, depth, options)
|
||||
}
|
||||
}
|
||||
|
||||
// #endregion
|
||||
122
packages/toon/src/encode/normalize.ts
Normal file
122
packages/toon/src/encode/normalize.ts
Normal file
@@ -0,0 +1,122 @@
|
||||
import type { JsonArray, JsonObject, JsonPrimitive, JsonValue } from '../types'
|
||||
|
||||
// #region Normalization (unknown → JsonValue)
|
||||
|
||||
export function normalizeValue(value: unknown): JsonValue {
|
||||
// null
|
||||
if (value === null) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Primitives
|
||||
if (typeof value === 'string' || typeof value === 'boolean') {
|
||||
return value
|
||||
}
|
||||
|
||||
// Numbers: canonicalize -0 to 0, handle NaN and Infinity
|
||||
if (typeof value === 'number') {
|
||||
if (Object.is(value, -0)) {
|
||||
return 0
|
||||
}
|
||||
if (!Number.isFinite(value)) {
|
||||
return null
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// BigInt → number (if safe) or string
|
||||
if (typeof value === 'bigint') {
|
||||
// Try to convert to number if within safe integer range
|
||||
if (value >= Number.MIN_SAFE_INTEGER && value <= Number.MAX_SAFE_INTEGER) {
|
||||
return Number(value)
|
||||
}
|
||||
// Otherwise convert to string (will be unquoted as it looks numeric)
|
||||
return value.toString()
|
||||
}
|
||||
|
||||
// Date → ISO string
|
||||
if (value instanceof Date) {
|
||||
return value.toISOString()
|
||||
}
|
||||
|
||||
// Array
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(normalizeValue)
|
||||
}
|
||||
|
||||
// Set → array
|
||||
if (value instanceof Set) {
|
||||
return Array.from(value).map(normalizeValue)
|
||||
}
|
||||
|
||||
// Map → object
|
||||
if (value instanceof Map) {
|
||||
return Object.fromEntries(
|
||||
Array.from(value, ([k, v]) => [String(k), normalizeValue(v)]),
|
||||
)
|
||||
}
|
||||
|
||||
// Plain object
|
||||
if (isPlainObject(value)) {
|
||||
const result: Record<string, JsonValue> = {}
|
||||
|
||||
for (const key in value) {
|
||||
if (Object.prototype.hasOwnProperty.call(value, key)) {
|
||||
result[key] = normalizeValue(value[key])
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// Fallback: function, symbol, undefined, or other → null
|
||||
return null
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Type guards
|
||||
|
||||
export function isJsonPrimitive(value: unknown): value is JsonPrimitive {
|
||||
return (
|
||||
value === null
|
||||
|| typeof value === 'string'
|
||||
|| typeof value === 'number'
|
||||
|| typeof value === 'boolean'
|
||||
)
|
||||
}
|
||||
|
||||
export function isJsonArray(value: unknown): value is JsonArray {
|
||||
return Array.isArray(value)
|
||||
}
|
||||
|
||||
export function isJsonObject(value: unknown): value is JsonObject {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
||||
}
|
||||
|
||||
export function isPlainObject(value: unknown): value is Record<string, unknown> {
|
||||
if (value === null || typeof value !== 'object') {
|
||||
return false
|
||||
}
|
||||
|
||||
const prototype = Object.getPrototypeOf(value)
|
||||
return prototype === null || prototype === Object.prototype
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Array type detection
|
||||
|
||||
export function isArrayOfPrimitives(value: JsonArray): value is readonly JsonPrimitive[] {
|
||||
return value.every(item => isJsonPrimitive(item))
|
||||
}
|
||||
|
||||
export function isArrayOfArrays(value: JsonArray): value is readonly JsonArray[] {
|
||||
return value.every(item => isJsonArray(item))
|
||||
}
|
||||
|
||||
export function isArrayOfObjects(value: JsonArray): value is readonly JsonObject[] {
|
||||
return value.every(item => isJsonObject(item))
|
||||
}
|
||||
|
||||
// #endregion
|
||||
89
packages/toon/src/encode/primitives.ts
Normal file
89
packages/toon/src/encode/primitives.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import type { JsonPrimitive } from '../types'
|
||||
import { COMMA, DEFAULT_DELIMITER, DOUBLE_QUOTE, NULL_LITERAL } from '../constants'
|
||||
import { escapeString } from '../shared/string-utils'
|
||||
import { isSafeUnquoted, isValidUnquotedKey } from '../shared/validation'
|
||||
|
||||
// #region Primitive encoding
|
||||
|
||||
export function encodePrimitive(value: JsonPrimitive, delimiter?: string): string {
|
||||
if (value === null) {
|
||||
return NULL_LITERAL
|
||||
}
|
||||
|
||||
if (typeof value === 'boolean') {
|
||||
return String(value)
|
||||
}
|
||||
|
||||
if (typeof value === 'number') {
|
||||
return String(value)
|
||||
}
|
||||
|
||||
return encodeStringLiteral(value, delimiter)
|
||||
}
|
||||
|
||||
export function encodeStringLiteral(value: string, delimiter: string = COMMA): string {
|
||||
if (isSafeUnquoted(value, delimiter)) {
|
||||
return value
|
||||
}
|
||||
|
||||
return `${DOUBLE_QUOTE}${escapeString(value)}${DOUBLE_QUOTE}`
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Key encoding
|
||||
|
||||
export function encodeKey(key: string): string {
|
||||
if (isValidUnquotedKey(key)) {
|
||||
return key
|
||||
}
|
||||
|
||||
return `${DOUBLE_QUOTE}${escapeString(key)}${DOUBLE_QUOTE}`
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Value joining
|
||||
|
||||
export function encodeAndJoinPrimitives(values: readonly JsonPrimitive[], delimiter: string = COMMA): string {
|
||||
return values.map(v => encodePrimitive(v, delimiter)).join(delimiter)
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Header formatters
|
||||
|
||||
export function formatHeader(
|
||||
length: number,
|
||||
options?: {
|
||||
key?: string
|
||||
fields?: readonly string[]
|
||||
delimiter?: string
|
||||
lengthMarker?: '#' | false
|
||||
},
|
||||
): string {
|
||||
const key = options?.key
|
||||
const fields = options?.fields
|
||||
const delimiter = options?.delimiter ?? COMMA
|
||||
const lengthMarker = options?.lengthMarker ?? false
|
||||
|
||||
let header = ''
|
||||
|
||||
if (key) {
|
||||
header += encodeKey(key)
|
||||
}
|
||||
|
||||
// Only include delimiter if it's not the default (comma)
|
||||
header += `[${lengthMarker || ''}${length}${delimiter !== DEFAULT_DELIMITER ? delimiter : ''}]`
|
||||
|
||||
if (fields) {
|
||||
const quotedFields = fields.map(f => encodeKey(f))
|
||||
header += `{${quotedFields.join(delimiter)}}`
|
||||
}
|
||||
|
||||
header += ':'
|
||||
|
||||
return header
|
||||
}
|
||||
|
||||
// #endregion
|
||||
24
packages/toon/src/encode/writer.ts
Normal file
24
packages/toon/src/encode/writer.ts
Normal file
@@ -0,0 +1,24 @@
|
||||
import type { Depth } from '../types'
|
||||
import { LIST_ITEM_PREFIX } from '../constants'
|
||||
|
||||
export class LineWriter {
|
||||
private readonly lines: string[] = []
|
||||
private readonly indentationString: string
|
||||
|
||||
constructor(indentSize: number) {
|
||||
this.indentationString = ' '.repeat(indentSize)
|
||||
}
|
||||
|
||||
push(depth: Depth, content: string): void {
|
||||
const indent = this.indentationString.repeat(depth)
|
||||
this.lines.push(indent + content)
|
||||
}
|
||||
|
||||
pushListItem(depth: Depth, content: string): void {
|
||||
this.push(depth, `${LIST_ITEM_PREFIX}${content}`)
|
||||
}
|
||||
|
||||
toString(): string {
|
||||
return this.lines.join('\n')
|
||||
}
|
||||
}
|
||||
53
packages/toon/src/index.ts
Normal file
53
packages/toon/src/index.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
import type { DecodeOptions, EncodeOptions, JsonValue, ResolvedDecodeOptions, ResolvedEncodeOptions } from './types'
|
||||
import { DEFAULT_DELIMITER } from './constants'
|
||||
import { decodeValueFromLines } from './decode/decoders'
|
||||
import { LineCursor, toParsedLines } from './decode/scanner'
|
||||
import { encodeValue } from './encode/encoders'
|
||||
import { normalizeValue } from './encode/normalize'
|
||||
|
||||
export { DEFAULT_DELIMITER, DELIMITERS } from './constants'
|
||||
export type {
|
||||
DecodeOptions,
|
||||
Delimiter,
|
||||
DelimiterKey,
|
||||
EncodeOptions,
|
||||
JsonArray,
|
||||
JsonObject,
|
||||
JsonPrimitive,
|
||||
JsonValue,
|
||||
ResolvedDecodeOptions,
|
||||
ResolvedEncodeOptions,
|
||||
} from './types'
|
||||
|
||||
export function encode(input: unknown, options?: EncodeOptions): string {
|
||||
const normalizedValue = normalizeValue(input)
|
||||
const resolvedOptions = resolveOptions(options)
|
||||
return encodeValue(normalizedValue, resolvedOptions)
|
||||
}
|
||||
|
||||
export function decode(input: string, options?: DecodeOptions): JsonValue {
|
||||
const resolvedOptions = resolveDecodeOptions(options)
|
||||
const scanResult = toParsedLines(input, resolvedOptions.indent, resolvedOptions.strict)
|
||||
|
||||
if (scanResult.lines.length === 0) {
|
||||
throw new TypeError('Cannot decode empty input: input must be a non-empty string')
|
||||
}
|
||||
|
||||
const cursor = new LineCursor(scanResult.lines, scanResult.blankLines)
|
||||
return decodeValueFromLines(cursor, resolvedOptions)
|
||||
}
|
||||
|
||||
function resolveOptions(options?: EncodeOptions): ResolvedEncodeOptions {
|
||||
return {
|
||||
indent: options?.indent ?? 2,
|
||||
delimiter: options?.delimiter ?? DEFAULT_DELIMITER,
|
||||
lengthMarker: options?.lengthMarker ?? false,
|
||||
}
|
||||
}
|
||||
|
||||
function resolveDecodeOptions(options?: DecodeOptions): ResolvedDecodeOptions {
|
||||
return {
|
||||
indent: options?.indent ?? 2,
|
||||
strict: options?.strict ?? true,
|
||||
}
|
||||
}
|
||||
28
packages/toon/src/shared/literal-utils.ts
Normal file
28
packages/toon/src/shared/literal-utils.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
import { FALSE_LITERAL, NULL_LITERAL, TRUE_LITERAL } from '../constants'
|
||||
|
||||
/**
|
||||
* Checks if a token is a boolean or null literal (`true`, `false`, `null`).
|
||||
*/
|
||||
export function isBooleanOrNullLiteral(token: string): boolean {
|
||||
return token === TRUE_LITERAL || token === FALSE_LITERAL || token === NULL_LITERAL
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a token represents a valid numeric literal.
|
||||
*
|
||||
* @remarks
|
||||
* Rejects numbers with leading zeros (except `"0"` itself or decimals like `"0.5"`).
|
||||
*/
|
||||
export function isNumericLiteral(token: string): boolean {
|
||||
if (!token)
|
||||
return false
|
||||
|
||||
// Must not have leading zeros (except for `"0"` itself or decimals like `"0.5"`)
|
||||
if (token.length > 1 && token[0] === '0' && token[1] !== '.') {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if it's a valid number
|
||||
const num = Number(token)
|
||||
return !Number.isNaN(num) && Number.isFinite(num)
|
||||
}
|
||||
127
packages/toon/src/shared/string-utils.ts
Normal file
127
packages/toon/src/shared/string-utils.ts
Normal file
@@ -0,0 +1,127 @@
|
||||
import { BACKSLASH, CARRIAGE_RETURN, DOUBLE_QUOTE, NEWLINE, TAB } from '../constants'
|
||||
|
||||
/**
|
||||
* Escapes special characters in a string for encoding.
|
||||
*
|
||||
* @remarks
|
||||
* Handles backslashes, quotes, newlines, carriage returns, and tabs.
|
||||
*/
|
||||
export function escapeString(value: string): string {
|
||||
return value
|
||||
.replace(/\\/g, `${BACKSLASH}${BACKSLASH}`)
|
||||
.replace(/"/g, `${BACKSLASH}${DOUBLE_QUOTE}`)
|
||||
.replace(/\n/g, `${BACKSLASH}n`)
|
||||
.replace(/\r/g, `${BACKSLASH}r`)
|
||||
.replace(/\t/g, `${BACKSLASH}t`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Unescapes a string by processing escape sequences.
|
||||
*
|
||||
* @remarks
|
||||
* Handles `\n`, `\t`, `\r`, `\\`, and `\"` escape sequences.
|
||||
*/
|
||||
export function unescapeString(value: string): string {
|
||||
let result = ''
|
||||
let i = 0
|
||||
|
||||
while (i < value.length) {
|
||||
if (value[i] === BACKSLASH) {
|
||||
if (i + 1 >= value.length) {
|
||||
throw new SyntaxError('Invalid escape sequence: backslash at end of string')
|
||||
}
|
||||
|
||||
const next = value[i + 1]
|
||||
if (next === 'n') {
|
||||
result += NEWLINE
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
if (next === 't') {
|
||||
result += TAB
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
if (next === 'r') {
|
||||
result += CARRIAGE_RETURN
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
if (next === BACKSLASH) {
|
||||
result += BACKSLASH
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
if (next === DOUBLE_QUOTE) {
|
||||
result += DOUBLE_QUOTE
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
|
||||
throw new SyntaxError(`Invalid escape sequence: \\${next}`)
|
||||
}
|
||||
|
||||
result += value[i]
|
||||
i++
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the index of the closing double quote in a string, accounting for escape sequences.
|
||||
*
|
||||
* @param content The string to search in
|
||||
* @param start The index of the opening quote
|
||||
* @returns The index of the closing quote, or -1 if not found
|
||||
*/
|
||||
export function findClosingQuote(content: string, start: number): number {
|
||||
let i = start + 1
|
||||
while (i < content.length) {
|
||||
if (content[i] === BACKSLASH && i + 1 < content.length) {
|
||||
// Skip escaped character
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
if (content[i] === DOUBLE_QUOTE) {
|
||||
return i
|
||||
}
|
||||
i++
|
||||
}
|
||||
return -1 // Not found
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the index of a specific character outside of quoted sections.
|
||||
*
|
||||
* @param content The string to search in
|
||||
* @param char The character to look for
|
||||
* @param start Optional starting index (defaults to 0)
|
||||
* @returns The index of the character, or -1 if not found outside quotes
|
||||
*/
|
||||
export function findUnquotedChar(content: string, char: string, start = 0): number {
|
||||
let inQuotes = false
|
||||
let i = start
|
||||
|
||||
while (i < content.length) {
|
||||
if (content[i] === BACKSLASH && i + 1 < content.length && inQuotes) {
|
||||
// Skip escaped character
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
|
||||
if (content[i] === DOUBLE_QUOTE) {
|
||||
inQuotes = !inQuotes
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if (content[i] === char && !inQuotes) {
|
||||
return i
|
||||
}
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
84
packages/toon/src/shared/validation.ts
Normal file
84
packages/toon/src/shared/validation.ts
Normal file
@@ -0,0 +1,84 @@
|
||||
import { COMMA, LIST_ITEM_MARKER } from '../constants'
|
||||
import { isBooleanOrNullLiteral } from './literal-utils'
|
||||
|
||||
/**
|
||||
* Checks if a key can be used without quotes.
|
||||
*
|
||||
* @remarks
|
||||
* Valid unquoted keys must start with a letter or underscore,
|
||||
* followed by letters, digits, underscores, or dots.
|
||||
*/
|
||||
export function isValidUnquotedKey(key: string): boolean {
|
||||
return /^[A-Z_][\w.]*$/i.test(key)
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if a string value can be safely encoded without quotes.
|
||||
*
|
||||
* @remarks
|
||||
* A string needs quoting if it:
|
||||
* - Is empty
|
||||
* - Has leading or trailing whitespace
|
||||
* - Could be confused with a literal (boolean, null, number)
|
||||
* - Contains structural characters (colons, brackets, braces)
|
||||
* - Contains quotes or backslashes (need escaping)
|
||||
* - Contains control characters (newlines, tabs, etc.)
|
||||
* - Contains the active delimiter
|
||||
* - Starts with a list marker (hyphen)
|
||||
*/
|
||||
export function isSafeUnquoted(value: string, delimiter: string = COMMA): boolean {
|
||||
if (!value) {
|
||||
return false
|
||||
}
|
||||
|
||||
if (value !== value.trim()) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if it looks like any literal value (boolean, null, or numeric)
|
||||
if (isBooleanOrNullLiteral(value) || isNumericLike(value)) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for colon (always structural)
|
||||
if (value.includes(':')) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for quotes and backslash (always need escaping)
|
||||
if (value.includes('"') || value.includes('\\')) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for brackets and braces (always structural)
|
||||
if (/[[\]{}]/.test(value)) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for control characters (newline, carriage return, tab - always need quoting/escaping)
|
||||
if (/[\n\r\t]/.test(value)) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for the active delimiter
|
||||
if (value.includes(delimiter)) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for hyphen at start (list marker)
|
||||
if (value.startsWith(LIST_ITEM_MARKER)) {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a string looks like a number.
|
||||
*
|
||||
* @remarks
|
||||
* Match numbers like `42`, `-3.14`, `1e-6`, `05`, etc.
|
||||
*/
|
||||
function isNumericLike(value: string): boolean {
|
||||
return /^-?\d+(?:\.\d+)?(?:e[+-]?\d+)?$/i.test(value) || /^0\d+$/.test(value)
|
||||
}
|
||||
84
packages/toon/src/types.ts
Normal file
84
packages/toon/src/types.ts
Normal file
@@ -0,0 +1,84 @@
|
||||
// #region JSON types
|
||||
|
||||
import type { Delimiter, DelimiterKey } from './constants'
|
||||
|
||||
export type JsonPrimitive = string | number | boolean | null
|
||||
export type JsonObject = { [Key in string]: JsonValue } & { [Key in string]?: JsonValue | undefined }
|
||||
export type JsonArray = JsonValue[] | readonly JsonValue[]
|
||||
export type JsonValue = JsonPrimitive | JsonObject | JsonArray
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Encoder options
|
||||
|
||||
export type { Delimiter, DelimiterKey }
|
||||
|
||||
export interface EncodeOptions {
|
||||
/**
|
||||
* Number of spaces per indentation level.
|
||||
* @default 2
|
||||
*/
|
||||
indent?: number
|
||||
/**
|
||||
* Delimiter to use for tabular array rows and inline primitive arrays.
|
||||
* @default DELIMITERS.comma
|
||||
*/
|
||||
delimiter?: Delimiter
|
||||
/**
|
||||
* Optional marker to prefix array lengths in headers.
|
||||
* When set to `#`, arrays render as [#N] instead of [N].
|
||||
* @default false
|
||||
*/
|
||||
lengthMarker?: '#' | false
|
||||
}
|
||||
|
||||
export type ResolvedEncodeOptions = Readonly<Required<EncodeOptions>>
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Decoder options
|
||||
|
||||
export interface DecodeOptions {
|
||||
/**
|
||||
* Number of spaces per indentation level.
|
||||
* @default 2
|
||||
*/
|
||||
indent?: number
|
||||
/**
|
||||
* When true, enforce strict validation of array lengths and tabular row counts.
|
||||
* @default true
|
||||
*/
|
||||
strict?: boolean
|
||||
}
|
||||
|
||||
export type ResolvedDecodeOptions = Readonly<Required<DecodeOptions>>
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Decoder parsing types
|
||||
|
||||
export interface ArrayHeaderInfo {
|
||||
key?: string
|
||||
length: number
|
||||
delimiter: Delimiter
|
||||
fields?: string[]
|
||||
hasLengthMarker: boolean
|
||||
}
|
||||
|
||||
export interface ParsedLine {
|
||||
raw: string
|
||||
depth: Depth
|
||||
indent: number
|
||||
content: string
|
||||
lineNumber: number
|
||||
}
|
||||
|
||||
export interface BlankLineInfo {
|
||||
lineNumber: number
|
||||
indent: number
|
||||
depth: Depth
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
export type Depth = number
|
||||
686
packages/toon/test/decode.test.ts
Normal file
686
packages/toon/test/decode.test.ts
Normal file
@@ -0,0 +1,686 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { decode } from '../src/index'
|
||||
|
||||
describe('primitives', () => {
|
||||
it('decodes safe unquoted strings', () => {
|
||||
expect(decode('hello')).toBe('hello')
|
||||
expect(decode('Ada_99')).toBe('Ada_99')
|
||||
})
|
||||
|
||||
it('decodes quoted strings and unescapes control characters', () => {
|
||||
expect(decode('""')).toBe('')
|
||||
expect(decode('"line1\\nline2"')).toBe('line1\nline2')
|
||||
expect(decode('"tab\\there"')).toBe('tab\there')
|
||||
expect(decode('"return\\rcarriage"')).toBe('return\rcarriage')
|
||||
expect(decode('"C:\\\\Users\\\\path"')).toBe('C:\\Users\\path')
|
||||
expect(decode('"say \\"hello\\""')).toBe('say "hello"')
|
||||
})
|
||||
|
||||
it('decodes unicode and emoji', () => {
|
||||
expect(decode('café')).toBe('café')
|
||||
expect(decode('你好')).toBe('你好')
|
||||
expect(decode('🚀')).toBe('🚀')
|
||||
expect(decode('hello 👋 world')).toBe('hello 👋 world')
|
||||
})
|
||||
|
||||
it('decodes numbers, booleans and null', () => {
|
||||
expect(decode('42')).toBe(42)
|
||||
expect(decode('3.14')).toBe(3.14)
|
||||
expect(decode('-7')).toBe(-7)
|
||||
expect(decode('true')).toBe(true)
|
||||
expect(decode('false')).toBe(false)
|
||||
expect(decode('null')).toBe(null)
|
||||
})
|
||||
|
||||
it('treats unquoted invalid numeric formats as strings', () => {
|
||||
expect(decode('05')).toBe('05')
|
||||
expect(decode('007')).toBe('007')
|
||||
expect(decode('0123')).toBe('0123')
|
||||
expect(decode('a: 05')).toEqual({ a: '05' })
|
||||
expect(decode('nums[3]: 05,007,0123')).toEqual({ nums: ['05', '007', '0123'] })
|
||||
})
|
||||
|
||||
it('respects ambiguity quoting (quoted primitives remain strings)', () => {
|
||||
expect(decode('"true"')).toBe('true')
|
||||
expect(decode('"false"')).toBe('false')
|
||||
expect(decode('"null"')).toBe('null')
|
||||
expect(decode('"42"')).toBe('42')
|
||||
expect(decode('"-3.14"')).toBe('-3.14')
|
||||
expect(decode('"1e-6"')).toBe('1e-6')
|
||||
expect(decode('"05"')).toBe('05')
|
||||
})
|
||||
})
|
||||
|
||||
describe('objects (simple)', () => {
|
||||
it('parses objects with primitive values', () => {
|
||||
const toon = 'id: 123\nname: Ada\nactive: true'
|
||||
expect(decode(toon)).toEqual({ id: 123, name: 'Ada', active: true })
|
||||
})
|
||||
|
||||
it('parses null values in objects', () => {
|
||||
const toon = 'id: 123\nvalue: null'
|
||||
expect(decode(toon)).toEqual({ id: 123, value: null })
|
||||
})
|
||||
|
||||
it('parses empty nested object header', () => {
|
||||
expect(decode('user:')).toEqual({ user: {} })
|
||||
})
|
||||
|
||||
it('parses quoted object values with special characters and escapes', () => {
|
||||
expect(decode('note: "a:b"')).toEqual({ note: 'a:b' })
|
||||
expect(decode('note: "a,b"')).toEqual({ note: 'a,b' })
|
||||
expect(decode('text: "line1\\nline2"')).toEqual({ text: 'line1\nline2' })
|
||||
expect(decode('text: "say \\"hello\\""')).toEqual({ text: 'say "hello"' })
|
||||
expect(decode('text: " padded "')).toEqual({ text: ' padded ' })
|
||||
expect(decode('text: " "')).toEqual({ text: ' ' })
|
||||
expect(decode('v: "true"')).toEqual({ v: 'true' })
|
||||
expect(decode('v: "42"')).toEqual({ v: '42' })
|
||||
expect(decode('v: "-7.5"')).toEqual({ v: '-7.5' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('objects (keys)', () => {
|
||||
it('parses quoted keys with special characters and escapes', () => {
|
||||
expect(decode('"order:id": 7')).toEqual({ 'order:id': 7 })
|
||||
expect(decode('"[index]": 5')).toEqual({ '[index]': 5 })
|
||||
expect(decode('"{key}": 5')).toEqual({ '{key}': 5 })
|
||||
expect(decode('"a,b": 1')).toEqual({ 'a,b': 1 })
|
||||
expect(decode('"full name": Ada')).toEqual({ 'full name': 'Ada' })
|
||||
expect(decode('"-lead": 1')).toEqual({ '-lead': 1 })
|
||||
expect(decode('" a ": 1')).toEqual({ ' a ': 1 })
|
||||
expect(decode('"123": x')).toEqual({ 123: 'x' })
|
||||
expect(decode('"": 1')).toEqual({ '': 1 })
|
||||
})
|
||||
|
||||
it('parses dotted keys as identifiers', () => {
|
||||
expect(decode('user.name: Ada')).toEqual({ 'user.name': 'Ada' })
|
||||
expect(decode('_private: 1')).toEqual({ _private: 1 })
|
||||
expect(decode('user_name: 1')).toEqual({ user_name: 1 })
|
||||
})
|
||||
|
||||
it('unescapes control characters and quotes in keys', () => {
|
||||
expect(decode('"line\\nbreak": 1')).toEqual({ 'line\nbreak': 1 })
|
||||
expect(decode('"tab\\there": 2')).toEqual({ 'tab\there': 2 })
|
||||
expect(decode('"he said \\"hi\\"": 1')).toEqual({ 'he said "hi"': 1 })
|
||||
})
|
||||
})
|
||||
|
||||
describe('nested objects', () => {
|
||||
it('parses deeply nested objects with indentation', () => {
|
||||
const toon = 'a:\n b:\n c: deep'
|
||||
expect(decode(toon)).toEqual({ a: { b: { c: 'deep' } } })
|
||||
})
|
||||
})
|
||||
|
||||
describe('arrays of primitives', () => {
|
||||
it('parses string arrays inline', () => {
|
||||
const toon = 'tags[3]: reading,gaming,coding'
|
||||
expect(decode(toon)).toEqual({ tags: ['reading', 'gaming', 'coding'] })
|
||||
})
|
||||
|
||||
it('parses number arrays inline', () => {
|
||||
const toon = 'nums[3]: 1,2,3'
|
||||
expect(decode(toon)).toEqual({ nums: [1, 2, 3] })
|
||||
})
|
||||
|
||||
it('parses mixed primitive arrays inline', () => {
|
||||
const toon = 'data[4]: x,y,true,10'
|
||||
expect(decode(toon)).toEqual({ data: ['x', 'y', true, 10] })
|
||||
})
|
||||
|
||||
it('parses empty arrays', () => {
|
||||
expect(decode('items[0]:')).toEqual({ items: [] })
|
||||
})
|
||||
|
||||
it('parses quoted strings in arrays including empty and whitespace-only', () => {
|
||||
expect(decode('items[1]: ""')).toEqual({ items: [''] })
|
||||
expect(decode('items[3]: a,"",b')).toEqual({ items: ['a', '', 'b'] })
|
||||
expect(decode('items[2]: " "," "')).toEqual({ items: [' ', ' '] })
|
||||
})
|
||||
|
||||
it('parses strings with delimiters and structural tokens in arrays', () => {
|
||||
expect(decode('items[3]: a,"b,c","d:e"')).toEqual({ items: ['a', 'b,c', 'd:e'] })
|
||||
expect(decode('items[4]: x,"true","42","-3.14"')).toEqual({ items: ['x', 'true', '42', '-3.14'] })
|
||||
expect(decode('items[3]: "[5]","- item","{key}"')).toEqual({ items: ['[5]', '- item', '{key}'] })
|
||||
})
|
||||
})
|
||||
|
||||
describe('arrays of objects (tabular and list items)', () => {
|
||||
it('parses tabular arrays of uniform objects', () => {
|
||||
const toon = 'items[2]{sku,qty,price}:\n A1,2,9.99\n B2,1,14.5'
|
||||
expect(decode(toon)).toEqual({
|
||||
items: [
|
||||
{ sku: 'A1', qty: 2, price: 9.99 },
|
||||
{ sku: 'B2', qty: 1, price: 14.5 },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('parses nulls and quoted values in tabular rows', () => {
|
||||
const toon = 'items[2]{id,value}:\n 1,null\n 2,"test"'
|
||||
expect(decode(toon)).toEqual({
|
||||
items: [
|
||||
{ id: 1, value: null },
|
||||
{ id: 2, value: 'test' },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('parses quoted header keys in tabular arrays', () => {
|
||||
const toon = 'items[2]{"order:id","full name"}:\n 1,Ada\n 2,Bob'
|
||||
expect(decode(toon)).toEqual({
|
||||
items: [
|
||||
{ 'order:id': 1, 'full name': 'Ada' },
|
||||
{ 'order:id': 2, 'full name': 'Bob' },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('parses list arrays for non-uniform objects', () => {
|
||||
const toon
|
||||
= 'items[2]:\n'
|
||||
+ ' - id: 1\n'
|
||||
+ ' name: First\n'
|
||||
+ ' - id: 2\n'
|
||||
+ ' name: Second\n'
|
||||
+ ' extra: true'
|
||||
expect(decode(toon)).toEqual({
|
||||
items: [
|
||||
{ id: 1, name: 'First' },
|
||||
{ id: 2, name: 'Second', extra: true },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('parses objects with nested values inside list items', () => {
|
||||
const toon
|
||||
= 'items[1]:\n'
|
||||
+ ' - id: 1\n'
|
||||
+ ' nested:\n'
|
||||
+ ' x: 1'
|
||||
expect(decode(toon)).toEqual({
|
||||
items: [{ id: 1, nested: { x: 1 } }],
|
||||
})
|
||||
})
|
||||
|
||||
it('parses nested tabular arrays as first field on hyphen line', () => {
|
||||
const toon
|
||||
= 'items[1]:\n'
|
||||
+ ' - users[2]{id,name}:\n'
|
||||
+ ' 1,Ada\n'
|
||||
+ ' 2,Bob\n'
|
||||
+ ' status: active'
|
||||
expect(decode(toon)).toEqual({
|
||||
items: [
|
||||
{
|
||||
users: [
|
||||
{ id: 1, name: 'Ada' },
|
||||
{ id: 2, name: 'Bob' },
|
||||
],
|
||||
status: 'active',
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('parses objects containing arrays (including empty arrays) in list format', () => {
|
||||
const toon
|
||||
= 'items[1]:\n'
|
||||
+ ' - name: test\n'
|
||||
+ ' data[0]:'
|
||||
expect(decode(toon)).toEqual({
|
||||
items: [{ name: 'test', data: [] }],
|
||||
})
|
||||
})
|
||||
|
||||
it('parses arrays of arrays within objects', () => {
|
||||
const toon
|
||||
= 'items[1]:\n'
|
||||
+ ' - matrix[2]:\n'
|
||||
+ ' - [2]: 1,2\n'
|
||||
+ ' - [2]: 3,4\n'
|
||||
+ ' name: grid'
|
||||
expect(decode(toon)).toEqual({
|
||||
items: [{ matrix: [[1, 2], [3, 4]], name: 'grid' }],
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('arrays of arrays (primitives only)', () => {
|
||||
it('parses nested arrays of primitives', () => {
|
||||
const toon = 'pairs[2]:\n - [2]: a,b\n - [2]: c,d'
|
||||
expect(decode(toon)).toEqual({ pairs: [['a', 'b'], ['c', 'd']] })
|
||||
})
|
||||
|
||||
it('parses quoted strings and mixed lengths in nested arrays', () => {
|
||||
const toon = 'pairs[2]:\n - [2]: a,b\n - [3]: "c,d","e:f","true"'
|
||||
expect(decode(toon)).toEqual({ pairs: [['a', 'b'], ['c,d', 'e:f', 'true']] })
|
||||
})
|
||||
|
||||
it('parses empty inner arrays', () => {
|
||||
const toon = 'pairs[2]:\n - [0]:\n - [0]:'
|
||||
expect(decode(toon)).toEqual({ pairs: [[], []] })
|
||||
})
|
||||
|
||||
it('parses mixed-length inner arrays', () => {
|
||||
const toon = 'pairs[2]:\n - [1]: 1\n - [2]: 2,3'
|
||||
expect(decode(toon)).toEqual({ pairs: [[1], [2, 3]] })
|
||||
})
|
||||
})
|
||||
|
||||
describe('root arrays', () => {
|
||||
it('parses root arrays of primitives (inline)', () => {
|
||||
const toon = '[5]: x,y,"true",true,10'
|
||||
expect(decode(toon)).toEqual(['x', 'y', 'true', true, 10])
|
||||
})
|
||||
|
||||
it('parses root arrays of uniform objects in tabular format', () => {
|
||||
const toon = '[2]{id}:\n 1\n 2'
|
||||
expect(decode(toon)).toEqual([{ id: 1 }, { id: 2 }])
|
||||
})
|
||||
|
||||
it('parses root arrays of non-uniform objects in list format', () => {
|
||||
const toon = '[2]:\n - id: 1\n - id: 2\n name: Ada'
|
||||
expect(decode(toon)).toEqual([{ id: 1 }, { id: 2, name: 'Ada' }])
|
||||
})
|
||||
|
||||
it('parses empty root arrays', () => {
|
||||
expect(decode('[0]:')).toEqual([])
|
||||
})
|
||||
|
||||
it('parses root arrays of arrays', () => {
|
||||
const toon = '[2]:\n - [2]: 1,2\n - [0]:'
|
||||
expect(decode(toon)).toEqual([[1, 2], []])
|
||||
})
|
||||
})
|
||||
|
||||
describe('complex structures', () => {
|
||||
it('parses mixed objects with arrays and nested objects', () => {
|
||||
const toon
|
||||
= 'user:\n'
|
||||
+ ' id: 123\n'
|
||||
+ ' name: Ada\n'
|
||||
+ ' tags[2]: reading,gaming\n'
|
||||
+ ' active: true\n'
|
||||
+ ' prefs[0]:'
|
||||
expect(decode(toon)).toEqual({
|
||||
user: {
|
||||
id: 123,
|
||||
name: 'Ada',
|
||||
tags: ['reading', 'gaming'],
|
||||
active: true,
|
||||
prefs: [],
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('mixed arrays', () => {
|
||||
it('parses arrays mixing primitives, objects and strings (list format)', () => {
|
||||
const toon
|
||||
= 'items[3]:\n'
|
||||
+ ' - 1\n'
|
||||
+ ' - a: 1\n'
|
||||
+ ' - text'
|
||||
expect(decode(toon)).toEqual({ items: [1, { a: 1 }, 'text'] })
|
||||
})
|
||||
|
||||
it('parses arrays mixing objects and arrays', () => {
|
||||
const toon
|
||||
= 'items[2]:\n'
|
||||
+ ' - a: 1\n'
|
||||
+ ' - [2]: 1,2'
|
||||
expect(decode(toon)).toEqual({ items: [{ a: 1 }, [1, 2]] })
|
||||
})
|
||||
})
|
||||
|
||||
describe('delimiter options', () => {
|
||||
describe('basic delimiter usage', () => {
|
||||
it.each([
|
||||
{ delimiter: '\t' as const, name: 'tab', header: '[3\t]', joined: 'reading\tgaming\tcoding' },
|
||||
{ delimiter: '|' as const, name: 'pipe', header: '[3|]', joined: 'reading|gaming|coding' },
|
||||
{ delimiter: ',' as const, name: 'comma', header: '[3]', joined: 'reading,gaming,coding' },
|
||||
])('parses primitive arrays with $name delimiter', ({ header, joined }) => {
|
||||
const toon = `tags${header}: ${joined}`
|
||||
expect(decode(toon)).toEqual({ tags: ['reading', 'gaming', 'coding'] })
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ delimiter: '\t' as const, name: 'tab', header: '[2\t]{sku\tqty\tprice}', rows: ['A1\t2\t9.99', 'B2\t1\t14.5'] },
|
||||
{ delimiter: '|' as const, name: 'pipe', header: '[2|]{sku|qty|price}', rows: ['A1|2|9.99', 'B2|1|14.5'] },
|
||||
])('parses tabular arrays with $name delimiter', ({ header, rows }) => {
|
||||
const toon = `items${header}:\n ${rows[0]}\n ${rows[1]}`
|
||||
expect(decode(toon)).toEqual({
|
||||
items: [
|
||||
{ sku: 'A1', qty: 2, price: 9.99 },
|
||||
{ sku: 'B2', qty: 1, price: 14.5 },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ header: '[2\t]', inner: '[2\t]', a: 'a\tb', b: 'c\td' },
|
||||
{ header: '[2|]', inner: '[2|]', a: 'a|b', b: 'c|d' },
|
||||
])('parses nested arrays with custom delimiters', ({ header, inner, a, b }) => {
|
||||
const toon = `pairs${header}:\n - ${inner}: ${a}\n - ${inner}: ${b}`
|
||||
expect(decode(toon)).toEqual({ pairs: [['a', 'b'], ['c', 'd']] })
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ parent: '[1\t]', nested: '[3]', values: 'a,b,c' },
|
||||
{ parent: '[1|]', nested: '[3]', values: 'a,b,c' },
|
||||
])('nested arrays inside list items default to comma delimiter', ({ parent, nested, values }) => {
|
||||
const toon = `items${parent}:\n - tags${nested}: ${values}`
|
||||
expect(decode(toon)).toEqual({ items: [{ tags: ['a', 'b', 'c'] }] })
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ header: '[3\t]', joined: 'x\ty\tz' },
|
||||
{ header: '[3|]', joined: 'x|y|z' },
|
||||
])('parses root arrays of primitives with custom delimiters', ({ header, joined }) => {
|
||||
const toon = `${header}: ${joined}`
|
||||
expect(decode(toon)).toEqual(['x', 'y', 'z'])
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ header: '[2\t]{id}', rows: ['1', '2'] },
|
||||
{ header: '[2|]{id}', rows: ['1', '2'] },
|
||||
])('parses root arrays of objects with custom delimiters', ({ header, rows }) => {
|
||||
const toon = `${header}:\n ${rows[0]}\n ${rows[1]}`
|
||||
expect(decode(toon)).toEqual([{ id: 1 }, { id: 2 }])
|
||||
})
|
||||
})
|
||||
|
||||
describe('delimiter-aware quoting', () => {
|
||||
it.each([
|
||||
{ header: '[3\t]', joined: 'a\t"b\\tc"\td', expected: ['a', 'b\tc', 'd'] },
|
||||
{ header: '[3|]', joined: 'a|"b|c"|d', expected: ['a', 'b|c', 'd'] },
|
||||
])('parses values containing the active delimiter when quoted', ({ header, joined, expected }) => {
|
||||
const toon = `items${header}: ${joined}`
|
||||
expect(decode(toon)).toEqual({ items: expected })
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ header: '[2\t]', joined: 'a,b\tc,d' },
|
||||
{ header: '[2|]', joined: 'a,b|c,d' },
|
||||
])('does not split on commas when using non-comma delimiter', ({ header, joined }) => {
|
||||
const toon = `items${header}: ${joined}`
|
||||
expect(decode(toon)).toEqual({ items: ['a,b', 'c,d'] })
|
||||
})
|
||||
|
||||
it('parses tabular values containing the active delimiter correctly', () => {
|
||||
const comma = 'items[2]{id,note}:\n 1,"a,b"\n 2,"c,d"'
|
||||
expect(decode(comma)).toEqual({ items: [{ id: 1, note: 'a,b' }, { id: 2, note: 'c,d' }] })
|
||||
|
||||
const tab = 'items[2\t]{id\tnote}:\n 1\ta,b\n 2\tc,d'
|
||||
expect(decode(tab)).toEqual({ items: [{ id: 1, note: 'a,b' }, { id: 2, note: 'c,d' }] })
|
||||
})
|
||||
|
||||
it('does not require quoting commas in object values when using non-comma delimiter elsewhere', () => {
|
||||
expect(decode('note: a,b')).toEqual({ note: 'a,b' })
|
||||
})
|
||||
|
||||
it('parses nested array values containing the active delimiter', () => {
|
||||
expect(decode('pairs[1|]:\n - [2|]: a|"b|c"')).toEqual({ pairs: [['a', 'b|c']] })
|
||||
expect(decode('pairs[1\t]:\n - [2\t]: a\t"b\\tc"')).toEqual({ pairs: [['a', 'b\tc']] })
|
||||
})
|
||||
})
|
||||
|
||||
describe('delimiter-independent quoting rules', () => {
|
||||
it('preserves quoted ambiguity regardless of delimiter', () => {
|
||||
expect(decode('items[3|]: "true"|"42"|"-3.14"')).toEqual({ items: ['true', '42', '-3.14'] })
|
||||
expect(decode('items[3\t]: "true"\t"42"\t"-3.14"')).toEqual({ items: ['true', '42', '-3.14'] })
|
||||
})
|
||||
|
||||
it('parses structural-looking strings when quoted', () => {
|
||||
expect(decode('items[3|]: "[5]"|"{key}"|"- item"')).toEqual({ items: ['[5]', '{key}', '- item'] })
|
||||
expect(decode('items[3\t]: "[5]"\t"{key}"\t"- item"')).toEqual({ items: ['[5]', '{key}', '- item'] })
|
||||
})
|
||||
|
||||
it('parses tabular headers with keys containing the active delimiter', () => {
|
||||
const toon = 'items[2|]{"a|b"}:\n 1\n 2'
|
||||
expect(decode(toon)).toEqual({ items: [{ 'a|b': 1 }, { 'a|b': 2 }] })
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('length marker option', () => {
|
||||
it('accepts length marker on primitive arrays', () => {
|
||||
expect(decode('tags[#3]: reading,gaming,coding')).toEqual({ tags: ['reading', 'gaming', 'coding'] })
|
||||
})
|
||||
|
||||
it('accepts length marker on empty arrays', () => {
|
||||
expect(decode('items[#0]:')).toEqual({ items: [] })
|
||||
})
|
||||
|
||||
it('accepts length marker on tabular arrays', () => {
|
||||
const toon = 'items[#2]{sku,qty,price}:\n A1,2,9.99\n B2,1,14.5'
|
||||
expect(decode(toon)).toEqual({
|
||||
items: [
|
||||
{ sku: 'A1', qty: 2, price: 9.99 },
|
||||
{ sku: 'B2', qty: 1, price: 14.5 },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('accepts length marker on nested arrays', () => {
|
||||
const toon = 'pairs[#2]:\n - [#2]: a,b\n - [#2]: c,d'
|
||||
expect(decode(toon)).toEqual({ pairs: [['a', 'b'], ['c', 'd']] })
|
||||
})
|
||||
|
||||
it('works with custom delimiters and length marker', () => {
|
||||
expect(decode('tags[#3|]: reading|gaming|coding')).toEqual({ tags: ['reading', 'gaming', 'coding'] })
|
||||
})
|
||||
})
|
||||
|
||||
describe('validation and error handling', () => {
|
||||
describe('length and structure errors', () => {
|
||||
it('throws on array length mismatch (inline primitives)', () => {
|
||||
const toon = 'tags[2]: a,b,c'
|
||||
expect(() => decode(toon)).toThrow()
|
||||
})
|
||||
|
||||
it('throws on array length mismatch (list format)', () => {
|
||||
const toon = 'items[1]:\n - 1\n - 2'
|
||||
expect(() => decode(toon)).toThrow()
|
||||
})
|
||||
|
||||
it('throws when tabular row value count does not match header field count', () => {
|
||||
const toon = 'items[2]{id,name}:\n 1,Ada\n 2'
|
||||
expect(() => decode(toon)).toThrow()
|
||||
})
|
||||
|
||||
it('throws when tabular row count does not match header length', () => {
|
||||
const toon = '[1]{id}:\n 1\n 2'
|
||||
expect(() => decode(toon)).toThrow()
|
||||
})
|
||||
|
||||
it('throws on invalid escape sequences', () => {
|
||||
expect(() => decode('"a\\x"')).toThrow()
|
||||
expect(() => decode('"unterminated')).toThrow()
|
||||
})
|
||||
|
||||
it('throws on missing colon in key-value context', () => {
|
||||
expect(() => decode('a:\n user')).toThrow()
|
||||
})
|
||||
|
||||
it('throws on delimiter mismatch', () => {
|
||||
const toon = 'items[2\t]{a\tb}:\n 1,2\n 3,4'
|
||||
expect(() => decode(toon)).toThrow()
|
||||
})
|
||||
})
|
||||
|
||||
describe('strict mode: indentation validation', () => {
|
||||
describe('non-multiple indentation errors', () => {
|
||||
it('throws when object field has non-multiple indentation', () => {
|
||||
const toon = 'a:\n b: 1' // 3 spaces with indent=2
|
||||
expect(() => decode(toon)).toThrow(/indentation/i)
|
||||
expect(() => decode(toon)).toThrow(/exact multiple/i)
|
||||
})
|
||||
|
||||
it('throws when list item has non-multiple indentation', () => {
|
||||
const toon = 'items[2]:\n - id: 1\n - id: 2' // 3 spaces
|
||||
expect(() => decode(toon)).toThrow(/indentation/i)
|
||||
})
|
||||
|
||||
it('throws with custom indent size when non-multiple', () => {
|
||||
const toon = 'a:\n b: 1' // 3 spaces with indent=4
|
||||
expect(() => decode(toon, { indent: 4 })).toThrow(/exact multiple of 4/i)
|
||||
})
|
||||
|
||||
it('accepts correct indentation with custom indent size', () => {
|
||||
const toon = 'a:\n b: 1' // 4 spaces with indent=4
|
||||
expect(decode(toon, { indent: 4 })).toEqual({ a: { b: 1 } })
|
||||
})
|
||||
})
|
||||
|
||||
describe('tab character errors', () => {
|
||||
it('throws when tab character used in indentation', () => {
|
||||
const toon = 'a:\n\tb: 1'
|
||||
expect(() => decode(toon)).toThrow(/tab/i)
|
||||
})
|
||||
|
||||
it('throws when mixed tabs and spaces in indentation', () => {
|
||||
const toon = 'a:\n \tb: 1' // space + tab
|
||||
expect(() => decode(toon)).toThrow(/tab/i)
|
||||
})
|
||||
|
||||
it('throws when tab at start of line', () => {
|
||||
const toon = '\ta: 1'
|
||||
expect(() => decode(toon)).toThrow(/tab/i)
|
||||
})
|
||||
})
|
||||
|
||||
describe('tabs in quoted strings are allowed', () => {
|
||||
it('accepts tabs in quoted string values', () => {
|
||||
const toon = 'text: "hello\tworld"'
|
||||
expect(decode(toon)).toEqual({ text: 'hello\tworld' })
|
||||
})
|
||||
|
||||
it('accepts tabs in quoted keys', () => {
|
||||
const toon = '"key\ttab": value'
|
||||
expect(decode(toon)).toEqual({ 'key\ttab': 'value' })
|
||||
})
|
||||
|
||||
it('accepts tabs in quoted array elements', () => {
|
||||
const toon = 'items[2]: "a\tb","c\td"'
|
||||
expect(decode(toon)).toEqual({ items: ['a\tb', 'c\td'] })
|
||||
})
|
||||
})
|
||||
|
||||
describe('non-strict mode', () => {
|
||||
it('accepts non-multiple indentation when strict=false', () => {
|
||||
const toon = 'a:\n b: 1' // 3 spaces with indent=2
|
||||
expect(decode(toon, { strict: false })).toEqual({ a: { b: 1 } })
|
||||
})
|
||||
|
||||
it('accepts tab indentation when strict=false', () => {
|
||||
const toon = 'a:\n\tb: 1'
|
||||
// Tabs are ignored in indentation counting, so depth=0, "b: 1" at root
|
||||
expect(decode(toon, { strict: false })).toEqual({ a: {}, b: 1 })
|
||||
})
|
||||
|
||||
it('accepts deeply nested non-multiples when strict=false', () => {
|
||||
const toon = 'a:\n b:\n c: 1' // 3 and 5 spaces
|
||||
expect(decode(toon, { strict: false })).toEqual({ a: { b: { c: 1 } } })
|
||||
})
|
||||
})
|
||||
|
||||
describe('edge cases', () => {
|
||||
it('empty lines do not trigger validation errors', () => {
|
||||
const toon = 'a: 1\n\nb: 2'
|
||||
expect(decode(toon)).toEqual({ a: 1, b: 2 })
|
||||
})
|
||||
|
||||
it('root-level content (0 indentation) is always valid', () => {
|
||||
const toon = 'a: 1\nb: 2\nc: 3'
|
||||
expect(decode(toon)).toEqual({ a: 1, b: 2, c: 3 })
|
||||
})
|
||||
|
||||
it('lines with only spaces are not validated if empty', () => {
|
||||
const toon = 'a: 1\n \nb: 2'
|
||||
expect(decode(toon)).toEqual({ a: 1, b: 2 })
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('strict mode: blank lines in arrays', () => {
|
||||
describe('errors on blank lines inside arrays', () => {
|
||||
it('throws on blank line inside list array', () => {
|
||||
const teon = 'items[3]:\n - a\n\n - b\n - c'
|
||||
expect(() => decode(teon)).toThrow(/blank line/i)
|
||||
expect(() => decode(teon)).toThrow(/list array/i)
|
||||
})
|
||||
|
||||
it('throws on blank line inside tabular array', () => {
|
||||
const teon = 'items[2]{id}:\n 1\n\n 2'
|
||||
expect(() => decode(teon)).toThrow(/blank line/i)
|
||||
expect(() => decode(teon)).toThrow(/tabular array/i)
|
||||
})
|
||||
|
||||
it('throws on multiple blank lines inside array', () => {
|
||||
const teon = 'items[2]:\n - a\n\n\n - b'
|
||||
expect(() => decode(teon)).toThrow(/blank line/i)
|
||||
})
|
||||
|
||||
it('throws on blank line with spaces inside array', () => {
|
||||
const teon = 'items[2]:\n - a\n \n - b'
|
||||
expect(() => decode(teon)).toThrow(/blank line/i)
|
||||
})
|
||||
|
||||
it('throws on blank line in nested list array', () => {
|
||||
const teon = 'outer[2]:\n - inner[2]:\n - a\n\n - b\n - x'
|
||||
expect(() => decode(teon)).toThrow(/blank line/i)
|
||||
})
|
||||
})
|
||||
|
||||
describe('accepts blank lines outside arrays', () => {
|
||||
it('accepts blank line between root-level fields', () => {
|
||||
const teon = 'a: 1\n\nb: 2'
|
||||
expect(decode(teon)).toEqual({ a: 1, b: 2 })
|
||||
})
|
||||
|
||||
it('accepts trailing newline at end of file', () => {
|
||||
const teon = 'a: 1\n'
|
||||
expect(decode(teon)).toEqual({ a: 1 })
|
||||
})
|
||||
|
||||
it('accepts multiple trailing newlines', () => {
|
||||
const teon = 'a: 1\n\n\n'
|
||||
expect(decode(teon)).toEqual({ a: 1 })
|
||||
})
|
||||
|
||||
it('accepts blank line after array ends', () => {
|
||||
const teon = 'items[1]:\n - a\n\nb: 2'
|
||||
expect(decode(teon)).toEqual({ items: ['a'], b: 2 })
|
||||
})
|
||||
|
||||
it('accepts blank line between nested object fields', () => {
|
||||
const teon = 'a:\n b: 1\n\n c: 2'
|
||||
expect(decode(teon)).toEqual({ a: { b: 1, c: 2 } })
|
||||
})
|
||||
})
|
||||
|
||||
describe('non-strict mode: ignores blank lines', () => {
|
||||
it('ignores blank lines inside list array', () => {
|
||||
const teon = 'items[3]:\n - a\n\n - b\n - c'
|
||||
expect(decode(teon, { strict: false })).toEqual({ items: ['a', 'b', 'c'] })
|
||||
})
|
||||
|
||||
it('ignores blank lines inside tabular array', () => {
|
||||
const teon = 'items[2]{id,name}:\n 1,Alice\n\n 2,Bob'
|
||||
expect(decode(teon, { strict: false })).toEqual({
|
||||
items: [
|
||||
{ id: 1, name: 'Alice' },
|
||||
{ id: 2, name: 'Bob' },
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('ignores multiple blank lines in arrays', () => {
|
||||
const teon = 'items[2]:\n - a\n\n\n - b'
|
||||
expect(decode(teon, { strict: false })).toEqual({ items: ['a', 'b'] })
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
777
packages/toon/test/encode.test.ts
Normal file
777
packages/toon/test/encode.test.ts
Normal file
@@ -0,0 +1,777 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { decode, encode } from '../src/index'
|
||||
|
||||
describe('primitives', () => {
|
||||
it('encodes safe strings without quotes', () => {
|
||||
expect(encode('hello')).toBe('hello')
|
||||
expect(encode('Ada_99')).toBe('Ada_99')
|
||||
})
|
||||
|
||||
it('quotes empty string', () => {
|
||||
expect(encode('')).toBe('""')
|
||||
})
|
||||
|
||||
it('quotes strings that look like booleans or numbers', () => {
|
||||
expect(encode('true')).toBe('"true"')
|
||||
expect(encode('false')).toBe('"false"')
|
||||
expect(encode('null')).toBe('"null"')
|
||||
expect(encode('42')).toBe('"42"')
|
||||
expect(encode('-3.14')).toBe('"-3.14"')
|
||||
expect(encode('1e-6')).toBe('"1e-6"')
|
||||
expect(encode('05')).toBe('"05"')
|
||||
})
|
||||
|
||||
it('escapes control characters in strings', () => {
|
||||
expect(encode('line1\nline2')).toBe('"line1\\nline2"')
|
||||
expect(encode('tab\there')).toBe('"tab\\there"')
|
||||
expect(encode('return\rcarriage')).toBe('"return\\rcarriage"')
|
||||
expect(encode('C:\\Users\\path')).toBe('"C:\\\\Users\\\\path"')
|
||||
})
|
||||
|
||||
it('quotes strings with structural characters', () => {
|
||||
expect(encode('[3]: x,y')).toBe('"[3]: x,y"')
|
||||
expect(encode('- item')).toBe('"- item"')
|
||||
expect(encode('[test]')).toBe('"[test]"')
|
||||
expect(encode('{key}')).toBe('"{key}"')
|
||||
})
|
||||
|
||||
it('handles Unicode and emoji', () => {
|
||||
expect(encode('café')).toBe('café')
|
||||
expect(encode('你好')).toBe('你好')
|
||||
expect(encode('🚀')).toBe('🚀')
|
||||
expect(encode('hello 👋 world')).toBe('hello 👋 world')
|
||||
})
|
||||
|
||||
it('encodes numbers', () => {
|
||||
expect(encode(42)).toBe('42')
|
||||
expect(encode(3.14)).toBe('3.14')
|
||||
expect(encode(-7)).toBe('-7')
|
||||
expect(encode(0)).toBe('0')
|
||||
})
|
||||
|
||||
it('handles special numeric values', () => {
|
||||
expect(encode(-0)).toBe('0')
|
||||
expect(encode(1e6)).toBe('1000000')
|
||||
expect(encode(1e-6)).toBe('0.000001')
|
||||
expect(encode(1e20)).toBe('100000000000000000000')
|
||||
expect(encode(Number.MAX_SAFE_INTEGER)).toBe('9007199254740991')
|
||||
})
|
||||
|
||||
it('preserves precision for repeating decimals', () => {
|
||||
const value = 1 / 3
|
||||
const encodedValue = encode({ value })
|
||||
const decodedValue = decode(encodedValue)
|
||||
expect((decodedValue as Record<string, unknown>)?.value).toBe(value) // Round-trip fidelity
|
||||
expect(encodedValue).toContain('0.3333333333333333') // Default JS precision
|
||||
})
|
||||
|
||||
it('encodes booleans', () => {
|
||||
expect(encode(true)).toBe('true')
|
||||
expect(encode(false)).toBe('false')
|
||||
})
|
||||
|
||||
it('encodes null', () => {
|
||||
expect(encode(null)).toBe('null')
|
||||
})
|
||||
})
|
||||
|
||||
describe('objects (simple)', () => {
|
||||
it('preserves key order in objects', () => {
|
||||
const obj = {
|
||||
id: 123,
|
||||
name: 'Ada',
|
||||
active: true,
|
||||
}
|
||||
expect(encode(obj)).toBe('id: 123\nname: Ada\nactive: true')
|
||||
})
|
||||
|
||||
it('encodes null values in objects', () => {
|
||||
const obj = { id: 123, value: null }
|
||||
expect(encode(obj)).toBe('id: 123\nvalue: null')
|
||||
})
|
||||
|
||||
it('encodes empty objects as empty string', () => {
|
||||
expect(encode({})).toBe('')
|
||||
})
|
||||
|
||||
it('quotes string values with special characters', () => {
|
||||
expect(encode({ note: 'a:b' })).toBe('note: "a:b"')
|
||||
expect(encode({ note: 'a,b' })).toBe('note: "a,b"')
|
||||
expect(encode({ text: 'line1\nline2' })).toBe('text: "line1\\nline2"')
|
||||
expect(encode({ text: 'say "hello"' })).toBe('text: "say \\"hello\\""')
|
||||
})
|
||||
|
||||
it('quotes string values with leading/trailing spaces', () => {
|
||||
expect(encode({ text: ' padded ' })).toBe('text: " padded "')
|
||||
expect(encode({ text: ' ' })).toBe('text: " "')
|
||||
})
|
||||
|
||||
it('quotes string values that look like booleans/numbers', () => {
|
||||
expect(encode({ v: 'true' })).toBe('v: "true"')
|
||||
expect(encode({ v: '42' })).toBe('v: "42"')
|
||||
expect(encode({ v: '-7.5' })).toBe('v: "-7.5"')
|
||||
})
|
||||
})
|
||||
|
||||
describe('objects (keys)', () => {
|
||||
it('quotes keys with special characters', () => {
|
||||
expect(encode({ 'order:id': 7 })).toBe('"order:id": 7')
|
||||
expect(encode({ '[index]': 5 })).toBe('"[index]": 5')
|
||||
expect(encode({ '{key}': 5 })).toBe('"{key}": 5')
|
||||
expect(encode({ 'a,b': 1 })).toBe('"a,b": 1')
|
||||
})
|
||||
|
||||
it('quotes keys with spaces or leading hyphens', () => {
|
||||
expect(encode({ 'full name': 'Ada' })).toBe('"full name": Ada')
|
||||
expect(encode({ '-lead': 1 })).toBe('"-lead": 1')
|
||||
expect(encode({ ' a ': 1 })).toBe('" a ": 1')
|
||||
})
|
||||
|
||||
it('quotes numeric keys', () => {
|
||||
expect(encode({ 123: 'x' })).toBe('"123": x')
|
||||
})
|
||||
|
||||
it('quotes empty string key', () => {
|
||||
expect(encode({ '': 1 })).toBe('"": 1')
|
||||
})
|
||||
|
||||
it('escapes control characters in keys', () => {
|
||||
expect(encode({ 'line\nbreak': 1 })).toBe('"line\\nbreak": 1')
|
||||
expect(encode({ 'tab\there': 2 })).toBe('"tab\\there": 2')
|
||||
})
|
||||
|
||||
it('escapes quotes in keys', () => {
|
||||
expect(encode({ 'he said "hi"': 1 })).toBe('"he said \\"hi\\"": 1')
|
||||
})
|
||||
})
|
||||
|
||||
describe('nested objects', () => {
|
||||
it('encodes deeply nested objects', () => {
|
||||
const obj = {
|
||||
a: {
|
||||
b: {
|
||||
c: 'deep',
|
||||
},
|
||||
},
|
||||
}
|
||||
expect(encode(obj)).toBe('a:\n b:\n c: deep')
|
||||
})
|
||||
|
||||
it('encodes empty nested object', () => {
|
||||
expect(encode({ user: {} })).toBe('user:')
|
||||
})
|
||||
})
|
||||
|
||||
describe('arrays of primitives', () => {
|
||||
it('encodes string arrays inline', () => {
|
||||
const obj = { tags: ['reading', 'gaming'] }
|
||||
expect(encode(obj)).toBe('tags[2]: reading,gaming')
|
||||
})
|
||||
|
||||
it('encodes number arrays inline', () => {
|
||||
const obj = { nums: [1, 2, 3] }
|
||||
expect(encode(obj)).toBe('nums[3]: 1,2,3')
|
||||
})
|
||||
|
||||
it('encodes mixed primitive arrays inline', () => {
|
||||
const obj = { data: ['x', 'y', true, 10] }
|
||||
expect(encode(obj)).toBe('data[4]: x,y,true,10')
|
||||
})
|
||||
|
||||
it('encodes empty arrays', () => {
|
||||
const obj = { items: [] }
|
||||
expect(encode(obj)).toBe('items[0]:')
|
||||
})
|
||||
|
||||
it('handles empty string in arrays', () => {
|
||||
const obj = { items: [''] }
|
||||
expect(encode(obj)).toBe('items[1]: ""')
|
||||
const obj2 = { items: ['a', '', 'b'] }
|
||||
expect(encode(obj2)).toBe('items[3]: a,"",b')
|
||||
})
|
||||
|
||||
it('handles whitespace-only strings in arrays', () => {
|
||||
const obj = { items: [' ', ' '] }
|
||||
expect(encode(obj)).toBe('items[2]: " "," "')
|
||||
})
|
||||
|
||||
it('quotes array strings with special characters', () => {
|
||||
const obj = { items: ['a', 'b,c', 'd:e'] }
|
||||
expect(encode(obj)).toBe('items[3]: a,"b,c","d:e"')
|
||||
})
|
||||
|
||||
it('quotes strings that look like booleans/numbers in arrays', () => {
|
||||
const obj = { items: ['x', 'true', '42', '-3.14'] }
|
||||
expect(encode(obj)).toBe('items[4]: x,"true","42","-3.14"')
|
||||
})
|
||||
|
||||
it('quotes strings with structural meanings in arrays', () => {
|
||||
const obj = { items: ['[5]', '- item', '{key}'] }
|
||||
expect(encode(obj)).toBe('items[3]: "[5]","- item","{key}"')
|
||||
})
|
||||
})
|
||||
|
||||
describe('arrays of objects (tabular and list items)', () => {
|
||||
it('encodes arrays of similar objects in tabular format', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ sku: 'A1', qty: 2, price: 9.99 },
|
||||
{ sku: 'B2', qty: 1, price: 14.5 },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe('items[2]{sku,qty,price}:\n A1,2,9.99\n B2,1,14.5')
|
||||
})
|
||||
|
||||
it('handles null values in tabular format', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ id: 1, value: null },
|
||||
{ id: 2, value: 'test' },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe('items[2]{id,value}:\n 1,null\n 2,test')
|
||||
})
|
||||
|
||||
it('quotes strings containing delimiters in tabular rows', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ sku: 'A,1', desc: 'cool', qty: 2 },
|
||||
{ sku: 'B2', desc: 'wip: test', qty: 1 },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe('items[2]{sku,desc,qty}:\n "A,1",cool,2\n B2,"wip: test",1')
|
||||
})
|
||||
|
||||
it('quotes ambiguous strings in tabular rows', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ id: 1, status: 'true' },
|
||||
{ id: 2, status: 'false' },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe('items[2]{id,status}:\n 1,"true"\n 2,"false"')
|
||||
})
|
||||
|
||||
it('handles tabular arrays with keys needing quotes', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ 'order:id': 1, 'full name': 'Ada' },
|
||||
{ 'order:id': 2, 'full name': 'Bob' },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe('items[2]{"order:id","full name"}:\n 1,Ada\n 2,Bob')
|
||||
})
|
||||
|
||||
it('uses list format for objects with different fields', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ id: 1, name: 'First' },
|
||||
{ id: 2, name: 'Second', extra: true },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe(
|
||||
'items[2]:\n'
|
||||
+ ' - id: 1\n'
|
||||
+ ' name: First\n'
|
||||
+ ' - id: 2\n'
|
||||
+ ' name: Second\n'
|
||||
+ ' extra: true',
|
||||
)
|
||||
})
|
||||
|
||||
it('uses list format for objects with nested values', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ id: 1, nested: { x: 1 } },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe(
|
||||
'items[1]:\n'
|
||||
+ ' - id: 1\n'
|
||||
+ ' nested:\n'
|
||||
+ ' x: 1',
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves field order in list items', () => {
|
||||
const obj = { items: [{ nums: [1, 2, 3], name: 'test' }] }
|
||||
expect(encode(obj)).toBe(
|
||||
'items[1]:\n'
|
||||
+ ' - nums[3]: 1,2,3\n'
|
||||
+ ' name: test',
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves field order when primitive appears first', () => {
|
||||
const obj = { items: [{ name: 'test', nums: [1, 2, 3] }] }
|
||||
expect(encode(obj)).toBe(
|
||||
'items[1]:\n'
|
||||
+ ' - name: test\n'
|
||||
+ ' nums[3]: 1,2,3',
|
||||
)
|
||||
})
|
||||
|
||||
it('uses list format for objects containing arrays of arrays', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ matrix: [[1, 2], [3, 4]], name: 'grid' },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe(
|
||||
'items[1]:\n'
|
||||
+ ' - matrix[2]:\n'
|
||||
+ ' - [2]: 1,2\n'
|
||||
+ ' - [2]: 3,4\n'
|
||||
+ ' name: grid',
|
||||
)
|
||||
})
|
||||
|
||||
it('uses tabular format for nested uniform object arrays', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ users: [{ id: 1, name: 'Ada' }, { id: 2, name: 'Bob' }], status: 'active' },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe(
|
||||
'items[1]:\n'
|
||||
+ ' - users[2]{id,name}:\n'
|
||||
+ ' 1,Ada\n'
|
||||
+ ' 2,Bob\n'
|
||||
+ ' status: active',
|
||||
)
|
||||
})
|
||||
|
||||
it('uses list format for nested object arrays with mismatched keys', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ users: [{ id: 1, name: 'Ada' }, { id: 2 }], status: 'active' },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe(
|
||||
'items[1]:\n'
|
||||
+ ' - users[2]:\n'
|
||||
+ ' - id: 1\n'
|
||||
+ ' name: Ada\n'
|
||||
+ ' - id: 2\n'
|
||||
+ ' status: active',
|
||||
)
|
||||
})
|
||||
|
||||
it('uses list format for objects with multiple array fields', () => {
|
||||
const obj = { items: [{ nums: [1, 2], tags: ['a', 'b'], name: 'test' }] }
|
||||
expect(encode(obj)).toBe(
|
||||
'items[1]:\n'
|
||||
+ ' - nums[2]: 1,2\n'
|
||||
+ ' tags[2]: a,b\n'
|
||||
+ ' name: test',
|
||||
)
|
||||
})
|
||||
|
||||
it('uses list format for objects with only array fields', () => {
|
||||
const obj = { items: [{ nums: [1, 2, 3], tags: ['a', 'b'] }] }
|
||||
expect(encode(obj)).toBe(
|
||||
'items[1]:\n'
|
||||
+ ' - nums[3]: 1,2,3\n'
|
||||
+ ' tags[2]: a,b',
|
||||
)
|
||||
})
|
||||
|
||||
it('handles objects with empty arrays in list format', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ name: 'test', data: [] },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe(
|
||||
'items[1]:\n'
|
||||
+ ' - name: test\n'
|
||||
+ ' data[0]:',
|
||||
)
|
||||
})
|
||||
|
||||
it('places first field of nested tabular arrays on hyphen line', () => {
|
||||
const obj = { items: [{ users: [{ id: 1 }, { id: 2 }], note: 'x' }] }
|
||||
expect(encode(obj)).toBe(
|
||||
'items[1]:\n'
|
||||
+ ' - users[2]{id}:\n'
|
||||
+ ' 1\n'
|
||||
+ ' 2\n'
|
||||
+ ' note: x',
|
||||
)
|
||||
})
|
||||
|
||||
it('places empty arrays on hyphen line when first', () => {
|
||||
const obj = { items: [{ data: [], name: 'x' }] }
|
||||
expect(encode(obj)).toBe(
|
||||
'items[1]:\n'
|
||||
+ ' - data[0]:\n'
|
||||
+ ' name: x',
|
||||
)
|
||||
})
|
||||
|
||||
it('uses field order from first object for tabular headers', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ a: 1, b: 2, c: 3 },
|
||||
{ c: 30, b: 20, a: 10 },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe('items[2]{a,b,c}:\n 1,2,3\n 10,20,30')
|
||||
})
|
||||
|
||||
it('uses list format for one object with nested column', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ id: 1, data: 'string' },
|
||||
{ id: 2, data: { nested: true } },
|
||||
],
|
||||
}
|
||||
expect(encode(obj)).toBe(
|
||||
'items[2]:\n'
|
||||
+ ' - id: 1\n'
|
||||
+ ' data: string\n'
|
||||
+ ' - id: 2\n'
|
||||
+ ' data:\n'
|
||||
+ ' nested: true',
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('arrays of arrays (primitives only)', () => {
|
||||
it('encodes nested arrays of primitives', () => {
|
||||
const obj = {
|
||||
pairs: [['a', 'b'], ['c', 'd']],
|
||||
}
|
||||
expect(encode(obj)).toBe('pairs[2]:\n - [2]: a,b\n - [2]: c,d')
|
||||
})
|
||||
|
||||
it('quotes strings containing delimiters in nested arrays', () => {
|
||||
const obj = {
|
||||
pairs: [['a', 'b'], ['c,d', 'e:f', 'true']],
|
||||
}
|
||||
expect(encode(obj)).toBe('pairs[2]:\n - [2]: a,b\n - [3]: "c,d","e:f","true"')
|
||||
})
|
||||
|
||||
it('handles empty inner arrays', () => {
|
||||
const obj = {
|
||||
pairs: [[], []],
|
||||
}
|
||||
expect(encode(obj)).toBe('pairs[2]:\n - [0]:\n - [0]:')
|
||||
})
|
||||
|
||||
it('handles mixed-length inner arrays', () => {
|
||||
const obj = {
|
||||
pairs: [[1], [2, 3]],
|
||||
}
|
||||
expect(encode(obj)).toBe('pairs[2]:\n - [1]: 1\n - [2]: 2,3')
|
||||
})
|
||||
})
|
||||
|
||||
describe('root arrays', () => {
|
||||
it('encodes arrays of primitives at root level', () => {
|
||||
const arr = ['x', 'y', 'true', true, 10]
|
||||
expect(encode(arr)).toBe('[5]: x,y,"true",true,10')
|
||||
})
|
||||
|
||||
it('encodes arrays of similar objects in tabular format', () => {
|
||||
const arr = [{ id: 1 }, { id: 2 }]
|
||||
expect(encode(arr)).toBe('[2]{id}:\n 1\n 2')
|
||||
})
|
||||
|
||||
it('encodes arrays of different objects in list format', () => {
|
||||
const arr = [{ id: 1 }, { id: 2, name: 'Ada' }]
|
||||
expect(encode(arr)).toBe('[2]:\n - id: 1\n - id: 2\n name: Ada')
|
||||
})
|
||||
|
||||
it('encodes empty arrays at root level', () => {
|
||||
expect(encode([])).toBe('[0]:')
|
||||
})
|
||||
|
||||
it('encodes arrays of arrays at root level', () => {
|
||||
const arr = [[1, 2], []]
|
||||
expect(encode(arr)).toBe('[2]:\n - [2]: 1,2\n - [0]:')
|
||||
})
|
||||
})
|
||||
|
||||
describe('complex structures', () => {
|
||||
it('encodes objects with mixed arrays and nested objects', () => {
|
||||
const obj = {
|
||||
user: {
|
||||
id: 123,
|
||||
name: 'Ada',
|
||||
tags: ['reading', 'gaming'],
|
||||
active: true,
|
||||
prefs: [],
|
||||
},
|
||||
}
|
||||
expect(encode(obj)).toBe(
|
||||
'user:\n'
|
||||
+ ' id: 123\n'
|
||||
+ ' name: Ada\n'
|
||||
+ ' tags[2]: reading,gaming\n'
|
||||
+ ' active: true\n'
|
||||
+ ' prefs[0]:',
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('mixed arrays', () => {
|
||||
it('uses list format for arrays mixing primitives and objects', () => {
|
||||
const obj = {
|
||||
items: [1, { a: 1 }, 'text'],
|
||||
}
|
||||
expect(encode(obj)).toBe(
|
||||
'items[3]:\n'
|
||||
+ ' - 1\n'
|
||||
+ ' - a: 1\n'
|
||||
+ ' - text',
|
||||
)
|
||||
})
|
||||
|
||||
it('uses list format for arrays mixing objects and arrays', () => {
|
||||
const obj = {
|
||||
items: [{ a: 1 }, [1, 2]],
|
||||
}
|
||||
expect(encode(obj)).toBe(
|
||||
'items[2]:\n'
|
||||
+ ' - a: 1\n'
|
||||
+ ' - [2]: 1,2',
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('whitespace and formatting invariants', () => {
|
||||
it('produces no trailing spaces at end of lines', () => {
|
||||
const obj = {
|
||||
user: {
|
||||
id: 123,
|
||||
name: 'Ada',
|
||||
},
|
||||
items: ['a', 'b'],
|
||||
}
|
||||
const result = encode(obj)
|
||||
const lines = result.split('\n')
|
||||
for (const line of lines) {
|
||||
expect(line).not.toMatch(/ $/)
|
||||
}
|
||||
})
|
||||
|
||||
it('produces no trailing newline at end of output', () => {
|
||||
const obj = { id: 123 }
|
||||
const result = encode(obj)
|
||||
expect(result).not.toMatch(/\n$/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('non-JSON-serializable values', () => {
|
||||
it('converts BigInt to string', () => {
|
||||
expect(encode(BigInt(123))).toBe('123')
|
||||
expect(encode({ id: BigInt(456) })).toBe('id: 456')
|
||||
})
|
||||
|
||||
it('converts Date to ISO string', () => {
|
||||
const date = new Date('2025-01-01T00:00:00.000Z')
|
||||
expect(encode(date)).toBe('"2025-01-01T00:00:00.000Z"')
|
||||
expect(encode({ created: date })).toBe('created: "2025-01-01T00:00:00.000Z"')
|
||||
})
|
||||
|
||||
it('converts undefined to null', () => {
|
||||
expect(encode(undefined)).toBe('null')
|
||||
expect(encode({ value: undefined })).toBe('value: null')
|
||||
})
|
||||
|
||||
it('converts non-finite numbers to null', () => {
|
||||
expect(encode(Infinity)).toBe('null')
|
||||
expect(encode(-Infinity)).toBe('null')
|
||||
expect(encode(Number.NaN)).toBe('null')
|
||||
})
|
||||
|
||||
it('converts functions to null', () => {
|
||||
expect(encode(() => {})).toBe('null')
|
||||
expect(encode({ fn: () => {} })).toBe('fn: null')
|
||||
})
|
||||
|
||||
it('converts symbols to null', () => {
|
||||
expect(encode(Symbol('test'))).toBe('null')
|
||||
expect(encode({ sym: Symbol('test') })).toBe('sym: null')
|
||||
})
|
||||
})
|
||||
|
||||
describe('delimiter options', () => {
|
||||
describe('basic delimiter usage', () => {
|
||||
it.each([
|
||||
{ delimiter: '\t' as const, name: 'tab', expected: 'reading\tgaming\tcoding' },
|
||||
{ delimiter: '|' as const, name: 'pipe', expected: 'reading|gaming|coding' },
|
||||
{ delimiter: ',' as const, name: 'comma', expected: 'reading,gaming,coding' },
|
||||
])('encodes primitive arrays with $name', ({ delimiter, expected }) => {
|
||||
const obj = { tags: ['reading', 'gaming', 'coding'] }
|
||||
expect(encode(obj, { delimiter })).toBe(`tags[3${delimiter !== ',' ? delimiter : ''}]: ${expected}`)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ delimiter: '\t' as const, name: 'tab', expected: 'items[2\t]{sku\tqty\tprice}:\n A1\t2\t9.99\n B2\t1\t14.5' },
|
||||
{ delimiter: '|' as const, name: 'pipe', expected: 'items[2|]{sku|qty|price}:\n A1|2|9.99\n B2|1|14.5' },
|
||||
])('encodes tabular arrays with $name', ({ delimiter, expected }) => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ sku: 'A1', qty: 2, price: 9.99 },
|
||||
{ sku: 'B2', qty: 1, price: 14.5 },
|
||||
],
|
||||
}
|
||||
expect(encode(obj, { delimiter })).toBe(expected)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ delimiter: '\t' as const, name: 'tab', expected: 'pairs[2\t]:\n - [2\t]: a\tb\n - [2\t]: c\td' },
|
||||
{ delimiter: '|' as const, name: 'pipe', expected: 'pairs[2|]:\n - [2|]: a|b\n - [2|]: c|d' },
|
||||
])('encodes nested arrays with $name', ({ delimiter, expected }) => {
|
||||
const obj = { pairs: [['a', 'b'], ['c', 'd']] }
|
||||
expect(encode(obj, { delimiter })).toBe(expected)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ delimiter: '\t' as const, name: 'tab' },
|
||||
{ delimiter: '|' as const, name: 'pipe' },
|
||||
])('encodes root arrays with $name', ({ delimiter }) => {
|
||||
const arr = ['x', 'y', 'z']
|
||||
expect(encode(arr, { delimiter })).toBe(`[3${delimiter}]: x${delimiter}y${delimiter}z`)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ delimiter: '\t' as const, name: 'tab', expected: '[2\t]{id}:\n 1\n 2' },
|
||||
{ delimiter: '|' as const, name: 'pipe', expected: '[2|]{id}:\n 1\n 2' },
|
||||
])('encodes root arrays of objects with $name', ({ delimiter, expected }) => {
|
||||
const arr = [{ id: 1 }, { id: 2 }]
|
||||
expect(encode(arr, { delimiter })).toBe(expected)
|
||||
})
|
||||
})
|
||||
|
||||
describe('delimiter-aware quoting', () => {
|
||||
it.each([
|
||||
{ delimiter: '\t' as const, name: 'tab', char: '\t', input: ['a', 'b\tc', 'd'], expected: 'a\t"b\\tc"\td' },
|
||||
{ delimiter: '|' as const, name: 'pipe', char: '|', input: ['a', 'b|c', 'd'], expected: 'a|"b|c"|d' },
|
||||
])('quotes strings containing $name', ({ delimiter, input, expected }) => {
|
||||
expect(encode({ items: input }, { delimiter })).toBe(`items[${input.length}${delimiter}]: ${expected}`)
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ delimiter: '\t' as const, name: 'tab', input: ['a,b', 'c,d'], expected: 'a,b\tc,d' },
|
||||
{ delimiter: '|' as const, name: 'pipe', input: ['a,b', 'c,d'], expected: 'a,b|c,d' },
|
||||
])('does not quote commas with $name', ({ delimiter, input, expected }) => {
|
||||
expect(encode({ items: input }, { delimiter })).toBe(`items[${input.length}${delimiter}]: ${expected}`)
|
||||
})
|
||||
|
||||
it('quotes tabular values containing the delimiter', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ id: 1, note: 'a,b' },
|
||||
{ id: 2, note: 'c,d' },
|
||||
],
|
||||
}
|
||||
expect(encode(obj, { delimiter: ',' })).toBe('items[2]{id,note}:\n 1,"a,b"\n 2,"c,d"')
|
||||
expect(encode(obj, { delimiter: '\t' })).toBe('items[2\t]{id\tnote}:\n 1\ta,b\n 2\tc,d')
|
||||
})
|
||||
|
||||
it('does not quote commas in object values with non-comma delimiter', () => {
|
||||
expect(encode({ note: 'a,b' }, { delimiter: '|' })).toBe('note: a,b')
|
||||
expect(encode({ note: 'a,b' }, { delimiter: '\t' })).toBe('note: a,b')
|
||||
})
|
||||
|
||||
it('quotes nested array values containing the delimiter', () => {
|
||||
expect(encode({ pairs: [['a', 'b|c']] }, { delimiter: '|' })).toBe('pairs[1|]:\n - [2|]: a|"b|c"')
|
||||
expect(encode({ pairs: [['a', 'b\tc']] }, { delimiter: '\t' })).toBe('pairs[1\t]:\n - [2\t]: a\t"b\\tc"')
|
||||
})
|
||||
})
|
||||
|
||||
describe('delimiter-independent quoting rules', () => {
|
||||
it('preserves ambiguity quoting regardless of delimiter', () => {
|
||||
const obj = { items: ['true', '42', '-3.14'] }
|
||||
expect(encode(obj, { delimiter: '|' })).toBe('items[3|]: "true"|"42"|"-3.14"')
|
||||
expect(encode(obj, { delimiter: '\t' })).toBe('items[3\t]: "true"\t"42"\t"-3.14"')
|
||||
})
|
||||
|
||||
it('preserves structural quoting regardless of delimiter', () => {
|
||||
const obj = { items: ['[5]', '{key}', '- item'] }
|
||||
expect(encode(obj, { delimiter: '|' })).toBe('items[3|]: "[5]"|"{key}"|"- item"')
|
||||
expect(encode(obj, { delimiter: '\t' })).toBe('items[3\t]: "[5]"\t"{key}"\t"- item"')
|
||||
})
|
||||
|
||||
it('quotes keys containing the delimiter', () => {
|
||||
expect(encode({ 'a|b': 1 }, { delimiter: '|' })).toBe('"a|b": 1')
|
||||
expect(encode({ 'a\tb': 1 }, { delimiter: '\t' })).toBe('"a\\tb": 1')
|
||||
})
|
||||
|
||||
it('quotes tabular headers containing the delimiter', () => {
|
||||
const obj = { items: [{ 'a|b': 1 }, { 'a|b': 2 }] }
|
||||
expect(encode(obj, { delimiter: '|' })).toBe('items[2|]{"a|b"}:\n 1\n 2')
|
||||
})
|
||||
|
||||
it('header uses the active delimiter', () => {
|
||||
const obj = { items: [{ a: 1, b: 2 }, { a: 3, b: 4 }] }
|
||||
expect(encode(obj, { delimiter: '|' })).toBe('items[2|]{a|b}:\n 1|2\n 3|4')
|
||||
expect(encode(obj, { delimiter: '\t' })).toBe('items[2\t]{a\tb}:\n 1\t2\n 3\t4')
|
||||
})
|
||||
})
|
||||
|
||||
describe('formatting invariants with delimiters', () => {
|
||||
it.each([
|
||||
{ delimiter: '\t' as const, name: 'tab' },
|
||||
{ delimiter: '|' as const, name: 'pipe' },
|
||||
])('produces no trailing spaces with $name', ({ delimiter }) => {
|
||||
const obj = {
|
||||
user: { id: 123, name: 'Ada' },
|
||||
items: ['a', 'b'],
|
||||
}
|
||||
const result = encode(obj, { delimiter })
|
||||
const lines = result.split('\n')
|
||||
for (const line of lines) {
|
||||
expect(line).not.toMatch(/ $/)
|
||||
}
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ delimiter: '\t' as const, name: 'tab' },
|
||||
{ delimiter: '|' as const, name: 'pipe' },
|
||||
])('produces no trailing newline with $name', ({ delimiter }) => {
|
||||
const obj = { id: 123 }
|
||||
const result = encode(obj, { delimiter })
|
||||
expect(result).not.toMatch(/\n$/)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('length marker option', () => {
|
||||
it('adds length marker to primitive arrays', () => {
|
||||
const obj = { tags: ['reading', 'gaming', 'coding'] }
|
||||
expect(encode(obj, { lengthMarker: '#' })).toBe('tags[#3]: reading,gaming,coding')
|
||||
})
|
||||
|
||||
it('handles empty arrays', () => {
|
||||
expect(encode({ items: [] }, { lengthMarker: '#' })).toBe('items[#0]:')
|
||||
})
|
||||
|
||||
it('adds length marker to tabular arrays', () => {
|
||||
const obj = {
|
||||
items: [
|
||||
{ sku: 'A1', qty: 2, price: 9.99 },
|
||||
{ sku: 'B2', qty: 1, price: 14.5 },
|
||||
],
|
||||
}
|
||||
expect(encode(obj, { lengthMarker: '#' })).toBe('items[#2]{sku,qty,price}:\n A1,2,9.99\n B2,1,14.5')
|
||||
})
|
||||
|
||||
it('adds length marker to nested arrays', () => {
|
||||
const obj = { pairs: [['a', 'b'], ['c', 'd']] }
|
||||
expect(encode(obj, { lengthMarker: '#' })).toBe('pairs[#2]:\n - [#2]: a,b\n - [#2]: c,d')
|
||||
})
|
||||
|
||||
it('works with delimiter option', () => {
|
||||
const obj = { tags: ['reading', 'gaming', 'coding'] }
|
||||
expect(encode(obj, { lengthMarker: '#', delimiter: '|' })).toBe('tags[#3|]: reading|gaming|coding')
|
||||
})
|
||||
|
||||
it('default is false (no length marker)', () => {
|
||||
const obj = { tags: ['reading', 'gaming', 'coding'] }
|
||||
expect(encode(obj)).toBe('tags[3]: reading,gaming,coding')
|
||||
})
|
||||
})
|
||||
9
packages/toon/tsdown.config.ts
Normal file
9
packages/toon/tsdown.config.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
import type { UserConfig, UserConfigFn } from 'tsdown/config'
|
||||
import { defineConfig } from 'tsdown/config'
|
||||
|
||||
const config: UserConfig | UserConfigFn = defineConfig({
|
||||
entry: 'src/index.ts',
|
||||
dts: true,
|
||||
})
|
||||
|
||||
export default config
|
||||
Reference in New Issue
Block a user