From 7a05d03e730dfb18bc75d21bce7395dd05025a44 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Sat, 22 Nov 2025 08:53:47 +0100 Subject: [PATCH] test(cli) add streaming events coverage --- packages/cli/test/index.test.ts | 284 +++++++++++++ packages/cli/test/json-from-events.test.ts | 423 +++++++++++++++++++ packages/toon/src/encode/normalize.ts | 2 +- packages/toon/test/decodeStream.test.ts | 43 +- packages/toon/test/decodeStreamAsync.test.ts | 261 ++++++++++++ packages/toon/test/encodeLines.test.ts | 10 +- 6 files changed, 995 insertions(+), 28 deletions(-) create mode 100644 packages/cli/test/json-from-events.test.ts create mode 100644 packages/toon/test/decodeStreamAsync.test.ts diff --git a/packages/cli/test/index.test.ts b/packages/cli/test/index.test.ts index 0465a39..49d1b7a 100644 --- a/packages/cli/test/index.test.ts +++ b/packages/cli/test/index.test.ts @@ -303,6 +303,184 @@ describe('toon CLI', () => { }) }) + describe('encode options', () => { + it('encodes with --key-folding safe', async () => { + const data = { + data: { + metadata: { + items: ['a', 'b'], + }, + }, + } + + const context = await createCliTestContext({ + 'input.json': JSON.stringify(data), + }) + + try { + await context.run(['input.json', '--keyFolding', 'safe', '--output', 'output.toon']) + + const output = await context.read('output.toon') + const expected = encode(data, { keyFolding: 'safe' }) + + expect(output).toBe(expected) + } + finally { + await context.cleanup() + } + }) + + it('encodes with --flatten-depth', async () => { + const data = { + level1: { + level2: { + level3: { + value: 'deep', + }, + }, + }, + } + + const context = await createCliTestContext({ + 'input.json': JSON.stringify(data), + }) + + try { + await context.run(['input.json', '--keyFolding', 'safe', '--flattenDepth', '2', '--output', 'output.toon']) + + const output = await context.read('output.toon') + const expected = encode(data, { keyFolding: 'safe', flattenDepth: 2 }) + + expect(output).toBe(expected) + } + finally { + await context.cleanup() + } + }) + }) + + describe('decode options', () => { + it('decodes with --expand-paths safe', async () => { + const data = { + data: { + metadata: { + items: ['a', 'b'], + }, + }, + } + const toonInput = encode(data, { keyFolding: 'safe' }) + + const context = await createCliTestContext({ + 'input.toon': toonInput, + }) + + try { + await context.run(['input.toon', '--decode', '--expandPaths', 'safe', '--output', 'output.json']) + + const output = await context.read('output.json') + const result = JSON.parse(output) + + expect(result).toEqual(data) + } + finally { + await context.cleanup() + } + }) + + it('decodes with --indent for JSON formatting', async () => { + const data = { + a: 1, + b: [2, 3], + c: { nested: true }, + } + const toonInput = encode(data, { indent: 4 }) + + const context = await createCliTestContext({ + 'input.toon': toonInput, + }) + + try { + await context.run(['input.toon', '--decode', '--indent', '4', '--output', 'output.json']) + + const output = await context.read('output.json') + const result = JSON.parse(output) + + expect(result).toEqual(data) + expect(output).toContain(' ') // Should have 4-space indentation + } + finally { + await context.cleanup() + } + }) + + it('decodes root primitive number', async () => { + const toonInput = '42' + + const cleanup = mockStdin(toonInput) + + const writeChunks: string[] = [] + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + writeChunks.push(String(chunk)) + return true + }) + + try { + await runCli({ rawArgs: ['--decode'] }) + + const fullOutput = writeChunks.join('') + expect(fullOutput).toBe('42\n') + } + finally { + cleanup() + } + }) + + it('decodes root primitive string', async () => { + const toonInput = '"Hello World"' + + const cleanup = mockStdin(toonInput) + + const writeChunks: string[] = [] + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + writeChunks.push(String(chunk)) + return true + }) + + try { + await runCli({ rawArgs: ['--decode'] }) + + const fullOutput = writeChunks.join('') + const jsonOutput = fullOutput.endsWith('\n') ? fullOutput.slice(0, -1) : fullOutput + expect(JSON.parse(jsonOutput)).toBe('Hello World') + } + finally { + cleanup() + } + }) + + it('decodes root primitive boolean', async () => { + const toonInput = 'true' + + const cleanup = mockStdin(toonInput) + + const writeChunks: string[] = [] + vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + writeChunks.push(String(chunk)) + return true + }) + + try { + await runCli({ rawArgs: ['--decode'] }) + + const fullOutput = writeChunks.join('') + expect(fullOutput).toBe('true\n') + } + finally { + cleanup() + } + }) + }) + describe('streaming output', () => { it('streams large JSON to TOON file with identical output', async () => { const data = { @@ -337,6 +515,40 @@ describe('toon CLI', () => { } }) + it('streams large TOON to JSON file with streaming decode', async () => { + const data = { + records: Array.from({ length: 1000 }, (_, i) => ({ + id: i, + title: `Record ${i}`, + score: Math.random() * 100, + })), + } + + const toonContent = encode(data, { + delimiter: DEFAULT_DELIMITER, + indent: 2, + }) + + const context = await createCliTestContext({ + 'large-input.toon': toonContent, + }) + + const consolaSuccess = vi.spyOn(consola, 'success').mockImplementation(() => undefined) + + try { + await context.run(['large-input.toon', '--decode', '--output', 'output.json']) + + const output = await context.read('output.json') + const result = JSON.parse(output) + + expect(result).toEqual(data) + expect(consolaSuccess).toHaveBeenCalledWith(expect.stringMatching(/Decoded .* → .*/)) + } + finally { + await context.cleanup() + } + }) + it('streams to stdout using process.stdout.write', async () => { const data = { users: [ @@ -500,5 +712,77 @@ describe('toon CLI', () => { await context.cleanup() } }) + + it('rejects invalid --key-folding value', async () => { + const context = await createCliTestContext({ + 'input.json': JSON.stringify({ value: 1 }), + }) + + const consolaError = vi.spyOn(consola, 'error').mockImplementation(() => undefined) + const exitSpy = vi.mocked(process.exit) + + try { + await context.run(['input.json', '--keyFolding', 'invalid']) + + expect(exitSpy).toHaveBeenCalledWith(1) + + const errorCall = consolaError.mock.calls.at(0) + expect(errorCall).toBeDefined() + const [error] = errorCall! + expect(error).toBeInstanceOf(Error) + expect(error.message).toContain('Invalid keyFolding value') + } + finally { + await context.cleanup() + } + }) + + it('rejects invalid --expandPaths value', async () => { + const context = await createCliTestContext({ + 'input.toon': 'key: value', + }) + + const consolaError = vi.spyOn(consola, 'error').mockImplementation(() => undefined) + const exitSpy = vi.mocked(process.exit) + + try { + await context.run(['input.toon', '--decode', '--expandPaths', 'invalid']) + + expect(exitSpy).toHaveBeenCalledWith(1) + + const errorCall = consolaError.mock.calls.at(0) + expect(errorCall).toBeDefined() + const [error] = errorCall! + expect(error).toBeInstanceOf(Error) + expect(error.message).toContain('Invalid expandPaths value') + } + finally { + await context.cleanup() + } + }) + + it('rejects invalid --flattenDepth value', async () => { + const context = await createCliTestContext({ + 'input.json': JSON.stringify({ value: 1 }), + }) + + const consolaError = vi.spyOn(consola, 'error').mockImplementation(() => undefined) + const exitSpy = vi.mocked(process.exit) + + try { + await context.run(['input.json', '--flattenDepth', '-1']) + + expect(exitSpy).toHaveBeenCalledWith(1) + + const errorCall = consolaError.mock.calls.at(0) + expect(errorCall).toBeDefined() + const [error] = errorCall! + expect(error).toBeInstanceOf(Error) + expect(error.message).toContain('Invalid flattenDepth value') + } + finally { + await context.cleanup() + } + }) }) }) diff --git a/packages/cli/test/json-from-events.test.ts b/packages/cli/test/json-from-events.test.ts new file mode 100644 index 0000000..19b3fcb --- /dev/null +++ b/packages/cli/test/json-from-events.test.ts @@ -0,0 +1,423 @@ +import type { JsonStreamEvent } from '../../toon/src/types' +import { describe, expect, it } from 'vitest' +import { jsonStreamFromEvents } from '../src/json-from-events' + +describe('jsonStreamFromEvents', () => { + describe('primitives', () => { + it('converts null event', async () => { + const events = [ + { type: 'primitive' as const, value: null }, + ] + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(null)) + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(null, null, 2)) + }) + + it('converts boolean events', async () => { + const eventsTrue = [{ type: 'primitive' as const, value: true }] + const eventsFalse = [{ type: 'primitive' as const, value: false }] + + expect(await join(jsonStreamFromEvents(asyncEvents(eventsTrue), 0))).toBe(JSON.stringify(true)) + expect(await join(jsonStreamFromEvents(asyncEvents(eventsFalse), 0))).toBe(JSON.stringify(false)) + expect(await join(jsonStreamFromEvents(asyncEvents(eventsTrue), 2))).toBe(JSON.stringify(true, null, 2)) + }) + + it('converts number events', async () => { + const events0 = [{ type: 'primitive' as const, value: 0 }] + const events42 = [{ type: 'primitive' as const, value: 42 }] + const eventsNeg = [{ type: 'primitive' as const, value: -17 }] + const eventsFloat = [{ type: 'primitive' as const, value: 3.14159 }] + + expect(await join(jsonStreamFromEvents(asyncEvents(events0), 0))).toBe(JSON.stringify(0)) + expect(await join(jsonStreamFromEvents(asyncEvents(events42), 0))).toBe(JSON.stringify(42)) + expect(await join(jsonStreamFromEvents(asyncEvents(eventsNeg), 0))).toBe(JSON.stringify(-17)) + expect(await join(jsonStreamFromEvents(asyncEvents(eventsFloat), 0))).toBe(JSON.stringify(3.14159)) + expect(await join(jsonStreamFromEvents(asyncEvents(events42), 2))).toBe(JSON.stringify(42, null, 2)) + }) + + it('converts string events', async () => { + const eventsEmpty = [{ type: 'primitive' as const, value: '' }] + const eventsHello = [{ type: 'primitive' as const, value: 'hello' }] + const eventsQuotes = [{ type: 'primitive' as const, value: 'with "quotes"' }] + + expect(await join(jsonStreamFromEvents(asyncEvents(eventsEmpty), 0))).toBe(JSON.stringify('')) + expect(await join(jsonStreamFromEvents(asyncEvents(eventsHello), 0))).toBe(JSON.stringify('hello')) + expect(await join(jsonStreamFromEvents(asyncEvents(eventsQuotes), 0))).toBe(JSON.stringify('with "quotes"')) + }) + }) + + describe('empty containers', () => { + it('converts empty array events', async () => { + const events = [ + { type: 'startArray' as const, length: 0 }, + { type: 'endArray' as const }, + ] + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify([], null, 0)) + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify([], null, 2)) + }) + + it('converts empty object events', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'endObject' as const }, + ] + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify({}, null, 0)) + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify({}, null, 2)) + }) + }) + + describe('arrays', () => { + it('converts simple array events with compact formatting', async () => { + const events = [ + { type: 'startArray' as const, length: 3 }, + { type: 'primitive' as const, value: 1 }, + { type: 'primitive' as const, value: 2 }, + { type: 'primitive' as const, value: 3 }, + { type: 'endArray' as const }, + ] + const value = [1, 2, 3] + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0)) + }) + + it('converts simple array events with pretty formatting', async () => { + const events = [ + { type: 'startArray' as const, length: 3 }, + { type: 'primitive' as const, value: 1 }, + { type: 'primitive' as const, value: 2 }, + { type: 'primitive' as const, value: 3 }, + { type: 'endArray' as const }, + ] + const value = [1, 2, 3] + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2)) + }) + + it('converts mixed-type array events', async () => { + const events = [ + { type: 'startArray' as const, length: 5 }, + { type: 'primitive' as const, value: 1 }, + { type: 'primitive' as const, value: 'two' }, + { type: 'primitive' as const, value: true }, + { type: 'primitive' as const, value: null }, + { type: 'startObject' as const }, + { type: 'key' as const, key: 'key' }, + { type: 'primitive' as const, value: 'value' }, + { type: 'endObject' as const }, + { type: 'endArray' as const }, + ] + const value = [1, 'two', true, null, { key: 'value' }] + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0)) + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2)) + }) + + it('converts nested array events', async () => { + const events = [ + { type: 'startArray' as const, length: 3 }, + { type: 'startArray' as const, length: 2 }, + { type: 'primitive' as const, value: 1 }, + { type: 'primitive' as const, value: 2 }, + { type: 'endArray' as const }, + { type: 'startArray' as const, length: 2 }, + { type: 'primitive' as const, value: 3 }, + { type: 'primitive' as const, value: 4 }, + { type: 'endArray' as const }, + { type: 'startArray' as const, length: 2 }, + { type: 'primitive' as const, value: 5 }, + { type: 'primitive' as const, value: 6 }, + { type: 'endArray' as const }, + { type: 'endArray' as const }, + ] + const value = [[1, 2], [3, 4], [5, 6]] + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0)) + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2)) + }) + }) + + describe('objects', () => { + it('converts simple object events with compact formatting', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'key' as const, key: 'a' }, + { type: 'primitive' as const, value: 1 }, + { type: 'key' as const, key: 'b' }, + { type: 'primitive' as const, value: 2 }, + { type: 'key' as const, key: 'c' }, + { type: 'primitive' as const, value: 3 }, + { type: 'endObject' as const }, + ] + const value = { a: 1, b: 2, c: 3 } + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0)) + }) + + it('converts simple object events with pretty formatting', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'key' as const, key: 'a' }, + { type: 'primitive' as const, value: 1 }, + { type: 'key' as const, key: 'b' }, + { type: 'primitive' as const, value: 2 }, + { type: 'key' as const, key: 'c' }, + { type: 'primitive' as const, value: 3 }, + { type: 'endObject' as const }, + ] + const value = { a: 1, b: 2, c: 3 } + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2)) + }) + + it('converts object events with mixed value types', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'key' as const, key: 'num' }, + { type: 'primitive' as const, value: 42 }, + { type: 'key' as const, key: 'str' }, + { type: 'primitive' as const, value: 'hello' }, + { type: 'key' as const, key: 'bool' }, + { type: 'primitive' as const, value: true }, + { type: 'key' as const, key: 'nil' }, + { type: 'primitive' as const, value: null }, + { type: 'key' as const, key: 'arr' }, + { type: 'startArray' as const, length: 3 }, + { type: 'primitive' as const, value: 1 }, + { type: 'primitive' as const, value: 2 }, + { type: 'primitive' as const, value: 3 }, + { type: 'endArray' as const }, + { type: 'endObject' as const }, + ] + const value = { + num: 42, + str: 'hello', + bool: true, + nil: null, + arr: [1, 2, 3], + } + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0)) + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2)) + }) + + it('converts nested object events', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'key' as const, key: 'level1' }, + { type: 'startObject' as const }, + { type: 'key' as const, key: 'level2' }, + { type: 'startObject' as const }, + { type: 'key' as const, key: 'level3' }, + { type: 'primitive' as const, value: 'deep' }, + { type: 'endObject' as const }, + { type: 'endObject' as const }, + { type: 'endObject' as const }, + ] + const value = { + level1: { + level2: { + level3: 'deep', + }, + }, + } + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0)) + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2)) + }) + + it('handles special characters in keys', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'key' as const, key: 'normal-key' }, + { type: 'primitive' as const, value: 1 }, + { type: 'key' as const, key: 'key with spaces' }, + { type: 'primitive' as const, value: 2 }, + { type: 'key' as const, key: 'key:with:colons' }, + { type: 'primitive' as const, value: 3 }, + { type: 'key' as const, key: 'key"with"quotes' }, + { type: 'primitive' as const, value: 4 }, + { type: 'endObject' as const }, + ] + const value = { + 'normal-key': 1, + 'key with spaces': 2, + 'key:with:colons': 3, + 'key"with"quotes': 4, + } + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0)) + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2)) + }) + }) + + describe('complex nested structures', () => { + it('converts object containing arrays', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'key' as const, key: 'name' }, + { type: 'primitive' as const, value: 'Alice' }, + { type: 'key' as const, key: 'scores' }, + { type: 'startArray' as const, length: 3 }, + { type: 'primitive' as const, value: 95 }, + { type: 'primitive' as const, value: 87 }, + { type: 'primitive' as const, value: 92 }, + { type: 'endArray' as const }, + { type: 'key' as const, key: 'metadata' }, + { type: 'startObject' as const }, + { type: 'key' as const, key: 'tags' }, + { type: 'startArray' as const, length: 2 }, + { type: 'primitive' as const, value: 'math' }, + { type: 'primitive' as const, value: 'science' }, + { type: 'endArray' as const }, + { type: 'endObject' as const }, + { type: 'endObject' as const }, + ] + const value = { + name: 'Alice', + scores: [95, 87, 92], + metadata: { + tags: ['math', 'science'], + }, + } + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0)) + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2)) + }) + + it('converts array of objects', async () => { + const events = [ + { type: 'startArray' as const, length: 3 }, + { type: 'startObject' as const }, + { type: 'key' as const, key: 'id' }, + { type: 'primitive' as const, value: 1 }, + { type: 'key' as const, key: 'name' }, + { type: 'primitive' as const, value: 'Alice' }, + { type: 'endObject' as const }, + { type: 'startObject' as const }, + { type: 'key' as const, key: 'id' }, + { type: 'primitive' as const, value: 2 }, + { type: 'key' as const, key: 'name' }, + { type: 'primitive' as const, value: 'Bob' }, + { type: 'endObject' as const }, + { type: 'startObject' as const }, + { type: 'key' as const, key: 'id' }, + { type: 'primitive' as const, value: 3 }, + { type: 'key' as const, key: 'name' }, + { type: 'primitive' as const, value: 'Charlie' }, + { type: 'endObject' as const }, + { type: 'endArray' as const }, + ] + const value = [ + { id: 1, name: 'Alice' }, + { id: 2, name: 'Bob' }, + { id: 3, name: 'Charlie' }, + ] + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0)) + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2)) + }) + }) + + describe('indentation levels', () => { + const events = [ + { type: 'startObject' as const }, + { type: 'key' as const, key: 'a' }, + { type: 'startArray' as const, length: 2 }, + { type: 'primitive' as const, value: 1 }, + { type: 'primitive' as const, value: 2 }, + { type: 'endArray' as const }, + { type: 'key' as const, key: 'b' }, + { type: 'startObject' as const }, + { type: 'key' as const, key: 'c' }, + { type: 'primitive' as const, value: 3 }, + { type: 'endObject' as const }, + { type: 'endObject' as const }, + ] + const value = { a: [1, 2], b: { c: 3 } } + + it('handles indent=0 (compact)', async () => { + expect(await join(jsonStreamFromEvents(asyncEvents(events), 0))).toBe(JSON.stringify(value, null, 0)) + }) + + it('handles indent=2', async () => { + expect(await join(jsonStreamFromEvents(asyncEvents(events), 2))).toBe(JSON.stringify(value, null, 2)) + }) + + it('handles indent=4', async () => { + expect(await join(jsonStreamFromEvents(asyncEvents(events), 4))).toBe(JSON.stringify(value, null, 4)) + }) + + it('handles indent=8', async () => { + expect(await join(jsonStreamFromEvents(asyncEvents(events), 8))).toBe(JSON.stringify(value, null, 8)) + }) + }) + + describe('error handling', () => { + it('throws on mismatched endObject event', async () => { + const events = [ + { type: 'startArray' as const, length: 0 }, + { type: 'endObject' as const }, // Wrong closing event + ] + + await expect(async () => { + await join(jsonStreamFromEvents(asyncEvents(events), 0)) + }).rejects.toThrow('Mismatched endObject event') + }) + + it('throws on mismatched endArray event', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'endArray' as const }, // Wrong closing event + ] + + await expect(async () => { + await join(jsonStreamFromEvents(asyncEvents(events), 0)) + }).rejects.toThrow('Mismatched endArray event') + }) + + it('throws on key event outside object context', async () => { + const events = [ + { type: 'key' as const, key: 'invalid' }, + { type: 'primitive' as const, value: 1 }, + ] + + await expect(async () => { + await join(jsonStreamFromEvents(asyncEvents(events), 0)) + }).rejects.toThrow('Key event outside of object context') + }) + + it('throws on primitive in object without preceding key', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'primitive' as const, value: 'invalid' }, // No key before primitive + { type: 'endObject' as const }, + ] + + await expect(async () => { + await join(jsonStreamFromEvents(asyncEvents(events), 0)) + }).rejects.toThrow('Primitive event in object without preceding key') + }) + + it('throws on incomplete event stream', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'key' as const, key: 'name' }, + { type: 'primitive' as const, value: 'Alice' }, + // Missing `endObject` + ] + + await expect(async () => { + await join(jsonStreamFromEvents(asyncEvents(events), 0)) + }).rejects.toThrow('Incomplete event stream: unclosed objects or arrays') + }) + }) +}) + +/** + * Converts array of events to async iterable. + */ +async function* asyncEvents(events: JsonStreamEvent[]): AsyncIterable { + for (const event of events) { + await Promise.resolve() + yield event + } +} + +/** + * Joins chunks from an async iterable into a single string. + */ +async function join(iter: AsyncIterable): Promise { + const chunks: string[] = [] + for await (const chunk of iter) { + chunks.push(chunk) + } + return chunks.join('') +} diff --git a/packages/toon/src/encode/normalize.ts b/packages/toon/src/encode/normalize.ts index 53538a2..89d85ba 100644 --- a/packages/toon/src/encode/normalize.ts +++ b/packages/toon/src/encode/normalize.ts @@ -30,7 +30,7 @@ export function normalizeValue(value: unknown): JsonValue { if (value >= Number.MIN_SAFE_INTEGER && value <= Number.MAX_SAFE_INTEGER) { return Number(value) } - // Otherwise convert to string (will be unquoted as it looks numeric) + // Otherwise convert to string (will be quoted in output) return value.toString() } diff --git a/packages/toon/test/decodeStream.test.ts b/packages/toon/test/decodeStream.test.ts index 8744255..b9ba669 100644 --- a/packages/toon/test/decodeStream.test.ts +++ b/packages/toon/test/decodeStream.test.ts @@ -4,7 +4,7 @@ import { decode, decodeFromLines, decodeStreamSync } from '../src/index' describe('streaming decode', () => { describe('decodeStreamSync', () => { - it('should decode simple object', () => { + it('decode simple object', () => { const input = 'name: Alice\nage: 30' const lines = input.split('\n') const events = Array.from(decodeStreamSync(lines)) @@ -19,7 +19,7 @@ describe('streaming decode', () => { ]) }) - it('should decode nested object', () => { + it('decode nested object', () => { const input = 'user:\n name: Alice\n age: 30' const lines = input.split('\n') const events = Array.from(decodeStreamSync(lines)) @@ -37,7 +37,7 @@ describe('streaming decode', () => { ]) }) - it('should decode inline primitive array', () => { + it('decode inline primitive array', () => { const input = 'scores[3]: 95, 87, 92' const lines = input.split('\n') const events = Array.from(decodeStreamSync(lines)) @@ -54,7 +54,7 @@ describe('streaming decode', () => { ]) }) - it('should decode list array', () => { + it('decode list array', () => { const input = 'items[2]:\n - Apple\n - Banana' const lines = input.split('\n') const events = Array.from(decodeStreamSync(lines)) @@ -70,7 +70,7 @@ describe('streaming decode', () => { ]) }) - it('should decode tabular array', () => { + it('decode tabular array', () => { const input = 'users[2]{name,age}:\n Alice, 30\n Bob, 25' const lines = input.split('\n') const events = Array.from(decodeStreamSync(lines)) @@ -96,7 +96,7 @@ describe('streaming decode', () => { ]) }) - it('should decode root primitive', () => { + it('decode root primitive', () => { const input = 'Hello World' const lines = input.split('\n') const events = Array.from(decodeStreamSync(lines)) @@ -106,7 +106,7 @@ describe('streaming decode', () => { ]) }) - it('should decode root array', () => { + it('decode root array', () => { const input = '[2]:\n - Apple\n - Banana' const lines = input.split('\n') const events = Array.from(decodeStreamSync(lines)) @@ -119,7 +119,7 @@ describe('streaming decode', () => { ]) }) - it('should decode empty input as empty object', () => { + it('decode empty input as empty object', () => { const lines: string[] = [] const events = Array.from(decodeStreamSync(lines)) @@ -129,7 +129,7 @@ describe('streaming decode', () => { ]) }) - it('should throw on expandPaths option', () => { + it('throw on expandPaths option', () => { const input = 'name: Alice' const lines = input.split('\n') @@ -137,7 +137,7 @@ describe('streaming decode', () => { .toThrow('expandPaths is not supported in streaming decode') }) - it('should enforce strict mode validation', () => { + it('enforce strict mode validation', () => { const input = 'items[2]:\n - Apple' const lines = input.split('\n') @@ -145,7 +145,7 @@ describe('streaming decode', () => { .toThrow() }) - it('should allow count mismatch in non-strict mode', () => { + it('allow count mismatch in non-strict mode', () => { const input = 'items[2]:\n - Apple' const lines = input.split('\n') @@ -158,7 +158,7 @@ describe('streaming decode', () => { }) describe('buildValueFromEvents', () => { - it('should build object from events', () => { + it('build object from events', () => { const events = [ { type: 'startObject' as const }, { type: 'key' as const, key: 'name' }, @@ -173,7 +173,7 @@ describe('streaming decode', () => { expect(result).toEqual({ name: 'Alice', age: 30 }) }) - it('should build nested object from events', () => { + it('build nested object from events', () => { const events = [ { type: 'startObject' as const }, { type: 'key' as const, key: 'user' }, @@ -189,7 +189,7 @@ describe('streaming decode', () => { expect(result).toEqual({ user: { name: 'Alice' } }) }) - it('should build array from events', () => { + it('build array from events', () => { const events = [ { type: 'startArray' as const, length: 3 }, { type: 'primitive' as const, value: 1 }, @@ -203,7 +203,7 @@ describe('streaming decode', () => { expect(result).toEqual([1, 2, 3]) }) - it('should build primitive from events', () => { + it('build primitive from events', () => { const events = [ { type: 'primitive' as const, value: 'Hello' }, ] @@ -213,11 +213,11 @@ describe('streaming decode', () => { expect(result).toEqual('Hello') }) - it('should throw on incomplete event stream', () => { + it('throw on incomplete event stream', () => { const events = [ { type: 'startObject' as const }, { type: 'key' as const, key: 'name' }, - // Missing primitive and endObject + // Missing primitive and `endObject` ] expect(() => buildValueFromEvents(events)) @@ -226,7 +226,7 @@ describe('streaming decode', () => { }) describe('decodeFromLines', () => { - it('should produce same result as decode', () => { + it('produce same result as decode', () => { const input = 'name: Alice\nage: 30\nscores[3]: 95, 87, 92' const lines = input.split('\n') @@ -236,7 +236,7 @@ describe('streaming decode', () => { expect(fromLines).toEqual(fromString) }) - it('should support expandPaths option', () => { + it('support expandPaths option', () => { const input = 'user.name: Alice\nuser.age: 30' const lines = input.split('\n') @@ -250,7 +250,7 @@ describe('streaming decode', () => { }) }) - it('should handle complex nested structures', () => { + it('handle complex nested structures', () => { const input = [ 'users[2]:', ' - name: Alice', @@ -271,7 +271,7 @@ describe('streaming decode', () => { }) }) - it('should handle tabular arrays', () => { + it('handle tabular arrays', () => { const input = [ 'users[3]{name,age,city}:', ' Alice, 30, NYC', @@ -294,7 +294,6 @@ describe('streaming decode', () => { }) describe('streaming equivalence', () => { - // Test that streaming produces same results as non-streaming for various inputs const testCases = [ { name: 'simple object', diff --git a/packages/toon/test/decodeStreamAsync.test.ts b/packages/toon/test/decodeStreamAsync.test.ts new file mode 100644 index 0000000..5a794b9 --- /dev/null +++ b/packages/toon/test/decodeStreamAsync.test.ts @@ -0,0 +1,261 @@ +import { describe, expect, it } from 'vitest' +import { buildValueFromEventsAsync } from '../src/decode/event-builder' +import { decodeStream } from '../src/index' + +describe('async streaming decode', () => { + describe('decodeStream (async)', () => { + it('decodes simple object', async () => { + const input = 'name: Alice\nage: 30' + const lines = input.split('\n') + const events = await collect(decodeStream(asyncLines(lines))) + + expect(events).toEqual([ + { type: 'startObject' }, + { type: 'key', key: 'name' }, + { type: 'primitive', value: 'Alice' }, + { type: 'key', key: 'age' }, + { type: 'primitive', value: 30 }, + { type: 'endObject' }, + ]) + }) + + it('decodes nested object', async () => { + const input = 'user:\n name: Alice\n age: 30' + const lines = input.split('\n') + const events = await collect(decodeStream(asyncLines(lines))) + + expect(events).toEqual([ + { type: 'startObject' }, + { type: 'key', key: 'user' }, + { type: 'startObject' }, + { type: 'key', key: 'name' }, + { type: 'primitive', value: 'Alice' }, + { type: 'key', key: 'age' }, + { type: 'primitive', value: 30 }, + { type: 'endObject' }, + { type: 'endObject' }, + ]) + }) + + it('decodes inline primitive array', async () => { + const input = 'scores[3]: 95, 87, 92' + const lines = input.split('\n') + const events = await collect(decodeStream(asyncLines(lines))) + + expect(events).toEqual([ + { type: 'startObject' }, + { type: 'key', key: 'scores' }, + { type: 'startArray', length: 3 }, + { type: 'primitive', value: 95 }, + { type: 'primitive', value: 87 }, + { type: 'primitive', value: 92 }, + { type: 'endArray' }, + { type: 'endObject' }, + ]) + }) + + it('decodes list array', async () => { + const input = 'items[2]:\n - Apple\n - Banana' + const lines = input.split('\n') + const events = await collect(decodeStream(asyncLines(lines))) + + expect(events).toEqual([ + { type: 'startObject' }, + { type: 'key', key: 'items' }, + { type: 'startArray', length: 2 }, + { type: 'primitive', value: 'Apple' }, + { type: 'primitive', value: 'Banana' }, + { type: 'endArray' }, + { type: 'endObject' }, + ]) + }) + + it('decodes tabular array', async () => { + const input = 'users[2]{name,age}:\n Alice, 30\n Bob, 25' + const lines = input.split('\n') + const events = await collect(decodeStream(asyncLines(lines))) + + expect(events).toEqual([ + { type: 'startObject' }, + { type: 'key', key: 'users' }, + { type: 'startArray', length: 2 }, + { type: 'startObject' }, + { type: 'key', key: 'name' }, + { type: 'primitive', value: 'Alice' }, + { type: 'key', key: 'age' }, + { type: 'primitive', value: 30 }, + { type: 'endObject' }, + { type: 'startObject' }, + { type: 'key', key: 'name' }, + { type: 'primitive', value: 'Bob' }, + { type: 'key', key: 'age' }, + { type: 'primitive', value: 25 }, + { type: 'endObject' }, + { type: 'endArray' }, + { type: 'endObject' }, + ]) + }) + + it('decodes root primitive', async () => { + const input = 'Hello World' + const lines = input.split('\n') + const events = await collect(decodeStream(asyncLines(lines))) + + expect(events).toEqual([ + { type: 'primitive', value: 'Hello World' }, + ]) + }) + + it('decodes root array', async () => { + const input = '[2]:\n - Apple\n - Banana' + const lines = input.split('\n') + const events = await collect(decodeStream(asyncLines(lines))) + + expect(events).toEqual([ + { type: 'startArray', length: 2 }, + { type: 'primitive', value: 'Apple' }, + { type: 'primitive', value: 'Banana' }, + { type: 'endArray' }, + ]) + }) + + it('decodes empty input as empty object', async () => { + const lines: string[] = [] + const events = await collect(decodeStream(asyncLines(lines))) + + expect(events).toEqual([ + { type: 'startObject' }, + { type: 'endObject' }, + ]) + }) + + it('throws on expandPaths option', async () => { + const input = 'name: Alice' + const lines = input.split('\n') + + await expect(async () => { + await collect(decodeStream(asyncLines(lines), { expandPaths: 'safe' } as any)) + }).rejects.toThrow('expandPaths is not supported in streaming decode') + }) + + it('enforces strict mode validation', async () => { + const input = 'items[2]:\n - Apple' + const lines = input.split('\n') + + await expect(async () => { + await collect(decodeStream(asyncLines(lines), { strict: true })) + }).rejects.toThrow() + }) + + it('allows count mismatch in non-strict mode', async () => { + const input = 'items[2]:\n - Apple' + const lines = input.split('\n') + + // Should not throw in non-strict mode + const events = await collect(decodeStream(asyncLines(lines), { strict: false })) + + expect(events).toBeDefined() + expect(events[0]).toEqual({ type: 'startObject' }) + }) + }) + + describe('buildValueFromEventsAsync', () => { + it('builds object from events', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'key' as const, key: 'name' }, + { type: 'primitive' as const, value: 'Alice' }, + { type: 'key' as const, key: 'age' }, + { type: 'primitive' as const, value: 30 }, + { type: 'endObject' as const }, + ] + + const result = await buildValueFromEventsAsync(asyncEvents(events)) + + expect(result).toEqual({ name: 'Alice', age: 30 }) + }) + + it('builds nested object from events', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'key' as const, key: 'user' }, + { type: 'startObject' as const }, + { type: 'key' as const, key: 'name' }, + { type: 'primitive' as const, value: 'Alice' }, + { type: 'endObject' as const }, + { type: 'endObject' as const }, + ] + + const result = await buildValueFromEventsAsync(asyncEvents(events)) + + expect(result).toEqual({ user: { name: 'Alice' } }) + }) + + it('builds array from events', async () => { + const events = [ + { type: 'startArray' as const, length: 3 }, + { type: 'primitive' as const, value: 1 }, + { type: 'primitive' as const, value: 2 }, + { type: 'primitive' as const, value: 3 }, + { type: 'endArray' as const }, + ] + + const result = await buildValueFromEventsAsync(asyncEvents(events)) + + expect(result).toEqual([1, 2, 3]) + }) + + it('builds primitive from events', async () => { + const events = [ + { type: 'primitive' as const, value: 'Hello' }, + ] + + const result = await buildValueFromEventsAsync(asyncEvents(events)) + + expect(result).toEqual('Hello') + }) + + it('throws on incomplete event stream', async () => { + const events = [ + { type: 'startObject' as const }, + { type: 'key' as const, key: 'name' }, + // Missing primitive and `endObject` + ] + + await expect(async () => { + await buildValueFromEventsAsync(asyncEvents(events)) + }).rejects.toThrow('Incomplete event stream') + }) + }) +}) + +/** + * Collects all items from an async iterable into an array. + */ +async function collect(iterable: AsyncIterable): Promise { + const results: T[] = [] + for await (const item of iterable) { + results.push(item) + } + return results +} + +/** + * Converts array of lines to async iterable. + */ +async function* asyncLines(lines: string[]): AsyncGenerator { + for (const line of lines) { + await Promise.resolve() + yield line + } +} + +/** + * Converts array of events to async iterable. + */ +async function* asyncEvents(events: T[]): AsyncGenerator { + for (const event of events) { + await Promise.resolve() + yield event + } +} diff --git a/packages/toon/test/encodeLines.test.ts b/packages/toon/test/encodeLines.test.ts index 6366d6c..ed50186 100644 --- a/packages/toon/test/encodeLines.test.ts +++ b/packages/toon/test/encodeLines.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest' import { encodeLines } from '../src/index' describe('encodeLines', () => { - it('should yield lines without newline characters', () => { + it('yield lines without newline characters', () => { const value = { name: 'Alice', age: 30, city: 'Paris' } const lines = Array.from(encodeLines(value)) @@ -11,13 +11,13 @@ describe('encodeLines', () => { } }) - it('should yield zero lines for empty object', () => { + it('yield zero lines for empty object', () => { const lines = Array.from(encodeLines({})) expect(lines.length).toBe(0) }) - it('should be iterable with for-of loop', () => { + it('be iterable with for-of loop', () => { const value = { x: 10, y: 20 } const collectedLines: string[] = [] @@ -30,7 +30,7 @@ describe('encodeLines', () => { expect(collectedLines[1]).toBe('y: 20') }) - it('should not have trailing spaces in lines', () => { + it('not have trailing spaces in lines', () => { const value = { user: { name: 'Alice', @@ -47,7 +47,7 @@ describe('encodeLines', () => { } }) - it('should yield correct number of lines', () => { + it('yield correct number of lines', () => { const value = { a: 1, b: 2, c: 3 } const lines = Array.from(encodeLines(value))