github/toon
1
0
Fork 0
mirror of https://github.com/voson-wang/toon.git synced 2026-01-29 23:34:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: share event handling in AST builder

Browse Source
This commit is contained in:
Johann Schopplich
2025-11-24 15:36:03 +01:00
parent 05abb99a7e
commit 8f96b8f0c8
2 changed files with 192 additions and 317 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
packages/toon/src/decode/decoders.ts
View File

@@ -1,4 +1,4 @@
import type { ArrayHeaderInfo, DecodeStreamOptions, Depth, JsonStreamEvent, ParsedLine } from '../types' import type { ArrayHeaderInfo, DecodeStreamOptions, Depth, JsonPrimitive, JsonStreamEvent, ParsedLine } from '../types'
import type { StreamingScanState } from './scanner' import type { StreamingScanState } from './scanner'
import { COLON, DEFAULT_DELIMITER, LIST_ITEM_MARKER, LIST_ITEM_PREFIX } from '../constants' import { COLON, DEFAULT_DELIMITER, LIST_ITEM_MARKER, LIST_ITEM_PREFIX } from '../constants'
import { findClosingQuote } from '../shared/string-utils' import { findClosingQuote } from '../shared/string-utils'
@@ -320,13 +320,7 @@ function* decodeTabularArraySync(
assertExpectedCount(values.length, header.fields!.length, 'tabular row values', options) assertExpectedCount(values.length, header.fields!.length, 'tabular row values', options)
const primitives = mapRowValuesToPrimitives(values) const primitives = mapRowValuesToPrimitives(values)
yield* yieldObjectFromFields(header.fields!, primitives)
yield { type: 'startObject' }
for (let i = 0; i < header.fields!.length; i++) {
yield { type: 'key', key: header.fields![i]! }
yield { type: 'primitive', value: primitives[i]! }
}
yield { type: 'endObject' }
rowCount++ rowCount++
} }
@@ -747,13 +741,7 @@ async function* decodeTabularArrayAsync(
assertExpectedCount(values.length, header.fields!.length, 'tabular row values', options) assertExpectedCount(values.length, header.fields!.length, 'tabular row values', options)
const primitives = mapRowValuesToPrimitives(values) const primitives = mapRowValuesToPrimitives(values)
yield* yieldObjectFromFields(header.fields!, primitives)
yield { type: 'startObject' }
for (let i = 0; i < header.fields!.length; i++) {
yield { type: 'key', key: header.fields![i]! }
yield { type: 'primitive', value: primitives[i]! }
}
yield { type: 'endObject' }
rowCount++ rowCount++
} }
@@ -963,3 +951,19 @@ async function* decodeListItemAsync(
} }
// #endregion // #endregion
// #region Shared decoder helpers
function* yieldObjectFromFields(
fields: string[],
primitives: JsonPrimitive[],
): Generator<JsonStreamEvent> {
yield { type: 'startObject' }
for (let i = 0; i < fields.length; i++) {
yield { type: 'key', key: fields[i]! }
yield { type: 'primitive', value: primitives[i]! }
}
yield { type: 'endObject' }
}
// #endregion

475
packages/toon/src/decode/event-builder.ts
View File

@@ -3,174 +3,27 @@ import { QUOTED_KEY_MARKER } from './expand'
// #region Build context types // #region Build context types
/**
* Stack context for building JSON values from events.
*/
type BuildContext type BuildContext
= | { type: 'object', obj: JsonObject, currentKey?: string, quotedKeys: Set<string> } = | { type: 'object', obj: JsonObject, currentKey?: string, quotedKeys: Set<string> }
| { type: 'array', arr: JsonValue[] } | { type: 'array', arr: JsonValue[] }
interface BuildState {
stack: BuildContext[]
root: JsonValue | undefined
}
// #endregion // #endregion
// #region Synchronous AST builder // #region Synchronous AST builder
export function buildValueFromEvents(events: Iterable<JsonStreamEvent>): JsonValue { export function buildValueFromEvents(events: Iterable<JsonStreamEvent>): JsonValue {
const stack: BuildContext[] = [] const state: BuildState = { stack: [], root: undefined }
let root: JsonValue | undefined
for (const event of events) { for (const event of events) {
switch (event.type) { applyEvent(state, event)
case 'startObject': {
const obj: JsonObject = {}
const quotedKeys = new Set<string>()
if (stack.length === 0) {
// Root object
stack.push({ type: 'object', obj, quotedKeys })
}
else {
const parent = stack[stack.length - 1]!
if (parent.type === 'object') {
if (parent.currentKey === undefined) {
throw new Error('Object startObject event without preceding key')
}
parent.obj[parent.currentKey] = obj
parent.currentKey = undefined
}
else if (parent.type === 'array') {
parent.arr.push(obj)
}
stack.push({ type: 'object', obj, quotedKeys })
}
break
}
case 'endObject': {
if (stack.length === 0) {
throw new Error('Unexpected endObject event')
}
const context = stack.pop()!
if (context.type !== 'object') {
throw new Error('Mismatched endObject event')
}
// Attach quoted keys metadata if any keys were quoted
if (context.quotedKeys.size > 0) {
Object.defineProperty(context.obj, QUOTED_KEY_MARKER, {
value: context.quotedKeys,
enumerable: false,
writable: false,
configurable: false,
})
}
if (stack.length === 0) {
root = context.obj
}
break
}
case 'startArray': {
const arr: JsonValue[] = []
if (stack.length === 0) {
// Root array
stack.push({ type: 'array', arr })
}
else {
const parent = stack[stack.length - 1]!
if (parent.type === 'object') {
if (parent.currentKey === undefined) {
throw new Error('Array startArray event without preceding key')
}
parent.obj[parent.currentKey] = arr
parent.currentKey = undefined
}
else if (parent.type === 'array') {
parent.arr.push(arr)
}
stack.push({ type: 'array', arr })
}
break
}
case 'endArray': {
if (stack.length === 0) {
throw new Error('Unexpected endArray event')
}
const context = stack.pop()!
if (context.type !== 'array') {
throw new Error('Mismatched endArray event')
}
if (stack.length === 0) {
root = context.arr
}
break
}
case 'key': {
if (stack.length === 0) {
throw new Error('Key event outside of object context')
}
const parent = stack[stack.length - 1]!
if (parent.type !== 'object') {
throw new Error('Key event in non-object context')
}
parent.currentKey = event.key
// Track quoted keys for path expansion
if (event.wasQuoted) {
parent.quotedKeys.add(event.key)
}
break
}
case 'primitive': {
if (stack.length === 0) {
// Root primitive
root = event.value
}
else {
const parent = stack[stack.length - 1]!
if (parent.type === 'object') {
if (parent.currentKey === undefined) {
throw new Error('Primitive event without preceding key in object')
}
parent.obj[parent.currentKey] = event.value
parent.currentKey = undefined
}
else if (parent.type === 'array') {
parent.arr.push(event.value)
}
}
break
}
}
} }
if (stack.length !== 0) { return finalizeState(state)
throw new Error('Incomplete event stream: stack not empty at end')
}
if (root === undefined) {
throw new Error('No root value built from events')
}
return root
} }
// #endregion // #endregion
@@ -178,157 +31,175 @@ export function buildValueFromEvents(events: Iterable<JsonStreamEvent>): JsonVal
// #region Asynchronous AST builder // #region Asynchronous AST builder
export async function buildValueFromEventsAsync(events: AsyncIterable<JsonStreamEvent>): Promise<JsonValue> { export async function buildValueFromEventsAsync(events: AsyncIterable<JsonStreamEvent>): Promise<JsonValue> {
const stack: BuildContext[] = [] const state: BuildState = { stack: [], root: undefined }
let root: JsonValue | undefined
for await (const event of events) { for await (const event of events) {
switch (event.type) { applyEvent(state, event)
case 'startObject': {
const obj: JsonObject = {}
const quotedKeys = new Set<string>()
if (stack.length === 0) {
stack.push({ type: 'object', obj, quotedKeys })
}
else {
const parent = stack[stack.length - 1]!
if (parent.type === 'object') {
if (parent.currentKey === undefined) {
throw new Error('Object startObject event without preceding key')
}
parent.obj[parent.currentKey] = obj
parent.currentKey = undefined
}
else if (parent.type === 'array') {
parent.arr.push(obj)
}
stack.push({ type: 'object', obj, quotedKeys })
}
break
}
case 'endObject': {
if (stack.length === 0) {
throw new Error('Unexpected endObject event')
}
const context = stack.pop()!
if (context.type !== 'object') {
throw new Error('Mismatched endObject event')
}
// Attach quoted keys metadata if any keys were quoted
if (context.quotedKeys.size > 0) {
Object.defineProperty(context.obj, QUOTED_KEY_MARKER, {
value: context.quotedKeys,
enumerable: false,
writable: false,
configurable: false,
})
}
if (stack.length === 0) {
root = context.obj
}
break
}
case 'startArray': {
const arr: JsonValue[] = []
if (stack.length === 0) {
stack.push({ type: 'array', arr })
}
else {
const parent = stack[stack.length - 1]!
if (parent.type === 'object') {
if (parent.currentKey === undefined) {
throw new Error('Array startArray event without preceding key')
}
parent.obj[parent.currentKey] = arr
parent.currentKey = undefined
}
else if (parent.type === 'array') {
parent.arr.push(arr)
}
stack.push({ type: 'array', arr })
}
break
}
case 'endArray': {
if (stack.length === 0) {
throw new Error('Unexpected endArray event')
}
const context = stack.pop()!
if (context.type !== 'array') {
throw new Error('Mismatched endArray event')
}
if (stack.length === 0) {
root = context.arr
}
break
}
case 'key': {
if (stack.length === 0) {
throw new Error('Key event outside of object context')
}
const parent = stack[stack.length - 1]!
if (parent.type !== 'object') {
throw new Error('Key event in non-object context')
}
parent.currentKey = event.key
// Track quoted keys for path expansion
if (event.wasQuoted) {
parent.quotedKeys.add(event.key)
}
break
}
case 'primitive': {
if (stack.length === 0) {
root = event.value
}
else {
const parent = stack[stack.length - 1]!
if (parent.type === 'object') {
if (parent.currentKey === undefined) {
throw new Error('Primitive event without preceding key in object')
}
parent.obj[parent.currentKey] = event.value
parent.currentKey = undefined
}
else if (parent.type === 'array') {
parent.arr.push(event.value)
}
}
break
}
}
} }
if (stack.length !== 0) { return finalizeState(state)
throw new Error('Incomplete event stream: stack not empty at end') }
}
// #endregion
if (root === undefined) {
throw new Error('No root value built from events') // #region Shared event handlers
}
function applyEvent(state: BuildState, event: JsonStreamEvent): void {
return root const { stack } = state
switch (event.type) {
case 'startObject': {
const obj: JsonObject = {}
const quotedKeys = new Set<string>()
if (stack.length === 0) {
// Root object
stack.push({ type: 'object', obj, quotedKeys })
}
else {
const parent = stack[stack.length - 1]!
if (parent.type === 'object') {
if (parent.currentKey === undefined) {
throw new Error('Object startObject event without preceding key')
}
parent.obj[parent.currentKey] = obj
parent.currentKey = undefined
}
else if (parent.type === 'array') {
parent.arr.push(obj)
}
stack.push({ type: 'object', obj, quotedKeys })
}
break
}
case 'endObject': {
if (stack.length === 0) {
throw new Error('Unexpected endObject event')
}
const context = stack.pop()!
if (context.type !== 'object') {
throw new Error('Mismatched endObject event')
}
// Attach quoted keys metadata if any keys were quoted
if (context.quotedKeys.size > 0) {
Object.defineProperty(context.obj, QUOTED_KEY_MARKER, {
value: context.quotedKeys,
enumerable: false,
writable: false,
configurable: false,
})
}
if (stack.length === 0) {
state.root = context.obj
}
break
}
case 'startArray': {
const arr: JsonValue[] = []
if (stack.length === 0) {
// Root array
stack.push({ type: 'array', arr })
}
else {
const parent = stack[stack.length - 1]!
if (parent.type === 'object') {
if (parent.currentKey === undefined) {
throw new Error('Array startArray event without preceding key')
}
parent.obj[parent.currentKey] = arr
parent.currentKey = undefined
}
else if (parent.type === 'array') {
parent.arr.push(arr)
}
stack.push({ type: 'array', arr })
}
break
}
case 'endArray': {
if (stack.length === 0) {
throw new Error('Unexpected endArray event')
}
const context = stack.pop()!
if (context.type !== 'array') {
throw new Error('Mismatched endArray event')
}
if (stack.length === 0) {
state.root = context.arr
}
break
}
case 'key': {
if (stack.length === 0) {
throw new Error('Key event outside of object context')
}
const parent = stack[stack.length - 1]!
if (parent.type !== 'object') {
throw new Error('Key event in non-object context')
}
parent.currentKey = event.key
// Track quoted keys for path expansion
if (event.wasQuoted) {
parent.quotedKeys.add(event.key)
}
break
}
case 'primitive': {
if (stack.length === 0) {
// Root primitive
state.root = event.value
}
else {
const parent = stack[stack.length - 1]!
if (parent.type === 'object') {
if (parent.currentKey === undefined) {
throw new Error('Primitive event without preceding key in object')
}
parent.obj[parent.currentKey] = event.value
parent.currentKey = undefined
}
else if (parent.type === 'array') {
parent.arr.push(event.value)
}
}
break
}
}
}
function finalizeState(state: BuildState): JsonValue {
if (state.stack.length !== 0) {
throw new Error('Incomplete event stream: stack not empty at end')
}
if (state.root === undefined) {
throw new Error('No root value built from events')
}
return state.root
} }
// #endregion // #endregion