github/toon
1
0
Fork 0
mirror of https://github.com/voson-wang/toon.git synced 2026-01-29 15:24:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add replacer function for encoding transformations and filtering (closes #209)

Browse Source
This commit is contained in:
Johann Schopplich
2025-12-01 17:15:21 +01:00
parent 0974a58527
commit 2c51932d51
6 changed files with 788 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
docs/reference/api.md
View File

@@ -144,6 +144,124 @@ for (const line of encodeLines(data, { delimiter: '\t' })) {
stream.end()
```
### Replacer Function
The `replacer` option allows you to transform or filter values during encoding. It works similarly to `JSON.stringify`'s replacer parameter, but with path tracking for more precise control.
#### Type Signature
```typescript
type EncodeReplacer = (
key: string,
value: JsonValue,
path: readonly (string | number)[]
) => unknown
```
#### Parameters
| Parameter | Type | Description |
|-----------|------|-------------|
| `key` | `string` | Property name, array index (as string), or empty string for root |
| `value` | `JsonValue` | The normalized value at this location |
| `path` | `readonly (string \| number)[]` | Path from root to current value |
#### Return Value
- Return the value unchanged to keep it
- Return a different value to replace it (will be normalized)
- Return `undefined` to omit properties/array elements
- For root value, `undefined` means "no change" (root cannot be omitted)
#### Examples
**Filtering sensitive data:**
```typescript
import { encode } from '@toon-format/toon'
const data = {
user: { name: 'Alice', password: 'secret123', email: 'alice@example.com' }
}
function replacer(key, value) {
if (key === 'password')
return undefined
return value
}
console.log(encode(data, { replacer }))
```
**Output:**
```yaml
user:
name: Alice
email: alice@example.com
```
**Transforming values:**
```typescript
const data = { user: 'alice', role: 'admin' }
function replacer(key, value) {
if (typeof value === 'string')
return value.toUpperCase()
return value
}
console.log(encode(data, { replacer }))
```
**Output:**
```yaml
user: ALICE
role: ADMIN
```
**Path-based transformations:**
```typescript
const data = {
metadata: { created: '2025-01-01' },
user: { created: '2025-01-02' }
}
function replacer(key, value, path) {
// Add timezone info only to top-level metadata
if (path.length === 1 && path[0] === 'metadata' && key === 'created') {
return `${value}T00:00:00Z`
}
return value
}
console.log(encode(data, { replacer }))
```
**Output:**
```yaml
metadata:
created: 2025-01-01T00:00:00Z
user:
created: 2025-01-02
```
::: tip Replacer Execution Order
The replacer is called in a depth-first manner:
1. Root value first (key = `''`, path = `[]`)
2. Then each property/element (with proper key and path)
3. Values are re-normalized after replacement
4. Children are processed after parent transformation
:::
::: warning Array Indices as Strings
Following `JSON.stringify` behavior, array indices are passed as strings (`'0'`, `'1'`, `'2'`, etc.) to the replacer, not as numbers.
:::
## Decoding Functions
### `decode(input, options?)`
@@ -375,6 +493,7 @@ Configuration for [`encode()`](#encode-input-options) and [`encodeLines()`](#enc
| `delimiter` | `','` \| `'\t'` \| `'\|'` | `','` | Delimiter for array values and tabular rows |
| `keyFolding` | `'off'` \| `'safe'` | `'off'` | Enable key folding to collapse single-key wrapper chains into dotted paths |
| `flattenDepth` | `number` | `Infinity` | Maximum number of segments to fold when `keyFolding` is enabled (values 0-1 have no practical effect) |
| `replacer` | `EncodeReplacer` | `undefined` | Optional hook to transform or omit values before encoding (see [Replacer Function](#replacer-function)) |
**Delimiter options:**

26
packages/toon/README.md
View File

@@ -785,6 +785,32 @@ for (const line of encodeLines(largeData)) {
> [!TIP]
> For streaming decode APIs, see [`decodeFromLines()`](https://toonformat.dev/reference/api#decodefromlines-lines-options) and [`decodeStream()`](https://toonformat.dev/reference/api#decodestream-source-options).
**Transforming values with replacer:**
```ts
import { encode } from '@toon-format/toon'
// Remove sensitive fields
const user = { name: 'Alice', password: 'secret', email: 'alice@example.com' }
const safe = encode(user, {
replacer: (key, value) => key === 'password' ? undefined : value
})
// name: Alice
// email: alice@example.com
// Transform values
const data = { status: 'active', count: 5 }
const transformed = encode(data, {
replacer: (key, value) =>
typeof value === 'string' ? value.toUpperCase() : value
})
// status: ACTIVE
// count: 5
```
> [!TIP]
> The `replacer` function provides fine-grained control over encoding, similar to `JSON.stringify`'s replacer but with path tracking. See the [API Reference](https://toonformat.dev/reference/api#replacer-function) for more examples.
## Playgrounds
Experiment with TOON format interactively using these tools for token comparison, format conversion, and validation.

126
packages/toon/src/encode/replacer.ts Normal file
View File

@@ -0,0 +1,126 @@
import type { EncodeReplacer, JsonArray, JsonObject, JsonValue } from '../types'
import { isJsonArray, isJsonObject, normalizeValue } from './normalize'
/**
* Applies a replacer function to a `JsonValue` and all its descendants.
*
* The replacer is called for:
* - The root value (with key='', path=[])
* - Every object property (with the property name as key)
* - Every array element (with the string index as key: '0', '1', etc.)
*
* @param root - The normalized `JsonValue` to transform
* @param replacer - The replacer function to apply
* @returns The transformed `JsonValue`
*/
export function applyReplacer(root: JsonValue, replacer: EncodeReplacer): JsonValue {
// Call replacer on root with empty string key and empty path
const replacedRoot = replacer('', root, [])
// For root, undefined means "no change" (don't omit the root)
if (replacedRoot === undefined) {
return transformChildren(root, replacer, [])
}
// Normalize the replaced value (in case user returned non-JsonValue)
const normalizedRoot = normalizeValue(replacedRoot)
// Recursively transform children
return transformChildren(normalizedRoot, replacer, [])
}
/**
* Recursively transforms the children of a `JsonValue` using the replacer.
*
* @param value - The value whose children should be transformed
* @param replacer - The replacer function to apply
* @param path - Current path from root
* @returns The value with transformed children
*/
function transformChildren(
value: JsonValue,
replacer: EncodeReplacer,
path: readonly (string | number)[],
): JsonValue {
if (isJsonObject(value)) {
return transformObject(value, replacer, path)
}
if (isJsonArray(value)) {
return transformArray(value, replacer, path)
}
// Primitives have no children
return value
}
/**
* Transforms an object by applying the replacer to each property.
*
* @param obj - The object to transform
* @param replacer - The replacer function to apply
* @param path - Current path from root
* @returns A new object with transformed properties
*/
function transformObject(
obj: JsonObject,
replacer: EncodeReplacer,
path: readonly (string | number)[],
): JsonObject {
const result: Record<string, JsonValue> = {}
for (const [key, value] of Object.entries(obj)) {
// Call replacer with the property key and current path
const childPath = [...path, key]
const replacedValue = replacer(key, value, childPath)
// undefined means omit this property
if (replacedValue === undefined) {
continue
}
// Normalize the replaced value
const normalizedValue = normalizeValue(replacedValue)
// Recursively transform children of the replaced value
result[key] = transformChildren(normalizedValue, replacer, childPath)
}
return result
}
/**
* Transforms an array by applying the replacer to each element.
*
* @param arr - The array to transform
* @param replacer - The replacer function to apply
* @param path - Current path from root
* @returns A new array with transformed elements
*/
function transformArray(
arr: JsonArray,
replacer: EncodeReplacer,
path: readonly (string | number)[],
): JsonArray {
const result: JsonValue[] = []
for (let i = 0; i < arr.length; i++) {
const value = arr[i]!
// Call replacer with string index (`'0'`, `'1'`, etc.) to match `JSON.stringify` behavior
const childPath = [...path, i]
const replacedValue = replacer(String(i), value, childPath)
// undefined means omit this element
if (replacedValue === undefined) {
continue
}
// Normalize the replaced value
const normalizedValue = normalizeValue(replacedValue)
// Recursively transform children of the replaced value
result.push(transformChildren(normalizedValue, replacer, childPath))
}
return result
}

11
packages/toon/src/index.ts
View File

@@ -5,6 +5,7 @@ import { buildValueFromEvents } from './decode/event-builder'
import { expandPathsSafe } from './decode/expand'
import { encodeJsonValue } from './encode/encoders'
import { normalizeValue } from './encode/normalize'
import { applyReplacer } from './encode/replacer'
export { DEFAULT_DELIMITER, DELIMITERS } from './constants'
export type {
@@ -13,6 +14,7 @@ export type {
Delimiter,
DelimiterKey,
EncodeOptions,
EncodeReplacer,
JsonArray,
JsonObject,
JsonPrimitive,
@@ -97,7 +99,13 @@ export function decode(input: string, options?: DecodeOptions): JsonValue {
export function encodeLines(input: unknown, options?: EncodeOptions): Iterable<string> {
const normalizedValue = normalizeValue(input)
const resolvedOptions = resolveOptions(options)
return encodeJsonValue(normalizedValue, resolvedOptions, 0)
// Apply replacer if provided
const maybeReplacedValue = resolvedOptions.replacer
? applyReplacer(normalizedValue, resolvedOptions.replacer)
: normalizedValue
return encodeJsonValue(maybeReplacedValue, resolvedOptions, 0)
}
/**
@@ -210,6 +218,7 @@ function resolveOptions(options?: EncodeOptions): ResolvedEncodeOptions {
delimiter: options?.delimiter ?? DEFAULT_DELIMITER,
keyFolding: options?.keyFolding ?? 'off',
flattenDepth: options?.flattenDepth ?? Number.POSITIVE_INFINITY,
replacer: options?.replacer,
}
}

45
packages/toon/src/types.ts
View File

@@ -13,6 +13,42 @@ export type JsonValue = JsonPrimitive | JsonObject | JsonArray
export type { Delimiter, DelimiterKey }
/**
* A function that transforms or filters values during encoding.
*
* Called for every value (root, object properties, array elements) during the encoding process.
* Similar to `JSON.stringify`'s replacer, but with path tracking.
*
* @param key - The property key or array index (as string). Empty string (`''`) for root value.
* @param value - The normalized `JsonValue` at this location.
* @param path - Array representing the path from root to this value.
*
* @returns The replacement value (will be normalized again), or `undefined` to omit.
* For root value, returning `undefined` means "no change" (don't omit root).
*
* @example
* ```ts
* // Remove password fields
* const replacer = (key, value) => {
* if (key === 'password') return undefined
* return value
* }
*
* // Add timestamps
* const replacer = (key, value, path) => {
* if (path.length === 0 && typeof value === 'object' && value !== null) {
* return { ...value, _timestamp: Date.now() }
* }
* return value
* }
* ```
*/
export type EncodeReplacer = (
key: string,
value: JsonValue,
path: readonly (string | number)[],
) => unknown
export interface EncodeOptions {
/**
* Number of spaces per indentation level.
@@ -38,9 +74,16 @@ export interface EncodeOptions {
* @default Infinity
*/
flattenDepth?: number
/**
* A function to transform or filter values during encoding.
* Called for the root value and every nested property/element.
* Return `undefined` to omit properties/elements (root cannot be omitted).
* @default undefined
*/
replacer?: EncodeReplacer
}
export type ResolvedEncodeOptions = Readonly<Required<EncodeOptions>>
export type ResolvedEncodeOptions = Readonly<Required<Omit<EncodeOptions, 'replacer'>>> & Pick<EncodeOptions, 'replacer'>
// #endregion

463
packages/toon/test/replacer.test.ts Normal file
View File

@@ -0,0 +1,463 @@
import type { EncodeReplacer, JsonObject, JsonValue } from '../src/types'
import { describe, expect, it } from 'vitest'
import { decode, encode } from '../src/index'
describe('replacer function', () => {
describe('basic filtering', () => {
it('removes properties by returning undefined', () => {
const input = { name: 'Alice', password: 'secret', email: 'alice@example.com' }
const replacer: EncodeReplacer = (key, value) => {
if (key === 'password')
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({ name: 'Alice', email: 'alice@example.com' })
expect(decoded).not.toHaveProperty('password')
})
it('removes array elements by returning undefined', () => {
const input = [1, 2, 3, 4, 5]
const replacer: EncodeReplacer = (key, value) => {
// Remove even numbers (key is index as string)
if (typeof value === 'number' && value % 2 === 0)
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual([1, 3, 5])
})
it('handles deeply nested filtering', () => {
const input = {
users: [
{ name: 'Alice', password: 'secret1', role: 'admin' },
{ name: 'Bob', password: 'secret2', role: 'user' },
],
}
const replacer: EncodeReplacer = (key, value) => {
if (key === 'password')
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({
users: [
{ name: 'Alice', role: 'admin' },
{ name: 'Bob', role: 'user' },
],
})
})
})
describe('value transformation', () => {
it('transforms primitive values', () => {
const input = { name: 'alice', age: 30 }
const replacer: EncodeReplacer = (key, value) => {
// Uppercase all strings
if (typeof value === 'string')
return value.toUpperCase()
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({ name: 'ALICE', age: 30 })
})
it('transforms objects', () => {
const input = { user: { name: 'Alice' } }
const replacer: EncodeReplacer = (key, value, path) => {
// Add metadata to all objects at depth 1
if (path.length === 1 && typeof value === 'object' && value !== null && !Array.isArray(value)) {
return { ...value as object, _id: `${key}_123` }
}
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({
user: { name: 'Alice', _id: 'user_123' },
})
})
it('transforms arrays', () => {
const input = { numbers: [1, 2, 3] }
const replacer: EncodeReplacer = (key, value) => {
// Double all numbers
if (typeof value === 'number')
return value * 2
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({ numbers: [2, 4, 6] })
})
})
describe('root value handling', () => {
it('calls replacer on root value with empty string key', () => {
const input = { value: 42 }
let rootKeySeen = false
let rootPathSeen = false
const replacer: EncodeReplacer = (key, value, path) => {
if (key === '' && path.length === 0) {
rootKeySeen = true
rootPathSeen = true
}
return value
}
encode(input, { replacer })
expect(rootKeySeen).toBe(true)
expect(rootPathSeen).toBe(true)
})
it('transforms root object', () => {
const input = { name: 'Alice' }
const replacer: EncodeReplacer = (key, value, path) => {
if (path.length === 0) {
return { ...value as object, timestamp: 1234567890 }
}
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({ name: 'Alice', timestamp: 1234567890 })
})
it('does not omit root when replacer returns undefined', () => {
const input = { name: 'Alice' }
const replacer: EncodeReplacer = (key, value, path) => {
// Try to omit root (should be ignored)
if (path.length === 0)
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
// Root should still be encoded
expect(decoded).toEqual({ name: 'Alice' })
})
it('handles primitive root values', () => {
const input = 'hello'
const replacer: EncodeReplacer = (key, value) => {
if (typeof value === 'string')
return value.toUpperCase()
return value
}
const result = encode(input, { replacer })
expect(result).toBe('HELLO')
})
it('provides correct arguments to root call', () => {
const input = { data: 'test' }
const calls: { key: string, path: (string | number)[] }[] = []
const replacer: EncodeReplacer = (key, value, path) => {
calls.push({ key, path: [...path] })
return value
}
encode(input, { replacer })
// First call should be root
expect(calls[0]).toEqual({ key: '', path: [] })
})
})
describe('path tracking', () => {
it('provides correct paths for nested objects', () => {
const input = {
user: {
profile: {
name: 'Alice',
},
},
}
const paths: string[] = []
const replacer: EncodeReplacer = (key, value, path) => {
paths.push(path.join('.'))
return value
}
encode(input, { replacer })
expect(paths).toContain('') // root
expect(paths).toContain('user')
expect(paths).toContain('user.profile')
expect(paths).toContain('user.profile.name')
})
it('provides correct paths for arrays with string indices', () => {
const input = { items: ['a', 'b', 'c'] }
const seenKeys: string[] = []
const replacer: EncodeReplacer = (key, value, path) => {
if (path.length > 0 && path[path.length - 1] !== 'items') {
seenKeys.push(key)
}
return value
}
encode(input, { replacer })
// Array indices should be string '0', '1', '2'
expect(seenKeys).toEqual(['0', '1', '2'])
})
it('provides correct paths for nested arrays', () => {
const input = {
matrix: [
[1, 2],
[3, 4],
],
}
const paths: string[] = []
const replacer: EncodeReplacer = (key, value, path) => {
if (typeof value === 'number') {
paths.push(`${path.join('.')} (key="${key}")`)
}
return value
}
encode(input, { replacer })
expect(paths).toContain('matrix.0.0 (key="0")')
expect(paths).toContain('matrix.0.1 (key="1")')
expect(paths).toContain('matrix.1.0 (key="0")')
expect(paths).toContain('matrix.1.1 (key="1")')
})
})
describe('edge cases', () => {
it('handles empty objects', () => {
const input = {}
const replacer: EncodeReplacer = (key, value) => value
const result = encode(input, { replacer })
expect(result).toBe('')
})
it('handles empty arrays', () => {
const input: JsonValue[] = []
const replacer: EncodeReplacer = (key, value) => value
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual([])
})
it('handles null values', () => {
const input = { value: null }
const replacer: EncodeReplacer = (key, value) => {
if (value === null)
return 'NULL'
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({ value: 'NULL' })
})
it('re-normalizes non-JsonValue returns', () => {
const input = { date: '2025-01-01' }
const replacer: EncodeReplacer = (key, value) => {
// Return a Date object (will be normalized to ISO string)
if (key === 'date')
return new Date(value as string)
return value
}
const result = encode(input, { replacer })
const decoded = decode(result) as JsonObject
// Date should be normalized to ISO string
expect(typeof decoded.date).toBe('string')
expect(decoded.date).toMatch(/^\d{4}-\d{2}-\d{2}T/)
})
it('handles all properties being filtered out', () => {
const input = { a: 1, b: 2, c: 3 }
const replacer: EncodeReplacer = (key, value, path) => {
// Filter out all properties (but not root)
if (path.length > 0)
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
// Should result in empty object
expect(decoded).toEqual({})
})
it('handles all array elements being filtered out', () => {
const input = [1, 2, 3]
const replacer: EncodeReplacer = (key, value, path) => {
// Filter out all elements
if (path.length > 0)
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
// Should result in empty array
expect(decoded).toEqual([])
})
it('handles nested objects with mixed omissions', () => {
const input = {
keep: 'this',
remove: 'that',
nested: {
keep: 'nested keep',
remove: 'nested remove',
},
}
const replacer: EncodeReplacer = (key, value) => {
if (key === 'remove')
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({
keep: 'this',
nested: {
keep: 'nested keep',
},
})
})
it('handles arrays with some elements removed', () => {
const input = { items: [{ id: 1, keep: true }, { id: 2, keep: false }, { id: 3, keep: true }] }
const replacer: EncodeReplacer = (key, value) => {
// Remove objects where keep is false
if (typeof value === 'object' && value !== null && !Array.isArray(value) && 'keep' in value && value.keep === false) {
return undefined
}
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({
items: [{ id: 1, keep: true }, { id: 3, keep: true }],
})
})
})
describe('integration with other options', () => {
it('works with keyFolding', () => {
const input = {
user: {
profile: {
name: 'Alice',
},
},
}
const replacer: EncodeReplacer = (key, value) => {
if (typeof value === 'string')
return value.toUpperCase()
return value
}
const result = encode(input, { replacer, keyFolding: 'safe' })
expect(result).toContain('user.profile.name: ALICE')
})
it('works with custom delimiters', () => {
const input = { items: [1, 2, 3] }
const replacer: EncodeReplacer = (key, value) => {
if (typeof value === 'number')
return value * 10
return value
}
const result = encode(input, { replacer, delimiter: '\t' })
expect(result).toContain('10\t20\t30')
})
it('works with custom indent', () => {
const input = { user: { name: 'Alice' } }
const replacer: EncodeReplacer = (key, value) => value
const result = encode(input, { replacer, indent: 4 })
// Should use 4-space indent
expect(result).toContain(' name: Alice')
})
})
describe('comparison with JSON.stringify replacer', () => {
it('behaves similarly to JSON.stringify for filtering', () => {
const input = { name: 'Alice', password: 'secret' }
// TOON replacer
const toonReplacer: EncodeReplacer = (key, value) => {
if (key === 'password')
return undefined
return value
}
// JSON.stringify replacer
const jsonReplacer = (key: string, value: unknown) => {
if (key === 'password')
return undefined
return value
}
const toonResult = decode(encode(input, { replacer: toonReplacer }))
const jsonResult = JSON.parse(JSON.stringify(input, jsonReplacer))
expect(toonResult).toEqual(jsonResult)
})
it('uses string indices for arrays like JSON.stringify', () => {
const input = ['a', 'b', 'c']
const keys: string[] = []
const replacer: EncodeReplacer = (key, value, path) => {
if (path.length > 0)
keys.push(key)
return value
}
encode(input, { replacer })
// Should match JSON.stringify behavior (string indices)
expect(keys).toEqual(['0', '1', '2'])
})
})
})