feat: add replacer function for encoding transformations and filtering (closes #209)

This commit is contained in:
Johann Schopplich
2025-12-01 17:15:21 +01:00
parent 0974a58527
commit 2c51932d51
6 changed files with 788 additions and 2 deletions

View File

@@ -144,6 +144,124 @@ for (const line of encodeLines(data, { delimiter: '\t' })) {
stream.end() stream.end()
``` ```
### Replacer Function
The `replacer` option allows you to transform or filter values during encoding. It works similarly to `JSON.stringify`'s replacer parameter, but with path tracking for more precise control.
#### Type Signature
```typescript
type EncodeReplacer = (
key: string,
value: JsonValue,
path: readonly (string | number)[]
) => unknown
```
#### Parameters
| Parameter | Type | Description |
|-----------|------|-------------|
| `key` | `string` | Property name, array index (as string), or empty string for root |
| `value` | `JsonValue` | The normalized value at this location |
| `path` | `readonly (string \| number)[]` | Path from root to current value |
#### Return Value
- Return the value unchanged to keep it
- Return a different value to replace it (will be normalized)
- Return `undefined` to omit properties/array elements
- For root value, `undefined` means "no change" (root cannot be omitted)
#### Examples
**Filtering sensitive data:**
```typescript
import { encode } from '@toon-format/toon'
const data = {
user: { name: 'Alice', password: 'secret123', email: 'alice@example.com' }
}
function replacer(key, value) {
if (key === 'password')
return undefined
return value
}
console.log(encode(data, { replacer }))
```
**Output:**
```yaml
user:
name: Alice
email: alice@example.com
```
**Transforming values:**
```typescript
const data = { user: 'alice', role: 'admin' }
function replacer(key, value) {
if (typeof value === 'string')
return value.toUpperCase()
return value
}
console.log(encode(data, { replacer }))
```
**Output:**
```yaml
user: ALICE
role: ADMIN
```
**Path-based transformations:**
```typescript
const data = {
metadata: { created: '2025-01-01' },
user: { created: '2025-01-02' }
}
function replacer(key, value, path) {
// Add timezone info only to top-level metadata
if (path.length === 1 && path[0] === 'metadata' && key === 'created') {
return `${value}T00:00:00Z`
}
return value
}
console.log(encode(data, { replacer }))
```
**Output:**
```yaml
metadata:
created: 2025-01-01T00:00:00Z
user:
created: 2025-01-02
```
::: tip Replacer Execution Order
The replacer is called in a depth-first manner:
1. Root value first (key = `''`, path = `[]`)
2. Then each property/element (with proper key and path)
3. Values are re-normalized after replacement
4. Children are processed after parent transformation
:::
::: warning Array Indices as Strings
Following `JSON.stringify` behavior, array indices are passed as strings (`'0'`, `'1'`, `'2'`, etc.) to the replacer, not as numbers.
:::
## Decoding Functions ## Decoding Functions
### `decode(input, options?)` ### `decode(input, options?)`
@@ -375,6 +493,7 @@ Configuration for [`encode()`](#encode-input-options) and [`encodeLines()`](#enc
| `delimiter` | `','` \| `'\t'` \| `'\|'` | `','` | Delimiter for array values and tabular rows | | `delimiter` | `','` \| `'\t'` \| `'\|'` | `','` | Delimiter for array values and tabular rows |
| `keyFolding` | `'off'` \| `'safe'` | `'off'` | Enable key folding to collapse single-key wrapper chains into dotted paths | | `keyFolding` | `'off'` \| `'safe'` | `'off'` | Enable key folding to collapse single-key wrapper chains into dotted paths |
| `flattenDepth` | `number` | `Infinity` | Maximum number of segments to fold when `keyFolding` is enabled (values 0-1 have no practical effect) | | `flattenDepth` | `number` | `Infinity` | Maximum number of segments to fold when `keyFolding` is enabled (values 0-1 have no practical effect) |
| `replacer` | `EncodeReplacer` | `undefined` | Optional hook to transform or omit values before encoding (see [Replacer Function](#replacer-function)) |
**Delimiter options:** **Delimiter options:**

View File

@@ -785,6 +785,32 @@ for (const line of encodeLines(largeData)) {
> [!TIP] > [!TIP]
> For streaming decode APIs, see [`decodeFromLines()`](https://toonformat.dev/reference/api#decodefromlines-lines-options) and [`decodeStream()`](https://toonformat.dev/reference/api#decodestream-source-options). > For streaming decode APIs, see [`decodeFromLines()`](https://toonformat.dev/reference/api#decodefromlines-lines-options) and [`decodeStream()`](https://toonformat.dev/reference/api#decodestream-source-options).
**Transforming values with replacer:**
```ts
import { encode } from '@toon-format/toon'
// Remove sensitive fields
const user = { name: 'Alice', password: 'secret', email: 'alice@example.com' }
const safe = encode(user, {
replacer: (key, value) => key === 'password' ? undefined : value
})
// name: Alice
// email: alice@example.com
// Transform values
const data = { status: 'active', count: 5 }
const transformed = encode(data, {
replacer: (key, value) =>
typeof value === 'string' ? value.toUpperCase() : value
})
// status: ACTIVE
// count: 5
```
> [!TIP]
> The `replacer` function provides fine-grained control over encoding, similar to `JSON.stringify`'s replacer but with path tracking. See the [API Reference](https://toonformat.dev/reference/api#replacer-function) for more examples.
## Playgrounds ## Playgrounds
Experiment with TOON format interactively using these tools for token comparison, format conversion, and validation. Experiment with TOON format interactively using these tools for token comparison, format conversion, and validation.

View File

@@ -0,0 +1,126 @@
import type { EncodeReplacer, JsonArray, JsonObject, JsonValue } from '../types'
import { isJsonArray, isJsonObject, normalizeValue } from './normalize'
/**
* Applies a replacer function to a `JsonValue` and all its descendants.
*
* The replacer is called for:
* - The root value (with key='', path=[])
* - Every object property (with the property name as key)
* - Every array element (with the string index as key: '0', '1', etc.)
*
* @param root - The normalized `JsonValue` to transform
* @param replacer - The replacer function to apply
* @returns The transformed `JsonValue`
*/
export function applyReplacer(root: JsonValue, replacer: EncodeReplacer): JsonValue {
// Call replacer on root with empty string key and empty path
const replacedRoot = replacer('', root, [])
// For root, undefined means "no change" (don't omit the root)
if (replacedRoot === undefined) {
return transformChildren(root, replacer, [])
}
// Normalize the replaced value (in case user returned non-JsonValue)
const normalizedRoot = normalizeValue(replacedRoot)
// Recursively transform children
return transformChildren(normalizedRoot, replacer, [])
}
/**
* Recursively transforms the children of a `JsonValue` using the replacer.
*
* @param value - The value whose children should be transformed
* @param replacer - The replacer function to apply
* @param path - Current path from root
* @returns The value with transformed children
*/
function transformChildren(
value: JsonValue,
replacer: EncodeReplacer,
path: readonly (string | number)[],
): JsonValue {
if (isJsonObject(value)) {
return transformObject(value, replacer, path)
}
if (isJsonArray(value)) {
return transformArray(value, replacer, path)
}
// Primitives have no children
return value
}
/**
* Transforms an object by applying the replacer to each property.
*
* @param obj - The object to transform
* @param replacer - The replacer function to apply
* @param path - Current path from root
* @returns A new object with transformed properties
*/
function transformObject(
obj: JsonObject,
replacer: EncodeReplacer,
path: readonly (string | number)[],
): JsonObject {
const result: Record<string, JsonValue> = {}
for (const [key, value] of Object.entries(obj)) {
// Call replacer with the property key and current path
const childPath = [...path, key]
const replacedValue = replacer(key, value, childPath)
// undefined means omit this property
if (replacedValue === undefined) {
continue
}
// Normalize the replaced value
const normalizedValue = normalizeValue(replacedValue)
// Recursively transform children of the replaced value
result[key] = transformChildren(normalizedValue, replacer, childPath)
}
return result
}
/**
* Transforms an array by applying the replacer to each element.
*
* @param arr - The array to transform
* @param replacer - The replacer function to apply
* @param path - Current path from root
* @returns A new array with transformed elements
*/
function transformArray(
arr: JsonArray,
replacer: EncodeReplacer,
path: readonly (string | number)[],
): JsonArray {
const result: JsonValue[] = []
for (let i = 0; i < arr.length; i++) {
const value = arr[i]!
// Call replacer with string index (`'0'`, `'1'`, etc.) to match `JSON.stringify` behavior
const childPath = [...path, i]
const replacedValue = replacer(String(i), value, childPath)
// undefined means omit this element
if (replacedValue === undefined) {
continue
}
// Normalize the replaced value
const normalizedValue = normalizeValue(replacedValue)
// Recursively transform children of the replaced value
result.push(transformChildren(normalizedValue, replacer, childPath))
}
return result
}

View File

@@ -5,6 +5,7 @@ import { buildValueFromEvents } from './decode/event-builder'
import { expandPathsSafe } from './decode/expand' import { expandPathsSafe } from './decode/expand'
import { encodeJsonValue } from './encode/encoders' import { encodeJsonValue } from './encode/encoders'
import { normalizeValue } from './encode/normalize' import { normalizeValue } from './encode/normalize'
import { applyReplacer } from './encode/replacer'
export { DEFAULT_DELIMITER, DELIMITERS } from './constants' export { DEFAULT_DELIMITER, DELIMITERS } from './constants'
export type { export type {
@@ -13,6 +14,7 @@ export type {
Delimiter, Delimiter,
DelimiterKey, DelimiterKey,
EncodeOptions, EncodeOptions,
EncodeReplacer,
JsonArray, JsonArray,
JsonObject, JsonObject,
JsonPrimitive, JsonPrimitive,
@@ -97,7 +99,13 @@ export function decode(input: string, options?: DecodeOptions): JsonValue {
export function encodeLines(input: unknown, options?: EncodeOptions): Iterable<string> { export function encodeLines(input: unknown, options?: EncodeOptions): Iterable<string> {
const normalizedValue = normalizeValue(input) const normalizedValue = normalizeValue(input)
const resolvedOptions = resolveOptions(options) const resolvedOptions = resolveOptions(options)
return encodeJsonValue(normalizedValue, resolvedOptions, 0)
// Apply replacer if provided
const maybeReplacedValue = resolvedOptions.replacer
? applyReplacer(normalizedValue, resolvedOptions.replacer)
: normalizedValue
return encodeJsonValue(maybeReplacedValue, resolvedOptions, 0)
} }
/** /**
@@ -210,6 +218,7 @@ function resolveOptions(options?: EncodeOptions): ResolvedEncodeOptions {
delimiter: options?.delimiter ?? DEFAULT_DELIMITER, delimiter: options?.delimiter ?? DEFAULT_DELIMITER,
keyFolding: options?.keyFolding ?? 'off', keyFolding: options?.keyFolding ?? 'off',
flattenDepth: options?.flattenDepth ?? Number.POSITIVE_INFINITY, flattenDepth: options?.flattenDepth ?? Number.POSITIVE_INFINITY,
replacer: options?.replacer,
} }
} }

View File

@@ -13,6 +13,42 @@ export type JsonValue = JsonPrimitive | JsonObject | JsonArray
export type { Delimiter, DelimiterKey } export type { Delimiter, DelimiterKey }
/**
* A function that transforms or filters values during encoding.
*
* Called for every value (root, object properties, array elements) during the encoding process.
* Similar to `JSON.stringify`'s replacer, but with path tracking.
*
* @param key - The property key or array index (as string). Empty string (`''`) for root value.
* @param value - The normalized `JsonValue` at this location.
* @param path - Array representing the path from root to this value.
*
* @returns The replacement value (will be normalized again), or `undefined` to omit.
* For root value, returning `undefined` means "no change" (don't omit root).
*
* @example
* ```ts
* // Remove password fields
* const replacer = (key, value) => {
* if (key === 'password') return undefined
* return value
* }
*
* // Add timestamps
* const replacer = (key, value, path) => {
* if (path.length === 0 && typeof value === 'object' && value !== null) {
* return { ...value, _timestamp: Date.now() }
* }
* return value
* }
* ```
*/
export type EncodeReplacer = (
key: string,
value: JsonValue,
path: readonly (string | number)[],
) => unknown
export interface EncodeOptions { export interface EncodeOptions {
/** /**
* Number of spaces per indentation level. * Number of spaces per indentation level.
@@ -38,9 +74,16 @@ export interface EncodeOptions {
* @default Infinity * @default Infinity
*/ */
flattenDepth?: number flattenDepth?: number
/**
* A function to transform or filter values during encoding.
* Called for the root value and every nested property/element.
* Return `undefined` to omit properties/elements (root cannot be omitted).
* @default undefined
*/
replacer?: EncodeReplacer
} }
export type ResolvedEncodeOptions = Readonly<Required<EncodeOptions>> export type ResolvedEncodeOptions = Readonly<Required<Omit<EncodeOptions, 'replacer'>>> & Pick<EncodeOptions, 'replacer'>
// #endregion // #endregion

View File

@@ -0,0 +1,463 @@
import type { EncodeReplacer, JsonObject, JsonValue } from '../src/types'
import { describe, expect, it } from 'vitest'
import { decode, encode } from '../src/index'
describe('replacer function', () => {
describe('basic filtering', () => {
it('removes properties by returning undefined', () => {
const input = { name: 'Alice', password: 'secret', email: 'alice@example.com' }
const replacer: EncodeReplacer = (key, value) => {
if (key === 'password')
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({ name: 'Alice', email: 'alice@example.com' })
expect(decoded).not.toHaveProperty('password')
})
it('removes array elements by returning undefined', () => {
const input = [1, 2, 3, 4, 5]
const replacer: EncodeReplacer = (key, value) => {
// Remove even numbers (key is index as string)
if (typeof value === 'number' && value % 2 === 0)
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual([1, 3, 5])
})
it('handles deeply nested filtering', () => {
const input = {
users: [
{ name: 'Alice', password: 'secret1', role: 'admin' },
{ name: 'Bob', password: 'secret2', role: 'user' },
],
}
const replacer: EncodeReplacer = (key, value) => {
if (key === 'password')
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({
users: [
{ name: 'Alice', role: 'admin' },
{ name: 'Bob', role: 'user' },
],
})
})
})
describe('value transformation', () => {
it('transforms primitive values', () => {
const input = { name: 'alice', age: 30 }
const replacer: EncodeReplacer = (key, value) => {
// Uppercase all strings
if (typeof value === 'string')
return value.toUpperCase()
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({ name: 'ALICE', age: 30 })
})
it('transforms objects', () => {
const input = { user: { name: 'Alice' } }
const replacer: EncodeReplacer = (key, value, path) => {
// Add metadata to all objects at depth 1
if (path.length === 1 && typeof value === 'object' && value !== null && !Array.isArray(value)) {
return { ...value as object, _id: `${key}_123` }
}
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({
user: { name: 'Alice', _id: 'user_123' },
})
})
it('transforms arrays', () => {
const input = { numbers: [1, 2, 3] }
const replacer: EncodeReplacer = (key, value) => {
// Double all numbers
if (typeof value === 'number')
return value * 2
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({ numbers: [2, 4, 6] })
})
})
describe('root value handling', () => {
it('calls replacer on root value with empty string key', () => {
const input = { value: 42 }
let rootKeySeen = false
let rootPathSeen = false
const replacer: EncodeReplacer = (key, value, path) => {
if (key === '' && path.length === 0) {
rootKeySeen = true
rootPathSeen = true
}
return value
}
encode(input, { replacer })
expect(rootKeySeen).toBe(true)
expect(rootPathSeen).toBe(true)
})
it('transforms root object', () => {
const input = { name: 'Alice' }
const replacer: EncodeReplacer = (key, value, path) => {
if (path.length === 0) {
return { ...value as object, timestamp: 1234567890 }
}
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({ name: 'Alice', timestamp: 1234567890 })
})
it('does not omit root when replacer returns undefined', () => {
const input = { name: 'Alice' }
const replacer: EncodeReplacer = (key, value, path) => {
// Try to omit root (should be ignored)
if (path.length === 0)
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
// Root should still be encoded
expect(decoded).toEqual({ name: 'Alice' })
})
it('handles primitive root values', () => {
const input = 'hello'
const replacer: EncodeReplacer = (key, value) => {
if (typeof value === 'string')
return value.toUpperCase()
return value
}
const result = encode(input, { replacer })
expect(result).toBe('HELLO')
})
it('provides correct arguments to root call', () => {
const input = { data: 'test' }
const calls: { key: string, path: (string | number)[] }[] = []
const replacer: EncodeReplacer = (key, value, path) => {
calls.push({ key, path: [...path] })
return value
}
encode(input, { replacer })
// First call should be root
expect(calls[0]).toEqual({ key: '', path: [] })
})
})
describe('path tracking', () => {
it('provides correct paths for nested objects', () => {
const input = {
user: {
profile: {
name: 'Alice',
},
},
}
const paths: string[] = []
const replacer: EncodeReplacer = (key, value, path) => {
paths.push(path.join('.'))
return value
}
encode(input, { replacer })
expect(paths).toContain('') // root
expect(paths).toContain('user')
expect(paths).toContain('user.profile')
expect(paths).toContain('user.profile.name')
})
it('provides correct paths for arrays with string indices', () => {
const input = { items: ['a', 'b', 'c'] }
const seenKeys: string[] = []
const replacer: EncodeReplacer = (key, value, path) => {
if (path.length > 0 && path[path.length - 1] !== 'items') {
seenKeys.push(key)
}
return value
}
encode(input, { replacer })
// Array indices should be string '0', '1', '2'
expect(seenKeys).toEqual(['0', '1', '2'])
})
it('provides correct paths for nested arrays', () => {
const input = {
matrix: [
[1, 2],
[3, 4],
],
}
const paths: string[] = []
const replacer: EncodeReplacer = (key, value, path) => {
if (typeof value === 'number') {
paths.push(`${path.join('.')} (key="${key}")`)
}
return value
}
encode(input, { replacer })
expect(paths).toContain('matrix.0.0 (key="0")')
expect(paths).toContain('matrix.0.1 (key="1")')
expect(paths).toContain('matrix.1.0 (key="0")')
expect(paths).toContain('matrix.1.1 (key="1")')
})
})
describe('edge cases', () => {
it('handles empty objects', () => {
const input = {}
const replacer: EncodeReplacer = (key, value) => value
const result = encode(input, { replacer })
expect(result).toBe('')
})
it('handles empty arrays', () => {
const input: JsonValue[] = []
const replacer: EncodeReplacer = (key, value) => value
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual([])
})
it('handles null values', () => {
const input = { value: null }
const replacer: EncodeReplacer = (key, value) => {
if (value === null)
return 'NULL'
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({ value: 'NULL' })
})
it('re-normalizes non-JsonValue returns', () => {
const input = { date: '2025-01-01' }
const replacer: EncodeReplacer = (key, value) => {
// Return a Date object (will be normalized to ISO string)
if (key === 'date')
return new Date(value as string)
return value
}
const result = encode(input, { replacer })
const decoded = decode(result) as JsonObject
// Date should be normalized to ISO string
expect(typeof decoded.date).toBe('string')
expect(decoded.date).toMatch(/^\d{4}-\d{2}-\d{2}T/)
})
it('handles all properties being filtered out', () => {
const input = { a: 1, b: 2, c: 3 }
const replacer: EncodeReplacer = (key, value, path) => {
// Filter out all properties (but not root)
if (path.length > 0)
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
// Should result in empty object
expect(decoded).toEqual({})
})
it('handles all array elements being filtered out', () => {
const input = [1, 2, 3]
const replacer: EncodeReplacer = (key, value, path) => {
// Filter out all elements
if (path.length > 0)
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
// Should result in empty array
expect(decoded).toEqual([])
})
it('handles nested objects with mixed omissions', () => {
const input = {
keep: 'this',
remove: 'that',
nested: {
keep: 'nested keep',
remove: 'nested remove',
},
}
const replacer: EncodeReplacer = (key, value) => {
if (key === 'remove')
return undefined
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({
keep: 'this',
nested: {
keep: 'nested keep',
},
})
})
it('handles arrays with some elements removed', () => {
const input = { items: [{ id: 1, keep: true }, { id: 2, keep: false }, { id: 3, keep: true }] }
const replacer: EncodeReplacer = (key, value) => {
// Remove objects where keep is false
if (typeof value === 'object' && value !== null && !Array.isArray(value) && 'keep' in value && value.keep === false) {
return undefined
}
return value
}
const result = encode(input, { replacer })
const decoded = decode(result)
expect(decoded).toEqual({
items: [{ id: 1, keep: true }, { id: 3, keep: true }],
})
})
})
describe('integration with other options', () => {
it('works with keyFolding', () => {
const input = {
user: {
profile: {
name: 'Alice',
},
},
}
const replacer: EncodeReplacer = (key, value) => {
if (typeof value === 'string')
return value.toUpperCase()
return value
}
const result = encode(input, { replacer, keyFolding: 'safe' })
expect(result).toContain('user.profile.name: ALICE')
})
it('works with custom delimiters', () => {
const input = { items: [1, 2, 3] }
const replacer: EncodeReplacer = (key, value) => {
if (typeof value === 'number')
return value * 10
return value
}
const result = encode(input, { replacer, delimiter: '\t' })
expect(result).toContain('10\t20\t30')
})
it('works with custom indent', () => {
const input = { user: { name: 'Alice' } }
const replacer: EncodeReplacer = (key, value) => value
const result = encode(input, { replacer, indent: 4 })
// Should use 4-space indent
expect(result).toContain(' name: Alice')
})
})
describe('comparison with JSON.stringify replacer', () => {
it('behaves similarly to JSON.stringify for filtering', () => {
const input = { name: 'Alice', password: 'secret' }
// TOON replacer
const toonReplacer: EncodeReplacer = (key, value) => {
if (key === 'password')
return undefined
return value
}
// JSON.stringify replacer
const jsonReplacer = (key: string, value: unknown) => {
if (key === 'password')
return undefined
return value
}
const toonResult = decode(encode(input, { replacer: toonReplacer }))
const jsonResult = JSON.parse(JSON.stringify(input, jsonReplacer))
expect(toonResult).toEqual(jsonResult)
})
it('uses string indices for arrays like JSON.stringify', () => {
const input = ['a', 'b', 'c']
const keys: string[] = []
const replacer: EncodeReplacer = (key, value, path) => {
if (path.length > 0)
keys.push(key)
return value
}
encode(input, { replacer })
// Should match JSON.stringify behavior (string indices)
expect(keys).toEqual(['0', '1', '2'])
})
})
})