From 2c51932d51dc954dc80a93bc09cde1ce8614f6e7 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Mon, 1 Dec 2025 17:15:21 +0100 Subject: [PATCH] feat: add replacer function for encoding transformations and filtering (closes #209) --- docs/reference/api.md | 119 +++++++ packages/toon/README.md | 26 ++ packages/toon/src/encode/replacer.ts | 126 ++++++++ packages/toon/src/index.ts | 11 +- packages/toon/src/types.ts | 45 ++- packages/toon/test/replacer.test.ts | 463 +++++++++++++++++++++++++++ 6 files changed, 788 insertions(+), 2 deletions(-) create mode 100644 packages/toon/src/encode/replacer.ts create mode 100644 packages/toon/test/replacer.test.ts diff --git a/docs/reference/api.md b/docs/reference/api.md index 411f4fe..aab33c6 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -144,6 +144,124 @@ for (const line of encodeLines(data, { delimiter: '\t' })) { stream.end() ``` +### Replacer Function + +The `replacer` option allows you to transform or filter values during encoding. It works similarly to `JSON.stringify`'s replacer parameter, but with path tracking for more precise control. + +#### Type Signature + +```typescript +type EncodeReplacer = ( + key: string, + value: JsonValue, + path: readonly (string | number)[] +) => unknown +``` + +#### Parameters + +| Parameter | Type | Description | +|-----------|------|-------------| +| `key` | `string` | Property name, array index (as string), or empty string for root | +| `value` | `JsonValue` | The normalized value at this location | +| `path` | `readonly (string \| number)[]` | Path from root to current value | + +#### Return Value + +- Return the value unchanged to keep it +- Return a different value to replace it (will be normalized) +- Return `undefined` to omit properties/array elements +- For root value, `undefined` means "no change" (root cannot be omitted) + +#### Examples + +**Filtering sensitive data:** + +```typescript +import { encode } from '@toon-format/toon' + +const data = { + user: { name: 'Alice', password: 'secret123', email: 'alice@example.com' } +} + +function replacer(key, value) { + if (key === 'password') + return undefined + return value +} + +console.log(encode(data, { replacer })) +``` + +**Output:** + +```yaml +user: + name: Alice + email: alice@example.com +``` + +**Transforming values:** + +```typescript +const data = { user: 'alice', role: 'admin' } + +function replacer(key, value) { + if (typeof value === 'string') + return value.toUpperCase() + return value +} + +console.log(encode(data, { replacer })) +``` + +**Output:** + +```yaml +user: ALICE +role: ADMIN +``` + +**Path-based transformations:** + +```typescript +const data = { + metadata: { created: '2025-01-01' }, + user: { created: '2025-01-02' } +} + +function replacer(key, value, path) { + // Add timezone info only to top-level metadata + if (path.length === 1 && path[0] === 'metadata' && key === 'created') { + return `${value}T00:00:00Z` + } + return value +} + +console.log(encode(data, { replacer })) +``` + +**Output:** + +```yaml +metadata: + created: 2025-01-01T00:00:00Z +user: + created: 2025-01-02 +``` + +::: tip Replacer Execution Order +The replacer is called in a depth-first manner: +1. Root value first (key = `''`, path = `[]`) +2. Then each property/element (with proper key and path) +3. Values are re-normalized after replacement +4. Children are processed after parent transformation +::: + +::: warning Array Indices as Strings +Following `JSON.stringify` behavior, array indices are passed as strings (`'0'`, `'1'`, `'2'`, etc.) to the replacer, not as numbers. +::: + ## Decoding Functions ### `decode(input, options?)` @@ -375,6 +493,7 @@ Configuration for [`encode()`](#encode-input-options) and [`encodeLines()`](#enc | `delimiter` | `','` \| `'\t'` \| `'\|'` | `','` | Delimiter for array values and tabular rows | | `keyFolding` | `'off'` \| `'safe'` | `'off'` | Enable key folding to collapse single-key wrapper chains into dotted paths | | `flattenDepth` | `number` | `Infinity` | Maximum number of segments to fold when `keyFolding` is enabled (values 0-1 have no practical effect) | +| `replacer` | `EncodeReplacer` | `undefined` | Optional hook to transform or omit values before encoding (see [Replacer Function](#replacer-function)) | **Delimiter options:** diff --git a/packages/toon/README.md b/packages/toon/README.md index 00c078a..dc40bea 100644 --- a/packages/toon/README.md +++ b/packages/toon/README.md @@ -785,6 +785,32 @@ for (const line of encodeLines(largeData)) { > [!TIP] > For streaming decode APIs, see [`decodeFromLines()`](https://toonformat.dev/reference/api#decodefromlines-lines-options) and [`decodeStream()`](https://toonformat.dev/reference/api#decodestream-source-options). +**Transforming values with replacer:** + +```ts +import { encode } from '@toon-format/toon' + +// Remove sensitive fields +const user = { name: 'Alice', password: 'secret', email: 'alice@example.com' } +const safe = encode(user, { + replacer: (key, value) => key === 'password' ? undefined : value +}) +// name: Alice +// email: alice@example.com + +// Transform values +const data = { status: 'active', count: 5 } +const transformed = encode(data, { + replacer: (key, value) => + typeof value === 'string' ? value.toUpperCase() : value +}) +// status: ACTIVE +// count: 5 +``` + +> [!TIP] +> The `replacer` function provides fine-grained control over encoding, similar to `JSON.stringify`'s replacer but with path tracking. See the [API Reference](https://toonformat.dev/reference/api#replacer-function) for more examples. + ## Playgrounds Experiment with TOON format interactively using these tools for token comparison, format conversion, and validation. diff --git a/packages/toon/src/encode/replacer.ts b/packages/toon/src/encode/replacer.ts new file mode 100644 index 0000000..9d5e535 --- /dev/null +++ b/packages/toon/src/encode/replacer.ts @@ -0,0 +1,126 @@ +import type { EncodeReplacer, JsonArray, JsonObject, JsonValue } from '../types' +import { isJsonArray, isJsonObject, normalizeValue } from './normalize' + +/** + * Applies a replacer function to a `JsonValue` and all its descendants. + * + * The replacer is called for: + * - The root value (with key='', path=[]) + * - Every object property (with the property name as key) + * - Every array element (with the string index as key: '0', '1', etc.) + * + * @param root - The normalized `JsonValue` to transform + * @param replacer - The replacer function to apply + * @returns The transformed `JsonValue` + */ +export function applyReplacer(root: JsonValue, replacer: EncodeReplacer): JsonValue { + // Call replacer on root with empty string key and empty path + const replacedRoot = replacer('', root, []) + + // For root, undefined means "no change" (don't omit the root) + if (replacedRoot === undefined) { + return transformChildren(root, replacer, []) + } + + // Normalize the replaced value (in case user returned non-JsonValue) + const normalizedRoot = normalizeValue(replacedRoot) + + // Recursively transform children + return transformChildren(normalizedRoot, replacer, []) +} + +/** + * Recursively transforms the children of a `JsonValue` using the replacer. + * + * @param value - The value whose children should be transformed + * @param replacer - The replacer function to apply + * @param path - Current path from root + * @returns The value with transformed children + */ +function transformChildren( + value: JsonValue, + replacer: EncodeReplacer, + path: readonly (string | number)[], +): JsonValue { + if (isJsonObject(value)) { + return transformObject(value, replacer, path) + } + + if (isJsonArray(value)) { + return transformArray(value, replacer, path) + } + + // Primitives have no children + return value +} + +/** + * Transforms an object by applying the replacer to each property. + * + * @param obj - The object to transform + * @param replacer - The replacer function to apply + * @param path - Current path from root + * @returns A new object with transformed properties + */ +function transformObject( + obj: JsonObject, + replacer: EncodeReplacer, + path: readonly (string | number)[], +): JsonObject { + const result: Record = {} + + for (const [key, value] of Object.entries(obj)) { + // Call replacer with the property key and current path + const childPath = [...path, key] + const replacedValue = replacer(key, value, childPath) + + // undefined means omit this property + if (replacedValue === undefined) { + continue + } + + // Normalize the replaced value + const normalizedValue = normalizeValue(replacedValue) + + // Recursively transform children of the replaced value + result[key] = transformChildren(normalizedValue, replacer, childPath) + } + + return result +} + +/** + * Transforms an array by applying the replacer to each element. + * + * @param arr - The array to transform + * @param replacer - The replacer function to apply + * @param path - Current path from root + * @returns A new array with transformed elements + */ +function transformArray( + arr: JsonArray, + replacer: EncodeReplacer, + path: readonly (string | number)[], +): JsonArray { + const result: JsonValue[] = [] + + for (let i = 0; i < arr.length; i++) { + const value = arr[i]! + // Call replacer with string index (`'0'`, `'1'`, etc.) to match `JSON.stringify` behavior + const childPath = [...path, i] + const replacedValue = replacer(String(i), value, childPath) + + // undefined means omit this element + if (replacedValue === undefined) { + continue + } + + // Normalize the replaced value + const normalizedValue = normalizeValue(replacedValue) + + // Recursively transform children of the replaced value + result.push(transformChildren(normalizedValue, replacer, childPath)) + } + + return result +} diff --git a/packages/toon/src/index.ts b/packages/toon/src/index.ts index 754c8e5..17146f8 100644 --- a/packages/toon/src/index.ts +++ b/packages/toon/src/index.ts @@ -5,6 +5,7 @@ import { buildValueFromEvents } from './decode/event-builder' import { expandPathsSafe } from './decode/expand' import { encodeJsonValue } from './encode/encoders' import { normalizeValue } from './encode/normalize' +import { applyReplacer } from './encode/replacer' export { DEFAULT_DELIMITER, DELIMITERS } from './constants' export type { @@ -13,6 +14,7 @@ export type { Delimiter, DelimiterKey, EncodeOptions, + EncodeReplacer, JsonArray, JsonObject, JsonPrimitive, @@ -97,7 +99,13 @@ export function decode(input: string, options?: DecodeOptions): JsonValue { export function encodeLines(input: unknown, options?: EncodeOptions): Iterable { const normalizedValue = normalizeValue(input) const resolvedOptions = resolveOptions(options) - return encodeJsonValue(normalizedValue, resolvedOptions, 0) + + // Apply replacer if provided + const maybeReplacedValue = resolvedOptions.replacer + ? applyReplacer(normalizedValue, resolvedOptions.replacer) + : normalizedValue + + return encodeJsonValue(maybeReplacedValue, resolvedOptions, 0) } /** @@ -210,6 +218,7 @@ function resolveOptions(options?: EncodeOptions): ResolvedEncodeOptions { delimiter: options?.delimiter ?? DEFAULT_DELIMITER, keyFolding: options?.keyFolding ?? 'off', flattenDepth: options?.flattenDepth ?? Number.POSITIVE_INFINITY, + replacer: options?.replacer, } } diff --git a/packages/toon/src/types.ts b/packages/toon/src/types.ts index 9e38a2a..fc57e8e 100644 --- a/packages/toon/src/types.ts +++ b/packages/toon/src/types.ts @@ -13,6 +13,42 @@ export type JsonValue = JsonPrimitive | JsonObject | JsonArray export type { Delimiter, DelimiterKey } +/** + * A function that transforms or filters values during encoding. + * + * Called for every value (root, object properties, array elements) during the encoding process. + * Similar to `JSON.stringify`'s replacer, but with path tracking. + * + * @param key - The property key or array index (as string). Empty string (`''`) for root value. + * @param value - The normalized `JsonValue` at this location. + * @param path - Array representing the path from root to this value. + * + * @returns The replacement value (will be normalized again), or `undefined` to omit. + * For root value, returning `undefined` means "no change" (don't omit root). + * + * @example + * ```ts + * // Remove password fields + * const replacer = (key, value) => { + * if (key === 'password') return undefined + * return value + * } + * + * // Add timestamps + * const replacer = (key, value, path) => { + * if (path.length === 0 && typeof value === 'object' && value !== null) { + * return { ...value, _timestamp: Date.now() } + * } + * return value + * } + * ``` + */ +export type EncodeReplacer = ( + key: string, + value: JsonValue, + path: readonly (string | number)[], +) => unknown + export interface EncodeOptions { /** * Number of spaces per indentation level. @@ -38,9 +74,16 @@ export interface EncodeOptions { * @default Infinity */ flattenDepth?: number + /** + * A function to transform or filter values during encoding. + * Called for the root value and every nested property/element. + * Return `undefined` to omit properties/elements (root cannot be omitted). + * @default undefined + */ + replacer?: EncodeReplacer } -export type ResolvedEncodeOptions = Readonly> +export type ResolvedEncodeOptions = Readonly>> & Pick // #endregion diff --git a/packages/toon/test/replacer.test.ts b/packages/toon/test/replacer.test.ts new file mode 100644 index 0000000..eb1a070 --- /dev/null +++ b/packages/toon/test/replacer.test.ts @@ -0,0 +1,463 @@ +import type { EncodeReplacer, JsonObject, JsonValue } from '../src/types' +import { describe, expect, it } from 'vitest' +import { decode, encode } from '../src/index' + +describe('replacer function', () => { + describe('basic filtering', () => { + it('removes properties by returning undefined', () => { + const input = { name: 'Alice', password: 'secret', email: 'alice@example.com' } + const replacer: EncodeReplacer = (key, value) => { + if (key === 'password') + return undefined + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + + expect(decoded).toEqual({ name: 'Alice', email: 'alice@example.com' }) + expect(decoded).not.toHaveProperty('password') + }) + + it('removes array elements by returning undefined', () => { + const input = [1, 2, 3, 4, 5] + const replacer: EncodeReplacer = (key, value) => { + // Remove even numbers (key is index as string) + if (typeof value === 'number' && value % 2 === 0) + return undefined + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + + expect(decoded).toEqual([1, 3, 5]) + }) + + it('handles deeply nested filtering', () => { + const input = { + users: [ + { name: 'Alice', password: 'secret1', role: 'admin' }, + { name: 'Bob', password: 'secret2', role: 'user' }, + ], + } + const replacer: EncodeReplacer = (key, value) => { + if (key === 'password') + return undefined + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + + expect(decoded).toEqual({ + users: [ + { name: 'Alice', role: 'admin' }, + { name: 'Bob', role: 'user' }, + ], + }) + }) + }) + + describe('value transformation', () => { + it('transforms primitive values', () => { + const input = { name: 'alice', age: 30 } + const replacer: EncodeReplacer = (key, value) => { + // Uppercase all strings + if (typeof value === 'string') + return value.toUpperCase() + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + + expect(decoded).toEqual({ name: 'ALICE', age: 30 }) + }) + + it('transforms objects', () => { + const input = { user: { name: 'Alice' } } + const replacer: EncodeReplacer = (key, value, path) => { + // Add metadata to all objects at depth 1 + if (path.length === 1 && typeof value === 'object' && value !== null && !Array.isArray(value)) { + return { ...value as object, _id: `${key}_123` } + } + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + + expect(decoded).toEqual({ + user: { name: 'Alice', _id: 'user_123' }, + }) + }) + + it('transforms arrays', () => { + const input = { numbers: [1, 2, 3] } + const replacer: EncodeReplacer = (key, value) => { + // Double all numbers + if (typeof value === 'number') + return value * 2 + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + + expect(decoded).toEqual({ numbers: [2, 4, 6] }) + }) + }) + + describe('root value handling', () => { + it('calls replacer on root value with empty string key', () => { + const input = { value: 42 } + let rootKeySeen = false + let rootPathSeen = false + + const replacer: EncodeReplacer = (key, value, path) => { + if (key === '' && path.length === 0) { + rootKeySeen = true + rootPathSeen = true + } + return value + } + + encode(input, { replacer }) + + expect(rootKeySeen).toBe(true) + expect(rootPathSeen).toBe(true) + }) + + it('transforms root object', () => { + const input = { name: 'Alice' } + const replacer: EncodeReplacer = (key, value, path) => { + if (path.length === 0) { + return { ...value as object, timestamp: 1234567890 } + } + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + + expect(decoded).toEqual({ name: 'Alice', timestamp: 1234567890 }) + }) + + it('does not omit root when replacer returns undefined', () => { + const input = { name: 'Alice' } + const replacer: EncodeReplacer = (key, value, path) => { + // Try to omit root (should be ignored) + if (path.length === 0) + return undefined + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + + // Root should still be encoded + expect(decoded).toEqual({ name: 'Alice' }) + }) + + it('handles primitive root values', () => { + const input = 'hello' + const replacer: EncodeReplacer = (key, value) => { + if (typeof value === 'string') + return value.toUpperCase() + return value + } + + const result = encode(input, { replacer }) + expect(result).toBe('HELLO') + }) + + it('provides correct arguments to root call', () => { + const input = { data: 'test' } + const calls: { key: string, path: (string | number)[] }[] = [] + + const replacer: EncodeReplacer = (key, value, path) => { + calls.push({ key, path: [...path] }) + return value + } + + encode(input, { replacer }) + + // First call should be root + expect(calls[0]).toEqual({ key: '', path: [] }) + }) + }) + + describe('path tracking', () => { + it('provides correct paths for nested objects', () => { + const input = { + user: { + profile: { + name: 'Alice', + }, + }, + } + const paths: string[] = [] + + const replacer: EncodeReplacer = (key, value, path) => { + paths.push(path.join('.')) + return value + } + + encode(input, { replacer }) + + expect(paths).toContain('') // root + expect(paths).toContain('user') + expect(paths).toContain('user.profile') + expect(paths).toContain('user.profile.name') + }) + + it('provides correct paths for arrays with string indices', () => { + const input = { items: ['a', 'b', 'c'] } + const seenKeys: string[] = [] + + const replacer: EncodeReplacer = (key, value, path) => { + if (path.length > 0 && path[path.length - 1] !== 'items') { + seenKeys.push(key) + } + return value + } + + encode(input, { replacer }) + + // Array indices should be string '0', '1', '2' + expect(seenKeys).toEqual(['0', '1', '2']) + }) + + it('provides correct paths for nested arrays', () => { + const input = { + matrix: [ + [1, 2], + [3, 4], + ], + } + const paths: string[] = [] + + const replacer: EncodeReplacer = (key, value, path) => { + if (typeof value === 'number') { + paths.push(`${path.join('.')} (key="${key}")`) + } + return value + } + + encode(input, { replacer }) + + expect(paths).toContain('matrix.0.0 (key="0")') + expect(paths).toContain('matrix.0.1 (key="1")') + expect(paths).toContain('matrix.1.0 (key="0")') + expect(paths).toContain('matrix.1.1 (key="1")') + }) + }) + + describe('edge cases', () => { + it('handles empty objects', () => { + const input = {} + const replacer: EncodeReplacer = (key, value) => value + + const result = encode(input, { replacer }) + expect(result).toBe('') + }) + + it('handles empty arrays', () => { + const input: JsonValue[] = [] + const replacer: EncodeReplacer = (key, value) => value + + const result = encode(input, { replacer }) + const decoded = decode(result) + expect(decoded).toEqual([]) + }) + + it('handles null values', () => { + const input = { value: null } + const replacer: EncodeReplacer = (key, value) => { + if (value === null) + return 'NULL' + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + expect(decoded).toEqual({ value: 'NULL' }) + }) + + it('re-normalizes non-JsonValue returns', () => { + const input = { date: '2025-01-01' } + const replacer: EncodeReplacer = (key, value) => { + // Return a Date object (will be normalized to ISO string) + if (key === 'date') + return new Date(value as string) + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) as JsonObject + + // Date should be normalized to ISO string + expect(typeof decoded.date).toBe('string') + expect(decoded.date).toMatch(/^\d{4}-\d{2}-\d{2}T/) + }) + + it('handles all properties being filtered out', () => { + const input = { a: 1, b: 2, c: 3 } + const replacer: EncodeReplacer = (key, value, path) => { + // Filter out all properties (but not root) + if (path.length > 0) + return undefined + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + + // Should result in empty object + expect(decoded).toEqual({}) + }) + + it('handles all array elements being filtered out', () => { + const input = [1, 2, 3] + const replacer: EncodeReplacer = (key, value, path) => { + // Filter out all elements + if (path.length > 0) + return undefined + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + + // Should result in empty array + expect(decoded).toEqual([]) + }) + + it('handles nested objects with mixed omissions', () => { + const input = { + keep: 'this', + remove: 'that', + nested: { + keep: 'nested keep', + remove: 'nested remove', + }, + } + const replacer: EncodeReplacer = (key, value) => { + if (key === 'remove') + return undefined + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + + expect(decoded).toEqual({ + keep: 'this', + nested: { + keep: 'nested keep', + }, + }) + }) + + it('handles arrays with some elements removed', () => { + const input = { items: [{ id: 1, keep: true }, { id: 2, keep: false }, { id: 3, keep: true }] } + const replacer: EncodeReplacer = (key, value) => { + // Remove objects where keep is false + if (typeof value === 'object' && value !== null && !Array.isArray(value) && 'keep' in value && value.keep === false) { + return undefined + } + return value + } + + const result = encode(input, { replacer }) + const decoded = decode(result) + + expect(decoded).toEqual({ + items: [{ id: 1, keep: true }, { id: 3, keep: true }], + }) + }) + }) + + describe('integration with other options', () => { + it('works with keyFolding', () => { + const input = { + user: { + profile: { + name: 'Alice', + }, + }, + } + const replacer: EncodeReplacer = (key, value) => { + if (typeof value === 'string') + return value.toUpperCase() + return value + } + + const result = encode(input, { replacer, keyFolding: 'safe' }) + expect(result).toContain('user.profile.name: ALICE') + }) + + it('works with custom delimiters', () => { + const input = { items: [1, 2, 3] } + const replacer: EncodeReplacer = (key, value) => { + if (typeof value === 'number') + return value * 10 + return value + } + + const result = encode(input, { replacer, delimiter: '\t' }) + expect(result).toContain('10\t20\t30') + }) + + it('works with custom indent', () => { + const input = { user: { name: 'Alice' } } + const replacer: EncodeReplacer = (key, value) => value + + const result = encode(input, { replacer, indent: 4 }) + // Should use 4-space indent + expect(result).toContain(' name: Alice') + }) + }) + + describe('comparison with JSON.stringify replacer', () => { + it('behaves similarly to JSON.stringify for filtering', () => { + const input = { name: 'Alice', password: 'secret' } + + // TOON replacer + const toonReplacer: EncodeReplacer = (key, value) => { + if (key === 'password') + return undefined + return value + } + + // JSON.stringify replacer + const jsonReplacer = (key: string, value: unknown) => { + if (key === 'password') + return undefined + return value + } + + const toonResult = decode(encode(input, { replacer: toonReplacer })) + const jsonResult = JSON.parse(JSON.stringify(input, jsonReplacer)) + + expect(toonResult).toEqual(jsonResult) + }) + + it('uses string indices for arrays like JSON.stringify', () => { + const input = ['a', 'b', 'c'] + const keys: string[] = [] + + const replacer: EncodeReplacer = (key, value, path) => { + if (path.length > 0) + keys.push(key) + return value + } + + encode(input, { replacer }) + + // Should match JSON.stringify behavior (string indices) + expect(keys).toEqual(['0', '1', '2']) + }) + }) +})