feat: add replacer function for encoding transformations and filtering (closes #209)

packages/toon/README.md

@@ -785,6 +785,32 @@ for (const line of encodeLines(largeData)) {
 > [!TIP]
 > For streaming decode APIs, see [`decodeFromLines()`](https://toonformat.dev/reference/api#decodefromlines-lines-options) and [`decodeStream()`](https://toonformat.dev/reference/api#decodestream-source-options).
 
+**Transforming values with replacer:**
+
+```ts
+import { encode } from '@toon-format/toon'
+
+// Remove sensitive fields
+const user = { name: 'Alice', password: 'secret', email: 'alice@example.com' }
+const safe = encode(user, {
+  replacer: (key, value) => key === 'password' ? undefined : value
+})
+// name: Alice
+// email: alice@example.com
+
+// Transform values
+const data = { status: 'active', count: 5 }
+const transformed = encode(data, {
+  replacer: (key, value) =>
+    typeof value === 'string' ? value.toUpperCase() : value
+})
+// status: ACTIVE
+// count: 5
+```
+
+> [!TIP]
+> The `replacer` function provides fine-grained control over encoding, similar to `JSON.stringify`'s replacer but with path tracking. See the [API Reference](https://toonformat.dev/reference/api#replacer-function) for more examples.
+
 ## Playgrounds
 
 Experiment with TOON format interactively using these tools for token comparison, format conversion, and validation.

packages/toon/src/encode/replacer.ts

@@ -0,0 +1,126 @@
+import type { EncodeReplacer, JsonArray, JsonObject, JsonValue } from '../types'
+import { isJsonArray, isJsonObject, normalizeValue } from './normalize'
+
+/**
+ * Applies a replacer function to a `JsonValue` and all its descendants.
+ *
+ * The replacer is called for:
+ * - The root value (with key='', path=[])
+ * - Every object property (with the property name as key)
+ * - Every array element (with the string index as key: '0', '1', etc.)
+ *
+ * @param root - The normalized `JsonValue` to transform
+ * @param replacer - The replacer function to apply
+ * @returns The transformed `JsonValue`
+ */
+export function applyReplacer(root: JsonValue, replacer: EncodeReplacer): JsonValue {
+  // Call replacer on root with empty string key and empty path
+  const replacedRoot = replacer('', root, [])
+
+  // For root, undefined means "no change" (don't omit the root)
+  if (replacedRoot === undefined) {
+    return transformChildren(root, replacer, [])
+  }
+
+  // Normalize the replaced value (in case user returned non-JsonValue)
+  const normalizedRoot = normalizeValue(replacedRoot)
+
+  // Recursively transform children
+  return transformChildren(normalizedRoot, replacer, [])
+}
+
+/**
+ * Recursively transforms the children of a `JsonValue` using the replacer.
+ *
+ * @param value - The value whose children should be transformed
+ * @param replacer - The replacer function to apply
+ * @param path - Current path from root
+ * @returns The value with transformed children
+ */
+function transformChildren(
+  value: JsonValue,
+  replacer: EncodeReplacer,
+  path: readonly (string | number)[],
+): JsonValue {
+  if (isJsonObject(value)) {
+    return transformObject(value, replacer, path)
+  }
+
+  if (isJsonArray(value)) {
+    return transformArray(value, replacer, path)
+  }
+
+  // Primitives have no children
+  return value
+}
+
+/**
+ * Transforms an object by applying the replacer to each property.
+ *
+ * @param obj - The object to transform
+ * @param replacer - The replacer function to apply
+ * @param path - Current path from root
+ * @returns A new object with transformed properties
+ */
+function transformObject(
+  obj: JsonObject,
+  replacer: EncodeReplacer,
+  path: readonly (string | number)[],
+): JsonObject {
+  const result: Record<string, JsonValue> = {}
+
+  for (const [key, value] of Object.entries(obj)) {
+    // Call replacer with the property key and current path
+    const childPath = [...path, key]
+    const replacedValue = replacer(key, value, childPath)
+
+    // undefined means omit this property
+    if (replacedValue === undefined) {
+      continue
+    }
+
+    // Normalize the replaced value
+    const normalizedValue = normalizeValue(replacedValue)
+
+    // Recursively transform children of the replaced value
+    result[key] = transformChildren(normalizedValue, replacer, childPath)
+  }
+
+  return result
+}
+
+/**
+ * Transforms an array by applying the replacer to each element.
+ *
+ * @param arr - The array to transform
+ * @param replacer - The replacer function to apply
+ * @param path - Current path from root
+ * @returns A new array with transformed elements
+ */
+function transformArray(
+  arr: JsonArray,
+  replacer: EncodeReplacer,
+  path: readonly (string | number)[],
+): JsonArray {
+  const result: JsonValue[] = []
+
+  for (let i = 0; i < arr.length; i++) {
+    const value = arr[i]!
+    // Call replacer with string index (`'0'`, `'1'`, etc.) to match `JSON.stringify` behavior
+    const childPath = [...path, i]
+    const replacedValue = replacer(String(i), value, childPath)
+
+    // undefined means omit this element
+    if (replacedValue === undefined) {
+      continue
+    }
+
+    // Normalize the replaced value
+    const normalizedValue = normalizeValue(replacedValue)
+
+    // Recursively transform children of the replaced value
+    result.push(transformChildren(normalizedValue, replacer, childPath))
+  }
+
+  return result
+}

packages/toon/src/index.ts

@@ -5,6 +5,7 @@ import { buildValueFromEvents } from './decode/event-builder'
 import { expandPathsSafe } from './decode/expand'
 import { encodeJsonValue } from './encode/encoders'
 import { normalizeValue } from './encode/normalize'
+import { applyReplacer } from './encode/replacer'
 
 export { DEFAULT_DELIMITER, DELIMITERS } from './constants'
 export type {
@@ -13,6 +14,7 @@ export type {
   Delimiter,
   DelimiterKey,
   EncodeOptions,
+  EncodeReplacer,
   JsonArray,
   JsonObject,
   JsonPrimitive,
@@ -97,7 +99,13 @@ export function decode(input: string, options?: DecodeOptions): JsonValue {
 export function encodeLines(input: unknown, options?: EncodeOptions): Iterable<string> {
   const normalizedValue = normalizeValue(input)
   const resolvedOptions = resolveOptions(options)
-  return encodeJsonValue(normalizedValue, resolvedOptions, 0)
+
+  // Apply replacer if provided
+  const maybeReplacedValue = resolvedOptions.replacer
+    ? applyReplacer(normalizedValue, resolvedOptions.replacer)
+    : normalizedValue
+
+  return encodeJsonValue(maybeReplacedValue, resolvedOptions, 0)
 }
 
 /**
@@ -210,6 +218,7 @@ function resolveOptions(options?: EncodeOptions): ResolvedEncodeOptions {
     delimiter: options?.delimiter ?? DEFAULT_DELIMITER,
     keyFolding: options?.keyFolding ?? 'off',
     flattenDepth: options?.flattenDepth ?? Number.POSITIVE_INFINITY,
+    replacer: options?.replacer,
   }
 }
 

packages/toon/src/types.ts

@@ -13,6 +13,42 @@ export type JsonValue = JsonPrimitive | JsonObject | JsonArray
 
 export type { Delimiter, DelimiterKey }
 
+/**
+ * A function that transforms or filters values during encoding.
+ *
+ * Called for every value (root, object properties, array elements) during the encoding process.
+ * Similar to `JSON.stringify`'s replacer, but with path tracking.
+ *
+ * @param key - The property key or array index (as string). Empty string (`''`) for root value.
+ * @param value - The normalized `JsonValue` at this location.
+ * @param path - Array representing the path from root to this value.
+ *
+ * @returns The replacement value (will be normalized again), or `undefined` to omit.
+ *          For root value, returning `undefined` means "no change" (don't omit root).
+ *
+ * @example
+ * ```ts
+ * // Remove password fields
+ * const replacer = (key, value) => {
+ *   if (key === 'password') return undefined
+ *   return value
+ * }
+ *
+ * // Add timestamps
+ * const replacer = (key, value, path) => {
+ *   if (path.length === 0 && typeof value === 'object' && value !== null) {
+ *     return { ...value, _timestamp: Date.now() }
+ *   }
+ *   return value
+ * }
+ * ```
+ */
+export type EncodeReplacer = (
+  key: string,
+  value: JsonValue,
+  path: readonly (string | number)[],
+) => unknown
+
 export interface EncodeOptions {
   /**
    * Number of spaces per indentation level.
@@ -38,9 +74,16 @@ export interface EncodeOptions {
    * @default Infinity
    */
   flattenDepth?: number
+  /**
+   * A function to transform or filter values during encoding.
+   * Called for the root value and every nested property/element.
+   * Return `undefined` to omit properties/elements (root cannot be omitted).
+   * @default undefined
+   */
+  replacer?: EncodeReplacer
 }
 
-export type ResolvedEncodeOptions = Readonly<Required<EncodeOptions>>
+export type ResolvedEncodeOptions = Readonly<Required<Omit<EncodeOptions, 'replacer'>>> & Pick<EncodeOptions, 'replacer'>
 
 // #endregion
 

packages/toon/test/replacer.test.ts

@@ -0,0 +1,463 @@
+import type { EncodeReplacer, JsonObject, JsonValue } from '../src/types'
+import { describe, expect, it } from 'vitest'
+import { decode, encode } from '../src/index'
+
+describe('replacer function', () => {
+  describe('basic filtering', () => {
+    it('removes properties by returning undefined', () => {
+      const input = { name: 'Alice', password: 'secret', email: 'alice@example.com' }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (key === 'password')
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({ name: 'Alice', email: 'alice@example.com' })
+      expect(decoded).not.toHaveProperty('password')
+    })
+
+    it('removes array elements by returning undefined', () => {
+      const input = [1, 2, 3, 4, 5]
+      const replacer: EncodeReplacer = (key, value) => {
+        // Remove even numbers (key is index as string)
+        if (typeof value === 'number' && value % 2 === 0)
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual([1, 3, 5])
+    })
+
+    it('handles deeply nested filtering', () => {
+      const input = {
+        users: [
+          { name: 'Alice', password: 'secret1', role: 'admin' },
+          { name: 'Bob', password: 'secret2', role: 'user' },
+        ],
+      }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (key === 'password')
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({
+        users: [
+          { name: 'Alice', role: 'admin' },
+          { name: 'Bob', role: 'user' },
+        ],
+      })
+    })
+  })
+
+  describe('value transformation', () => {
+    it('transforms primitive values', () => {
+      const input = { name: 'alice', age: 30 }
+      const replacer: EncodeReplacer = (key, value) => {
+        // Uppercase all strings
+        if (typeof value === 'string')
+          return value.toUpperCase()
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({ name: 'ALICE', age: 30 })
+    })
+
+    it('transforms objects', () => {
+      const input = { user: { name: 'Alice' } }
+      const replacer: EncodeReplacer = (key, value, path) => {
+        // Add metadata to all objects at depth 1
+        if (path.length === 1 && typeof value === 'object' && value !== null && !Array.isArray(value)) {
+          return { ...value as object, _id: `${key}_123` }
+        }
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({
+        user: { name: 'Alice', _id: 'user_123' },
+      })
+    })
+
+    it('transforms arrays', () => {
+      const input = { numbers: [1, 2, 3] }
+      const replacer: EncodeReplacer = (key, value) => {
+        // Double all numbers
+        if (typeof value === 'number')
+          return value * 2
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({ numbers: [2, 4, 6] })
+    })
+  })
+
+  describe('root value handling', () => {
+    it('calls replacer on root value with empty string key', () => {
+      const input = { value: 42 }
+      let rootKeySeen = false
+      let rootPathSeen = false
+
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (key === '' && path.length === 0) {
+          rootKeySeen = true
+          rootPathSeen = true
+        }
+        return value
+      }
+
+      encode(input, { replacer })
+
+      expect(rootKeySeen).toBe(true)
+      expect(rootPathSeen).toBe(true)
+    })
+
+    it('transforms root object', () => {
+      const input = { name: 'Alice' }
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (path.length === 0) {
+          return { ...value as object, timestamp: 1234567890 }
+        }
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({ name: 'Alice', timestamp: 1234567890 })
+    })
+
+    it('does not omit root when replacer returns undefined', () => {
+      const input = { name: 'Alice' }
+      const replacer: EncodeReplacer = (key, value, path) => {
+        // Try to omit root (should be ignored)
+        if (path.length === 0)
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      // Root should still be encoded
+      expect(decoded).toEqual({ name: 'Alice' })
+    })
+
+    it('handles primitive root values', () => {
+      const input = 'hello'
+      const replacer: EncodeReplacer = (key, value) => {
+        if (typeof value === 'string')
+          return value.toUpperCase()
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      expect(result).toBe('HELLO')
+    })
+
+    it('provides correct arguments to root call', () => {
+      const input = { data: 'test' }
+      const calls: { key: string, path: (string | number)[] }[] = []
+
+      const replacer: EncodeReplacer = (key, value, path) => {
+        calls.push({ key, path: [...path] })
+        return value
+      }
+
+      encode(input, { replacer })
+
+      // First call should be root
+      expect(calls[0]).toEqual({ key: '', path: [] })
+    })
+  })
+
+  describe('path tracking', () => {
+    it('provides correct paths for nested objects', () => {
+      const input = {
+        user: {
+          profile: {
+            name: 'Alice',
+          },
+        },
+      }
+      const paths: string[] = []
+
+      const replacer: EncodeReplacer = (key, value, path) => {
+        paths.push(path.join('.'))
+        return value
+      }
+
+      encode(input, { replacer })
+
+      expect(paths).toContain('') // root
+      expect(paths).toContain('user')
+      expect(paths).toContain('user.profile')
+      expect(paths).toContain('user.profile.name')
+    })
+
+    it('provides correct paths for arrays with string indices', () => {
+      const input = { items: ['a', 'b', 'c'] }
+      const seenKeys: string[] = []
+
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (path.length > 0 && path[path.length - 1] !== 'items') {
+          seenKeys.push(key)
+        }
+        return value
+      }
+
+      encode(input, { replacer })
+
+      // Array indices should be string '0', '1', '2'
+      expect(seenKeys).toEqual(['0', '1', '2'])
+    })
+
+    it('provides correct paths for nested arrays', () => {
+      const input = {
+        matrix: [
+          [1, 2],
+          [3, 4],
+        ],
+      }
+      const paths: string[] = []
+
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (typeof value === 'number') {
+          paths.push(`${path.join('.')} (key="${key}")`)
+        }
+        return value
+      }
+
+      encode(input, { replacer })
+
+      expect(paths).toContain('matrix.0.0 (key="0")')
+      expect(paths).toContain('matrix.0.1 (key="1")')
+      expect(paths).toContain('matrix.1.0 (key="0")')
+      expect(paths).toContain('matrix.1.1 (key="1")')
+    })
+  })
+
+  describe('edge cases', () => {
+    it('handles empty objects', () => {
+      const input = {}
+      const replacer: EncodeReplacer = (key, value) => value
+
+      const result = encode(input, { replacer })
+      expect(result).toBe('')
+    })
+
+    it('handles empty arrays', () => {
+      const input: JsonValue[] = []
+      const replacer: EncodeReplacer = (key, value) => value
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+      expect(decoded).toEqual([])
+    })
+
+    it('handles null values', () => {
+      const input = { value: null }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (value === null)
+          return 'NULL'
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+      expect(decoded).toEqual({ value: 'NULL' })
+    })
+
+    it('re-normalizes non-JsonValue returns', () => {
+      const input = { date: '2025-01-01' }
+      const replacer: EncodeReplacer = (key, value) => {
+        // Return a Date object (will be normalized to ISO string)
+        if (key === 'date')
+          return new Date(value as string)
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result) as JsonObject
+
+      // Date should be normalized to ISO string
+      expect(typeof decoded.date).toBe('string')
+      expect(decoded.date).toMatch(/^\d{4}-\d{2}-\d{2}T/)
+    })
+
+    it('handles all properties being filtered out', () => {
+      const input = { a: 1, b: 2, c: 3 }
+      const replacer: EncodeReplacer = (key, value, path) => {
+        // Filter out all properties (but not root)
+        if (path.length > 0)
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      // Should result in empty object
+      expect(decoded).toEqual({})
+    })
+
+    it('handles all array elements being filtered out', () => {
+      const input = [1, 2, 3]
+      const replacer: EncodeReplacer = (key, value, path) => {
+        // Filter out all elements
+        if (path.length > 0)
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      // Should result in empty array
+      expect(decoded).toEqual([])
+    })
+
+    it('handles nested objects with mixed omissions', () => {
+      const input = {
+        keep: 'this',
+        remove: 'that',
+        nested: {
+          keep: 'nested keep',
+          remove: 'nested remove',
+        },
+      }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (key === 'remove')
+          return undefined
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({
+        keep: 'this',
+        nested: {
+          keep: 'nested keep',
+        },
+      })
+    })
+
+    it('handles arrays with some elements removed', () => {
+      const input = { items: [{ id: 1, keep: true }, { id: 2, keep: false }, { id: 3, keep: true }] }
+      const replacer: EncodeReplacer = (key, value) => {
+        // Remove objects where keep is false
+        if (typeof value === 'object' && value !== null && !Array.isArray(value) && 'keep' in value && value.keep === false) {
+          return undefined
+        }
+        return value
+      }
+
+      const result = encode(input, { replacer })
+      const decoded = decode(result)
+
+      expect(decoded).toEqual({
+        items: [{ id: 1, keep: true }, { id: 3, keep: true }],
+      })
+    })
+  })
+
+  describe('integration with other options', () => {
+    it('works with keyFolding', () => {
+      const input = {
+        user: {
+          profile: {
+            name: 'Alice',
+          },
+        },
+      }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (typeof value === 'string')
+          return value.toUpperCase()
+        return value
+      }
+
+      const result = encode(input, { replacer, keyFolding: 'safe' })
+      expect(result).toContain('user.profile.name: ALICE')
+    })
+
+    it('works with custom delimiters', () => {
+      const input = { items: [1, 2, 3] }
+      const replacer: EncodeReplacer = (key, value) => {
+        if (typeof value === 'number')
+          return value * 10
+        return value
+      }
+
+      const result = encode(input, { replacer, delimiter: '\t' })
+      expect(result).toContain('10\t20\t30')
+    })
+
+    it('works with custom indent', () => {
+      const input = { user: { name: 'Alice' } }
+      const replacer: EncodeReplacer = (key, value) => value
+
+      const result = encode(input, { replacer, indent: 4 })
+      // Should use 4-space indent
+      expect(result).toContain('    name: Alice')
+    })
+  })
+
+  describe('comparison with JSON.stringify replacer', () => {
+    it('behaves similarly to JSON.stringify for filtering', () => {
+      const input = { name: 'Alice', password: 'secret' }
+
+      // TOON replacer
+      const toonReplacer: EncodeReplacer = (key, value) => {
+        if (key === 'password')
+          return undefined
+        return value
+      }
+
+      // JSON.stringify replacer
+      const jsonReplacer = (key: string, value: unknown) => {
+        if (key === 'password')
+          return undefined
+        return value
+      }
+
+      const toonResult = decode(encode(input, { replacer: toonReplacer }))
+      const jsonResult = JSON.parse(JSON.stringify(input, jsonReplacer))
+
+      expect(toonResult).toEqual(jsonResult)
+    })
+
+    it('uses string indices for arrays like JSON.stringify', () => {
+      const input = ['a', 'b', 'c']
+      const keys: string[] = []
+
+      const replacer: EncodeReplacer = (key, value, path) => {
+        if (path.length > 0)
+          keys.push(key)
+        return value
+      }
+
+      encode(input, { replacer })
+
+      // Should match JSON.stringify behavior (string indices)
+      expect(keys).toEqual(['0', '1', '2'])
+    })
+  })
+})