github/toon
1
0
Fork 0
mirror of https://github.com/voson-wang/toon.git synced 2026-01-29 15:24:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: initial commit

Browse Source
This commit is contained in:
Johann Schopplich
2025-10-22 20:16:02 +02:00
commit f105551c3e
24 changed files with 6983 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.editorconfig Normal file
View File

@@ -0,0 +1,9 @@
root = true
[*]
charset = utf-8
indent_style = space
indent_size = 2
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true

35
.github/workflows/ci.yml vendored Normal file
View File

@@ -0,0 +1,35 @@
name: CI
on:
push:
branches:
- main
pull_request:
branches:
- main
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
- uses: actions/setup-node@v4
with:
node-version: 24
cache: pnpm
- run: pnpm install
- run: pnpm run lint
typecheck:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
- uses: actions/setup-node@v4
with:
node-version: 24
cache: pnpm
- run: pnpm install
- run: pnpm run test:types

37
.github/workflows/release.yml vendored Normal file
View File

@@ -0,0 +1,37 @@
name: Release
on:
push:
tags:
- 'v*'
permissions:
id-token: write
contents: write
jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: pnpm/action-setup@v4
- uses: actions/setup-node@v4
with:
node-version: 24
registry-url: https://registry.npmjs.org/
cache: pnpm
- name: Publish changelog
run: npx changelogithub
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: pnpm install
- run: pnpm run build
- name: Publish to npm
run: npm install -g npm@latest && npm publish --access public
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

3
.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
dist
node_modules
.DS_Store

1
.npmrc Normal file
View File

@@ -0,0 +1 @@
shamefully-hoist=true

5
.vscode/extensions.json vendored Normal file
View File

@@ -0,0 +1,5 @@
{
"recommendations": [
"dbaeumer.vscode-eslint"
]
}

42
.vscode/settings.json vendored Normal file
View File

@@ -0,0 +1,42 @@
{
// Enable the ESLint flat config support
"eslint.useFlatConfig": true,
// Disable the default formatter, use ESLint instead
"prettier.enable": false,
"editor.formatOnSave": false,
// Auto-fix
"editor.codeActionsOnSave": {
"source.fixAll.eslint": "explicit",
"source.organizeImports": "never"
},
// Silent the stylistic rules in you IDE, but still auto-fix them
"eslint.rules.customizations": [
{ "rule": "style/*", "severity": "off" },
{ "rule": "format/*", "severity": "off" },
{ "rule": "*-indent", "severity": "off" },
{ "rule": "*-spacing", "severity": "off" },
{ "rule": "*-spaces", "severity": "off" },
{ "rule": "*-order", "severity": "off" },
{ "rule": "*-dangle", "severity": "off" },
{ "rule": "*-newline", "severity": "off" },
{ "rule": "*quotes", "severity": "off" },
{ "rule": "*semi", "severity": "off" }
],
// Enable ESLint for all supported languages
"eslint.validate": [
"javascript",
"javascriptreact",
"typescript",
"typescriptreact",
"vue",
"html",
"markdown",
"json",
"jsonc",
"yaml"
]
}

21
LICENSE Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025-PRESENT Johann Schopplich
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

602
README.md Normal file
View File

@@ -0,0 +1,602 @@
# Token-Oriented Object Notation (TOON)
AI is becoming cheaper and more accessible, but larger context windows allow for larger data inputs as well. **LLM tokens still cost money** – this is where TOON comes in.
**Token-Oriented Object Notation** is a compact, human-readable format designed for passing structured data to Large Language Models. It reduces token usage compared to JSON by:
- Removing redundant punctuation (braces/brackets, most quotes)
- Using indentation for structure
- Tabularizing arrays of objects
- Writing inline primitive arrays without spaces
## Token Benchmarks
<!-- automd:file src="./docs/benchmarks.md" -->
| Example | JSON | TOON | Saved | Reduction |
|---------|------|------|-------|-----------|
| 👤 Simple user object | 31 | 18 | 13 | **41.9%** |
| 🏷️ User with tags | 48 | 28 | 20 | **41.7%** |
| 📦 Small product catalog | 117 | 49 | 68 | **58.1%** |
| 👥 API response with users | 123 | 53 | 70 | **56.9%** |
| ⚙️ Nested configuration | 67 | 41 | 26 | **38.8%** |
| 🛒 E-commerce order | 163 | 94 | 69 | **42.3%** |
| 📊 Analytics data | 209 | 94 | 115 | **55.0%** |
| 📈 Large dataset (50 records) | 2159 | 762 | 1397 | **64.7%** |
| **Total** | **2917** | **1139** | **1778** | **61.0%** |
<details>
<summary><strong>View detailed examples</strong></summary>
### 📦 Small product catalog
**Savings: 68 tokens (58.1% reduction)**
**JSON** (117 tokens):
```json
{
"items": [
{
"sku": "A1",
"name": "Widget",
"qty": 2,
"price": 9.99
},
{
"sku": "B2",
"name": "Gadget",
"qty": 1,
"price": 14.5
},
{
"sku": "C3",
"name": "Doohickey",
"qty": 5,
"price": 7.25
}
]
}
```
**TOON** (49 tokens):
```
items[3]{sku,name,qty,price}:
A1,Widget,2,9.99
B2,Gadget,1,14.5
C3,Doohickey,5,7.25
```
---
### 👥 API response with users
**Savings: 70 tokens (56.9% reduction)**
**JSON** (123 tokens):
```json
{
"users": [
{
"id": 1,
"name": "Alice",
"email": "alice@example.com",
"active": true
},
{
"id": 2,
"name": "Bob",
"email": "bob@example.com",
"active": true
},
{
"id": 3,
"name": "Charlie",
"email": "charlie@example.com",
"active": false
}
],
"total": 3,
"page": 1
}
```
**TOON** (53 tokens):
```
users[3]{id,name,email,active}:
1,Alice,alice@example.com,true
2,Bob,bob@example.com,true
3,Charlie,charlie@example.com,false
total: 3
page: 1
```
---
### 📊 Analytics data
**Savings: 115 tokens (55.0% reduction)**
**JSON** (209 tokens):
```json
{
"metrics": [
{
"date": "2025-01-01",
"views": 1234,
"clicks": 89,
"conversions": 12
},
{
"date": "2025-01-02",
"views": 2345,
"clicks": 156,
"conversions": 23
},
{
"date": "2025-01-03",
"views": 1890,
"clicks": 123,
"conversions": 18
},
{
"date": "2025-01-04",
"views": 3456,
"clicks": 234,
"conversions": 34
},
{
"date": "2025-01-05",
"views": 2789,
"clicks": 178,
"conversions": 27
}
]
}
```
**TOON** (94 tokens):
```
metrics[5]{date,views,clicks,conversions}:
2025-01-01,1234,89,12
2025-01-02,2345,156,23
2025-01-03,1890,123,18
2025-01-04,3456,234,34
2025-01-05,2789,178,27
```
</details>
<!-- /automd -->
> [!NOTE]
> Measured with [`gpt-tokenizer`](https://github.com/niieani/gpt-tokenizer) using `o200k_base` encoding (used by GPT-5 and other modern models). Savings will vary across models and tokenizers.
## Why TOON?
Standard JSON is verbose and token-expensive in LLM contexts:
```json
{
"users": [
{ "id": 1, "name": "Alice", "role": "admin" },
{ "id": 2, "name": "Bob", "role": "user" }
]
}
```
TOON conveys the same information with **fewer tokens**:
```
users[2]{id,name,role}:
1,Alice,admin
2,Bob,user
```
## Key Features
- 📉 **Token-efficient:** typically 30–60% fewer tokens vs JSON on GPT-style tokenizers
- 📊 **Tabular arrays:** write object keys once, list rows beneath
- ✂️ **Minimal quoting:** only when required (e.g., commas, colons, ambiguous primitives)
- 📐 **Indentation-based structure:** no braces/brackets for objects
- 🎯 **Inline primitive arrays:** written without spaces after commas
- 🎲 **Deterministic:** stable key order, no trailing spaces/newline
## Installation
```bash
# npm
npm install toon
# pnpm
pnpm add toon
# yarn
yarn add toon
```
## Quick Start
```ts
import { encode } from 'toon'
const data = {
user: {
id: 123,
name: 'Ada',
tags: ['admin', 'ops'],
active: true
}
}
console.log(encode(data))
```
Output:
```
user:
id: 123
name: Ada
tags[2]: admin,ops
active: true
```
## Canonical Formatting Rules
TOON formatting is deterministic and minimal:
- **Indentation**: 2 spaces per nesting level.
- **Lines**:
- `key: value` for primitives (single space after colon).
- `key:` for nested/empty objects (no trailing space on that line).
- **Arrays**:
- Primitive arrays inline: `key[N]: v1,v2` (no spaces after commas).
- List items: two spaces, hyphen, space (`" - …"`).
- **Whitespace invariants**:
- No trailing spaces at end of any line.
- No trailing newline at end of output.
## Format Overview
### Objects
Simple objects with primitive values:
```ts
encode({
id: 123,
name: 'Ada',
active: true
})
```
```
id: 123
name: Ada
active: true
```
Nested objects:
```ts
encode({
user: {
id: 123,
name: 'Ada'
}
})
```
```
user:
id: 123
name: Ada
```
### Arrays
#### Primitive Arrays (Inline)
```ts
encode({
tags: ['admin', 'ops', 'dev']
})
```
```
tags[3]: admin,ops,dev
```
#### Arrays of Objects (Tabular)
When all objects share the same primitive fields, TOON uses an efficient **tabular format**:
```ts
encode({
items: [
{ sku: 'A1', qty: 2, price: 9.99 },
{ sku: 'B2', qty: 1, price: 14.5 }
]
})
```
```
items[2]{sku,qty,price}:
A1,2,9.99
B2,1,14.5
```
#### Mixed and Non-Uniform Arrays
Arrays that don't meet the tabular requirements use list format:
```
items[3]:
- 1
- a: 1
- text
```
When objects appear in list format, the first field is placed on the hyphen line:
```
items[2]:
- id: 1
name: First
- id: 2
name: Second
extra: true
```
#### Arrays of Arrays
When you have arrays containing primitive inner arrays:
```ts
encode({
pairs: [
[1, 2],
[3, 4]
]
})
```
```
pairs[2]:
- [2]: 1,2
- [2]: 3,4
```
#### Empty Arrays and Objects
Empty containers have special representations:
```ts
encode({ items: [] }) // items[0]:
encode([]) // [0]:
encode({}) // (empty output)
encode({ config: {} }) // config:
```
### Quoting Rules
TOON quotes strings **only when necessary** to maximize token efficiency. Inner spaces are allowed; leading or trailing spaces force quotes. Unicode and emoji are safe unquoted.
#### Keys
Keys are quoted when any of the following is true:
| Condition | Examples |
|---|---|
| Contains spaces, commas, colons, quotes, control chars | `"full name"`, `"a,b"`, `"order:id"`, `"tab\there"` |
| Contains brackets or braces | `"[index]"`, `"{key}"` |
| Leading hyphen | `"-lead"` |
| Numeric-only key | `"123"` |
| Empty key | `""` |
**Notes:**
- Quotes and control characters in keys are escaped (e.g., `"he said \"hi\""`, `"line\nbreak"`).
#### String Values
String values are quoted when any of the following is true:
| Condition | Examples |
|---|---|
| Empty string | `""` |
| Contains comma, colon, quote, backslash, or control chars | `"a,b"`, `"a:b"`, `"say \"hi\""`, `"C:\\Users"`, `"line1\\nline2"` |
| Leading or trailing spaces | `" padded "`, `" "` |
| Looks like boolean/number/null | `"true"`, `"false"`, `"null"`, `"42"`, `"-3.14"`, `"1e-6"`, `"05"` |
| Starts with `"- "` (list-like) | `"- item"` |
| Looks like structural token | `"[5]"`, `"{key}"`, `"[3]: x,y"` |
#### Examples
```
note: "hello, world"
items[3]: x,"true","- item"
hello 👋 world // unquoted
" padded " // quoted
value: null // null value
name: "" // empty string (quoted)
text: "line1\nline2" // multi-line string (escaped)
```
### Tabular Format Requirements
For arrays of objects to use the efficient tabular format, all of the following must be true:
| Requirement | Detail |
|---|---|
| All elements are objects | No primitives in the array |
| Identical key sets | No missing or extra keys across rows |
| Primitive values only | No nested arrays or objects |
| Header key order | Taken from the first object |
| Header key quoting | Same rules as object keys |
| Row value quoting | Same rules as string values |
If any condition fails, TOON falls back to list format.
## Type Conversions
Some non-JSON types are automatically normalized for LLM-safe output:
| Input | Output |
|---|---|
| Number (finite) | Decimal form, no scientific notation; `-0` → `0` |
| Number (`NaN`, `±Infinity`) | `null` |
| `BigInt` | Decimal digits (no quotes) |
| `Date` | ISO string in quotes (e.g., `"2025-01-01T00:00:00.000Z"`) |
| `undefined` | `null` |
| `function` | `null` |
| `symbol` | `null` |
Number normalization examples:
```
-0 → 0
1e6 → 1000000
1e-6 → 0.000001
```
## API
### `encode(value: unknown): string`
Converts any JSON-serializable value to TOON format.
**Parameters:**
- `value` – Any JSON-serializable value (object, array, primitive, or nested structure). Non-JSON-serializable values (functions, symbols, undefined, non-finite numbers) are converted to `null`. Dates are converted to ISO strings, and BigInts are emitted as decimal integers (no quotes).
**Returns:**
A TOON-formatted string with no trailing newline or spaces.
**Example:**
```ts
import { encode } from 'toon'
const items = [
{ sku: 'A1', qty: 2, price: 9.99 },
{ sku: 'B2', qty: 1, price: 14.5 }
]
console.log(encode({ items }))
```
**Output:**
```
items[2]{sku,qty,price}:
A1,2,9.99
B2,1,14.5
```
## Using TOON in LLM Prompts
When incorporating TOON into your LLM workflows:
- Wrap TOON data in a fenced code block in your prompt.
- Tell the model: "Do not add extra punctuation or spaces; follow the exact TOON format."
- When asking the model to generate TOON, specify the same rules (2-space indentation, no trailing spaces, quoting rules).
## Token Savings Example
Here's a realistic API response to illustrate the token savings:
**JSON:**
```json
{
"users": [
{ "id": 1, "name": "Alice", "email": "alice@example.com", "active": true },
{ "id": 2, "name": "Bob", "email": "bob@example.com", "active": true },
{ "id": 3, "name": "Charlie", "email": "charlie@example.com", "active": false }
]
}
```
**TOON:**
```
users[3]{id,name,email,active}:
1,Alice,alice@example.com,true
2,Bob,bob@example.com,true
3,Charlie,charlie@example.com,false
```
Typical savings vs JSON are in the **30–60% range** on GPT-style tokenizers, driven by:
- Tabular arrays of objects (keys written once)
- No structural braces/brackets
- Minimal quoting
- No spaces after commas
## Notes and Limitations
- **Token counts vary by tokenizer and model.** Benchmarks use a GPT-style tokenizer (cl100k/o200k); actual savings will differ with other models (e.g., SentencePiece).
- **TOON is designed for LLM contexts** where human readability and token efficiency matter. It's **not** a drop-in replacement for JSON in APIs or storage.
- **Tabular arrays** require all objects to have exactly the same keys with primitive values only. Arrays with mixed types (primitives + objects/arrays), non-uniform objects, or nested structures will use a more verbose list format.
- **Object key order** is preserved from the input. In tabular arrays, header order follows the first object's keys.
- **Arrays mixing primitives and objects/arrays** always use list form:
```
items[2]:
- a: 1
- [2]: 1,2
```
- **Deterministic formatting:** 2-space indentation, stable key order, no trailing spaces/newline.
## Quick Reference
```
// Object
{ id: 1, name: 'Ada' } → id: 1
name: Ada
// Nested object
{ user: { id: 1 } } → user:
id: 1
// Primitive array (inline)
{ tags: ['a', 'b'] } → tags[2]: a,b
// Tabular array (uniform objects)
{ items: [ → items[2]{id,qty}:
{ id: 1, qty: 5 }, 1,5
{ id: 2, qty: 3 } 2,3
]}
// Mixed / non-uniform (list)
{ items: [1, { a: 1 }, 'x'] } → items[3]:
- 1
- a: 1
- x
// Array of arrays
{ pairs: [[1, 2], [3, 4]] } → pairs[2]:
- [2]: 1,2
- [2]: 3,4
// Root array
['x', 'y'] → [2]: x,y
// Empty containers
{} → (empty output)
{ items: [] } → items[0]:
// Special quoting
{ note: 'hello, world' } → note: "hello, world"
{ items: ['true', true] } → items[2]: "true",true
```
## License
[MIT](./LICENSE) License © 2025-PRESENT [Johann Schopplich](https://github.com/johannschopplich)

158
docs/benchmarks.md Normal file
View File

@@ -0,0 +1,158 @@
| Example | JSON | TOON | Saved | Reduction |
|---------|------|------|-------|-----------|
| 👤 Simple user object | 31 | 18 | 13 | **41.9%** |
| 🏷️ User with tags | 48 | 28 | 20 | **41.7%** |
| 📦 Small product catalog | 117 | 49 | 68 | **58.1%** |
| 👥 API response with users | 123 | 53 | 70 | **56.9%** |
| ⚙️ Nested configuration | 67 | 41 | 26 | **38.8%** |
| 🛒 E-commerce order | 163 | 94 | 69 | **42.3%** |
| 📊 Analytics data | 209 | 94 | 115 | **55.0%** |
| 📈 Large dataset (50 records) | 2159 | 762 | 1397 | **64.7%** |
| **Total** | **2917** | **1139** | **1778** | **61.0%** |
<details>
<summary><strong>View detailed examples</strong></summary>
### 📦 Small product catalog
**Savings: 68 tokens (58.1% reduction)**
**JSON** (117 tokens):
```json
{
"items": [
{
"sku": "A1",
"name": "Widget",
"qty": 2,
"price": 9.99
},
{
"sku": "B2",
"name": "Gadget",
"qty": 1,
"price": 14.5
},
{
"sku": "C3",
"name": "Doohickey",
"qty": 5,
"price": 7.25
}
]
}
```
**TOON** (49 tokens):
```
items[3]{sku,name,qty,price}:
A1,Widget,2,9.99
B2,Gadget,1,14.5
C3,Doohickey,5,7.25
```
---
### 👥 API response with users
**Savings: 70 tokens (56.9% reduction)**
**JSON** (123 tokens):
```json
{
"users": [
{
"id": 1,
"name": "Alice",
"email": "alice@example.com",
"active": true
},
{
"id": 2,
"name": "Bob",
"email": "bob@example.com",
"active": true
},
{
"id": 3,
"name": "Charlie",
"email": "charlie@example.com",
"active": false
}
],
"total": 3,
"page": 1
}
```
**TOON** (53 tokens):
```
users[3]{id,name,email,active}:
1,Alice,alice@example.com,true
2,Bob,bob@example.com,true
3,Charlie,charlie@example.com,false
total: 3
page: 1
```
---
### 📊 Analytics data
**Savings: 115 tokens (55.0% reduction)**
**JSON** (209 tokens):
```json
{
"metrics": [
{
"date": "2025-01-01",
"views": 1234,
"clicks": 89,
"conversions": 12
},
{
"date": "2025-01-02",
"views": 2345,
"clicks": 156,
"conversions": 23
},
{
"date": "2025-01-03",
"views": 1890,
"clicks": 123,
"conversions": 18
},
{
"date": "2025-01-04",
"views": 3456,
"clicks": 234,
"conversions": 34
},
{
"date": "2025-01-05",
"views": 2789,
"clicks": 178,
"conversions": 27
}
]
}
```
**TOON** (94 tokens):
```
metrics[5]{date,views,clicks,conversions}:
2025-01-01,1234,89,12
2025-01-02,2345,156,23
2025-01-03,1890,123,18
2025-01-04,3456,234,34
2025-01-05,2789,178,27
```
</details>

4
eslint.config.mjs Normal file
View File

@@ -0,0 +1,4 @@
// @ts-check
import antfu from '@antfu/eslint-config'
export default antfu()

55
package.json Normal file
View File

@@ -0,0 +1,55 @@
{
"name": "@byjohann/toon",
"type": "module",
"version": "0.0.0",
"packageManager": "pnpm@10.19.0",
"description": "Token-Oriented Object Notation – a token-efficient data notation for LLM prompts",
"author": "Johann Schopplich <hello@johannschopplich.com>",
"license": "MIT",
"homepage": "https://github.com/johannschopplich/toon#readme",
"repository": {
"type": "git",
"url": "git+https://github.com/johannschopplich/toon.git"
},
"bugs": {
"url": "https://github.com/johannschopplich/toon/issues"
},
"sideEffects": false,
"exports": {
".": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
},
"types": "./dist/index.d.ts",
"files": [
"dist"
],
"scripts": {
"automd": "tsx scripts/generate-bench.ts && automd",
"build": "tsdown",
"lint": "eslint .",
"lint:fix": "eslint . --fix",
"test": "vitest",
"test:types": "tsc --noEmit",
"release": "bumpp"
},
"devDependencies": {
"@antfu/eslint-config": "^6.0.0",
"@types/node": "^24.9.1",
"automd": "^0.4.2",
"bumpp": "^10.3.1",
"eslint": "^9.38.0",
"gpt-tokenizer": "^3.2.0",
"tsdown": "^0.15.9",
"tsx": "^4.20.6",
"typescript": "^5.9.3",
"vitest": "^3.2.4"
},
"pnpm": {
"onlyBuiltDependencies": [
"@parcel/watcher",
"esbuild"
]
}
}

4557
pnpm-lock.yaml generated Normal file
View File

File diff suppressed because it is too large Load Diff

213
scripts/generate-bench.ts Normal file
View File

@@ -0,0 +1,213 @@
import * as fsp from 'node:fs/promises'
import * as path from 'node:path'
import * as url from 'node:url'
import { encode } from 'gpt-tokenizer' // o200k_base encoding (default)
import { encode as encodeToon } from '../src/index'
interface BenchmarkResult {
name: string
emoji: string
jsonTokens: number
toonTokens: number
savings: number
savingsPercent: string
}
const rootDir = url.fileURLToPath(new URL('../', import.meta.url))
const benchPath = path.join(rootDir, 'docs', 'benchmarks.md')
const BENCHMARK_EXAMPLES = [
{
name: 'Simple user object',
emoji: '👤',
data: {
id: 123,
name: 'Alice',
email: 'alice@example.com',
active: true,
},
},
{
name: 'User with tags',
emoji: '🏷️',
data: {
user: {
id: 123,
name: 'Ada',
tags: ['admin', 'ops', 'developer'],
active: true,
},
},
},
{
name: 'Small product catalog',
emoji: '📦',
data: {
items: [
{ sku: 'A1', name: 'Widget', qty: 2, price: 9.99 },
{ sku: 'B2', name: 'Gadget', qty: 1, price: 14.5 },
{ sku: 'C3', name: 'Doohickey', qty: 5, price: 7.25 },
],
},
},
{
name: 'API response with users',
emoji: '👥',
data: {
users: [
{ id: 1, name: 'Alice', email: 'alice@example.com', active: true },
{ id: 2, name: 'Bob', email: 'bob@example.com', active: true },
{ id: 3, name: 'Charlie', email: 'charlie@example.com', active: false },
],
total: 3,
page: 1,
},
},
{
name: 'Nested configuration',
emoji: '⚙️',
data: {
database: {
host: 'localhost',
port: 5432,
credentials: {
username: 'admin',
password: 'secret123',
},
},
cache: {
enabled: true,
ttl: 3600,
},
},
},
{
name: 'E-commerce order',
emoji: '🛒',
data: {
orderId: 'ORD-2025-001',
customer: {
id: 456,
name: 'Jane Doe',
email: 'jane@example.com',
},
items: [
{ sku: 'PROD-A', name: 'Premium Widget', quantity: 2, price: 29.99 },
{ sku: 'PROD-B', name: 'Deluxe Gadget', quantity: 1, price: 49.99 },
],
subtotal: 109.97,
tax: 10.99,
total: 120.96,
status: 'pending',
},
},
{
name: 'Analytics data',
emoji: '📊',
data: {
metrics: [
{ date: '2025-01-01', views: 1234, clicks: 89, conversions: 12 },
{ date: '2025-01-02', views: 2345, clicks: 156, conversions: 23 },
{ date: '2025-01-03', views: 1890, clicks: 123, conversions: 18 },
{ date: '2025-01-04', views: 3456, clicks: 234, conversions: 34 },
{ date: '2025-01-05', views: 2789, clicks: 178, conversions: 27 },
],
},
},
{
name: 'Large dataset (50 records)',
emoji: '📈',
data: {
records: Array.from({ length: 50 }, (_, i) => ({
id: i + 1,
name: `User ${i + 1}`,
email: `user${i + 1}@example.com`,
score: (i * 7) % 100,
active: i % 3 !== 0,
})),
},
},
] as const
const DETAILED_EXAMPLE_INDICES = [2, 3, 6] // Small product catalog, API response, Analytics data
// Calculate total savings
let totalJsonTokens = 0
let totalToonTokens = 0
const results: BenchmarkResult[] = []
for (const example of BENCHMARK_EXAMPLES) {
const jsonString = JSON.stringify(example.data, null, 2)
const toonString = encodeToon(example.data)
const jsonTokens = encode(jsonString).length
const toonTokens = encode(toonString).length
const savings = jsonTokens - toonTokens
const savingsPercent = ((savings / jsonTokens) * 100).toFixed(1)
totalJsonTokens += jsonTokens
totalToonTokens += toonTokens
results.push({
name: example.name,
emoji: example.emoji,
jsonTokens,
toonTokens,
savings,
savingsPercent,
})
}
const totalSavings = totalJsonTokens - totalToonTokens
const totalSavingsPercent = ((totalSavings / totalJsonTokens) * 100).toFixed(1)
// Generate markdown content matching README style
const summaryRows = results
.map(result => `| ${result.emoji} ${result.name} | ${result.jsonTokens} | ${result.toonTokens} | ${result.savings} | **${result.savingsPercent}%** |`)
.join('\n')
const detailedExamples = DETAILED_EXAMPLE_INDICES
.map((exampleIndex, i) => {
const example = BENCHMARK_EXAMPLES[exampleIndex]!
const result = results[exampleIndex]!
const separator = i < DETAILED_EXAMPLE_INDICES.length - 1 ? '\n\n---' : ''
return `### ${result.emoji} ${result.name}
**Savings: ${result.savings} tokens (${result.savingsPercent}% reduction)**
**JSON** (${result.jsonTokens} tokens):
\`\`\`json
${JSON.stringify(example.data, null, 2)}
\`\`\`
**TOON** (${result.toonTokens} tokens):
\`\`\`
${encodeToon(example.data)}
\`\`\`${separator}`
})
.join('\n\n')
const markdown = `
| Example | JSON | TOON | Saved | Reduction |
|---------|------|------|-------|-----------|
${summaryRows}
| **Total** | **${totalJsonTokens}** | **${totalToonTokens}** | **${totalSavings}** | **${totalSavingsPercent}%** |
<details>
<summary><strong>View detailed examples</strong></summary>
${detailedExamples}
</details>
`.trimStart()
console.log(markdown)
await fsp.mkdir(path.join(rootDir, 'docs'), { recursive: true })
await fsp.writeFile(benchPath, markdown, 'utf-8')
console.log(`✅ Benchmark written to ${benchPath}`)

41
src/constants.ts Normal file
View File

@@ -0,0 +1,41 @@
// #region List markers
export const LIST_ITEM_MARKER = '-'
export const LIST_ITEM_PREFIX = '- '
// #endregion
// #region Structural characters
export const COMMA = ','
export const COLON = ':'
export const SPACE = ' '
// #endregion
// #region Brackets and braces
export const OPEN_BRACKET = '['
export const CLOSE_BRACKET = ']'
export const OPEN_BRACE = '{'
export const CLOSE_BRACE = '}'
// #endregion
// #region Literals
export const NULL_LITERAL = 'null'
export const TRUE_LITERAL = 'true'
export const FALSE_LITERAL = 'false'
// #endregion
// #region Escape characters
export const BACKSLASH = '\\'
export const DOUBLE_QUOTE = '"'
export const NEWLINE = '\n'
export const CARRIAGE_RETURN = '\r'
export const TAB = '\t'
// #endregion

360
src/encoders.ts Normal file
View File

@@ -0,0 +1,360 @@
import type {
Depth,
JsonArray,
JsonObject,
JsonPrimitive,
JsonValue,
ResolvedEncodeOptions,
} from './types'
import { LIST_ITEM_MARKER, LIST_ITEM_PREFIX } from './constants'
import {
isArrayOfArrays,
isArrayOfObjects,
isArrayOfPrimitives,
isJsonArray,
isJsonObject,
isJsonPrimitive,
} from './normalize'
import {
encodeKey,
encodePrimitive,
formatArrayHeader,
formatKeyedArrayHeader,
formatKeyedTableHeader,
formatTabularHeader,
joinEncodedValues,
} from './primitives'
import { LineWriter } from './writer'
// #region Encode normalized JsonValue
export function encodeValue(value: JsonValue, options: ResolvedEncodeOptions): string {
if (isJsonPrimitive(value)) {
return encodePrimitive(value)
}
const writer = new LineWriter(options.indent)
if (isJsonArray(value)) {
encodeRootArray(value, writer)
}
else if (isJsonObject(value)) {
encodeObject(value, writer, 0)
}
return writer.toString()
}
// #endregion
// #region Object encoding
export function encodeObject(value: JsonObject, writer: LineWriter, depth: Depth): void {
const keys = Object.keys(value)
for (const key of keys) {
encodeKeyValuePair(key, value[key]!, writer, depth)
}
}
export function encodeKeyValuePair(key: string, value: JsonValue, writer: LineWriter, depth: Depth): void {
const encodedKey = encodeKey(key)
if (isJsonPrimitive(value)) {
writer.push(depth, `${encodedKey}: ${encodePrimitive(value)}`)
}
else if (isJsonArray(value)) {
encodeArrayProperty(key, value, writer, depth)
}
else if (isJsonObject(value)) {
const nestedKeys = Object.keys(value)
if (nestedKeys.length === 0) {
// Empty object
writer.push(depth, `${encodedKey}:`)
}
else {
writer.push(depth, `${encodedKey}:`)
encodeObject(value, writer, depth + 1)
}
}
}
// #endregion
// #region Array encoding
export function encodeRootArray(value: JsonArray, writer: LineWriter): void {
if (value.length === 0) {
writer.push(0, '[0]:')
return
}
// Primitive array
if (isArrayOfPrimitives(value)) {
encodeInlinePrimitiveArray(undefined, value, writer, 0)
return
}
// Array of arrays (all primitives)
if (isArrayOfArrays(value)) {
const allPrimitiveArrays = value.every(arr => isArrayOfPrimitives(arr))
if (allPrimitiveArrays) {
encodeArrayOfArraysAsListItems(undefined, value, writer, 0)
return
}
}
// Array of objects
if (isArrayOfObjects(value)) {
const header = detectTabularHeader(value)
if (header) {
encodeArrayOfObjectsAsTabular(undefined, value, header, writer, 0)
}
else {
encodeArrayOfObjectsAsListItems(undefined, value, writer, 0)
}
return
}
// Mixed array: fallback to expanded format (not in spec, but safe default)
encodeMixedArrayAsListItems(undefined, value, writer, 0)
}
export function encodeArrayProperty(key: string, value: JsonArray, writer: LineWriter, depth: Depth): void {
if (value.length === 0) {
const encodedKey = encodeKey(key)
writer.push(depth, `${encodedKey}[0]:`)
return
}
// Primitive array
if (isArrayOfPrimitives(value)) {
encodeInlinePrimitiveArray(key, value, writer, depth)
return
}
// Array of arrays (all primitives)
if (isArrayOfArrays(value)) {
const allPrimitiveArrays = value.every(arr => isArrayOfPrimitives(arr))
if (allPrimitiveArrays) {
encodeArrayOfArraysAsListItems(key, value, writer, depth)
return
}
}
// Array of objects
if (isArrayOfObjects(value)) {
const header = detectTabularHeader(value)
if (header) {
encodeArrayOfObjectsAsTabular(key, value, header, writer, depth)
}
else {
encodeArrayOfObjectsAsListItems(key, value, writer, depth)
}
return
}
// Mixed array: fallback to expanded format
encodeMixedArrayAsListItems(key, value, writer, depth)
}
// #endregion
// #region Primitive array encoding (inline)
export function encodeInlinePrimitiveArray(
prefix: string | undefined,
values: readonly JsonPrimitive[],
writer: LineWriter,
depth: Depth,
): void {
const header = prefix ? formatKeyedArrayHeader(prefix, values.length) : formatArrayHeader(values.length)
const joinedValue = joinEncodedValues(values)
// Only add space if there are values
if (values.length === 0) {
writer.push(depth, header)
}
else {
writer.push(depth, `${header} ${joinedValue}`)
}
}
// #endregion
// #region Array of arrays (expanded format)
export function encodeArrayOfArraysAsListItems(
prefix: string | undefined,
values: readonly JsonArray[],
writer: LineWriter,
depth: Depth,
): void {
const header = prefix ? formatKeyedArrayHeader(prefix, values.length) : formatArrayHeader(values.length)
writer.push(depth, header)
for (const arr of values) {
if (isArrayOfPrimitives(arr)) {
const inline = formatInlineArray(arr)
writer.push(depth + 1, `${LIST_ITEM_PREFIX}${inline}`)
}
}
}
export function formatInlineArray(values: readonly JsonPrimitive[]): string {
const header = formatArrayHeader(values.length)
const joinedValue = joinEncodedValues(values)
// Only add space if there are values
if (values.length === 0) {
return header
}
return `${header} ${joinedValue}`
}
// #endregion
// #region Array of objects (tabular format)
export function encodeArrayOfObjectsAsTabular(
prefix: string | undefined,
rows: readonly JsonObject[],
header: readonly string[],
writer: LineWriter,
depth: Depth,
): void {
const headerStr = prefix
? formatKeyedTableHeader(prefix, rows.length, header)
: formatTabularHeader(rows.length, header)
writer.push(depth, `${headerStr}`)
for (const row of rows) {
const values = header.map(key => row[key])
const joinedValue = joinEncodedValues(values as JsonPrimitive[])
writer.push(depth + 1, joinedValue)
}
}
export function detectTabularHeader(rows: readonly JsonObject[]): string[] | undefined {
if (rows.length === 0)
return
const firstRow = rows[0]!
const firstKeys = Object.keys(firstRow)
if (firstKeys.length === 0)
return
if (isTabularArray(rows, firstKeys)) {
return firstKeys
}
}
export function isTabularArray(
rows: readonly JsonObject[],
header: readonly string[],
): boolean {
for (const row of rows) {
const keys = Object.keys(row)
// All objects must have the same keys (but order can differ)
if (keys.length !== header.length) {
return false
}
// Check that all header keys exist in the row and all values are primitives
for (const key of header) {
if (!(key in row)) {
return false
}
if (!isJsonPrimitive(row[key])) {
return false
}
}
}
return true
}
// #endregion
// #region Array of objects (expanded format)
export function encodeMixedArrayAsListItems(
prefix: string | undefined,
items: readonly JsonValue[],
writer: LineWriter,
depth: Depth,
): void {
const header = prefix ? formatKeyedArrayHeader(prefix, items.length) : formatArrayHeader(items.length)
writer.push(depth, header)
for (const item of items) {
if (isJsonPrimitive(item)) {
// Direct primitive as list item
writer.push(depth + 1, `${LIST_ITEM_PREFIX}${encodePrimitive(item)}`)
}
else if (isJsonArray(item)) {
// Direct array as list item
if (isArrayOfPrimitives(item)) {
const inline = formatInlineArray(item)
writer.push(depth + 1, `${LIST_ITEM_PREFIX}${inline}`)
}
}
else if (isJsonObject(item)) {
// Object as list item
encodeObjectAsListItem(item, writer, depth + 1)
}
}
}
export function encodeArrayOfObjectsAsListItems(
prefix: string | undefined,
rows: readonly JsonObject[],
writer: LineWriter,
depth: Depth,
): void {
const header = prefix ? formatKeyedArrayHeader(prefix, rows.length) : formatArrayHeader(rows.length)
writer.push(depth, `${header}`)
for (const obj of rows) {
encodeObjectAsListItem(obj, writer, depth + 1)
}
}
export function encodeObjectAsListItem(obj: JsonObject, writer: LineWriter, depth: Depth): void {
const keys = Object.keys(obj)
if (keys.length === 0) {
writer.push(depth, LIST_ITEM_MARKER)
return
}
// First key-value on the same line as "- "
const firstKey = keys[0]!
const encodedKey = encodeKey(firstKey)
const firstValue = obj[firstKey]!
if (isJsonPrimitive(firstValue)) {
writer.push(depth, `${LIST_ITEM_PREFIX}${encodedKey}: ${encodePrimitive(firstValue)}`)
}
else if (isJsonArray(firstValue)) {
// For arrays, we need to put them on separate lines
writer.push(depth, `${LIST_ITEM_PREFIX}${encodedKey}[${firstValue.length}]:`)
// ... handle array encoding (simplified for now)
}
else if (isJsonObject(firstValue)) {
const nestedKeys = Object.keys(firstValue)
if (nestedKeys.length === 0) {
writer.push(depth, `${LIST_ITEM_PREFIX}${encodedKey}:`)
}
else {
writer.push(depth, `${LIST_ITEM_PREFIX}${encodedKey}:`)
encodeObject(firstValue, writer, depth + 2)
}
}
// Remaining keys on indented lines
for (let i = 1; i < keys.length; i++) {
const key = keys[i]!
encodeKeyValuePair(key, obj[key]!, writer, depth + 1)
}
}
// #endregion

27
src/index.ts Normal file
View File

@@ -0,0 +1,27 @@
import type {
EncodeOptions,
ResolvedEncodeOptions,
} from './types'
import { encodeValue } from './encoders'
import { normalizeValue } from './normalize'
export type {
EncodeOptions,
JsonArray,
JsonObject,
JsonPrimitive,
JsonValue,
ResolvedEncodeOptions,
} from './types'
export function encode(input: unknown, options?: EncodeOptions): string {
const normalizedValue = normalizeValue(input)
const resolvedOptions = resolveOptions(options)
return encodeValue(normalizedValue, resolvedOptions)
}
function resolveOptions(options?: EncodeOptions): ResolvedEncodeOptions {
return {
indent: options?.indent ?? 2,
}
}

143
src/normalize.ts Normal file
View File

@@ -0,0 +1,143 @@
import type {
JsonArray,
JsonObject,
JsonPrimitive,
JsonValue,
} from './types'
// #region Normalization (unknown → JsonValue)
export function normalizeValue(value: unknown): JsonValue {
// null
if (value === null) {
return null
}
// Primitives
if (typeof value === 'string' || typeof value === 'boolean') {
return value
}
// Numbers: canonicalize -0 to 0, handle NaN and Infinity
if (typeof value === 'number') {
if (Object.is(value, -0)) {
return 0
}
if (!Number.isFinite(value)) {
return null
}
return value
}
// BigInt → number (if safe) or string
if (typeof value === 'bigint') {
// Try to convert to number if within safe integer range
if (value >= Number.MIN_SAFE_INTEGER && value <= Number.MAX_SAFE_INTEGER) {
return Number(value)
}
// Otherwise convert to string (will be unquoted as it looks numeric)
return value.toString()
}
// Date → ISO string
if (value instanceof Date) {
return value.toISOString()
}
// Array
if (Array.isArray(value)) {
return normalizeArray(value)
}
// Set → array
if (value instanceof Set) {
return normalizeArray(Array.from(value))
}
// Map → object
if (value instanceof Map) {
return Object.fromEntries(
Array.from(value, ([k, v]) => [String(k), normalizeValue(v)]),
)
}
// Plain object
if (isPlainObject(value)) {
return normalizeObject(value)
}
// Fallback: function, symbol, undefined, or other → null
return null
}
export function normalizeArray(value: unknown): JsonArray {
if (!Array.isArray(value)) {
return []
}
return value.map(item => normalizeValue(item))
}
export function normalizeObject(value: unknown): JsonObject {
if (!isPlainObject(value)) {
return {}
}
const result: Record<string, JsonValue> = {}
for (const key in value) {
if (Object.prototype.hasOwnProperty.call(value, key)) {
result[key] = normalizeValue(value[key])
}
}
return result
}
// #endregion
// #region Type guards
export function isJsonPrimitive(value: unknown): value is JsonPrimitive {
return (
value === null
|| typeof value === 'string'
|| typeof value === 'number'
|| typeof value === 'boolean'
)
}
export function isJsonArray(value: unknown): value is JsonArray {
return Array.isArray(value)
}
export function isJsonObject(value: unknown): value is JsonObject {
return value !== null && typeof value === 'object' && !Array.isArray(value)
}
export function isPlainObject(value: unknown): value is Record<string, unknown> {
if (value === null || typeof value !== 'object') {
return false
}
const prototype = Object.getPrototypeOf(value)
return prototype === null || prototype === Object.prototype
}
// #endregion
// #region Array type detection
export function isArrayOfPrimitives(value: JsonArray): value is readonly JsonPrimitive[] {
return value.every(item => isJsonPrimitive(item))
}
export function isArrayOfArrays(value: JsonArray): value is readonly JsonArray[] {
return value.every(item => isJsonArray(item))
}
export function isArrayOfObjects(value: JsonArray): value is readonly JsonObject[] {
return value.every(item => isJsonObject(item))
}
// #endregion

129
src/primitives.ts Normal file
View File

@@ -0,0 +1,129 @@
import type { JsonPrimitive } from './types'
import {
BACKSLASH,
COMMA,
DOUBLE_QUOTE,
FALSE_LITERAL,
LIST_ITEM_MARKER,
NULL_LITERAL,
TRUE_LITERAL,
} from './constants'
// #region Primitive encoding
export function encodePrimitive(value: JsonPrimitive): string {
if (value === null) {
return NULL_LITERAL
}
if (typeof value === 'boolean') {
return String(value)
}
if (typeof value === 'number') {
return String(value)
}
return encodeStringLiteral(value)
}
export function encodeStringLiteral(value: string): string {
if (isSafeUnquoted(value)) {
return value
}
return `${DOUBLE_QUOTE}${escapeString(value)}${DOUBLE_QUOTE}`
}
export function escapeString(value: string): string {
return value
.replace(/\\/g, `${BACKSLASH}${BACKSLASH}`)
.replace(/"/g, `${BACKSLASH}${DOUBLE_QUOTE}`)
.replace(/\n/g, `${BACKSLASH}n`)
.replace(/\r/g, `${BACKSLASH}r`)
.replace(/\t/g, `${BACKSLASH}t`)
}
export function isSafeUnquoted(value: string): boolean {
if (!value) {
return false
}
if (isPaddedWithWhitespace(value)) {
return false
}
if (value === TRUE_LITERAL || value === FALSE_LITERAL || value === NULL_LITERAL) {
return false
}
if (isNumericLike(value)) {
return false
}
// Check for structural characters: comma, colon, brackets, braces, hyphen at start, newline, carriage return, tab, double-quote
if (/[,:\n\r\t"[\]{}]/.test(value) || value.startsWith(LIST_ITEM_MARKER)) {
return false
}
return true
}
export function isNumericLike(value: string): boolean {
// Match numbers like: 42, -3.14, 1e-6, 05, etc.
return /^-?\d+(?:\.\d+)?(?:e[+-]?\d+)?$/i.test(value) || /^0\d+$/.test(value)
}
export function isPaddedWithWhitespace(value: string): boolean {
return value !== value.trim()
}
// #endregion
// #region Key encoding
export function encodeKey(key: string): string {
if (isValidUnquotedKey(key)) {
return key
}
return `${DOUBLE_QUOTE}${escapeString(key)}${DOUBLE_QUOTE}`
}
function isValidUnquotedKey(key: string): boolean {
return /^[A-Z_][\w.]*$/i.test(key)
}
// #endregion
// #region Value joining
export function joinEncodedValues(values: readonly JsonPrimitive[]): string {
return values.map(v => encodePrimitive(v)).join(COMMA)
}
// #endregion
// #region Header formatters
export function formatArrayHeader(length: number): string {
return `[${length}]:`
}
export function formatTabularHeader(length: number, fields: readonly string[]): string {
const quotedFields = fields.map(f => encodeKey(f))
return `[${length}]{${quotedFields.join(',')}}:`
}
export function formatKeyedArrayHeader(key: string, length: number): string {
const encodedKey = encodeKey(key)
return `${encodedKey}[${length}]:`
}
export function formatKeyedTableHeader(key: string, length: number, fields: readonly string[]): string {
const encodedKey = encodeKey(key)
const quotedFields = fields.map(f => encodeKey(f))
return `${encodedKey}[${length}]{${quotedFields.join(',')}}:`
}
// #endregion

20
src/types.ts Normal file
View File

@@ -0,0 +1,20 @@
// #region JSON types
export type JsonPrimitive = string | number | boolean | null
export type JsonObject = { [Key in string]: JsonValue } & { [Key in string]?: JsonValue | undefined }
export type JsonArray = JsonValue[] | readonly JsonValue[]
export type JsonValue = JsonPrimitive | JsonObject | JsonArray
// #endregion
// #region Encoder options
export interface EncodeOptions {
indent?: number
}
export type ResolvedEncodeOptions = Readonly<Required<EncodeOptions>>
// #endregion
export type Depth = number

19
src/writer.ts Normal file
View File

@@ -0,0 +1,19 @@
import type { Depth } from './types'
export class LineWriter {
private readonly lines: string[] = []
private readonly indentationString: string
constructor(indentSize: number) {
this.indentationString = ' '.repeat(indentSize)
}
push(depth: Depth, content: string): void {
const indent = this.indentationString.repeat(depth)
this.lines.push(indent + content)
}
toString(): string {
return this.lines.join('\n')
}
}

474
test/index.test.ts Normal file
View File

@@ -0,0 +1,474 @@
import { describe, expect, it } from 'vitest'
import { encode } from '../src/index'
describe('primitives', () => {
it('encodes safe strings without quotes', () => {
expect(encode('hello')).toBe('hello')
expect(encode('Ada_99')).toBe('Ada_99')
})
it('quotes empty string', () => {
expect(encode('')).toBe('""')
})
it('quotes strings that look like booleans or numbers', () => {
expect(encode('true')).toBe('"true"')
expect(encode('false')).toBe('"false"')
expect(encode('null')).toBe('"null"')
expect(encode('42')).toBe('"42"')
expect(encode('-3.14')).toBe('"-3.14"')
expect(encode('1e-6')).toBe('"1e-6"')
expect(encode('05')).toBe('"05"')
})
it('escapes control characters in strings', () => {
expect(encode('line1\nline2')).toBe('"line1\\nline2"')
expect(encode('tab\there')).toBe('"tab\\there"')
expect(encode('return\rcarriage')).toBe('"return\\rcarriage"')
expect(encode('C:\\Users\\path')).toBe('"C:\\\\Users\\\\path"')
})
it('quotes strings with structural characters', () => {
expect(encode('[3]: x,y')).toBe('"[3]: x,y"')
expect(encode('- item')).toBe('"- item"')
expect(encode('[test]')).toBe('"[test]"')
expect(encode('{key}')).toBe('"{key}"')
})
it('handles Unicode and emoji', () => {
expect(encode('café')).toBe('café')
expect(encode('你好')).toBe('你好')
expect(encode('🚀')).toBe('🚀')
expect(encode('hello 👋 world')).toBe('hello 👋 world')
})
it('encodes numbers', () => {
expect(encode(42)).toBe('42')
expect(encode(3.14)).toBe('3.14')
expect(encode(-7)).toBe('-7')
expect(encode(0)).toBe('0')
})
it('handles special numeric values', () => {
expect(encode(-0)).toBe('0')
expect(encode(1e6)).toBe('1000000')
expect(encode(1e-6)).toBe('0.000001')
expect(encode(1e20)).toBe('100000000000000000000')
expect(encode(Number.MAX_SAFE_INTEGER)).toBe('9007199254740991')
})
it('encodes booleans', () => {
expect(encode(true)).toBe('true')
expect(encode(false)).toBe('false')
})
it('encodes null', () => {
expect(encode(null)).toBe('null')
})
})
describe('objects (simple)', () => {
it('preserves key order in objects', () => {
const obj = {
id: 123,
name: 'Ada',
active: true,
}
expect(encode(obj)).toBe('id: 123\nname: Ada\nactive: true')
})
it('encodes null values in objects', () => {
const obj = { id: 123, value: null }
expect(encode(obj)).toBe('id: 123\nvalue: null')
})
it('encodes empty objects as empty string', () => {
expect(encode({})).toBe('')
})
it('quotes string values with special characters', () => {
expect(encode({ note: 'a:b' })).toBe('note: "a:b"')
expect(encode({ note: 'a,b' })).toBe('note: "a,b"')
expect(encode({ text: 'line1\nline2' })).toBe('text: "line1\\nline2"')
expect(encode({ text: 'say "hello"' })).toBe('text: "say \\"hello\\""')
})
it('quotes string values with leading/trailing spaces', () => {
expect(encode({ text: ' padded ' })).toBe('text: " padded "')
expect(encode({ text: ' ' })).toBe('text: " "')
})
it('quotes string values that look like booleans/numbers', () => {
expect(encode({ v: 'true' })).toBe('v: "true"')
expect(encode({ v: '42' })).toBe('v: "42"')
expect(encode({ v: '-7.5' })).toBe('v: "-7.5"')
})
})
describe('objects (keys)', () => {
it('quotes keys with special characters', () => {
expect(encode({ 'order:id': 7 })).toBe('"order:id": 7')
expect(encode({ '[index]': 5 })).toBe('"[index]": 5')
expect(encode({ '{key}': 5 })).toBe('"{key}": 5')
expect(encode({ 'a,b': 1 })).toBe('"a,b": 1')
})
it('quotes keys with spaces or leading hyphens', () => {
expect(encode({ 'full name': 'Ada' })).toBe('"full name": Ada')
expect(encode({ '-lead': 1 })).toBe('"-lead": 1')
expect(encode({ ' a ': 1 })).toBe('" a ": 1')
})
it('quotes numeric keys', () => {
expect(encode({ 123: 'x' })).toBe('"123": x')
})
it('quotes empty string key', () => {
expect(encode({ '': 1 })).toBe('"": 1')
})
it('escapes control characters in keys', () => {
expect(encode({ 'line\nbreak': 1 })).toBe('"line\\nbreak": 1')
expect(encode({ 'tab\there': 2 })).toBe('"tab\\there": 2')
})
it('escapes quotes in keys', () => {
expect(encode({ 'he said "hi"': 1 })).toBe('"he said \\"hi\\"": 1')
})
})
describe('nested objects', () => {
it('encodes deeply nested objects', () => {
const obj = {
a: {
b: {
c: 'deep',
},
},
}
expect(encode(obj)).toBe('a:\n b:\n c: deep')
})
it('encodes empty nested object', () => {
expect(encode({ user: {} })).toBe('user:')
})
})
describe('arrays of primitives', () => {
it('encodes string arrays inline', () => {
const obj = { tags: ['admin', 'ops'] }
expect(encode(obj)).toBe('tags[2]: admin,ops')
})
it('encodes number arrays inline', () => {
const obj = { nums: [1, 2, 3] }
expect(encode(obj)).toBe('nums[3]: 1,2,3')
})
it('encodes mixed primitive arrays inline', () => {
const obj = { data: ['x', 'y', true, 10] }
expect(encode(obj)).toBe('data[4]: x,y,true,10')
})
it('encodes empty arrays', () => {
const obj = { items: [] }
expect(encode(obj)).toBe('items[0]:')
})
it('handles empty string in arrays', () => {
const obj = { items: [''] }
expect(encode(obj)).toBe('items[1]: ""')
const obj2 = { items: ['a', '', 'b'] }
expect(encode(obj2)).toBe('items[3]: a,"",b')
})
it('handles whitespace-only strings in arrays', () => {
const obj = { items: [' ', ' '] }
expect(encode(obj)).toBe('items[2]: " "," "')
})
it('quotes array strings with special characters', () => {
const obj = { items: ['a', 'b,c', 'd:e'] }
expect(encode(obj)).toBe('items[3]: a,"b,c","d:e"')
})
it('quotes strings that look like booleans/numbers in arrays', () => {
const obj = { items: ['x', 'true', '42', '-3.14'] }
expect(encode(obj)).toBe('items[4]: x,"true","42","-3.14"')
})
it('quotes strings with structural meanings in arrays', () => {
const obj = { items: ['[5]', '- item', '{key}'] }
expect(encode(obj)).toBe('items[3]: "[5]","- item","{key}"')
})
})
describe('arrays of objects (tabular and list items)', () => {
it('encodes arrays of similar objects in tabular format', () => {
const obj = {
items: [
{ sku: 'A1', qty: 2, price: 9.99 },
{ sku: 'B2', qty: 1, price: 14.5 },
],
}
expect(encode(obj)).toBe('items[2]{sku,qty,price}:\n A1,2,9.99\n B2,1,14.5')
})
it('handles null values in tabular format', () => {
const obj = {
items: [
{ id: 1, value: null },
{ id: 2, value: 'test' },
],
}
expect(encode(obj)).toBe('items[2]{id,value}:\n 1,null\n 2,test')
})
it('quotes strings in tabular rows when needed', () => {
const obj = {
items: [
{ sku: 'A,1', desc: 'cool', qty: 2 },
{ sku: 'B2', desc: 'wip: test', qty: 1 },
],
}
expect(encode(obj)).toBe('items[2]{sku,desc,qty}:\n "A,1",cool,2\n B2,"wip: test",1')
})
it('quotes ambiguous strings in tabular rows', () => {
const obj = {
items: [
{ id: 1, status: 'true' },
{ id: 2, status: 'false' },
],
}
expect(encode(obj)).toBe('items[2]{id,status}:\n 1,"true"\n 2,"false"')
})
it('handles tabular arrays with keys needing quotes', () => {
const obj = {
items: [
{ 'order:id': 1, 'full name': 'Ada' },
{ 'order:id': 2, 'full name': 'Bob' },
],
}
expect(encode(obj)).toBe('items[2]{"order:id","full name"}:\n 1,Ada\n 2,Bob')
})
it('uses list format for objects with different fields', () => {
const obj = {
items: [
{ id: 1, name: 'First' },
{ id: 2, name: 'Second', extra: true },
],
}
expect(encode(obj)).toBe(
'items[2]:\n'
+ ' - id: 1\n'
+ ' name: First\n'
+ ' - id: 2\n'
+ ' name: Second\n'
+ ' extra: true',
)
})
it('uses list format for objects with nested values', () => {
const obj = {
items: [
{ id: 1, nested: { x: 1 } },
],
}
expect(encode(obj)).toBe(
'items[1]:\n'
+ ' - id: 1\n'
+ ' nested:\n'
+ ' x: 1',
)
})
it('uses field order from first object for tabular headers', () => {
const obj = {
items: [
{ a: 1, b: 2, c: 3 },
{ c: 30, b: 20, a: 10 },
],
}
expect(encode(obj)).toBe('items[2]{a,b,c}:\n 1,2,3\n 10,20,30')
})
it('uses list format for one object with nested column', () => {
const obj = {
items: [
{ id: 1, data: 'string' },
{ id: 2, data: { nested: true } },
],
}
expect(encode(obj)).toBe(
'items[2]:\n'
+ ' - id: 1\n'
+ ' data: string\n'
+ ' - id: 2\n'
+ ' data:\n'
+ ' nested: true',
)
})
})
describe('arrays of arrays (primitives only)', () => {
it('encodes nested arrays of primitives', () => {
const obj = {
pairs: [['a', 'b'], ['c', 'd']],
}
expect(encode(obj)).toBe('pairs[2]:\n - [2]: a,b\n - [2]: c,d')
})
it('quotes nested array strings when needed', () => {
const obj = {
pairs: [['a', 'b'], ['c,d', 'e:f', 'true']],
}
expect(encode(obj)).toBe('pairs[2]:\n - [2]: a,b\n - [3]: "c,d","e:f","true"')
})
it('handles empty inner arrays', () => {
const obj = {
pairs: [[], []],
}
expect(encode(obj)).toBe('pairs[2]:\n - [0]:\n - [0]:')
})
it('handles mixed-length inner arrays', () => {
const obj = {
pairs: [[1], [2, 3]],
}
expect(encode(obj)).toBe('pairs[2]:\n - [1]: 1\n - [2]: 2,3')
})
})
describe('root arrays', () => {
it('encodes arrays of primitives at root level', () => {
const arr = ['x', 'y', 'true', true, 10]
expect(encode(arr)).toBe('[5]: x,y,"true",true,10')
})
it('encodes arrays of similar objects in tabular format', () => {
const arr = [{ id: 1 }, { id: 2 }]
expect(encode(arr)).toBe('[2]{id}:\n 1\n 2')
})
it('encodes arrays of different objects in list format', () => {
const arr = [{ id: 1 }, { id: 2, name: 'Ada' }]
expect(encode(arr)).toBe('[2]:\n - id: 1\n - id: 2\n name: Ada')
})
it('encodes empty arrays at root level', () => {
expect(encode([])).toBe('[0]:')
})
it('encodes root arrays of arrays', () => {
const arr = [[1, 2], []]
expect(encode(arr)).toBe('[2]:\n - [2]: 1,2\n - [0]:')
})
})
describe('complex structures', () => {
it('encodes objects with mixed arrays and nested objects', () => {
const obj = {
user: {
id: 123,
name: 'Ada',
tags: ['admin', 'ops'],
active: true,
prefs: [],
},
}
expect(encode(obj)).toBe(
'user:\n'
+ ' id: 123\n'
+ ' name: Ada\n'
+ ' tags[2]: admin,ops\n'
+ ' active: true\n'
+ ' prefs[0]:',
)
})
})
describe('mixed arrays', () => {
it('uses list format for arrays mixing primitives and objects', () => {
const obj = {
items: [1, { a: 1 }, 'text'],
}
expect(encode(obj)).toBe(
'items[3]:\n'
+ ' - 1\n'
+ ' - a: 1\n'
+ ' - text',
)
})
it('uses list format for arrays mixing objects and arrays', () => {
const obj = {
items: [{ a: 1 }, [1, 2]],
}
expect(encode(obj)).toBe(
'items[2]:\n'
+ ' - a: 1\n'
+ ' - [2]: 1,2',
)
})
})
describe('whitespace and formatting invariants', () => {
it('produces no trailing spaces at end of lines', () => {
const obj = {
user: {
id: 123,
name: 'Ada',
},
items: ['a', 'b'],
}
const result = encode(obj)
const lines = result.split('\n')
for (const line of lines) {
expect(line).not.toMatch(/ $/)
}
})
it('produces no trailing newline at end of output', () => {
const obj = { id: 123 }
const result = encode(obj)
expect(result).not.toMatch(/\n$/)
})
})
describe('non-JSON-serializable values', () => {
it('converts BigInt to string', () => {
expect(encode(BigInt(123))).toBe('123')
expect(encode({ id: BigInt(456) })).toBe('id: 456')
})
it('converts Date to ISO string', () => {
const date = new Date('2025-01-01T00:00:00.000Z')
expect(encode(date)).toBe('"2025-01-01T00:00:00.000Z"')
expect(encode({ created: date })).toBe('created: "2025-01-01T00:00:00.000Z"')
})
it('converts undefined to null', () => {
expect(encode(undefined)).toBe('null')
expect(encode({ value: undefined })).toBe('value: null')
})
it('converts non-finite numbers to null', () => {
expect(encode(Infinity)).toBe('null')
expect(encode(-Infinity)).toBe('null')
expect(encode(Number.NaN)).toBe('null')
})
it('converts functions to null', () => {
expect(encode(() => {})).toBe('null')
expect(encode({ fn: () => {} })).toBe('fn: null')
})
it('converts symbols to null', () => {
expect(encode(Symbol('test'))).toBe('null')
expect(encode({ sym: Symbol('test') })).toBe('sym: null')
})
})

19
tsconfig.json Normal file
View File

@@ -0,0 +1,19 @@
{
"compilerOptions": {
"target": "ESNext",
"rootDir": ".",
"module": "ESNext",
"moduleResolution": "Bundler",
"resolveJsonModule": true,
"strict": true,
"noUncheckedIndexedAccess": true,
"declaration": true,
"noEmit": true,
"esModuleInterop": true,
"isolatedDeclarations": true,
"isolatedModules": true,
"verbatimModuleSyntax": true,
"erasableSyntaxOnly": true,
"skipLibCheck": true
}
}

9
tsdown.config.ts Normal file
View File

@@ -0,0 +1,9 @@
import type { UserConfig, UserConfigFn } from 'tsdown/config'
import { defineConfig } from 'tsdown/config'
const config: UserConfig | UserConfigFn = defineConfig({
entry: ['src/index.ts'],
dts: true,
})
export default config