mirror of
https://github.com/voson-wang/toon.git
synced 2026-01-29 23:34:10 +08:00
feat!: publish to @toon-format/toon and @toon-format/cli
This commit is contained in:
2
.github/workflows/release.yml
vendored
2
.github/workflows/release.yml
vendored
@@ -32,6 +32,6 @@ jobs:
|
|||||||
- run: pnpm run build
|
- run: pnpm run build
|
||||||
|
|
||||||
- name: Publish to npm
|
- name: Publish to npm
|
||||||
run: npm install -g npm@latest && npm publish --access public
|
run: npm install -g npm@latest && pnpm -r publish --access public --no-git-checks
|
||||||
env:
|
env:
|
||||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||||
|
|||||||
34
README.md
34
README.md
@@ -3,9 +3,9 @@
|
|||||||
# Token-Oriented Object Notation (TOON)
|
# Token-Oriented Object Notation (TOON)
|
||||||
|
|
||||||
[](https://github.com/johannschopplich/toon/actions)
|
[](https://github.com/johannschopplich/toon/actions)
|
||||||
[](https://www.npmjs.com/package/@byjohann/toon)
|
[](https://www.npmjs.com/package/@toon-format/toon)
|
||||||
[](./SPEC.md)
|
[](./SPEC.md)
|
||||||
[](https://www.npmjs.com/package/@byjohann/toon)
|
[](https://www.npmjs.com/package/@toon-format/toon)
|
||||||
[](./LICENSE)
|
[](./LICENSE)
|
||||||
|
|
||||||
**Token-Oriented Object Notation** is a compact, human-readable format designed for passing structured data to Large Language Models with significantly reduced token usage. It's intended for LLM input, not output.
|
**Token-Oriented Object Notation** is a compact, human-readable format designed for passing structured data to Large Language Models with significantly reduced token usage. It's intended for LLM input, not output.
|
||||||
@@ -439,19 +439,19 @@ Four datasets designed to test different structural patterns (all contain arrays
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# npm
|
# npm
|
||||||
npm install @byjohann/toon
|
npm install @toon-format/toon
|
||||||
|
|
||||||
# pnpm
|
# pnpm
|
||||||
pnpm add @byjohann/toon
|
pnpm add @toon-format/toon
|
||||||
|
|
||||||
# yarn
|
# yarn
|
||||||
yarn add @byjohann/toon
|
yarn add @toon-format/toon
|
||||||
```
|
```
|
||||||
|
|
||||||
**Example usage:**
|
**Example usage:**
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
import { encode } from '@byjohann/toon'
|
import { encode } from '@toon-format/toon'
|
||||||
|
|
||||||
const data = {
|
const data = {
|
||||||
users: [
|
users: [
|
||||||
@@ -473,20 +473,20 @@ Command-line tool for converting between JSON and TOON formats.
|
|||||||
### Usage
|
### Usage
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
npx @byjohann/toon <input> [options]
|
npx @toon-format/cli <input> [options]
|
||||||
```
|
```
|
||||||
|
|
||||||
**Auto-detection:** The CLI automatically detects the operation based on file extension (`.json` → encode, `.toon` → decode).
|
**Auto-detection:** The CLI automatically detects the operation based on file extension (`.json` → encode, `.toon` → decode).
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Encode JSON to TOON (auto-detected)
|
# Encode JSON to TOON (auto-detected)
|
||||||
toon input.json -o output.toon
|
npx @toon-format/cli input.json -o output.toon
|
||||||
|
|
||||||
# Decode TOON to JSON (auto-detected)
|
# Decode TOON to JSON (auto-detected)
|
||||||
toon data.toon -o output.json
|
npx @toon-format/cli data.toon -o output.json
|
||||||
|
|
||||||
# Output to stdout
|
# Output to stdout
|
||||||
toon input.json
|
npx @toon-format/cli input.json
|
||||||
```
|
```
|
||||||
|
|
||||||
### Options
|
### Options
|
||||||
@@ -506,16 +506,16 @@ toon input.json
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Show token savings when encoding
|
# Show token savings when encoding
|
||||||
toon data.json --stats -o output.toon
|
npx @toon-format/cli data.json --stats -o output.toon
|
||||||
|
|
||||||
# Tab-separated output (often more token-efficient)
|
# Tab-separated output (often more token-efficient)
|
||||||
toon data.json --delimiter "\t" -o output.toon
|
npx @toon-format/cli data.json --delimiter "\t" -o output.toon
|
||||||
|
|
||||||
# Pipe-separated with length markers
|
# Pipe-separated with length markers
|
||||||
toon data.json --delimiter "|" --length-marker -o output.toon
|
npx @toon-format/cli data.json --delimiter "|" --length-marker -o output.toon
|
||||||
|
|
||||||
# Lenient decoding (skip validation)
|
# Lenient decoding (skip validation)
|
||||||
toon data.toon --no-strict -o output.json
|
npx @toon-format/cli data.toon --no-strict -o output.json
|
||||||
```
|
```
|
||||||
|
|
||||||
## Format Overview
|
## Format Overview
|
||||||
@@ -741,7 +741,7 @@ A TOON-formatted string with no trailing newline or spaces.
|
|||||||
**Example:**
|
**Example:**
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
import { encode } from '@byjohann/toon'
|
import { encode } from '@toon-format/toon'
|
||||||
|
|
||||||
const items = [
|
const items = [
|
||||||
{ sku: 'A1', qty: 2, price: 9.99 },
|
{ sku: 'A1', qty: 2, price: 9.99 },
|
||||||
@@ -862,7 +862,7 @@ A JavaScript value (object, array, or primitive) representing the parsed TOON da
|
|||||||
**Example:**
|
**Example:**
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
import { decode } from '@byjohann/toon'
|
import { decode } from '@toon-format/toon'
|
||||||
|
|
||||||
const toon = `
|
const toon = `
|
||||||
items[2]{sku,qty,price}:
|
items[2]{sku,qty,price}:
|
||||||
@@ -980,7 +980,7 @@ Task: Return only users with role "user" as TOON. Use the same header. Set [N] t
|
|||||||
## Other Implementations
|
## Other Implementations
|
||||||
|
|
||||||
> [!NOTE]
|
> [!NOTE]
|
||||||
> When implementing TOON in other languages, please follow the [SPEC.md](./SPEC.md) (currently v1.3) to ensure compatibility across implementations. The [TypeScript test suite](./test) provides comprehensive examples of encoding and decoding behavior that can serve as a reference implementation.
|
> When implementing TOON in other languages, please follow the [SPEC.md](./SPEC.md) (currently v1.3) to ensure compatibility across implementations. The [TypeScript test suite](./packages/toon/test) provides comprehensive examples of encoding and decoding behavior that can serve as a reference implementation.
|
||||||
|
|
||||||
- **.NET:** [ToonSharp](https://github.com/0xZunia/ToonSharp)
|
- **.NET:** [ToonSharp](https://github.com/0xZunia/ToonSharp)
|
||||||
- **Crystal:** [toon-crystal](https://github.com/mamantoha/toon-crystal)
|
- **Crystal:** [toon-crystal](https://github.com/mamantoha/toon-crystal)
|
||||||
|
|||||||
18
SPEC.md
18
SPEC.md
@@ -3,9 +3,13 @@
|
|||||||
## Token-Oriented Object Notation
|
## Token-Oriented Object Notation
|
||||||
|
|
||||||
**Version:** 1.3
|
**Version:** 1.3
|
||||||
|
|
||||||
**Date:** 2025-10-31
|
**Date:** 2025-10-31
|
||||||
|
|
||||||
**Status:** Working Draft
|
**Status:** Working Draft
|
||||||
|
|
||||||
**Author:** Johann Schopplich ([@johannschopplich](https://github.com/johannschopplich))
|
**Author:** Johann Schopplich ([@johannschopplich](https://github.com/johannschopplich))
|
||||||
|
|
||||||
**License:** MIT
|
**License:** MIT
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -137,9 +141,9 @@ This specification defines:
|
|||||||
### Type Terms
|
### Type Terms
|
||||||
|
|
||||||
- Primitive: string, number, boolean, or null.
|
- Primitive: string, number, boolean, or null.
|
||||||
- Object: Mapping from string keys to JsonValue.
|
- Object: Mapping from string keys to `JsonValue`.
|
||||||
- Array: Ordered sequence of JsonValue.
|
- Array: Ordered sequence of `JsonValue`.
|
||||||
- JsonValue: Primitive | Object | Array.
|
- `JsonValue`: Primitive | Object | Array.
|
||||||
|
|
||||||
### Conformance Terms
|
### Conformance Terms
|
||||||
|
|
||||||
@@ -153,9 +157,9 @@ This specification defines:
|
|||||||
## 2. Data Model
|
## 2. Data Model
|
||||||
|
|
||||||
- TOON models data as:
|
- TOON models data as:
|
||||||
- JsonPrimitive: string | number | boolean | null
|
- `JsonPrimitive`: string | number | boolean | null
|
||||||
- JsonObject: { [string]: JsonValue }
|
- `JsonObject`: { [string]: `JsonValue` }
|
||||||
- JsonArray: JsonValue[]
|
- `JsonArray`: `JsonValue`[]
|
||||||
- Ordering:
|
- Ordering:
|
||||||
- Array order MUST be preserved.
|
- Array order MUST be preserved.
|
||||||
- Object key order MUST be preserved as encountered by the encoder.
|
- Object key order MUST be preserved as encountered by the encoder.
|
||||||
@@ -959,7 +963,7 @@ These sketches illustrate structure and common decoding helpers. They are inform
|
|||||||
### Reference Test Suite
|
### Reference Test Suite
|
||||||
|
|
||||||
A reference test suite is maintained at:
|
A reference test suite is maintained at:
|
||||||
https://github.com/johannschopplich/toon/tree/main/test
|
https://github.com/johannschopplich/toon/tree/main/packages/toon/test
|
||||||
|
|
||||||
The test suite is versioned alongside this specification. Implementations are encouraged to validate against this test suite, but conformance is determined solely by adherence to the normative requirements in Sections 1-16 and Section 19 of this specification. Test coverage does not define the specification; the specification defines conformance.
|
The test suite is versioned alongside this specification. Implementations are encouraged to validate against this test suite, but conformance is determined solely by adherence to the normative requirements in Sections 1-16 and Section 19 of this specification. Test coverage does not define the specification; the specification defines conformance.
|
||||||
|
|
||||||
|
|||||||
@@ -1,13 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "@toon/cli",
|
|
||||||
"type": "module",
|
|
||||||
"private": true,
|
|
||||||
"scripts": {
|
|
||||||
"dev": "tsx ./src/index.ts"
|
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"citty": "^0.1.6",
|
|
||||||
"consola": "^3.4.2",
|
|
||||||
"tokenx": "^1.2.0"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
41
package.json
41
package.json
@@ -1,42 +1,17 @@
|
|||||||
{
|
{
|
||||||
"name": "@byjohann/toon",
|
"name": "@toon-format/monorepo",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"version": "0.6.0",
|
"version": "0.6.0",
|
||||||
|
"private": true,
|
||||||
"packageManager": "pnpm@10.19.0",
|
"packageManager": "pnpm@10.19.0",
|
||||||
"description": "Token-Oriented Object Notation – a token-efficient JSON alternative for LLM prompts",
|
|
||||||
"author": "Johann Schopplich <hello@johannschopplich.com>",
|
|
||||||
"license": "MIT",
|
|
||||||
"homepage": "https://github.com/johannschopplich/toon#readme",
|
|
||||||
"repository": {
|
|
||||||
"type": "git",
|
|
||||||
"url": "git+https://github.com/johannschopplich/toon.git"
|
|
||||||
},
|
|
||||||
"bugs": {
|
|
||||||
"url": "https://github.com/johannschopplich/toon/issues"
|
|
||||||
},
|
|
||||||
"sideEffects": false,
|
|
||||||
"exports": {
|
|
||||||
".": {
|
|
||||||
"types": "./dist/index.d.ts",
|
|
||||||
"default": "./dist/index.js"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"types": "./dist/index.d.ts",
|
|
||||||
"bin": {
|
|
||||||
"toon": "bin/toon.mjs"
|
|
||||||
},
|
|
||||||
"files": [
|
|
||||||
"bin",
|
|
||||||
"dist"
|
|
||||||
],
|
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
"build": "pnpm -r --filter=./packages/** run build",
|
||||||
"automd": "automd",
|
"automd": "automd",
|
||||||
"build": "tsdown",
|
|
||||||
"lint": "eslint .",
|
"lint": "eslint .",
|
||||||
"lint:fix": "eslint . --fix",
|
"lint:fix": "eslint . --fix",
|
||||||
"test": "vitest",
|
"test": "pnpm -r test",
|
||||||
"test:types": "tsc --noEmit",
|
"test:types": "tsc --noEmit",
|
||||||
"release": "bumpp"
|
"release": "bumpp -r"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@antfu/eslint-config": "^6.1.0",
|
"@antfu/eslint-config": "^6.1.0",
|
||||||
@@ -48,11 +23,5 @@
|
|||||||
"tsx": "^4.20.6",
|
"tsx": "^4.20.6",
|
||||||
"typescript": "^5.9.3",
|
"typescript": "^5.9.3",
|
||||||
"vitest": "^4.0.3"
|
"vitest": "^4.0.3"
|
||||||
},
|
|
||||||
"pnpm": {
|
|
||||||
"onlyBuiltDependencies": [
|
|
||||||
"@parcel/watcher",
|
|
||||||
"esbuild"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
2
bin/toon.mjs → packages/cli/bin/toon.mjs
Normal file → Executable file
2
bin/toon.mjs → packages/cli/bin/toon.mjs
Normal file → Executable file
@@ -1,3 +1,3 @@
|
|||||||
#!/usr/bin/env node
|
#!/usr/bin/env node
|
||||||
'use strict'
|
'use strict'
|
||||||
import('../dist/cli/index.js')
|
import('../dist/index.js')
|
||||||
41
packages/cli/package.json
Normal file
41
packages/cli/package.json
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"name": "@toon-format/cli",
|
||||||
|
"type": "module",
|
||||||
|
"version": "0.6.0",
|
||||||
|
"packageManager": "pnpm@10.19.0",
|
||||||
|
"description": "CLI for encoding and decoding Token-Oriented Object Notation (TOON)",
|
||||||
|
"author": "Johann Schopplich <hello@johannschopplich.com>",
|
||||||
|
"license": "MIT",
|
||||||
|
"homepage": "https://toonformat.dev",
|
||||||
|
"repository": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "git+https://github.com/johannschopplich/toon.git"
|
||||||
|
},
|
||||||
|
"bugs": {
|
||||||
|
"url": "https://github.com/johannschopplich/toon/issues"
|
||||||
|
},
|
||||||
|
"sideEffects": false,
|
||||||
|
"exports": {
|
||||||
|
".": {
|
||||||
|
"types": "./dist/index.d.ts",
|
||||||
|
"default": "./dist/index.js"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"types": "./dist/index.d.ts",
|
||||||
|
"bin": {
|
||||||
|
"toon": "bin/toon.mjs"
|
||||||
|
},
|
||||||
|
"files": [
|
||||||
|
"bin",
|
||||||
|
"dist"
|
||||||
|
],
|
||||||
|
"scripts": {
|
||||||
|
"dev": "tsx ./src/index.ts",
|
||||||
|
"build": "tsdown"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"citty": "^0.1.6",
|
||||||
|
"consola": "^3.4.2",
|
||||||
|
"tokenx": "^1.2.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,12 +1,12 @@
|
|||||||
import type { DecodeOptions, Delimiter, EncodeOptions } from '../../src'
|
import type { DecodeOptions, Delimiter, EncodeOptions } from '../../toon/src'
|
||||||
import * as fsp from 'node:fs/promises'
|
import * as fsp from 'node:fs/promises'
|
||||||
import * as path from 'node:path'
|
import * as path from 'node:path'
|
||||||
import process from 'node:process'
|
import process from 'node:process'
|
||||||
import { defineCommand, runMain } from 'citty'
|
import { defineCommand, runMain } from 'citty'
|
||||||
import { consola } from 'consola'
|
import { consola } from 'consola'
|
||||||
import { estimateTokenCount } from 'tokenx'
|
import { estimateTokenCount } from 'tokenx'
|
||||||
import { name, version } from '../../package.json' with { type: 'json' }
|
import { name, version } from '../../toon/package.json' with { type: 'json' }
|
||||||
import { decode, DEFAULT_DELIMITER, DELIMITERS, encode } from '../../src'
|
import { decode, DEFAULT_DELIMITER, DELIMITERS, encode } from '../../toon/src'
|
||||||
|
|
||||||
const main = defineCommand({
|
const main = defineCommand({
|
||||||
meta: {
|
meta: {
|
||||||
@@ -2,10 +2,7 @@ import type { UserConfig, UserConfigFn } from 'tsdown/config'
|
|||||||
import { defineConfig } from 'tsdown/config'
|
import { defineConfig } from 'tsdown/config'
|
||||||
|
|
||||||
const config: UserConfig | UserConfigFn = defineConfig({
|
const config: UserConfig | UserConfigFn = defineConfig({
|
||||||
entry: {
|
entry: 'src/index.ts',
|
||||||
'index': 'src/index.ts',
|
|
||||||
'cli/index': 'cli/src/index.ts',
|
|
||||||
},
|
|
||||||
dts: true,
|
dts: true,
|
||||||
})
|
})
|
||||||
|
|
||||||
32
packages/toon/package.json
Normal file
32
packages/toon/package.json
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
{
|
||||||
|
"name": "@toon-format/toon",
|
||||||
|
"type": "module",
|
||||||
|
"version": "0.6.0",
|
||||||
|
"packageManager": "pnpm@10.19.0",
|
||||||
|
"description": "Token-Oriented Object Notation (TOON) – a token-efficient JSON alternative for LLM prompts",
|
||||||
|
"author": "Johann Schopplich <hello@johannschopplich.com>",
|
||||||
|
"license": "MIT",
|
||||||
|
"homepage": "https://toonformat.dev",
|
||||||
|
"repository": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "git+https://github.com/johannschopplich/toon.git"
|
||||||
|
},
|
||||||
|
"bugs": {
|
||||||
|
"url": "https://github.com/johannschopplich/toon/issues"
|
||||||
|
},
|
||||||
|
"sideEffects": false,
|
||||||
|
"exports": {
|
||||||
|
".": {
|
||||||
|
"types": "./dist/index.d.ts",
|
||||||
|
"default": "./dist/index.js"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"types": "./dist/index.d.ts",
|
||||||
|
"files": [
|
||||||
|
"dist"
|
||||||
|
],
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsdown",
|
||||||
|
"test": "vitest"
|
||||||
|
}
|
||||||
|
}
|
||||||
9
packages/toon/tsdown.config.ts
Normal file
9
packages/toon/tsdown.config.ts
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
import type { UserConfig, UserConfigFn } from 'tsdown/config'
|
||||||
|
import { defineConfig } from 'tsdown/config'
|
||||||
|
|
||||||
|
const config: UserConfig | UserConfigFn = defineConfig({
|
||||||
|
entry: 'src/index.ts',
|
||||||
|
dts: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
export default config
|
||||||
4
pnpm-lock.yaml
generated
4
pnpm-lock.yaml
generated
@@ -87,7 +87,7 @@ importers:
|
|||||||
specifier: ^2.8.1
|
specifier: ^2.8.1
|
||||||
version: 2.8.1
|
version: 2.8.1
|
||||||
|
|
||||||
cli:
|
packages/cli:
|
||||||
dependencies:
|
dependencies:
|
||||||
citty:
|
citty:
|
||||||
specifier: ^0.1.6
|
specifier: ^0.1.6
|
||||||
@@ -99,6 +99,8 @@ importers:
|
|||||||
specifier: ^1.2.0
|
specifier: ^1.2.0
|
||||||
version: 1.2.0
|
version: 1.2.0
|
||||||
|
|
||||||
|
packages/toon: {}
|
||||||
|
|
||||||
packages:
|
packages:
|
||||||
|
|
||||||
'@ai-sdk/anthropic@2.0.37':
|
'@ai-sdk/anthropic@2.0.37':
|
||||||
|
|||||||
@@ -1,3 +1,7 @@
|
|||||||
packages:
|
packages:
|
||||||
- benchmarks
|
- benchmarks
|
||||||
- cli
|
- packages/*
|
||||||
|
|
||||||
|
onlyBuiltDependencies:
|
||||||
|
- '@parcel/watcher'
|
||||||
|
- esbuild
|
||||||
|
|||||||
Reference in New Issue
Block a user