mirror of
https://github.com/voson-wang/toon.git
synced 2026-01-29 15:24:10 +08:00
feat!: publish to @toon-format/toon and @toon-format/cli
This commit is contained in:
2
.github/workflows/release.yml
vendored
2
.github/workflows/release.yml
vendored
@@ -32,6 +32,6 @@ jobs:
|
||||
- run: pnpm run build
|
||||
|
||||
- name: Publish to npm
|
||||
run: npm install -g npm@latest && npm publish --access public
|
||||
run: npm install -g npm@latest && pnpm -r publish --access public --no-git-checks
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
|
||||
34
README.md
34
README.md
@@ -3,9 +3,9 @@
|
||||
# Token-Oriented Object Notation (TOON)
|
||||
|
||||
[](https://github.com/johannschopplich/toon/actions)
|
||||
[](https://www.npmjs.com/package/@byjohann/toon)
|
||||
[](https://www.npmjs.com/package/@toon-format/toon)
|
||||
[](./SPEC.md)
|
||||
[](https://www.npmjs.com/package/@byjohann/toon)
|
||||
[](https://www.npmjs.com/package/@toon-format/toon)
|
||||
[](./LICENSE)
|
||||
|
||||
**Token-Oriented Object Notation** is a compact, human-readable format designed for passing structured data to Large Language Models with significantly reduced token usage. It's intended for LLM input, not output.
|
||||
@@ -439,19 +439,19 @@ Four datasets designed to test different structural patterns (all contain arrays
|
||||
|
||||
```bash
|
||||
# npm
|
||||
npm install @byjohann/toon
|
||||
npm install @toon-format/toon
|
||||
|
||||
# pnpm
|
||||
pnpm add @byjohann/toon
|
||||
pnpm add @toon-format/toon
|
||||
|
||||
# yarn
|
||||
yarn add @byjohann/toon
|
||||
yarn add @toon-format/toon
|
||||
```
|
||||
|
||||
**Example usage:**
|
||||
|
||||
```ts
|
||||
import { encode } from '@byjohann/toon'
|
||||
import { encode } from '@toon-format/toon'
|
||||
|
||||
const data = {
|
||||
users: [
|
||||
@@ -473,20 +473,20 @@ Command-line tool for converting between JSON and TOON formats.
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
npx @byjohann/toon <input> [options]
|
||||
npx @toon-format/cli <input> [options]
|
||||
```
|
||||
|
||||
**Auto-detection:** The CLI automatically detects the operation based on file extension (`.json` → encode, `.toon` → decode).
|
||||
|
||||
```bash
|
||||
# Encode JSON to TOON (auto-detected)
|
||||
toon input.json -o output.toon
|
||||
npx @toon-format/cli input.json -o output.toon
|
||||
|
||||
# Decode TOON to JSON (auto-detected)
|
||||
toon data.toon -o output.json
|
||||
npx @toon-format/cli data.toon -o output.json
|
||||
|
||||
# Output to stdout
|
||||
toon input.json
|
||||
npx @toon-format/cli input.json
|
||||
```
|
||||
|
||||
### Options
|
||||
@@ -506,16 +506,16 @@ toon input.json
|
||||
|
||||
```bash
|
||||
# Show token savings when encoding
|
||||
toon data.json --stats -o output.toon
|
||||
npx @toon-format/cli data.json --stats -o output.toon
|
||||
|
||||
# Tab-separated output (often more token-efficient)
|
||||
toon data.json --delimiter "\t" -o output.toon
|
||||
npx @toon-format/cli data.json --delimiter "\t" -o output.toon
|
||||
|
||||
# Pipe-separated with length markers
|
||||
toon data.json --delimiter "|" --length-marker -o output.toon
|
||||
npx @toon-format/cli data.json --delimiter "|" --length-marker -o output.toon
|
||||
|
||||
# Lenient decoding (skip validation)
|
||||
toon data.toon --no-strict -o output.json
|
||||
npx @toon-format/cli data.toon --no-strict -o output.json
|
||||
```
|
||||
|
||||
## Format Overview
|
||||
@@ -741,7 +741,7 @@ A TOON-formatted string with no trailing newline or spaces.
|
||||
**Example:**
|
||||
|
||||
```ts
|
||||
import { encode } from '@byjohann/toon'
|
||||
import { encode } from '@toon-format/toon'
|
||||
|
||||
const items = [
|
||||
{ sku: 'A1', qty: 2, price: 9.99 },
|
||||
@@ -862,7 +862,7 @@ A JavaScript value (object, array, or primitive) representing the parsed TOON da
|
||||
**Example:**
|
||||
|
||||
```ts
|
||||
import { decode } from '@byjohann/toon'
|
||||
import { decode } from '@toon-format/toon'
|
||||
|
||||
const toon = `
|
||||
items[2]{sku,qty,price}:
|
||||
@@ -980,7 +980,7 @@ Task: Return only users with role "user" as TOON. Use the same header. Set [N] t
|
||||
## Other Implementations
|
||||
|
||||
> [!NOTE]
|
||||
> When implementing TOON in other languages, please follow the [SPEC.md](./SPEC.md) (currently v1.3) to ensure compatibility across implementations. The [TypeScript test suite](./test) provides comprehensive examples of encoding and decoding behavior that can serve as a reference implementation.
|
||||
> When implementing TOON in other languages, please follow the [SPEC.md](./SPEC.md) (currently v1.3) to ensure compatibility across implementations. The [TypeScript test suite](./packages/toon/test) provides comprehensive examples of encoding and decoding behavior that can serve as a reference implementation.
|
||||
|
||||
- **.NET:** [ToonSharp](https://github.com/0xZunia/ToonSharp)
|
||||
- **Crystal:** [toon-crystal](https://github.com/mamantoha/toon-crystal)
|
||||
|
||||
18
SPEC.md
18
SPEC.md
@@ -3,9 +3,13 @@
|
||||
## Token-Oriented Object Notation
|
||||
|
||||
**Version:** 1.3
|
||||
|
||||
**Date:** 2025-10-31
|
||||
|
||||
**Status:** Working Draft
|
||||
|
||||
**Author:** Johann Schopplich ([@johannschopplich](https://github.com/johannschopplich))
|
||||
|
||||
**License:** MIT
|
||||
|
||||
---
|
||||
@@ -137,9 +141,9 @@ This specification defines:
|
||||
### Type Terms
|
||||
|
||||
- Primitive: string, number, boolean, or null.
|
||||
- Object: Mapping from string keys to JsonValue.
|
||||
- Array: Ordered sequence of JsonValue.
|
||||
- JsonValue: Primitive | Object | Array.
|
||||
- Object: Mapping from string keys to `JsonValue`.
|
||||
- Array: Ordered sequence of `JsonValue`.
|
||||
- `JsonValue`: Primitive | Object | Array.
|
||||
|
||||
### Conformance Terms
|
||||
|
||||
@@ -153,9 +157,9 @@ This specification defines:
|
||||
## 2. Data Model
|
||||
|
||||
- TOON models data as:
|
||||
- JsonPrimitive: string | number | boolean | null
|
||||
- JsonObject: { [string]: JsonValue }
|
||||
- JsonArray: JsonValue[]
|
||||
- `JsonPrimitive`: string | number | boolean | null
|
||||
- `JsonObject`: { [string]: `JsonValue` }
|
||||
- `JsonArray`: `JsonValue`[]
|
||||
- Ordering:
|
||||
- Array order MUST be preserved.
|
||||
- Object key order MUST be preserved as encountered by the encoder.
|
||||
@@ -959,7 +963,7 @@ These sketches illustrate structure and common decoding helpers. They are inform
|
||||
### Reference Test Suite
|
||||
|
||||
A reference test suite is maintained at:
|
||||
https://github.com/johannschopplich/toon/tree/main/test
|
||||
https://github.com/johannschopplich/toon/tree/main/packages/toon/test
|
||||
|
||||
The test suite is versioned alongside this specification. Implementations are encouraged to validate against this test suite, but conformance is determined solely by adherence to the normative requirements in Sections 1-16 and Section 19 of this specification. Test coverage does not define the specification; the specification defines conformance.
|
||||
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
{
|
||||
"name": "@toon/cli",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "tsx ./src/index.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"citty": "^0.1.6",
|
||||
"consola": "^3.4.2",
|
||||
"tokenx": "^1.2.0"
|
||||
}
|
||||
}
|
||||
41
package.json
41
package.json
@@ -1,42 +1,17 @@
|
||||
{
|
||||
"name": "@byjohann/toon",
|
||||
"name": "@toon-format/monorepo",
|
||||
"type": "module",
|
||||
"version": "0.6.0",
|
||||
"private": true,
|
||||
"packageManager": "pnpm@10.19.0",
|
||||
"description": "Token-Oriented Object Notation – a token-efficient JSON alternative for LLM prompts",
|
||||
"author": "Johann Schopplich <hello@johannschopplich.com>",
|
||||
"license": "MIT",
|
||||
"homepage": "https://github.com/johannschopplich/toon#readme",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/johannschopplich/toon.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/johannschopplich/toon/issues"
|
||||
},
|
||||
"sideEffects": false,
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"types": "./dist/index.d.ts",
|
||||
"bin": {
|
||||
"toon": "bin/toon.mjs"
|
||||
},
|
||||
"files": [
|
||||
"bin",
|
||||
"dist"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "pnpm -r --filter=./packages/** run build",
|
||||
"automd": "automd",
|
||||
"build": "tsdown",
|
||||
"lint": "eslint .",
|
||||
"lint:fix": "eslint . --fix",
|
||||
"test": "vitest",
|
||||
"test": "pnpm -r test",
|
||||
"test:types": "tsc --noEmit",
|
||||
"release": "bumpp"
|
||||
"release": "bumpp -r"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@antfu/eslint-config": "^6.1.0",
|
||||
@@ -48,11 +23,5 @@
|
||||
"tsx": "^4.20.6",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^4.0.3"
|
||||
},
|
||||
"pnpm": {
|
||||
"onlyBuiltDependencies": [
|
||||
"@parcel/watcher",
|
||||
"esbuild"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
2
bin/toon.mjs → packages/cli/bin/toon.mjs
Normal file → Executable file
2
bin/toon.mjs → packages/cli/bin/toon.mjs
Normal file → Executable file
@@ -1,3 +1,3 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict'
|
||||
import('../dist/cli/index.js')
|
||||
import('../dist/index.js')
|
||||
41
packages/cli/package.json
Normal file
41
packages/cli/package.json
Normal file
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"name": "@toon-format/cli",
|
||||
"type": "module",
|
||||
"version": "0.6.0",
|
||||
"packageManager": "pnpm@10.19.0",
|
||||
"description": "CLI for encoding and decoding Token-Oriented Object Notation (TOON)",
|
||||
"author": "Johann Schopplich <hello@johannschopplich.com>",
|
||||
"license": "MIT",
|
||||
"homepage": "https://toonformat.dev",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/johannschopplich/toon.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/johannschopplich/toon/issues"
|
||||
},
|
||||
"sideEffects": false,
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"types": "./dist/index.d.ts",
|
||||
"bin": {
|
||||
"toon": "bin/toon.mjs"
|
||||
},
|
||||
"files": [
|
||||
"bin",
|
||||
"dist"
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "tsx ./src/index.ts",
|
||||
"build": "tsdown"
|
||||
},
|
||||
"dependencies": {
|
||||
"citty": "^0.1.6",
|
||||
"consola": "^3.4.2",
|
||||
"tokenx": "^1.2.0"
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,12 @@
|
||||
import type { DecodeOptions, Delimiter, EncodeOptions } from '../../src'
|
||||
import type { DecodeOptions, Delimiter, EncodeOptions } from '../../toon/src'
|
||||
import * as fsp from 'node:fs/promises'
|
||||
import * as path from 'node:path'
|
||||
import process from 'node:process'
|
||||
import { defineCommand, runMain } from 'citty'
|
||||
import { consola } from 'consola'
|
||||
import { estimateTokenCount } from 'tokenx'
|
||||
import { name, version } from '../../package.json' with { type: 'json' }
|
||||
import { decode, DEFAULT_DELIMITER, DELIMITERS, encode } from '../../src'
|
||||
import { name, version } from '../../toon/package.json' with { type: 'json' }
|
||||
import { decode, DEFAULT_DELIMITER, DELIMITERS, encode } from '../../toon/src'
|
||||
|
||||
const main = defineCommand({
|
||||
meta: {
|
||||
@@ -2,10 +2,7 @@ import type { UserConfig, UserConfigFn } from 'tsdown/config'
|
||||
import { defineConfig } from 'tsdown/config'
|
||||
|
||||
const config: UserConfig | UserConfigFn = defineConfig({
|
||||
entry: {
|
||||
'index': 'src/index.ts',
|
||||
'cli/index': 'cli/src/index.ts',
|
||||
},
|
||||
entry: 'src/index.ts',
|
||||
dts: true,
|
||||
})
|
||||
|
||||
32
packages/toon/package.json
Normal file
32
packages/toon/package.json
Normal file
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"name": "@toon-format/toon",
|
||||
"type": "module",
|
||||
"version": "0.6.0",
|
||||
"packageManager": "pnpm@10.19.0",
|
||||
"description": "Token-Oriented Object Notation (TOON) – a token-efficient JSON alternative for LLM prompts",
|
||||
"author": "Johann Schopplich <hello@johannschopplich.com>",
|
||||
"license": "MIT",
|
||||
"homepage": "https://toonformat.dev",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/johannschopplich/toon.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/johannschopplich/toon/issues"
|
||||
},
|
||||
"sideEffects": false,
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"types": "./dist/index.d.ts",
|
||||
"files": [
|
||||
"dist"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsdown",
|
||||
"test": "vitest"
|
||||
}
|
||||
}
|
||||
9
packages/toon/tsdown.config.ts
Normal file
9
packages/toon/tsdown.config.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
import type { UserConfig, UserConfigFn } from 'tsdown/config'
|
||||
import { defineConfig } from 'tsdown/config'
|
||||
|
||||
const config: UserConfig | UserConfigFn = defineConfig({
|
||||
entry: 'src/index.ts',
|
||||
dts: true,
|
||||
})
|
||||
|
||||
export default config
|
||||
4
pnpm-lock.yaml
generated
4
pnpm-lock.yaml
generated
@@ -87,7 +87,7 @@ importers:
|
||||
specifier: ^2.8.1
|
||||
version: 2.8.1
|
||||
|
||||
cli:
|
||||
packages/cli:
|
||||
dependencies:
|
||||
citty:
|
||||
specifier: ^0.1.6
|
||||
@@ -99,6 +99,8 @@ importers:
|
||||
specifier: ^1.2.0
|
||||
version: 1.2.0
|
||||
|
||||
packages/toon: {}
|
||||
|
||||
packages:
|
||||
|
||||
'@ai-sdk/anthropic@2.0.37':
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
packages:
|
||||
- benchmarks
|
||||
- cli
|
||||
- packages/*
|
||||
|
||||
onlyBuiltDependencies:
|
||||
- '@parcel/watcher'
|
||||
- esbuild
|
||||
|
||||
Reference in New Issue
Block a user