feat!: publish to @toon-format/toon and @toon-format/cli

This commit is contained in:
Johann Schopplich
2025-11-01 16:53:41 +01:00
parent 8bcbdb7315
commit 0710bd19e7
29 changed files with 129 additions and 84 deletions

View File

@@ -32,6 +32,6 @@ jobs:
- run: pnpm run build - run: pnpm run build
- name: Publish to npm - name: Publish to npm
run: npm install -g npm@latest && npm publish --access public run: npm install -g npm@latest && pnpm -r publish --access public --no-git-checks
env: env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

View File

@@ -3,9 +3,9 @@
# Token-Oriented Object Notation (TOON) # Token-Oriented Object Notation (TOON)
[![CI](https://github.com/johannschopplich/toon/actions/workflows/ci.yml/badge.svg)](https://github.com/johannschopplich/toon/actions) [![CI](https://github.com/johannschopplich/toon/actions/workflows/ci.yml/badge.svg)](https://github.com/johannschopplich/toon/actions)
[![npm version](https://img.shields.io/npm/v/@byjohann/toon.svg)](https://www.npmjs.com/package/@byjohann/toon) [![npm version](https://img.shields.io/npm/v/@toon-format/toon.svg)](https://www.npmjs.com/package/@toon-format/toon)
[![SPEC v1.3](https://img.shields.io/badge/spec-v1.3-lightgrey)](./SPEC.md) [![SPEC v1.3](https://img.shields.io/badge/spec-v1.3-lightgrey)](./SPEC.md)
[![npm downloads (total)](https://img.shields.io/npm/dt/@byjohann/toon.svg)](https://www.npmjs.com/package/@byjohann/toon) [![npm downloads (total)](https://img.shields.io/npm/dt/@toon-format/toon.svg)](https://www.npmjs.com/package/@toon-format/toon)
[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE)
**Token-Oriented Object Notation** is a compact, human-readable format designed for passing structured data to Large Language Models with significantly reduced token usage. It's intended for LLM input, not output. **Token-Oriented Object Notation** is a compact, human-readable format designed for passing structured data to Large Language Models with significantly reduced token usage. It's intended for LLM input, not output.
@@ -439,19 +439,19 @@ Four datasets designed to test different structural patterns (all contain arrays
```bash ```bash
# npm # npm
npm install @byjohann/toon npm install @toon-format/toon
# pnpm # pnpm
pnpm add @byjohann/toon pnpm add @toon-format/toon
# yarn # yarn
yarn add @byjohann/toon yarn add @toon-format/toon
``` ```
**Example usage:** **Example usage:**
```ts ```ts
import { encode } from '@byjohann/toon' import { encode } from '@toon-format/toon'
const data = { const data = {
users: [ users: [
@@ -473,20 +473,20 @@ Command-line tool for converting between JSON and TOON formats.
### Usage ### Usage
```bash ```bash
npx @byjohann/toon <input> [options] npx @toon-format/cli <input> [options]
``` ```
**Auto-detection:** The CLI automatically detects the operation based on file extension (`.json` → encode, `.toon` → decode). **Auto-detection:** The CLI automatically detects the operation based on file extension (`.json` → encode, `.toon` → decode).
```bash ```bash
# Encode JSON to TOON (auto-detected) # Encode JSON to TOON (auto-detected)
toon input.json -o output.toon npx @toon-format/cli input.json -o output.toon
# Decode TOON to JSON (auto-detected) # Decode TOON to JSON (auto-detected)
toon data.toon -o output.json npx @toon-format/cli data.toon -o output.json
# Output to stdout # Output to stdout
toon input.json npx @toon-format/cli input.json
``` ```
### Options ### Options
@@ -506,16 +506,16 @@ toon input.json
```bash ```bash
# Show token savings when encoding # Show token savings when encoding
toon data.json --stats -o output.toon npx @toon-format/cli data.json --stats -o output.toon
# Tab-separated output (often more token-efficient) # Tab-separated output (often more token-efficient)
toon data.json --delimiter "\t" -o output.toon npx @toon-format/cli data.json --delimiter "\t" -o output.toon
# Pipe-separated with length markers # Pipe-separated with length markers
toon data.json --delimiter "|" --length-marker -o output.toon npx @toon-format/cli data.json --delimiter "|" --length-marker -o output.toon
# Lenient decoding (skip validation) # Lenient decoding (skip validation)
toon data.toon --no-strict -o output.json npx @toon-format/cli data.toon --no-strict -o output.json
``` ```
## Format Overview ## Format Overview
@@ -741,7 +741,7 @@ A TOON-formatted string with no trailing newline or spaces.
**Example:** **Example:**
```ts ```ts
import { encode } from '@byjohann/toon' import { encode } from '@toon-format/toon'
const items = [ const items = [
{ sku: 'A1', qty: 2, price: 9.99 }, { sku: 'A1', qty: 2, price: 9.99 },
@@ -862,7 +862,7 @@ A JavaScript value (object, array, or primitive) representing the parsed TOON da
**Example:** **Example:**
```ts ```ts
import { decode } from '@byjohann/toon' import { decode } from '@toon-format/toon'
const toon = ` const toon = `
items[2]{sku,qty,price}: items[2]{sku,qty,price}:
@@ -980,7 +980,7 @@ Task: Return only users with role "user" as TOON. Use the same header. Set [N] t
## Other Implementations ## Other Implementations
> [!NOTE] > [!NOTE]
> When implementing TOON in other languages, please follow the [SPEC.md](./SPEC.md) (currently v1.3) to ensure compatibility across implementations. The [TypeScript test suite](./test) provides comprehensive examples of encoding and decoding behavior that can serve as a reference implementation. > When implementing TOON in other languages, please follow the [SPEC.md](./SPEC.md) (currently v1.3) to ensure compatibility across implementations. The [TypeScript test suite](./packages/toon/test) provides comprehensive examples of encoding and decoding behavior that can serve as a reference implementation.
- **.NET:** [ToonSharp](https://github.com/0xZunia/ToonSharp) - **.NET:** [ToonSharp](https://github.com/0xZunia/ToonSharp)
- **Crystal:** [toon-crystal](https://github.com/mamantoha/toon-crystal) - **Crystal:** [toon-crystal](https://github.com/mamantoha/toon-crystal)

18
SPEC.md
View File

@@ -3,9 +3,13 @@
## Token-Oriented Object Notation ## Token-Oriented Object Notation
**Version:** 1.3 **Version:** 1.3
**Date:** 2025-10-31 **Date:** 2025-10-31
**Status:** Working Draft **Status:** Working Draft
**Author:** Johann Schopplich ([@johannschopplich](https://github.com/johannschopplich)) **Author:** Johann Schopplich ([@johannschopplich](https://github.com/johannschopplich))
**License:** MIT **License:** MIT
--- ---
@@ -137,9 +141,9 @@ This specification defines:
### Type Terms ### Type Terms
- Primitive: string, number, boolean, or null. - Primitive: string, number, boolean, or null.
- Object: Mapping from string keys to JsonValue. - Object: Mapping from string keys to `JsonValue`.
- Array: Ordered sequence of JsonValue. - Array: Ordered sequence of `JsonValue`.
- JsonValue: Primitive | Object | Array. - `JsonValue`: Primitive | Object | Array.
### Conformance Terms ### Conformance Terms
@@ -153,9 +157,9 @@ This specification defines:
## 2. Data Model ## 2. Data Model
- TOON models data as: - TOON models data as:
- JsonPrimitive: string | number | boolean | null - `JsonPrimitive`: string | number | boolean | null
- JsonObject: { [string]: JsonValue } - `JsonObject`: { [string]: `JsonValue` }
- JsonArray: JsonValue[] - `JsonArray`: `JsonValue`[]
- Ordering: - Ordering:
- Array order MUST be preserved. - Array order MUST be preserved.
- Object key order MUST be preserved as encountered by the encoder. - Object key order MUST be preserved as encountered by the encoder.
@@ -959,7 +963,7 @@ These sketches illustrate structure and common decoding helpers. They are inform
### Reference Test Suite ### Reference Test Suite
A reference test suite is maintained at: A reference test suite is maintained at:
https://github.com/johannschopplich/toon/tree/main/test https://github.com/johannschopplich/toon/tree/main/packages/toon/test
The test suite is versioned alongside this specification. Implementations are encouraged to validate against this test suite, but conformance is determined solely by adherence to the normative requirements in Sections 1-16 and Section 19 of this specification. Test coverage does not define the specification; the specification defines conformance. The test suite is versioned alongside this specification. Implementations are encouraged to validate against this test suite, but conformance is determined solely by adherence to the normative requirements in Sections 1-16 and Section 19 of this specification. Test coverage does not define the specification; the specification defines conformance.

View File

@@ -1,13 +0,0 @@
{
"name": "@toon/cli",
"type": "module",
"private": true,
"scripts": {
"dev": "tsx ./src/index.ts"
},
"dependencies": {
"citty": "^0.1.6",
"consola": "^3.4.2",
"tokenx": "^1.2.0"
}
}

View File

@@ -1,42 +1,17 @@
{ {
"name": "@byjohann/toon", "name": "@toon-format/monorepo",
"type": "module", "type": "module",
"version": "0.6.0", "version": "0.6.0",
"private": true,
"packageManager": "pnpm@10.19.0", "packageManager": "pnpm@10.19.0",
"description": "Token-Oriented Object Notation a token-efficient JSON alternative for LLM prompts",
"author": "Johann Schopplich <hello@johannschopplich.com>",
"license": "MIT",
"homepage": "https://github.com/johannschopplich/toon#readme",
"repository": {
"type": "git",
"url": "git+https://github.com/johannschopplich/toon.git"
},
"bugs": {
"url": "https://github.com/johannschopplich/toon/issues"
},
"sideEffects": false,
"exports": {
".": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
},
"types": "./dist/index.d.ts",
"bin": {
"toon": "bin/toon.mjs"
},
"files": [
"bin",
"dist"
],
"scripts": { "scripts": {
"build": "pnpm -r --filter=./packages/** run build",
"automd": "automd", "automd": "automd",
"build": "tsdown",
"lint": "eslint .", "lint": "eslint .",
"lint:fix": "eslint . --fix", "lint:fix": "eslint . --fix",
"test": "vitest", "test": "pnpm -r test",
"test:types": "tsc --noEmit", "test:types": "tsc --noEmit",
"release": "bumpp" "release": "bumpp -r"
}, },
"devDependencies": { "devDependencies": {
"@antfu/eslint-config": "^6.1.0", "@antfu/eslint-config": "^6.1.0",
@@ -48,11 +23,5 @@
"tsx": "^4.20.6", "tsx": "^4.20.6",
"typescript": "^5.9.3", "typescript": "^5.9.3",
"vitest": "^4.0.3" "vitest": "^4.0.3"
},
"pnpm": {
"onlyBuiltDependencies": [
"@parcel/watcher",
"esbuild"
]
} }
} }

2
bin/toon.mjs → packages/cli/bin/toon.mjs Normal file → Executable file
View File

@@ -1,3 +1,3 @@
#!/usr/bin/env node #!/usr/bin/env node
'use strict' 'use strict'
import('../dist/cli/index.js') import('../dist/index.js')

41
packages/cli/package.json Normal file
View File

@@ -0,0 +1,41 @@
{
"name": "@toon-format/cli",
"type": "module",
"version": "0.6.0",
"packageManager": "pnpm@10.19.0",
"description": "CLI for encoding and decoding Token-Oriented Object Notation (TOON)",
"author": "Johann Schopplich <hello@johannschopplich.com>",
"license": "MIT",
"homepage": "https://toonformat.dev",
"repository": {
"type": "git",
"url": "git+https://github.com/johannschopplich/toon.git"
},
"bugs": {
"url": "https://github.com/johannschopplich/toon/issues"
},
"sideEffects": false,
"exports": {
".": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
},
"types": "./dist/index.d.ts",
"bin": {
"toon": "bin/toon.mjs"
},
"files": [
"bin",
"dist"
],
"scripts": {
"dev": "tsx ./src/index.ts",
"build": "tsdown"
},
"dependencies": {
"citty": "^0.1.6",
"consola": "^3.4.2",
"tokenx": "^1.2.0"
}
}

View File

@@ -1,12 +1,12 @@
import type { DecodeOptions, Delimiter, EncodeOptions } from '../../src' import type { DecodeOptions, Delimiter, EncodeOptions } from '../../toon/src'
import * as fsp from 'node:fs/promises' import * as fsp from 'node:fs/promises'
import * as path from 'node:path' import * as path from 'node:path'
import process from 'node:process' import process from 'node:process'
import { defineCommand, runMain } from 'citty' import { defineCommand, runMain } from 'citty'
import { consola } from 'consola' import { consola } from 'consola'
import { estimateTokenCount } from 'tokenx' import { estimateTokenCount } from 'tokenx'
import { name, version } from '../../package.json' with { type: 'json' } import { name, version } from '../../toon/package.json' with { type: 'json' }
import { decode, DEFAULT_DELIMITER, DELIMITERS, encode } from '../../src' import { decode, DEFAULT_DELIMITER, DELIMITERS, encode } from '../../toon/src'
const main = defineCommand({ const main = defineCommand({
meta: { meta: {

View File

@@ -2,10 +2,7 @@ import type { UserConfig, UserConfigFn } from 'tsdown/config'
import { defineConfig } from 'tsdown/config' import { defineConfig } from 'tsdown/config'
const config: UserConfig | UserConfigFn = defineConfig({ const config: UserConfig | UserConfigFn = defineConfig({
entry: { entry: 'src/index.ts',
'index': 'src/index.ts',
'cli/index': 'cli/src/index.ts',
},
dts: true, dts: true,
}) })

View File

@@ -0,0 +1,32 @@
{
"name": "@toon-format/toon",
"type": "module",
"version": "0.6.0",
"packageManager": "pnpm@10.19.0",
"description": "Token-Oriented Object Notation (TOON) a token-efficient JSON alternative for LLM prompts",
"author": "Johann Schopplich <hello@johannschopplich.com>",
"license": "MIT",
"homepage": "https://toonformat.dev",
"repository": {
"type": "git",
"url": "git+https://github.com/johannschopplich/toon.git"
},
"bugs": {
"url": "https://github.com/johannschopplich/toon/issues"
},
"sideEffects": false,
"exports": {
".": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
},
"types": "./dist/index.d.ts",
"files": [
"dist"
],
"scripts": {
"build": "tsdown",
"test": "vitest"
}
}

View File

@@ -0,0 +1,9 @@
import type { UserConfig, UserConfigFn } from 'tsdown/config'
import { defineConfig } from 'tsdown/config'
const config: UserConfig | UserConfigFn = defineConfig({
entry: 'src/index.ts',
dts: true,
})
export default config

4
pnpm-lock.yaml generated
View File

@@ -87,7 +87,7 @@ importers:
specifier: ^2.8.1 specifier: ^2.8.1
version: 2.8.1 version: 2.8.1
cli: packages/cli:
dependencies: dependencies:
citty: citty:
specifier: ^0.1.6 specifier: ^0.1.6
@@ -99,6 +99,8 @@ importers:
specifier: ^1.2.0 specifier: ^1.2.0
version: 1.2.0 version: 1.2.0
packages/toon: {}
packages: packages:
'@ai-sdk/anthropic@2.0.37': '@ai-sdk/anthropic@2.0.37':

View File

@@ -1,3 +1,7 @@
packages: packages:
- benchmarks - benchmarks
- cli - packages/*
onlyBuiltDependencies:
- '@parcel/watcher'
- esbuild