feat!: publish to @toon-format/toon and @toon-format/cli

This commit is contained in:
Johann Schopplich
2025-11-01 16:53:41 +01:00
parent 8bcbdb7315
commit 0710bd19e7
29 changed files with 129 additions and 84 deletions

View File

@@ -32,6 +32,6 @@ jobs:
- run: pnpm run build
- name: Publish to npm
run: npm install -g npm@latest && npm publish --access public
run: npm install -g npm@latest && pnpm -r publish --access public --no-git-checks
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

View File

@@ -3,9 +3,9 @@
# Token-Oriented Object Notation (TOON)
[![CI](https://github.com/johannschopplich/toon/actions/workflows/ci.yml/badge.svg)](https://github.com/johannschopplich/toon/actions)
[![npm version](https://img.shields.io/npm/v/@byjohann/toon.svg)](https://www.npmjs.com/package/@byjohann/toon)
[![npm version](https://img.shields.io/npm/v/@toon-format/toon.svg)](https://www.npmjs.com/package/@toon-format/toon)
[![SPEC v1.3](https://img.shields.io/badge/spec-v1.3-lightgrey)](./SPEC.md)
[![npm downloads (total)](https://img.shields.io/npm/dt/@byjohann/toon.svg)](https://www.npmjs.com/package/@byjohann/toon)
[![npm downloads (total)](https://img.shields.io/npm/dt/@toon-format/toon.svg)](https://www.npmjs.com/package/@toon-format/toon)
[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE)
**Token-Oriented Object Notation** is a compact, human-readable format designed for passing structured data to Large Language Models with significantly reduced token usage. It's intended for LLM input, not output.
@@ -439,19 +439,19 @@ Four datasets designed to test different structural patterns (all contain arrays
```bash
# npm
npm install @byjohann/toon
npm install @toon-format/toon
# pnpm
pnpm add @byjohann/toon
pnpm add @toon-format/toon
# yarn
yarn add @byjohann/toon
yarn add @toon-format/toon
```
**Example usage:**
```ts
import { encode } from '@byjohann/toon'
import { encode } from '@toon-format/toon'
const data = {
users: [
@@ -473,20 +473,20 @@ Command-line tool for converting between JSON and TOON formats.
### Usage
```bash
npx @byjohann/toon <input> [options]
npx @toon-format/cli <input> [options]
```
**Auto-detection:** The CLI automatically detects the operation based on file extension (`.json` → encode, `.toon` → decode).
```bash
# Encode JSON to TOON (auto-detected)
toon input.json -o output.toon
npx @toon-format/cli input.json -o output.toon
# Decode TOON to JSON (auto-detected)
toon data.toon -o output.json
npx @toon-format/cli data.toon -o output.json
# Output to stdout
toon input.json
npx @toon-format/cli input.json
```
### Options
@@ -506,16 +506,16 @@ toon input.json
```bash
# Show token savings when encoding
toon data.json --stats -o output.toon
npx @toon-format/cli data.json --stats -o output.toon
# Tab-separated output (often more token-efficient)
toon data.json --delimiter "\t" -o output.toon
npx @toon-format/cli data.json --delimiter "\t" -o output.toon
# Pipe-separated with length markers
toon data.json --delimiter "|" --length-marker -o output.toon
npx @toon-format/cli data.json --delimiter "|" --length-marker -o output.toon
# Lenient decoding (skip validation)
toon data.toon --no-strict -o output.json
npx @toon-format/cli data.toon --no-strict -o output.json
```
## Format Overview
@@ -741,7 +741,7 @@ A TOON-formatted string with no trailing newline or spaces.
**Example:**
```ts
import { encode } from '@byjohann/toon'
import { encode } from '@toon-format/toon'
const items = [
{ sku: 'A1', qty: 2, price: 9.99 },
@@ -862,7 +862,7 @@ A JavaScript value (object, array, or primitive) representing the parsed TOON da
**Example:**
```ts
import { decode } from '@byjohann/toon'
import { decode } from '@toon-format/toon'
const toon = `
items[2]{sku,qty,price}:
@@ -980,7 +980,7 @@ Task: Return only users with role "user" as TOON. Use the same header. Set [N] t
## Other Implementations
> [!NOTE]
> When implementing TOON in other languages, please follow the [SPEC.md](./SPEC.md) (currently v1.3) to ensure compatibility across implementations. The [TypeScript test suite](./test) provides comprehensive examples of encoding and decoding behavior that can serve as a reference implementation.
> When implementing TOON in other languages, please follow the [SPEC.md](./SPEC.md) (currently v1.3) to ensure compatibility across implementations. The [TypeScript test suite](./packages/toon/test) provides comprehensive examples of encoding and decoding behavior that can serve as a reference implementation.
- **.NET:** [ToonSharp](https://github.com/0xZunia/ToonSharp)
- **Crystal:** [toon-crystal](https://github.com/mamantoha/toon-crystal)

18
SPEC.md
View File

@@ -3,9 +3,13 @@
## Token-Oriented Object Notation
**Version:** 1.3
**Date:** 2025-10-31
**Status:** Working Draft
**Author:** Johann Schopplich ([@johannschopplich](https://github.com/johannschopplich))
**License:** MIT
---
@@ -137,9 +141,9 @@ This specification defines:
### Type Terms
- Primitive: string, number, boolean, or null.
- Object: Mapping from string keys to JsonValue.
- Array: Ordered sequence of JsonValue.
- JsonValue: Primitive | Object | Array.
- Object: Mapping from string keys to `JsonValue`.
- Array: Ordered sequence of `JsonValue`.
- `JsonValue`: Primitive | Object | Array.
### Conformance Terms
@@ -153,9 +157,9 @@ This specification defines:
## 2. Data Model
- TOON models data as:
- JsonPrimitive: string | number | boolean | null
- JsonObject: { [string]: JsonValue }
- JsonArray: JsonValue[]
- `JsonPrimitive`: string | number | boolean | null
- `JsonObject`: { [string]: `JsonValue` }
- `JsonArray`: `JsonValue`[]
- Ordering:
- Array order MUST be preserved.
- Object key order MUST be preserved as encountered by the encoder.
@@ -959,7 +963,7 @@ These sketches illustrate structure and common decoding helpers. They are inform
### Reference Test Suite
A reference test suite is maintained at:
https://github.com/johannschopplich/toon/tree/main/test
https://github.com/johannschopplich/toon/tree/main/packages/toon/test
The test suite is versioned alongside this specification. Implementations are encouraged to validate against this test suite, but conformance is determined solely by adherence to the normative requirements in Sections 1-16 and Section 19 of this specification. Test coverage does not define the specification; the specification defines conformance.

View File

@@ -1,13 +0,0 @@
{
"name": "@toon/cli",
"type": "module",
"private": true,
"scripts": {
"dev": "tsx ./src/index.ts"
},
"dependencies": {
"citty": "^0.1.6",
"consola": "^3.4.2",
"tokenx": "^1.2.0"
}
}

View File

@@ -1,42 +1,17 @@
{
"name": "@byjohann/toon",
"name": "@toon-format/monorepo",
"type": "module",
"version": "0.6.0",
"private": true,
"packageManager": "pnpm@10.19.0",
"description": "Token-Oriented Object Notation a token-efficient JSON alternative for LLM prompts",
"author": "Johann Schopplich <hello@johannschopplich.com>",
"license": "MIT",
"homepage": "https://github.com/johannschopplich/toon#readme",
"repository": {
"type": "git",
"url": "git+https://github.com/johannschopplich/toon.git"
},
"bugs": {
"url": "https://github.com/johannschopplich/toon/issues"
},
"sideEffects": false,
"exports": {
".": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
},
"types": "./dist/index.d.ts",
"bin": {
"toon": "bin/toon.mjs"
},
"files": [
"bin",
"dist"
],
"scripts": {
"build": "pnpm -r --filter=./packages/** run build",
"automd": "automd",
"build": "tsdown",
"lint": "eslint .",
"lint:fix": "eslint . --fix",
"test": "vitest",
"test": "pnpm -r test",
"test:types": "tsc --noEmit",
"release": "bumpp"
"release": "bumpp -r"
},
"devDependencies": {
"@antfu/eslint-config": "^6.1.0",
@@ -48,11 +23,5 @@
"tsx": "^4.20.6",
"typescript": "^5.9.3",
"vitest": "^4.0.3"
},
"pnpm": {
"onlyBuiltDependencies": [
"@parcel/watcher",
"esbuild"
]
}
}

2
bin/toon.mjs → packages/cli/bin/toon.mjs Normal file → Executable file
View File

@@ -1,3 +1,3 @@
#!/usr/bin/env node
'use strict'
import('../dist/cli/index.js')
import('../dist/index.js')

41
packages/cli/package.json Normal file
View File

@@ -0,0 +1,41 @@
{
"name": "@toon-format/cli",
"type": "module",
"version": "0.6.0",
"packageManager": "pnpm@10.19.0",
"description": "CLI for encoding and decoding Token-Oriented Object Notation (TOON)",
"author": "Johann Schopplich <hello@johannschopplich.com>",
"license": "MIT",
"homepage": "https://toonformat.dev",
"repository": {
"type": "git",
"url": "git+https://github.com/johannschopplich/toon.git"
},
"bugs": {
"url": "https://github.com/johannschopplich/toon/issues"
},
"sideEffects": false,
"exports": {
".": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
},
"types": "./dist/index.d.ts",
"bin": {
"toon": "bin/toon.mjs"
},
"files": [
"bin",
"dist"
],
"scripts": {
"dev": "tsx ./src/index.ts",
"build": "tsdown"
},
"dependencies": {
"citty": "^0.1.6",
"consola": "^3.4.2",
"tokenx": "^1.2.0"
}
}

View File

@@ -1,12 +1,12 @@
import type { DecodeOptions, Delimiter, EncodeOptions } from '../../src'
import type { DecodeOptions, Delimiter, EncodeOptions } from '../../toon/src'
import * as fsp from 'node:fs/promises'
import * as path from 'node:path'
import process from 'node:process'
import { defineCommand, runMain } from 'citty'
import { consola } from 'consola'
import { estimateTokenCount } from 'tokenx'
import { name, version } from '../../package.json' with { type: 'json' }
import { decode, DEFAULT_DELIMITER, DELIMITERS, encode } from '../../src'
import { name, version } from '../../toon/package.json' with { type: 'json' }
import { decode, DEFAULT_DELIMITER, DELIMITERS, encode } from '../../toon/src'
const main = defineCommand({
meta: {

View File

@@ -2,10 +2,7 @@ import type { UserConfig, UserConfigFn } from 'tsdown/config'
import { defineConfig } from 'tsdown/config'
const config: UserConfig | UserConfigFn = defineConfig({
entry: {
'index': 'src/index.ts',
'cli/index': 'cli/src/index.ts',
},
entry: 'src/index.ts',
dts: true,
})

View File

@@ -0,0 +1,32 @@
{
"name": "@toon-format/toon",
"type": "module",
"version": "0.6.0",
"packageManager": "pnpm@10.19.0",
"description": "Token-Oriented Object Notation (TOON) a token-efficient JSON alternative for LLM prompts",
"author": "Johann Schopplich <hello@johannschopplich.com>",
"license": "MIT",
"homepage": "https://toonformat.dev",
"repository": {
"type": "git",
"url": "git+https://github.com/johannschopplich/toon.git"
},
"bugs": {
"url": "https://github.com/johannschopplich/toon/issues"
},
"sideEffects": false,
"exports": {
".": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
}
},
"types": "./dist/index.d.ts",
"files": [
"dist"
],
"scripts": {
"build": "tsdown",
"test": "vitest"
}
}

View File

@@ -0,0 +1,9 @@
import type { UserConfig, UserConfigFn } from 'tsdown/config'
import { defineConfig } from 'tsdown/config'
const config: UserConfig | UserConfigFn = defineConfig({
entry: 'src/index.ts',
dts: true,
})
export default config

4
pnpm-lock.yaml generated
View File

@@ -87,7 +87,7 @@ importers:
specifier: ^2.8.1
version: 2.8.1
cli:
packages/cli:
dependencies:
citty:
specifier: ^0.1.6
@@ -99,6 +99,8 @@ importers:
specifier: ^1.2.0
version: 1.2.0
packages/toon: {}
packages:
'@ai-sdk/anthropic@2.0.37':

View File

@@ -1,3 +1,7 @@
packages:
- benchmarks
- cli
- packages/*
onlyBuiltDependencies:
- '@parcel/watcher'
- esbuild