From 0710bd19e770efe7eb9d9c1696d42865092ee4b6 Mon Sep 17 00:00:00 2001 From: Johann Schopplich Date: Sat, 1 Nov 2025 16:53:41 +0100 Subject: [PATCH] feat!: publish to `@toon-format/toon` and `@toon-format/cli` --- .github/workflows/release.yml | 2 +- README.md | 34 +++++++-------- SPEC.md | 18 ++++---- cli/package.json | 13 ------ package.json | 41 +++---------------- {bin => packages/cli/bin}/toon.mjs | 2 +- packages/cli/package.json | 41 +++++++++++++++++++ {cli => packages/cli}/src/index.ts | 6 +-- .../cli/tsdown.config.ts | 5 +-- packages/toon/package.json | 32 +++++++++++++++ {src => packages/toon/src}/constants.ts | 0 {src => packages/toon/src}/decode/decoders.ts | 0 {src => packages/toon/src}/decode/parser.ts | 0 {src => packages/toon/src}/decode/scanner.ts | 0 .../toon/src}/decode/validation.ts | 0 {src => packages/toon/src}/encode/encoders.ts | 0 .../toon/src}/encode/normalize.ts | 0 .../toon/src}/encode/primitives.ts | 0 {src => packages/toon/src}/encode/writer.ts | 0 {src => packages/toon/src}/index.ts | 0 .../toon/src}/shared/literal-utils.ts | 0 .../toon/src}/shared/string-utils.ts | 0 .../toon/src}/shared/validation.ts | 0 {src => packages/toon/src}/types.ts | 0 {test => packages/toon/test}/decode.test.ts | 0 {test => packages/toon/test}/encode.test.ts | 0 packages/toon/tsdown.config.ts | 9 ++++ pnpm-lock.yaml | 4 +- pnpm-workspace.yaml | 6 ++- 29 files changed, 129 insertions(+), 84 deletions(-) delete mode 100644 cli/package.json rename {bin => packages/cli/bin}/toon.mjs (51%) mode change 100644 => 100755 create mode 100644 packages/cli/package.json rename {cli => packages/cli}/src/index.ts (98%) rename tsdown.config.ts => packages/cli/tsdown.config.ts (71%) create mode 100644 packages/toon/package.json rename {src => packages/toon/src}/constants.ts (100%) rename {src => packages/toon/src}/decode/decoders.ts (100%) rename {src => packages/toon/src}/decode/parser.ts (100%) rename {src => packages/toon/src}/decode/scanner.ts (100%) rename {src => packages/toon/src}/decode/validation.ts (100%) rename {src => packages/toon/src}/encode/encoders.ts (100%) rename {src => packages/toon/src}/encode/normalize.ts (100%) rename {src => packages/toon/src}/encode/primitives.ts (100%) rename {src => packages/toon/src}/encode/writer.ts (100%) rename {src => packages/toon/src}/index.ts (100%) rename {src => packages/toon/src}/shared/literal-utils.ts (100%) rename {src => packages/toon/src}/shared/string-utils.ts (100%) rename {src => packages/toon/src}/shared/validation.ts (100%) rename {src => packages/toon/src}/types.ts (100%) rename {test => packages/toon/test}/decode.test.ts (100%) rename {test => packages/toon/test}/encode.test.ts (100%) create mode 100644 packages/toon/tsdown.config.ts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b0c3835..928b807 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,6 +32,6 @@ jobs: - run: pnpm run build - name: Publish to npm - run: npm install -g npm@latest && npm publish --access public + run: npm install -g npm@latest && pnpm -r publish --access public --no-git-checks env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/README.md b/README.md index 8f68a17..6793c52 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ # Token-Oriented Object Notation (TOON) [![CI](https://github.com/johannschopplich/toon/actions/workflows/ci.yml/badge.svg)](https://github.com/johannschopplich/toon/actions) -[![npm version](https://img.shields.io/npm/v/@byjohann/toon.svg)](https://www.npmjs.com/package/@byjohann/toon) +[![npm version](https://img.shields.io/npm/v/@toon-format/toon.svg)](https://www.npmjs.com/package/@toon-format/toon) [![SPEC v1.3](https://img.shields.io/badge/spec-v1.3-lightgrey)](./SPEC.md) -[![npm downloads (total)](https://img.shields.io/npm/dt/@byjohann/toon.svg)](https://www.npmjs.com/package/@byjohann/toon) +[![npm downloads (total)](https://img.shields.io/npm/dt/@toon-format/toon.svg)](https://www.npmjs.com/package/@toon-format/toon) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) **Token-Oriented Object Notation** is a compact, human-readable format designed for passing structured data to Large Language Models with significantly reduced token usage. It's intended for LLM input, not output. @@ -439,19 +439,19 @@ Four datasets designed to test different structural patterns (all contain arrays ```bash # npm -npm install @byjohann/toon +npm install @toon-format/toon # pnpm -pnpm add @byjohann/toon +pnpm add @toon-format/toon # yarn -yarn add @byjohann/toon +yarn add @toon-format/toon ``` **Example usage:** ```ts -import { encode } from '@byjohann/toon' +import { encode } from '@toon-format/toon' const data = { users: [ @@ -473,20 +473,20 @@ Command-line tool for converting between JSON and TOON formats. ### Usage ```bash -npx @byjohann/toon [options] +npx @toon-format/cli [options] ``` **Auto-detection:** The CLI automatically detects the operation based on file extension (`.json` → encode, `.toon` → decode). ```bash # Encode JSON to TOON (auto-detected) -toon input.json -o output.toon +npx @toon-format/cli input.json -o output.toon # Decode TOON to JSON (auto-detected) -toon data.toon -o output.json +npx @toon-format/cli data.toon -o output.json # Output to stdout -toon input.json +npx @toon-format/cli input.json ``` ### Options @@ -506,16 +506,16 @@ toon input.json ```bash # Show token savings when encoding -toon data.json --stats -o output.toon +npx @toon-format/cli data.json --stats -o output.toon # Tab-separated output (often more token-efficient) -toon data.json --delimiter "\t" -o output.toon +npx @toon-format/cli data.json --delimiter "\t" -o output.toon # Pipe-separated with length markers -toon data.json --delimiter "|" --length-marker -o output.toon +npx @toon-format/cli data.json --delimiter "|" --length-marker -o output.toon # Lenient decoding (skip validation) -toon data.toon --no-strict -o output.json +npx @toon-format/cli data.toon --no-strict -o output.json ``` ## Format Overview @@ -741,7 +741,7 @@ A TOON-formatted string with no trailing newline or spaces. **Example:** ```ts -import { encode } from '@byjohann/toon' +import { encode } from '@toon-format/toon' const items = [ { sku: 'A1', qty: 2, price: 9.99 }, @@ -862,7 +862,7 @@ A JavaScript value (object, array, or primitive) representing the parsed TOON da **Example:** ```ts -import { decode } from '@byjohann/toon' +import { decode } from '@toon-format/toon' const toon = ` items[2]{sku,qty,price}: @@ -980,7 +980,7 @@ Task: Return only users with role "user" as TOON. Use the same header. Set [N] t ## Other Implementations > [!NOTE] -> When implementing TOON in other languages, please follow the [SPEC.md](./SPEC.md) (currently v1.3) to ensure compatibility across implementations. The [TypeScript test suite](./test) provides comprehensive examples of encoding and decoding behavior that can serve as a reference implementation. +> When implementing TOON in other languages, please follow the [SPEC.md](./SPEC.md) (currently v1.3) to ensure compatibility across implementations. The [TypeScript test suite](./packages/toon/test) provides comprehensive examples of encoding and decoding behavior that can serve as a reference implementation. - **.NET:** [ToonSharp](https://github.com/0xZunia/ToonSharp) - **Crystal:** [toon-crystal](https://github.com/mamantoha/toon-crystal) diff --git a/SPEC.md b/SPEC.md index 51f181f..e44a40e 100644 --- a/SPEC.md +++ b/SPEC.md @@ -3,9 +3,13 @@ ## Token-Oriented Object Notation **Version:** 1.3 + **Date:** 2025-10-31 + **Status:** Working Draft + **Author:** Johann Schopplich ([@johannschopplich](https://github.com/johannschopplich)) + **License:** MIT --- @@ -137,9 +141,9 @@ This specification defines: ### Type Terms - Primitive: string, number, boolean, or null. -- Object: Mapping from string keys to JsonValue. -- Array: Ordered sequence of JsonValue. -- JsonValue: Primitive | Object | Array. +- Object: Mapping from string keys to `JsonValue`. +- Array: Ordered sequence of `JsonValue`. +- `JsonValue`: Primitive | Object | Array. ### Conformance Terms @@ -153,9 +157,9 @@ This specification defines: ## 2. Data Model - TOON models data as: - - JsonPrimitive: string | number | boolean | null - - JsonObject: { [string]: JsonValue } - - JsonArray: JsonValue[] + - `JsonPrimitive`: string | number | boolean | null + - `JsonObject`: { [string]: `JsonValue` } + - `JsonArray`: `JsonValue`[] - Ordering: - Array order MUST be preserved. - Object key order MUST be preserved as encountered by the encoder. @@ -959,7 +963,7 @@ These sketches illustrate structure and common decoding helpers. They are inform ### Reference Test Suite A reference test suite is maintained at: -https://github.com/johannschopplich/toon/tree/main/test +https://github.com/johannschopplich/toon/tree/main/packages/toon/test The test suite is versioned alongside this specification. Implementations are encouraged to validate against this test suite, but conformance is determined solely by adherence to the normative requirements in Sections 1-16 and Section 19 of this specification. Test coverage does not define the specification; the specification defines conformance. diff --git a/cli/package.json b/cli/package.json deleted file mode 100644 index 8526f51..0000000 --- a/cli/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "@toon/cli", - "type": "module", - "private": true, - "scripts": { - "dev": "tsx ./src/index.ts" - }, - "dependencies": { - "citty": "^0.1.6", - "consola": "^3.4.2", - "tokenx": "^1.2.0" - } -} diff --git a/package.json b/package.json index ffa4aaf..3987350 100644 --- a/package.json +++ b/package.json @@ -1,42 +1,17 @@ { - "name": "@byjohann/toon", + "name": "@toon-format/monorepo", "type": "module", "version": "0.6.0", + "private": true, "packageManager": "pnpm@10.19.0", - "description": "Token-Oriented Object Notation – a token-efficient JSON alternative for LLM prompts", - "author": "Johann Schopplich ", - "license": "MIT", - "homepage": "https://github.com/johannschopplich/toon#readme", - "repository": { - "type": "git", - "url": "git+https://github.com/johannschopplich/toon.git" - }, - "bugs": { - "url": "https://github.com/johannschopplich/toon/issues" - }, - "sideEffects": false, - "exports": { - ".": { - "types": "./dist/index.d.ts", - "default": "./dist/index.js" - } - }, - "types": "./dist/index.d.ts", - "bin": { - "toon": "bin/toon.mjs" - }, - "files": [ - "bin", - "dist" - ], "scripts": { + "build": "pnpm -r --filter=./packages/** run build", "automd": "automd", - "build": "tsdown", "lint": "eslint .", "lint:fix": "eslint . --fix", - "test": "vitest", + "test": "pnpm -r test", "test:types": "tsc --noEmit", - "release": "bumpp" + "release": "bumpp -r" }, "devDependencies": { "@antfu/eslint-config": "^6.1.0", @@ -48,11 +23,5 @@ "tsx": "^4.20.6", "typescript": "^5.9.3", "vitest": "^4.0.3" - }, - "pnpm": { - "onlyBuiltDependencies": [ - "@parcel/watcher", - "esbuild" - ] } } diff --git a/bin/toon.mjs b/packages/cli/bin/toon.mjs old mode 100644 new mode 100755 similarity index 51% rename from bin/toon.mjs rename to packages/cli/bin/toon.mjs index 37a89e8..a714f0c --- a/bin/toon.mjs +++ b/packages/cli/bin/toon.mjs @@ -1,3 +1,3 @@ #!/usr/bin/env node 'use strict' -import('../dist/cli/index.js') +import('../dist/index.js') diff --git a/packages/cli/package.json b/packages/cli/package.json new file mode 100644 index 0000000..130e802 --- /dev/null +++ b/packages/cli/package.json @@ -0,0 +1,41 @@ +{ + "name": "@toon-format/cli", + "type": "module", + "version": "0.6.0", + "packageManager": "pnpm@10.19.0", + "description": "CLI for encoding and decoding Token-Oriented Object Notation (TOON)", + "author": "Johann Schopplich ", + "license": "MIT", + "homepage": "https://toonformat.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/johannschopplich/toon.git" + }, + "bugs": { + "url": "https://github.com/johannschopplich/toon/issues" + }, + "sideEffects": false, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "types": "./dist/index.d.ts", + "bin": { + "toon": "bin/toon.mjs" + }, + "files": [ + "bin", + "dist" + ], + "scripts": { + "dev": "tsx ./src/index.ts", + "build": "tsdown" + }, + "dependencies": { + "citty": "^0.1.6", + "consola": "^3.4.2", + "tokenx": "^1.2.0" + } +} diff --git a/cli/src/index.ts b/packages/cli/src/index.ts similarity index 98% rename from cli/src/index.ts rename to packages/cli/src/index.ts index 4bbbf88..c791d7d 100644 --- a/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -1,12 +1,12 @@ -import type { DecodeOptions, Delimiter, EncodeOptions } from '../../src' +import type { DecodeOptions, Delimiter, EncodeOptions } from '../../toon/src' import * as fsp from 'node:fs/promises' import * as path from 'node:path' import process from 'node:process' import { defineCommand, runMain } from 'citty' import { consola } from 'consola' import { estimateTokenCount } from 'tokenx' -import { name, version } from '../../package.json' with { type: 'json' } -import { decode, DEFAULT_DELIMITER, DELIMITERS, encode } from '../../src' +import { name, version } from '../../toon/package.json' with { type: 'json' } +import { decode, DEFAULT_DELIMITER, DELIMITERS, encode } from '../../toon/src' const main = defineCommand({ meta: { diff --git a/tsdown.config.ts b/packages/cli/tsdown.config.ts similarity index 71% rename from tsdown.config.ts rename to packages/cli/tsdown.config.ts index f661973..b817157 100644 --- a/tsdown.config.ts +++ b/packages/cli/tsdown.config.ts @@ -2,10 +2,7 @@ import type { UserConfig, UserConfigFn } from 'tsdown/config' import { defineConfig } from 'tsdown/config' const config: UserConfig | UserConfigFn = defineConfig({ - entry: { - 'index': 'src/index.ts', - 'cli/index': 'cli/src/index.ts', - }, + entry: 'src/index.ts', dts: true, }) diff --git a/packages/toon/package.json b/packages/toon/package.json new file mode 100644 index 0000000..2a650c3 --- /dev/null +++ b/packages/toon/package.json @@ -0,0 +1,32 @@ +{ + "name": "@toon-format/toon", + "type": "module", + "version": "0.6.0", + "packageManager": "pnpm@10.19.0", + "description": "Token-Oriented Object Notation (TOON) – a token-efficient JSON alternative for LLM prompts", + "author": "Johann Schopplich ", + "license": "MIT", + "homepage": "https://toonformat.dev", + "repository": { + "type": "git", + "url": "git+https://github.com/johannschopplich/toon.git" + }, + "bugs": { + "url": "https://github.com/johannschopplich/toon/issues" + }, + "sideEffects": false, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "types": "./dist/index.d.ts", + "files": [ + "dist" + ], + "scripts": { + "build": "tsdown", + "test": "vitest" + } +} diff --git a/src/constants.ts b/packages/toon/src/constants.ts similarity index 100% rename from src/constants.ts rename to packages/toon/src/constants.ts diff --git a/src/decode/decoders.ts b/packages/toon/src/decode/decoders.ts similarity index 100% rename from src/decode/decoders.ts rename to packages/toon/src/decode/decoders.ts diff --git a/src/decode/parser.ts b/packages/toon/src/decode/parser.ts similarity index 100% rename from src/decode/parser.ts rename to packages/toon/src/decode/parser.ts diff --git a/src/decode/scanner.ts b/packages/toon/src/decode/scanner.ts similarity index 100% rename from src/decode/scanner.ts rename to packages/toon/src/decode/scanner.ts diff --git a/src/decode/validation.ts b/packages/toon/src/decode/validation.ts similarity index 100% rename from src/decode/validation.ts rename to packages/toon/src/decode/validation.ts diff --git a/src/encode/encoders.ts b/packages/toon/src/encode/encoders.ts similarity index 100% rename from src/encode/encoders.ts rename to packages/toon/src/encode/encoders.ts diff --git a/src/encode/normalize.ts b/packages/toon/src/encode/normalize.ts similarity index 100% rename from src/encode/normalize.ts rename to packages/toon/src/encode/normalize.ts diff --git a/src/encode/primitives.ts b/packages/toon/src/encode/primitives.ts similarity index 100% rename from src/encode/primitives.ts rename to packages/toon/src/encode/primitives.ts diff --git a/src/encode/writer.ts b/packages/toon/src/encode/writer.ts similarity index 100% rename from src/encode/writer.ts rename to packages/toon/src/encode/writer.ts diff --git a/src/index.ts b/packages/toon/src/index.ts similarity index 100% rename from src/index.ts rename to packages/toon/src/index.ts diff --git a/src/shared/literal-utils.ts b/packages/toon/src/shared/literal-utils.ts similarity index 100% rename from src/shared/literal-utils.ts rename to packages/toon/src/shared/literal-utils.ts diff --git a/src/shared/string-utils.ts b/packages/toon/src/shared/string-utils.ts similarity index 100% rename from src/shared/string-utils.ts rename to packages/toon/src/shared/string-utils.ts diff --git a/src/shared/validation.ts b/packages/toon/src/shared/validation.ts similarity index 100% rename from src/shared/validation.ts rename to packages/toon/src/shared/validation.ts diff --git a/src/types.ts b/packages/toon/src/types.ts similarity index 100% rename from src/types.ts rename to packages/toon/src/types.ts diff --git a/test/decode.test.ts b/packages/toon/test/decode.test.ts similarity index 100% rename from test/decode.test.ts rename to packages/toon/test/decode.test.ts diff --git a/test/encode.test.ts b/packages/toon/test/encode.test.ts similarity index 100% rename from test/encode.test.ts rename to packages/toon/test/encode.test.ts diff --git a/packages/toon/tsdown.config.ts b/packages/toon/tsdown.config.ts new file mode 100644 index 0000000..b817157 --- /dev/null +++ b/packages/toon/tsdown.config.ts @@ -0,0 +1,9 @@ +import type { UserConfig, UserConfigFn } from 'tsdown/config' +import { defineConfig } from 'tsdown/config' + +const config: UserConfig | UserConfigFn = defineConfig({ + entry: 'src/index.ts', + dts: true, +}) + +export default config diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 249cc55..d986688 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -87,7 +87,7 @@ importers: specifier: ^2.8.1 version: 2.8.1 - cli: + packages/cli: dependencies: citty: specifier: ^0.1.6 @@ -99,6 +99,8 @@ importers: specifier: ^1.2.0 version: 1.2.0 + packages/toon: {} + packages: '@ai-sdk/anthropic@2.0.37': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 4a20a92..d8c4b2e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,3 +1,7 @@ packages: - benchmarks - - cli + - packages/* + +onlyBuiltDependencies: + - '@parcel/watcher' + - esbuild