feat!: remove optional length marker option [#N] in favor of [N]

packages/cli/README.md

@@ -62,7 +62,6 @@ cat data.toon | toon --decode
 | `-d, --decode` | Force decode mode (overrides auto-detection) |
 | `--delimiter <char>` | Array delimiter: `,` (comma), `\t` (tab), `\|` (pipe) |
 | `--indent <number>` | Indentation size (default: `2`) |
-| `--length-marker` | Add `#` prefix to array lengths (e.g., `items[#3]`) |
 | `--stats` | Show token count estimates and savings (encode only) |
 | `--no-strict` | Disable strict validation when decoding |
 | `--key-folding <mode>` | Enable key folding: `off`, `safe` (default: `off`) |
@@ -122,7 +121,7 @@ cat large-dataset.json | toon --delimiter "\t" > output.toon
 jq '.results' data.json | toon > filtered.toon
 ```
 
-### Key Folding (spec v1.5)
+### Key Folding (Since v1.5)
 
 Collapse nested wrapper chains to reduce tokens:
 

packages/cli/src/conversion.ts

@@ -13,7 +13,6 @@ export async function encodeToToon(config: {
   output?: string
   indent: NonNullable<EncodeOptions['indent']>
   delimiter: NonNullable<EncodeOptions['delimiter']>
-  lengthMarker: NonNullable<EncodeOptions['lengthMarker']>
   keyFolding?: NonNullable<EncodeOptions['keyFolding']>
   flattenDepth?: number
   printStats: boolean
@@ -31,7 +30,6 @@ export async function encodeToToon(config: {
   const encodeOptions: EncodeOptions = {
     delimiter: config.delimiter,
     indent: config.indent,
-    lengthMarker: config.lengthMarker,
     keyFolding: config.keyFolding,
     flattenDepth: config.flattenDepth,
   }

packages/cli/src/index.ts

@@ -41,11 +41,6 @@ export const mainCommand: CommandDef<{
     description: string
     default: string
   }
-  lengthMarker: {
-    type: 'boolean'
-    description: string
-    default: false
-  }
   strict: {
     type: 'boolean'
     description: string
@@ -107,11 +102,6 @@ export const mainCommand: CommandDef<{
       description: 'Indentation size',
       default: '2',
     },
-    lengthMarker: {
-      type: 'boolean',
-      description: 'Use length marker (#) for arrays',
-      default: false,
-    },
     strict: {
       type: 'boolean',
       description: 'Enable strict mode for decoding',
@@ -187,10 +177,9 @@ export const mainCommand: CommandDef<{
           output: outputPath,
           delimiter: delimiter as Delimiter,
           indent,
-          lengthMarker: args.lengthMarker === true ? '#' : false,
-          printStats: args.stats === true,
           keyFolding: keyFolding as NonNullable<EncodeOptions['keyFolding']>,
           flattenDepth,
+          printStats: args.stats === true,
         })
       }
       else {

packages/cli/test/index.test.ts

@@ -44,7 +44,6 @@ describe('toon CLI', () => {
         const expected = encode(data, {
           delimiter: DEFAULT_DELIMITER,
           indent: 2,
-          lengthMarker: false,
         })
 
         expect(output).toBe(expected)

packages/toon/package.json

@@ -38,6 +38,6 @@
     "test": "vitest"
   },
   "devDependencies": {
-    "@toon-format/spec": "^1.5.2"
+    "@toon-format/spec": "^2.0.0"
   }
 }

packages/toon/src/constants.ts

@@ -11,7 +11,6 @@ export const COMMA = ','
 export const COLON = ':'
 export const SPACE = ' '
 export const PIPE = '|'
-export const HASH = '#'
 export const DOT = '.'
 
 // #endregion

packages/toon/src/decode/parser.ts

@@ -1,5 +1,5 @@
 import type { ArrayHeaderInfo, Delimiter, JsonPrimitive } from '../types'
-import { BACKSLASH, CLOSE_BRACE, CLOSE_BRACKET, COLON, DELIMITERS, DOUBLE_QUOTE, FALSE_LITERAL, HASH, NULL_LITERAL, OPEN_BRACE, OPEN_BRACKET, PIPE, TAB, TRUE_LITERAL } from '../constants'
+import { BACKSLASH, CLOSE_BRACE, CLOSE_BRACKET, COLON, DELIMITERS, DOUBLE_QUOTE, FALSE_LITERAL, NULL_LITERAL, OPEN_BRACE, OPEN_BRACKET, PIPE, TAB, TRUE_LITERAL } from '../constants'
 import { isBooleanOrNullLiteral, isNumericLiteral } from '../shared/literal-utils'
 import { findClosingQuote, findUnquotedChar, unescapeString } from '../shared/string-utils'
 
@@ -84,7 +84,7 @@ export function parseArrayHeaderLine(
     return
   }
 
-  const { length, delimiter, hasLengthMarker } = parsedBracket
+  const { length, delimiter } = parsedBracket
 
   // Check for fields segment
   let fields: string[] | undefined
@@ -102,7 +102,6 @@ export function parseArrayHeaderLine(
       length,
       delimiter,
       fields,
-      hasLengthMarker,
     },
     inlineValues: afterColon || undefined,
   }
@@ -111,16 +110,9 @@ export function parseArrayHeaderLine(
 export function parseBracketSegment(
   seg: string,
   defaultDelimiter: Delimiter,
-): { length: number, delimiter: Delimiter, hasLengthMarker: boolean } {
-  let hasLengthMarker = false
+): { length: number, delimiter: Delimiter } {
   let content = seg
 
-  // Check for length marker
-  if (content.startsWith(HASH)) {
-    hasLengthMarker = true
-    content = content.slice(1)
-  }
-
   // Check for delimiter suffix
   let delimiter = defaultDelimiter
   if (content.endsWith(TAB)) {
@@ -137,7 +129,7 @@ export function parseBracketSegment(
     throw new TypeError(`Invalid array length: ${seg}`)
   }
 
-  return { length, delimiter, hasLengthMarker }
+  return { length, delimiter }
 }
 
 // #endregion

packages/toon/src/encode/encoders.ts

@@ -113,15 +113,15 @@ export function encodeArray(
   options: ResolvedEncodeOptions,
 ): void {
   if (value.length === 0) {
-    const header = formatHeader(0, { key, delimiter: options.delimiter, lengthMarker: options.lengthMarker })
+    const header = formatHeader(0, { key, delimiter: options.delimiter })
     writer.push(depth, header)
     return
   }
 
   // Primitive array
   if (isArrayOfPrimitives(value)) {
-    const formatted = encodeInlineArrayLine(value, options.delimiter, key, options.lengthMarker)
-    writer.push(depth, formatted)
+    const arrayLine = encodeInlineArrayLine(value, options.delimiter, key)
+    writer.push(depth, arrayLine)
     return
   }
 
@@ -161,19 +161,19 @@ export function encodeArrayOfArraysAsListItems(
   depth: Depth,
   options: ResolvedEncodeOptions,
 ): void {
-  const header = formatHeader(values.length, { key: prefix, delimiter: options.delimiter, lengthMarker: options.lengthMarker })
+  const header = formatHeader(values.length, { key: prefix, delimiter: options.delimiter })
   writer.push(depth, header)
 
   for (const arr of values) {
     if (isArrayOfPrimitives(arr)) {
-      const inline = encodeInlineArrayLine(arr, options.delimiter, undefined, options.lengthMarker)
-      writer.pushListItem(depth + 1, inline)
+      const arrayLine = encodeInlineArrayLine(arr, options.delimiter)
+      writer.pushListItem(depth + 1, arrayLine)
     }
   }
 }
 
-export function encodeInlineArrayLine(values: readonly JsonPrimitive[], delimiter: string, prefix?: string, lengthMarker?: '#' | false): string {
-  const header = formatHeader(values.length, { key: prefix, delimiter, lengthMarker })
+export function encodeInlineArrayLine(values: readonly JsonPrimitive[], delimiter: string, prefix?: string): string {
+  const header = formatHeader(values.length, { key: prefix, delimiter })
   const joinedValue = encodeAndJoinPrimitives(values, delimiter)
   // Only add space if there are values
   if (values.length === 0) {
@@ -194,7 +194,7 @@ export function encodeArrayOfObjectsAsTabular(
   depth: Depth,
   options: ResolvedEncodeOptions,
 ): void {
-  const formattedHeader = formatHeader(rows.length, { key: prefix, fields: header, delimiter: options.delimiter, lengthMarker: options.lengthMarker })
+  const formattedHeader = formatHeader(rows.length, { key: prefix, fields: header, delimiter: options.delimiter })
   writer.push(depth, `${formattedHeader}`)
 
   writeTabularRows(rows, header, writer, depth + 1, options)
@@ -265,7 +265,7 @@ export function encodeMixedArrayAsListItems(
   depth: Depth,
   options: ResolvedEncodeOptions,
 ): void {
-  const header = formatHeader(items.length, { key: prefix, delimiter: options.delimiter, lengthMarker: options.lengthMarker })
+  const header = formatHeader(items.length, { key: prefix, delimiter: options.delimiter })
   writer.push(depth, header)
 
   for (const item of items) {
@@ -289,15 +289,15 @@ export function encodeObjectAsListItem(obj: JsonObject, writer: LineWriter, dept
   else if (isJsonArray(firstValue)) {
     if (isArrayOfPrimitives(firstValue)) {
       // Inline format for primitive arrays
-      const formatted = encodeInlineArrayLine(firstValue, options.delimiter, firstKey, options.lengthMarker)
-      writer.pushListItem(depth, formatted)
+      const arrayPropertyLine = encodeInlineArrayLine(firstValue, options.delimiter, firstKey)
+      writer.pushListItem(depth, arrayPropertyLine)
     }
     else if (isArrayOfObjects(firstValue)) {
       // Check if array of objects can use tabular format
       const header = extractTabularHeader(firstValue)
       if (header) {
         // Tabular format for uniform arrays of objects
-        const formattedHeader = formatHeader(firstValue.length, { key: firstKey, fields: header, delimiter: options.delimiter, lengthMarker: options.lengthMarker })
+        const formattedHeader = formatHeader(firstValue.length, { key: firstKey, fields: header, delimiter: options.delimiter })
         writer.pushListItem(depth, formattedHeader)
         writeTabularRows(firstValue, header, writer, depth + 1, options)
       }
@@ -347,8 +347,8 @@ function encodeListItemValue(
     writer.pushListItem(depth, encodePrimitive(value, options.delimiter))
   }
   else if (isJsonArray(value) && isArrayOfPrimitives(value)) {
-    const inline = encodeInlineArrayLine(value, options.delimiter, undefined, options.lengthMarker)
-    writer.pushListItem(depth, inline)
+    const arrayLine = encodeInlineArrayLine(value, options.delimiter)
+    writer.pushListItem(depth, arrayLine)
   }
   else if (isJsonObject(value)) {
     encodeObjectAsListItem(value, writer, depth, options)

packages/toon/src/encode/primitives.ts

@@ -59,13 +59,11 @@ export function formatHeader(
     key?: string
     fields?: readonly string[]
     delimiter?: string
-    lengthMarker?: '#' | false
   },
 ): string {
   const key = options?.key
   const fields = options?.fields
   const delimiter = options?.delimiter ?? COMMA
-  const lengthMarker = options?.lengthMarker ?? false
 
   let header = ''
 
@@ -74,7 +72,7 @@ export function formatHeader(
   }
 
   // Only include delimiter if it's not the default (comma)
-  header += `[${lengthMarker || ''}${length}${delimiter !== DEFAULT_DELIMITER ? delimiter : ''}]`
+  header += `[${length}${delimiter !== DEFAULT_DELIMITER ? delimiter : ''}]`
 
   if (fields) {
     const quotedFields = fields.map(f => encodeKey(f))

packages/toon/src/index.ts

@@ -88,7 +88,6 @@ function resolveOptions(options?: EncodeOptions): ResolvedEncodeOptions {
   return {
     indent: options?.indent ?? 2,
     delimiter: options?.delimiter ?? DEFAULT_DELIMITER,
-    lengthMarker: options?.lengthMarker ?? false,
     keyFolding: options?.keyFolding ?? 'off',
     flattenDepth: options?.flattenDepth ?? Number.POSITIVE_INFINITY,
   }

packages/toon/src/types.ts

@@ -24,12 +24,6 @@ export interface EncodeOptions {
    * @default DELIMITERS.comma
    */
   delimiter?: Delimiter
-  /**
-   * Optional marker to prefix array lengths in headers.
-   * When set to `#`, arrays render as [#N] instead of [N].
-   * @default false
-   */
-  lengthMarker?: '#' | false
   /**
    * Enable key folding to collapse single-key wrapper chains.
    * When set to 'safe', nested objects with single keys are collapsed into dotted paths
@@ -84,7 +78,6 @@ export interface ArrayHeaderInfo {
   length: number
   delimiter: Delimiter
   fields?: string[]
-  hasLengthMarker: boolean
 }
 
 export interface ParsedLine {

packages/toon/test/encode.test.ts

@@ -7,7 +7,6 @@ import arraysTabular from '@toon-format/spec/tests/fixtures/encode/arrays-tabula
 import delimiters from '@toon-format/spec/tests/fixtures/encode/delimiters.json'
 import keyFolding from '@toon-format/spec/tests/fixtures/encode/key-folding.json'
 import objects from '@toon-format/spec/tests/fixtures/encode/objects.json'
-import options from '@toon-format/spec/tests/fixtures/encode/options.json'
 import primitives from '@toon-format/spec/tests/fixtures/encode/primitives.json'
 import whitespace from '@toon-format/spec/tests/fixtures/encode/whitespace.json'
 import { describe, expect, it } from 'vitest'
@@ -23,7 +22,6 @@ const fixtureFiles = [
   keyFolding,
   delimiters,
   whitespace,
-  options,
 ] as Fixtures[]
 
 for (const fixtures of fixtureFiles) {
@@ -49,7 +47,6 @@ function resolveEncodeOptions(options?: TestCase['options']): ResolvedEncodeOpti
   return {
     indent: options?.indent ?? 2,
     delimiter: options?.delimiter ?? DEFAULT_DELIMITER,
-    lengthMarker: options?.lengthMarker === '#' ? '#' : false,
     keyFolding: options?.keyFolding ?? 'off',
     flattenDepth: options?.flattenDepth ?? Number.POSITIVE_INFINITY,
   }

packages/toon/test/types.ts

@@ -13,7 +13,6 @@ export interface TestCase {
   options?: {
     delimiter?: ',' | '\t' | '|'
     indent?: number
-    lengthMarker?: '#'
     strict?: boolean
     keyFolding?: 'off' | 'safe'
     flattenDepth?: number