import process from 'node:process' import * as url from 'node:url' export const ROOT_DIR: string = url.fileURLToPath(new URL('../../', import.meta.url)) export const BENCHMARKS_DIR: string = url.fileURLToPath(new URL('../', import.meta.url)) /** * Default concurrency for parallel evaluations to prevent bursting */ export const DEFAULT_CONCURRENCY = 10 /** * Enable dry run mode for quick testing with limited AI requests * * @remarks * Set via environment variable: `DRY_RUN=true`. */ export const DRY_RUN: boolean = process.env.DRY_RUN === 'true' /** * Limits applied during dry run mode */ export const DRY_RUN_LIMITS = { /** Maximum number of questions to evaluate */ maxQuestions: 10, } /** * Model-specific RPM (requests per minute) limits to handle API quotas * * @remarks * Set `undefined` for models without specific limits. */ /// keep-sorted export const MODEL_RPM_LIMITS: Record = { 'claude-haiku-4-5-20251001': 50, 'gemini-2.5-flash': 25, 'gpt-5-nano': 50, 'grok-4-fast-non-reasoning': 50, } /** * Display names for data format types */ export const FORMATTER_DISPLAY_NAMES: Record = { 'json-pretty': 'JSON', 'json-compact': 'JSON compact', 'toon': 'TOON', 'csv': 'CSV', 'xml': 'XML', 'yaml': 'YAML', } as const /** * Question type identifiers */ export const QUESTION_TYPES = [ 'field-retrieval', 'retrieval', 'aggregation', 'filtering', 'structure-awareness', 'structural-validation', ] as const /** * Display names for question types */ export const QUESTION_TYPE_LABELS = { 'field-retrieval': 'Field Retrieval', 'retrieval': 'Retrieval', 'aggregation': 'Aggregation', 'filtering': 'Filtering', 'structure-awareness': 'Structure Awareness', 'structural-validation': 'Structural Validation', } as const /** * Dataset identifiers */ export const DATASET_NAMES = [ 'tabular', 'nested', 'analytics', 'github', 'event-logs', 'nested-config', 'large-uniform', 'structural-validation-control', 'structural-validation-truncated', 'structural-validation-extra-rows', 'structural-validation-width-mismatch', 'structural-validation-missing-fields', ] as const /** * Structure class identifiers */ export const STRUCTURE_CLASSES = [ 'uniform', 'semi-uniform', 'nested', 'deep', ] as const /** * Threshold values for filtering and aggregation questions */ export const QUESTION_THRESHOLDS = { tabular: { salaryRanges: [60000, 80000, 100000], experienceYears: [5, 10, 15, 20], departmentSalaryThreshold: 80000, departmentExperienceThreshold: 10, }, nested: { highValueOrders: [200, 400, 600], statusValueThreshold: 300, itemCountThreshold: 3, totalThresholdsForItems: [300, 500], }, analytics: { views: [6000], conversions: [20], viewsForFiltering: [6000, 7000], conversionsForFiltering: 15, revenueThresholds: [1000, 1500, 2000], viewsThresholdForRevenue: 6000, clicksForFiltering: [250, 400], conversionsForClickFiltering: 15, revenueForBounceRate: [1000, 1500], bounceRateThreshold: 0.5, }, github: { stars: [100000, 150000, 200000], forks: [20000, 35000], watchers: [8000], starForkCombinations: [ { stars: 75000, forks: 15000 }, { stars: 100000, forks: 20000 }, { stars: 150000, forks: 30000 }, { stars: 200000, forks: 45000 }, ], starWatcherCombinations: [ { stars: 100000, watchers: 7000 }, { stars: 150000, watchers: 9000 }, ], }, } as const /** * Question generation configuration */ export const QUESTION_LIMITS = { tabular: { fieldRetrieval: 12, aggregationDepartments: 3, filteringMultiConditionDepartments: 5, filteringExperience: 3, filteringDepartmentExp: 3, filteringDepartmentActive: 2, }, nested: { fieldRetrievalOrders: 8, fieldRetrievalCustomers: 8, aggregationStatuses: 3, filteringStatusAndValue: 4, filteringStatusAndItems: 3, }, analytics: { fieldRetrievalDates: 9, }, github: { fieldRetrievalRepos: 11, aggregationBranches: 2, filteringStarsAndForks: 3, }, eventLogs: { fieldRetrieval: 10, aggregationEndpoints: 2, filteringLevelAndStatus: 3, filteringEndpointAndStatus: 3, filteringEndpointRetryable: 2, }, nestedConfig: { fieldRetrieval: 10, filteringComplex: 5, }, } as const