konnosDashboard
ratchet-eslint-rules.ts425 lines · main
1/* eslint-disable turbo/no-undeclared-env-vars */
2/**
3 * Ratchet ESLint violations for selected rules.
4 *
5 * Examples:
6 * # Initialize baselines for two rules
7 * tsx scripts/ratchet-eslint-rules.ts --init \
8 * --rule react-hooks/exhaustive-deps --rule no-console
9 *
10 * # Compare current counts vs baselines
11 * tsx scripts/ratchet-eslint-rules.ts \
12 * --rule react-hooks/exhaustive-deps --rule no-console
13 *
14 # Decrease baselines when improvements occur
15 * tsx scripts/ratchet-eslint-rules.ts \
16 * --rule react-hooks/exhaustive-deps --rule no-console \
17 * --decrease-baselines
18 *
19 * Flags:
20 * --metadata <path> Path to baseline file (default .github/eslint-rule-baselines.json)
21 * --init Write current counts for the provided --rule(s) into metadata and exit 0
22 * --eslint "<cmd>" ESLint command to run (default "npx eslint"). Do not pass untrusted input.
23 * --eslint-args "<...>" Extra args/paths for ESLint (e.g., "."). Do not pass untrusted input.
24 * --rule <id>[,<id>...] Rule id(s). Repeat flag or comma-separate. REQUIRED.
25 * --decrease-baselines When improvements occur, lower stored baselines to match the new counts.
26 *
27 * Notes:
28 * - Counts occurrences regardless of severity (warn/error).
29 * - Fails if any selected rule has currentCount > baselineCount.
30 */
31
32import { spawnSync } from 'node:child_process'
33import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
34import path from 'node:path'
35import { pathToFileURL } from 'node:url'
36
37interface Args {
38 metadata: string
39 init: boolean
40 eslint: string
41 eslintArgs: string
42 decreaseBaselines: boolean
43 rules: string[]
44}
45
46interface ESLintMessage {
47 ruleId?: string | null
48}
49
50interface ESLintResult {
51 filePath?: string
52 messages?: ESLintMessage[]
53}
54
55interface ESLintExecutionResult {
56 results: ESLintResult[]
57 stderr: string
58}
59
60interface BaselineData {
61 rules: Record<string, number>
62 ruleFiles?: Record<string, Record<string, number>>
63}
64
65interface RuleSnapshot {
66 total: number
67 files: Record<string, number>
68}
69
70function parseArgs(argv: string[]): Args {
71 const args: Args = {
72 metadata: '.github/eslint-rule-baselines.json',
73 init: false,
74 eslint: 'npx eslint',
75 eslintArgs: '',
76 decreaseBaselines: false,
77 rules: [],
78 }
79
80 for (let i = 2; i < argv.length; i += 1) {
81 const a = argv[i]
82 if (a === '--init') {
83 args.init = true
84 } else if (a === '--metadata') {
85 args.metadata = argv[++i]
86 } else if (a === '--eslint') {
87 args.eslint = argv[++i]
88 } else if (a === '--eslint-args') {
89 args.eslintArgs = argv[++i]
90 } else if (a === '--rule') {
91 const val = (argv[++i] ?? '').trim()
92 if (val) {
93 args.rules.push(
94 ...val
95 .split(',')
96 .map((s) => s.trim())
97 .filter(Boolean)
98 )
99 }
100 } else if (a === '--decrease-baselines') {
101 args.decreaseBaselines = true
102 } else {
103 console.warn(`Unknown argument: ${a}`)
104 }
105 }
106
107 if (args.rules.length === 0) {
108 console.error('Error: You must provide at least one --rule <rule-id>.')
109 console.error('Example: --rule exhaustive-deps --rule no-console')
110 process.exit(2)
111 }
112
113 const dedupedRules = new Set(args.rules)
114 args.rules = Array.from(dedupedRules)
115
116 return args
117}
118
119/**
120 * SECURITY:
121 * Directly spawns a command from its arguments. Should not be called with
122 * untrusted input.
123 */
124function dangerouslyRunEsLint(eslintCmd: string, eslintArgs: string): ESLintExecutionResult {
125 const fullCmd = `${eslintCmd} ${eslintArgs || ''} --format json`.trim()
126 const proc = spawnSync(fullCmd, {
127 shell: true,
128 encoding: 'utf8',
129 stdio: ['ignore', 'pipe', 'pipe'],
130 env: process.env,
131 maxBuffer: 32 * 1024 * 1024, // allow large ESLint JSON payloads
132 })
133
134 const stdout = typeof proc.stdout === 'string' ? proc.stdout : ''
135 const stderr = typeof proc.stderr === 'string' ? proc.stderr : ''
136
137 if (!stdout.trim()) {
138 console.error('ESLint did not produce JSON output. stderr:\n', stderr)
139 process.exit(2)
140 }
141
142 let results: ESLintResult[]
143 try {
144 results = JSON.parse(stdout) as ESLintResult[]
145 } catch (e) {
146 console.error('Failed to parse ESLint JSON output:', e)
147 console.error('Raw output (truncated to 4k):\n', stdout.slice(0, 4096))
148 process.exit(2)
149 }
150
151 return { results, stderr }
152}
153
154function normalizeFilePath(filePath?: string | null): string | null {
155 if (!filePath) return null
156 const rel = path.relative(process.cwd(), filePath)
157 const normalized = rel || path.basename(filePath)
158 return normalized.split(path.sep).join('/')
159}
160
161function collectRuleSnapshots(
162 results: ESLintResult[],
163 ruleIds: string[]
164): Record<string, RuleSnapshot> {
165 const checkedIds = new Set(ruleIds)
166 const snapshots: Record<string, RuleSnapshot> = {}
167
168 for (const id of ruleIds) {
169 snapshots[id] = { total: 0, files: {} }
170 }
171
172 for (const file of results) {
173 if (!file || !Array.isArray(file.messages)) continue
174 const normalizedPath = normalizeFilePath(file.filePath)
175 for (const msg of file.messages) {
176 const id = msg?.ruleId ?? ''
177 if (id && checkedIds.has(id)) {
178 const snapshot = snapshots[id] ?? { total: 0, files: {} }
179 snapshot.total += 1
180 if (normalizedPath) {
181 snapshot.files[normalizedPath] = (snapshot.files[normalizedPath] ?? 0) + 1
182 }
183 snapshots[id] = snapshot
184 }
185 }
186 }
187
188 return snapshots
189}
190
191function readBaselines(fp: string): BaselineData {
192 if (!existsSync(fp)) return { rules: {}, ruleFiles: {} }
193 try {
194 const data = JSON.parse(readFileSync(fp, 'utf8')) as Partial<BaselineData>
195 if (data && typeof data === 'object' && data.rules && typeof data.rules === 'object') {
196 return { rules: data.rules, ruleFiles: data.ruleFiles ?? {} }
197 }
198 } catch {
199 // ignore invalid metadata files and fall back to blank baselines
200 }
201 return { rules: {}, ruleFiles: {} }
202}
203
204function writeBaselines(fp: string, updates: Record<string, RuleSnapshot>, merge = true): void {
205 const dir = path.dirname(fp)
206 mkdirSync(dir, { recursive: true })
207
208 let current: BaselineData = { rules: {}, ruleFiles: {} }
209 if (merge && existsSync(fp)) {
210 current = readBaselines(fp)
211 }
212
213 const nextRules = merge ? { ...current.rules } : {}
214 const nextRuleFiles = merge ? { ...(current.ruleFiles ?? {}) } : {}
215
216 for (const [rule, snapshot] of Object.entries(updates)) {
217 nextRules[rule] = snapshot.total
218 nextRuleFiles[rule] = snapshot.files
219 }
220
221 const next: BaselineData = { rules: nextRules, ruleFiles: nextRuleFiles }
222 writeFileSync(fp, `${JSON.stringify(next, null, 2)}\n`, 'utf8')
223}
224
225function writeSummary(markdown: string): void {
226 const summaryFile = process.env.GITHUB_STEP_SUMMARY
227 if (summaryFile) {
228 try {
229 appendFileSync(summaryFile, `${markdown}\n`, 'utf8')
230 } catch {
231 // ignore summary write errors because they shouldn't block the script
232 }
233 }
234}
235
236export function runRatchet(argv: string[], runEslint = dangerouslyRunEsLint): number {
237 const args = parseArgs(argv)
238
239 // SECURITY:
240 // Offloaded to user. Must document that they should not pass untrusted input
241 // via --eslint or --eslint-args.
242 const { results, stderr } = runEslint(args.eslint, args.eslintArgs)
243
244 // Filter out test files.
245 const filteredResults = results.filter((result) => !result.filePath?.includes('.test.'))
246
247 const currentSnapshots = collectRuleSnapshots(filteredResults, args.rules)
248 const currentCounts: Record<string, number> = {}
249 for (const rule of args.rules) {
250 currentCounts[rule] = currentSnapshots[rule]?.total ?? 0
251 }
252
253 if (args.init) {
254 writeBaselines(args.metadata, currentSnapshots, true)
255
256 const rows = Object.entries(currentCounts)
257 .map(([rule, count]) => `| \`${rule}\` | **${count}** |`)
258 .join('\n')
259
260 writeSummary(
261 [
262 `### ESLint rule baselines initialized`,
263 `Metadata: \`${args.metadata}\``,
264 ``,
265 `| Rule | Baseline |`,
266 `| --- | ---: |`,
267 rows,
268 ``,
269 ].join('\n')
270 )
271
272 console.log(
273 `Initialized/updated baselines for: ${args.rules.join(', ')} (saved to ${args.metadata}).`
274 )
275 return 0
276 }
277
278 const baselineData = readBaselines(args.metadata)
279 const baselineRules = baselineData.rules || {}
280 const baselineRuleFiles = baselineData.ruleFiles || {}
281
282 const missing = args.rules.filter((r) => typeof baselineRules[r] !== 'number')
283 if (missing.length) {
284 const msg = `Missing baselines for: ${missing.join(', ')} in ${args.metadata}. Run with --init to set them.`
285 console.error(msg)
286 writeSummary(`### ESLint rule ratchet\n${msg}`)
287 console.log(`::error title=Missing baselines::${msg}`)
288 return 2
289 }
290
291 let failed = false
292 const tableRows: string[] = []
293 const improvedRules: string[] = []
294 const decreasedBaselines: Record<string, { from: number; to: number; snapshot: RuleSnapshot }> =
295 {}
296 for (const rule of args.rules) {
297 const baseline = baselineRules[rule] ?? 0
298 const current = currentCounts[rule] ?? 0
299 const delta = current - baseline
300 const currentSnapshot = currentSnapshots[rule] ?? { total: 0, files: {} }
301 const baselineFiles = baselineRuleFiles[rule] ?? {}
302
303 tableRows.push(
304 `| \`${rule}\` | **${baseline}** | **${current}** | ${delta >= 0 ? '+' : '-'}${delta} |`
305 )
306
307 if (current > baseline) {
308 failed = true
309 const delta = current - baseline
310 const baselineHasFiles = Object.hasOwn(baselineRuleFiles, rule)
311 const fileSummary = describeFileRegression(
312 baselineFiles,
313 currentSnapshot.files,
314 baselineHasFiles
315 )
316 const msgParts = [
317 `You added ${delta === 1 ? 'a new violation' : `${delta} new violations`} of ${rule}. Please fix it: baseline=${baseline}, current=${current}`,
318 ]
319 if (fileSummary) {
320 msgParts.push(
321 `Affected files: ${fileSummary}${baselineHasFiles ? '' : ' (baseline missing file breakdown; rerun with --init to capture it)'}`
322 )
323 }
324 const msg = msgParts.join(' ')
325 console.error(msg)
326 console.log(`::error title=New violations::${msg}`)
327 } else if (current < baseline) {
328 improvedRules.push(rule)
329 if (args.decreaseBaselines) {
330 decreasedBaselines[rule] = { from: baseline, to: current, snapshot: currentSnapshot }
331 }
332 }
333 }
334
335 const summaryLines = [
336 `### ESLint rule ratchet`,
337 `Metadata: \`${args.metadata}\``,
338 ``,
339 `| Rule | Baseline | Current | Δ |`,
340 `| --- | ---: | ---: | ---: |`,
341 ...tableRows,
342 ``,
343 ]
344
345 if (args.decreaseBaselines && Object.keys(decreasedBaselines).length > 0) {
346 const updates: Record<string, RuleSnapshot> = {}
347 const details: string[] = []
348 const logParts: string[] = []
349 for (const [rule, { from, to, snapshot }] of Object.entries(decreasedBaselines)) {
350 updates[rule] = snapshot
351 details.push(`- \`${rule}\`: ${from} -> ${to}`)
352 logParts.push(`${rule}: ${from} -> ${to}`)
353 }
354 writeBaselines(args.metadata, updates, true)
355 summaryLines.push('', 'Baselines decreased for improved rules:', ...details, '')
356 console.log(`Baselines decreased for improved rules: ${logParts.join(', ')}`)
357 }
358
359 writeSummary(summaryLines.join('\n'))
360
361 if (failed) {
362 if (stderr && stderr.trim()) console.error('\nESLint stderr:\n', stderr)
363 return 1
364 } else {
365 console.log(
366 improvedRules.length > 0
367 ? 'Nice! Some rules improved.'
368 : 'Stable: No regressions for selected rules.'
369 )
370 return 0
371 }
372}
373
374function main(): void {
375 const exitCode = runRatchet(process.argv, dangerouslyRunEsLint)
376 process.exit(exitCode)
377}
378
379if (process.argv[1]) {
380 const invokedPath = pathToFileURL(path.resolve(process.argv[1])).href
381 if (import.meta.url === invokedPath) {
382 main()
383 }
384}
385
386function describeFileRegression(
387 baselineFiles: Record<string, number>,
388 currentFiles: Record<string, number>,
389 baselineHasFiles: boolean
390): string {
391 const MAX_FILES = 5
392 if (baselineHasFiles) {
393 const entries = Object.entries(currentFiles)
394 .map(([file, count]) => ({
395 file,
396 delta: count - (baselineFiles[file] ?? 0),
397 }))
398 .filter(({ delta }) => delta > 0)
399 .sort((a, b) => b.delta - a.delta || a.file.localeCompare(b.file))
400
401 if (!entries.length) return ''
402
403 return formatFileList(
404 entries.map(({ file, delta }) => `${file} (+${delta})`),
405 MAX_FILES
406 )
407 }
408
409 const currentEntries = Object.entries(currentFiles)
410 .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
411 .map(([file, count]) => `${file} (${count} current)`)
412
413 if (!currentEntries.length) return ''
414
415 return formatFileList(currentEntries, MAX_FILES)
416}
417
418function formatFileList(entries: string[], maxFiles: number): string {
419 if (entries.length <= maxFiles) {
420 return entries.join(', ')
421 }
422 const remainder = entries.length - maxFiles
423 const plural = remainder === 1 ? 'file' : 'files'
424 return `${entries.slice(0, maxFiles).join(', ')}, +${remainder} more ${plural}`
425}