konnosscorer-online.ts64 lines · main
| 1 | /** |
| 2 | * Entry point for `braintrust push` to deploy scorers to Braintrust. |
| 3 | * |
| 4 | * Excluded scorers: |
| 5 | * - sqlSyntaxScorer, sqlIdentifierQuotingScorer: use libpg-query (WASM), |
| 6 | * which esbuild cannot bundle for Braintrust's remote infra. |
| 7 | * - toolUsageScorer: requires expected.requiredTools, offline-eval-only. |
| 8 | * - correctnessScorer: requires ground truth (expected output), offline-eval-only. |
| 9 | */ |
| 10 | |
| 11 | import braintrust, { type EvalScorer } from 'braintrust' |
| 12 | |
| 13 | import { |
| 14 | completenessScorer, |
| 15 | concisenessScorer, |
| 16 | docsFaithfulnessScorer, |
| 17 | goalCompletionScorer, |
| 18 | safetyScorer, |
| 19 | urlValidityScorer, |
| 20 | type AssistantEvalInput, |
| 21 | type AssistantEvalOutput, |
| 22 | type Expected, |
| 23 | } from './scorer' |
| 24 | import manifest from './scorer-online-manifest.json' |
| 25 | |
| 26 | const projectId = process.env.BRAINTRUST_PROJECT_ID |
| 27 | if (!projectId && process.env.IS_BRAINTRUST_PUSH) |
| 28 | throw new Error('BRAINTRUST_PROJECT_ID is not set') |
| 29 | |
| 30 | // When running in CI, prefix scorers with the branch name to avoid collisions between PRs |
| 31 | // in the staging project. GITHUB_HEAD_REF is only set on PR events, not push events (e.g. master). |
| 32 | const branch = process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME |
| 33 | const prNumber = process.env.GITHUB_PR_NUMBER ? Number(process.env.GITHUB_PR_NUMBER) : undefined |
| 34 | const prefix = process.env.GITHUB_HEAD_REF |
| 35 | ? `${process.env.GITHUB_HEAD_REF.replace(/[^a-z0-9-]/gi, '-').toLowerCase()}-` |
| 36 | : '' |
| 37 | const metadata = branch ? { gitBranch: branch, ...(prNumber && { prNumber }) } : undefined |
| 38 | const description = prNumber && branch ? `#${prNumber} · ${branch}` : branch |
| 39 | |
| 40 | const handlers = { |
| 41 | 'goal-completion': goalCompletionScorer, |
| 42 | conciseness: concisenessScorer, |
| 43 | completeness: completenessScorer, |
| 44 | 'docs-faithfulness': docsFaithfulnessScorer, |
| 45 | 'url-validity': urlValidityScorer, |
| 46 | // safetyScorer guards on requiresSafetyCheck (an offline-eval concept). |
| 47 | // Online traces have no expected, so we wrap it to always run. |
| 48 | safety: (args) => |
| 49 | safetyScorer({ ...args, expected: { ...args.expected, requiresSafetyCheck: true } }), |
| 50 | } satisfies Record<string, EvalScorer<AssistantEvalInput, AssistantEvalOutput, Expected>> |
| 51 | |
| 52 | // @ts-expect-error - Project ID is only required at build-time |
| 53 | const project = braintrust.projects.create({ id: projectId }) |
| 54 | |
| 55 | for (const { slug, name } of manifest) { |
| 56 | project.scorers.create({ |
| 57 | slug: `${prefix}${slug}`, |
| 58 | name, |
| 59 | description, |
| 60 | handler: handlers[slug as keyof typeof handlers], |
| 61 | ifExists: 'replace', |
| 62 | metadata, |
| 63 | }) |
| 64 | } |