assistant.eval.ts62 lines · main

1	import assert from 'node:assert'
2	import { Eval } from 'braintrust'
3
4	import { dataset } from './dataset'
5	import {
6	completenessScorer,
7	concisenessScorer,
8	correctnessScorer,
9	docsFaithfulnessScorer,
10	goalCompletionScorer,
11	knowledgeUsageScorer,
12	safetyScorer,
13	toolUsageScorer,
14	urlValidityScorer,
15	} from './scorer'
16	import { sqlIdentifierQuotingScorer, sqlSyntaxScorer } from './scorer-wasm'
17	import { generateAssistantResponse } from '@/lib/ai/generate-assistant-response'
18	import { getModel } from '@/lib/ai/model'
19	import { DEFAULT_ASSISTANT_BASE_MODEL_ID, getAssistantModelEntry } from '@/lib/ai/model.utils'
20	import { getMockTools } from '@/lib/ai/tools/mock-tools'
21
22	assert(process.env.BRAINTRUST_PROJECT_ID, 'BRAINTRUST_PROJECT_ID is not set')
23	assert(process.env.OPENAI_API_KEY, 'OPENAI_API_KEY is not set')
24
25	Eval('Assistant', {
26	projectId: process.env.BRAINTRUST_PROJECT_ID,
27	trialCount: process.env.CI ? 3 : 1,
28	data: () => dataset,
29	task: async (input) => {
30	const modelEntry = getAssistantModelEntry(DEFAULT_ASSISTANT_BASE_MODEL_ID)
31	const modelResponse = await getModel({ provider: 'openai', modelEntry })
32	if (modelResponse.error) throw modelResponse.error
33
34	const result = await generateAssistantResponse({
35	...modelResponse.modelParams,
36	messages: [
37	{
38	id: '1',
39	role: 'user',
40	parts: [{ type: 'text', text: input.prompt }],
41	},
42	],
43	tools: await getMockTools(input.mockTables ? { list_tables: input.mockTables } : undefined),
44	})
45
46	const finishReason = await result.finishReason
47	return { finishReason }
48	},
49	scores: [
50	toolUsageScorer,
51	knowledgeUsageScorer,
52	sqlSyntaxScorer,
53	sqlIdentifierQuotingScorer,
54	goalCompletionScorer,
55	concisenessScorer,
56	completenessScorer,
57	docsFaithfulnessScorer,
58	correctnessScorer,
59	safetyScorer,
60	urlValidityScorer,
61	],
62	})