mongodb-chatbot-eval

Interfaces

• BaseGeneratedData
• BaseTestCase
• CheckResponseQuality
• CheckResponseQualityParams
• CommandMetadataStore
• CommandRunMetadata
• ConversationEvalData
• ConversationGeneratedData
• ConversationTestCase
• EvalConfig
• EvalResult
• EvaluateConversationQualityParams
• EvaluationStore
• GenerateDataAndMetadataParams
• GenerateDataFuncParams
• GeneratedDataStore
• MakeGenerateConversationDataParams
• MakeMongoDbCommandMetadataStoreParams
• MakeMongoDbEvaluationStoreParams
• MakeMongoDbGeneratedDataStoreParams
• MakeMongoDbReportStoreParams
• MongoDbEvaluationStore
• MongoDbGeneratedDataStore
• MongoDbReportStore
• Report
• ReportStore
• ResponseQualityExample

Type Aliases

ConfigConstructor

Ƭ ConfigConstructor: () => Promise<EvalConfig>

Type declaration

▸ (): Promise<EvalConfig>

Returns

Promise<EvalConfig>

Defined in

packages/mongodb-chatbot-eval/src/EvalConfig.ts:43

---

ConversationTestCaseData

Ƭ ConversationTestCaseData: z.infer<typeof ConversationTestCaseDataSchema>

Defined in

packages/mongodb-chatbot-eval/src/generate/TestCase.ts:23

---

EvaluateQualityFunc

Ƭ EvaluateQualityFunc: (params: EvaluateQualityFuncParams) => Promise<EvalResult>

Type declaration

▸ (params): Promise<EvalResult>

Parameters

Name	Type
params	EvaluateQualityFuncParams

Returns

Promise<EvalResult>

Defined in

packages/mongodb-chatbot-eval/src/evaluate/EvaluateQualityFunc.ts:10

---

GenerateDataFunc

Ƭ GenerateDataFunc: (params: GenerateDataFuncParams) => Promise<{ failedCases: SomeTestCase[] ; generatedData: SomeGeneratedData[] }>

Type declaration

▸ (params): Promise<{ failedCases: SomeTestCase[] ; generatedData: SomeGeneratedData[] }>

Parameters

Name	Type
params	GenerateDataFuncParams

Returns

Promise<{ failedCases: SomeTestCase[] ; generatedData: SomeGeneratedData[] }>

Defined in

packages/mongodb-chatbot-eval/src/generate/GenerateDataFunc.ts:16

---

LoadConfigArgs

Ƭ LoadConfigArgs: Object

Type declaration

Name	Type
config?	string

Defined in

packages/mongodb-chatbot-eval/src/withConfig.ts:6

---

ReportEvalFunc

Ƭ ReportEvalFunc: (evalution: EvalResult) => Promise<Report>

Type declaration

▸ (evalution): Promise<Report>

Parameters

Name	Type
evalution	EvalResult

Returns

Promise<Report>

Defined in

packages/mongodb-chatbot-eval/src/report/ReportEvalFunc.ts:3

---

SomeGeneratedData

Ƭ SomeGeneratedData: ConversationGeneratedData | BaseGeneratedData

Defined in

packages/mongodb-chatbot-eval/src/generate/GeneratedDataStore.ts:17

---

SomeTestCase

Ƭ SomeTestCase: ConversationTestCase | BaseTestCase

Defined in

packages/mongodb-chatbot-eval/src/generate/TestCase.ts:32

Variables

ConversationTestCaseDataSchema

• Const ConversationTestCaseDataSchema: ZodObject<{ expectation: ZodString ; messages: ZodArray<ZodObject<{ content: ZodString ; role: ZodEnum<["assistant", "user"]> }, "strip", ZodTypeAny, { content: string ; role: "assistant" | "user" }, { content: string ; role: "assistant" | "user" }>, "many"> ; name: ZodString ; skip: ZodOptional<ZodBoolean> ; tags: ZodOptional<ZodArray<ZodString, "many">> }, "strip", ZodTypeAny, { expectation: string ; messages: { role: "assistant" | "user"; content: string; }[] ; name: string ; skip?: boolean ; tags?: string[] }, { expectation: string ; messages: { role: "assistant" | "user"; content: string; }[] ; name: string ; skip?: boolean ; tags?: string[] }>

Defined in

packages/mongodb-chatbot-eval/src/generate/TestCase.ts:8

---

mongodbResponseQualityExamples

• Const mongodbResponseQualityExamples: ResponseQualityExample[]

Defined in

packages/mongodb-chatbot-eval/src/evaluate/checkResponseQuality.ts:129

Functions

checkResponseQuality

▸ checkResponseQuality(«destructured»): Promise<CheckResponseQuality>

Parameters

Name	Type
«destructured»	CheckResponseQualityParams

Returns

Promise<CheckResponseQuality>

Defined in

packages/mongodb-chatbot-eval/src/evaluate/checkResponseQuality.ts:28

---

convertCommandRunMetadataToJson

▸ convertCommandRunMetadataToJson(command): Object

Parameters

Name	Type
command	CommandRunMetadata

Returns

Object

Name	Type
_id	string
command	"generate" | "evaluate" | "report"
endTime	number
name	string
startTime	number

Defined in

packages/mongodb-chatbot-eval/src/CommandMetadataStore.ts:16

---

generateDataAndMetadata

▸ generateDataAndMetadata(«destructured»): Promise<{ failedCases: SomeTestCase[] ; generatedData: SomeGeneratedData[] ; metadata: { _id: ObjectId = runId; command: "generate" = "generate"; endTime: Date ; name: string ; startTime: Date } }>

Generate data for test cases and store metadata about the generation.

Parameters

Name	Type
«destructured»	GenerateDataAndMetadataParams

Returns

Promise<{ failedCases: SomeTestCase[] ; generatedData: SomeGeneratedData[] ; metadata: { _id: ObjectId = runId; command: "generate" = "generate"; endTime: Date ; name: string ; startTime: Date } }>

Defined in

packages/mongodb-chatbot-eval/src/generate/generateDataAndMetadata.ts:40

---

getConversationsTestCasesFromYaml

▸ getConversationsTestCasesFromYaml(yamlData): ConversationTestCase[]

Get conversation test cases from YAML file. Throws if the YAML is not correctly formatted.

Parameters

Name	Type
yamlData	string

Returns

ConversationTestCase[]

Defined in

packages/mongodb-chatbot-eval/src/generate/getConversationsTestCasesFromYaml.ts:12

---

isConversationTestCase

▸ isConversationTestCase(testCase): testCase is ConversationTestCase

Parameters

Name	Type
testCase	SomeTestCase

Returns

testCase is ConversationTestCase

Defined in

packages/mongodb-chatbot-eval/src/generate/TestCase.ts:34

---

loadConfig

▸ loadConfig(«destructured»): Promise<EvalConfig>

Parameters

Name	Type
«destructured»	LoadConfigArgs

Returns

Promise<EvalConfig>

Defined in

packages/mongodb-chatbot-eval/src/withConfig.ts:10

---

makeEvaluateConversationQuality

▸ makeEvaluateConversationQuality(«destructured»): EvaluateQualityFunc

Construct a a EvaluateQualityFunc that evaluates the quality of a conversation using an OpenAI ChatGPT LLM.

The returned EvalResult has the following properties:

• In EvalResult.result, 1 if the conversation meets quality standards and 0 if it does not.
• In EvalResult.metadata, reason for the result, as generated by the LLM.

Parameters

Name	Type
«destructured»	EvaluateConversationQualityParams

Returns

EvaluateQualityFunc

Defined in

packages/mongodb-chatbot-eval/src/evaluate/evaluateConversationQuality.ts:43

---

makeGenerateConversationData

▸ makeGenerateConversationData(«destructured»): GenerateDataFunc

Generate conversation data from test cases.

Parameters

Name	Type
«destructured»	MakeGenerateConversationDataParams

Returns

GenerateDataFunc

Defined in

packages/mongodb-chatbot-eval/src/generate/generateConversationData.ts:48

---

makeMongoDbCommandMetadataStore

▸ makeMongoDbCommandMetadataStore(«destructured»): CommandMetadataStore

Parameters

Name	Type
«destructured»	MakeMongoDbCommandMetadataStoreParams

Returns

CommandMetadataStore

Defined in

packages/mongodb-chatbot-eval/src/CommandMetadataStore.ts:36

---

makeMongoDbEvaluationStore

▸ makeMongoDbEvaluationStore(«destructured»): MongoDbEvaluationStore

Parameters

Name	Type
«destructured»	MakeMongoDbEvaluationStoreParams

Returns

MongoDbEvaluationStore

Defined in

packages/mongodb-chatbot-eval/src/evaluate/EvaluationStore.ts:36

---

makeMongoDbGeneratedDataStore

▸ makeMongoDbGeneratedDataStore(«destructured»): MongoDbGeneratedDataStore

Parameters

Name	Type
«destructured»	MakeMongoDbGeneratedDataStoreParams

Returns

MongoDbGeneratedDataStore

Defined in

packages/mongodb-chatbot-eval/src/generate/GeneratedDataStore.ts:59

---

makeMongoDbReportStore

▸ makeMongoDbReportStore(«destructured»): MongoDbReportStore

Parameters

Name	Type
«destructured»	MakeMongoDbReportStoreParams

Returns

MongoDbReportStore

Defined in

packages/mongodb-chatbot-eval/src/report/ReportStore.ts:29

---

stringifyConversation

▸ stringifyConversation(messages): string

Parameters

Name	Type
messages	{ content: string ; role: string }[]

Returns

string

Defined in

packages/mongodb-chatbot-eval/src/evaluate/stringifyConversation.ts:1

---

withConfig

▸ withConfig<T>(action, args): Promise<void>

Type parameters

Name
T

Parameters

Name	Type
action	(config: EvalConfig, args: T) => Promise<void>
args	LoadConfigArgs & T

Returns

Promise<void>

Defined in

packages/mongodb-chatbot-eval/src/withConfig.ts:62

---

withConfigOptions

▸ withConfigOptions<T>(args): Argv<T & LoadConfigArgs>

Apply config options to CLI command.

Type parameters

Name
T

Parameters

Name	Type
args	Argv<T>

Returns

Argv<T & LoadConfigArgs>

Defined in

packages/mongodb-chatbot-eval/src/withConfig.ts:88