feat: add speech clarity and turn-taking voice scorers

js/llm.test.ts

@@ -8,7 +8,9 @@ import {
   buildClassificationTools,
   LLMClassifierFromTemplate,
   OpenAIClassifier,
+  SpeechClarity,
   templateUsesThreadVariables,
+  VoiceTaskSuccess,
 } from "../js/llm";
 import {
   openaiClassifierShouldEvaluateArithmeticExpressions,
@@ -662,3 +664,78 @@ Issue Description: {{page_content}}
     init({ client });
   });
 });
+
+describe("SpeechClarity", () => {
+  test("sends audio as a file to gemini-3.8-flash through chat completions", async () => {
+    let body: any;
+    server.use(
+      http.post(
+        "https://api.openai.com/v1/chat/completions",
+        async ({ request }) => {
+          body = await request.json();
+          return HttpResponse.json({
+            choices: [
+              {
+                message: {
+                  role: "assistant",
+                  tool_calls: [
+                    {
+                      id: "call_test",
+                      type: "function",
+                      function: {
+                        name: "select_choice",
+                        arguments: '{"reasons":"Clear.","choice":"A"}',
+                      },
+                    },
+                  ],
+                },
+              },
+            ],
+          });
+        },
+      ),
+    );
+
+    const score = await SpeechClarity({
+      input: {
+        audio: {
+          data: new Uint8Array([1, 2, 3]),
+          content_type: "audio/ogg",
+        },
+      },
+      output: undefined,
+      openAiApiKey: "test-api-key",
+    });
+
+    expect(score.score).toBe(1);
+    expect(body.model).toBe("gemini-3.8-flash");
+    expect(body.messages[0].content[0].type).toBe("text");
+    expect(body.messages[0].content[1].file.file_data).toBe(
+      "data:audio/ogg;base64,AQID",
+    );
+  });
+
+  test("skips when there is no audio", async () => {
+    const score = await SpeechClarity({
+      input: { text: "hello" },
+      output: undefined,
+      openAiApiKey: "test-api-key",
+    });
+    expect(score.score).toBeNull();
+  });
+});
+
+describe("VoiceTaskSuccess", () => {
+  test("skips when the conversation is empty", async () => {
+    const score = await VoiceTaskSuccess({
+      output: undefined,
+      trace: {
+        async getThread() {
+          return [{ role: "system", content: "Be helpful." }];
+        },
+      },
+      openAiApiKey: "test-api-key",
+    });
+    expect(score.score).toBeNull();
+  });
+});

js/llm.ts

@@ -7,6 +7,7 @@ import {
 } from "./oai";
 import { ModelGradedSpec, templates } from "./templates";
 import {
+  ChatCompletionContentPart,
   ChatCompletionMessage,
   ChatCompletionMessageParam,
   ChatCompletionTool,
@@ -62,6 +63,42 @@ function filterSystemMessagesFromThread(thread: unknown[]): unknown[] {
   });
 }
 
+function getPath(args: unknown, path: string): unknown {
+  let value = args;
+  for (const key of path.split(".")) {
+    value =
+      value && typeof value === "object" && !Array.isArray(value)
+        ? Reflect.get(value, key)
+        : undefined;
+  }
+  return value;
+}
+
+export type Audio = { data: Uint8Array; content_type: string };
+
+function audioPart(audio: unknown): ChatCompletionContentPart.File {
+  const data = Reflect.get(Object(audio), "data");
+  if (!(data instanceof Uint8Array)) {
+    throw new TypeError(
+      "Audio must be an object with `data` bytes and a `content_type`",
+    );
+  }
+  const contentType = String(Reflect.get(Object(audio), "content_type") ?? "")
+    .split(";")[0]
+    .trim()
+    .toLowerCase();
+  if (!contentType.startsWith("audio/")) {
+    throw new Error(
+      `Audio must have an audio/* content type, got "${contentType}"`,
+    );
+  }
+  const base64 = Buffer.from(data).toString("base64");
+  return {
+    type: "file",
+    file: { file_data: `data:${contentType};base64,${base64}` },
+  };
+}
+
 const NO_COT_SUFFIX =
   "Answer the question by calling `select_choice` with a single choice from {{__choices}}.";
 
@@ -142,6 +179,7 @@ export type OpenAIClassifierArgs<RenderArgs> = {
   messages: ChatCompletionMessageParam[];
   choiceScores: Record<string, number>;
   classificationTools: ChatCompletionTool[];
+  audioFile?: ChatCompletionContentPart.File;
   cache?: ChatCache;
 } & LLMArgs &
   RenderArgs;
@@ -174,6 +212,7 @@ export async function OpenAIClassifier<RenderArgs, Output>(
     reasoningEnabled,
     reasoningBudget,
     useResponsesApi,
+    audioFile,
     cache,
     ...remainingRenderArgs
   } = remaining;
@@ -212,6 +251,13 @@ export async function OpenAIClassifier<RenderArgs, Output>(
   };
 
   const messages = renderMessages(messagesArg, renderArgs);
+  if (audioFile) {
+    const last = messages[messages.length - 1];
+    messages[messages.length - 1] = {
+      role: "user",
+      content: [{ type: "text", text: String(last.content) }, audioFile],
+    };
+  }
 
   const resp = await cachedChatCompletion(
     {
@@ -306,6 +352,7 @@ export function LLMClassifierFromTemplate<RenderArgs>({
   reasoningEnabled,
   reasoningBudget,
   useResponsesApi,
+  audio,
 }: {
   name: string;
   promptTemplate: string;
@@ -318,11 +365,17 @@ export function LLMClassifierFromTemplate<RenderArgs>({
   reasoningEnabled?: boolean;
   reasoningBudget?: number;
   useResponsesApi?: boolean;
+  audio?: string;
 }): Scorer<string, LLMClassifierArgs<RenderArgs>> {
   const choiceStrings = Object.keys(choiceScores);
   const ret = async (
     runtimeArgs: ScorerArgs<string, LLMClassifierArgs<RenderArgs>>,
   ) => {
+    const audioValue = audio ? getPath(runtimeArgs, audio) : undefined;
+    if (audio && audioValue == null) {
+      return { name, score: null };
+    }
+
     const useCoT = runtimeArgs.useCoT ?? useCoTArg ?? true;
     // Use runtime model > template model > configured default model
     const model = runtimeArgs.model ?? modelArg ?? getDefaultModel();
@@ -343,6 +396,12 @@ export function LLMClassifierFromTemplate<RenderArgs>({
       }
     }
 
+    const threadCount =
+      Reflect.get(runtimeArgs, "thread_count") ?? threadVars.thread_count;
+    if (templateUsesThreadVariables(promptTemplate) && threadCount === 0) {
+      return { name, score: null };
+    }
+
     const prompt =
       promptTemplate + "\n" + (useCoT ? COT_SUFFIX : NO_COT_SUFFIX);
 
@@ -374,6 +433,7 @@ export function LLMClassifierFromTemplate<RenderArgs>({
       // Since the logic is a bit funky for computing this, include
       // it at the end to prevent overrides
       useCoT,
+      audioFile: audio ? audioPart(audioValue) : undefined,
     };
 
     return await OpenAIClassifier(classifierArgs);
@@ -398,6 +458,7 @@ export function LLMClassifierFromSpec<RenderArgs>(
     useCoT: spec.use_cot,
     temperature: spec.temperature,
     maxTokens: spec.max_tokens,
+    audio: spec.audio,
   });
 }
 
@@ -409,10 +470,10 @@ export function LLMClassifierFromSpecFile<RenderArgs>(
   return LLMClassifierFromSpec(name, doc);
 }
 
-function buildLLMClassifier<RenderArgs>(
+function buildLLMClassifier<RenderArgs, Output = string>(
   name: string,
   templateName: keyof typeof templates,
-): ScorerWithPartial<string, LLMClassifierArgs<RenderArgs>> {
+): ScorerWithPartial<Output, LLMClassifierArgs<RenderArgs>> {
   if (!(templateName in templates)) {
     throw new Error(`Model template ${name} not found`);
   }
@@ -492,3 +553,29 @@ export const Translation = buildLLMClassifier<{
   language: string;
   input: string;
 }>("Translation", "translation");
+
+/**
+ * Test how clearly an agent speaks, from a recording of the whole conversation (`input.audio`).
+ */
+export const SpeechClarity = buildLLMClassifier<
+  { input: { audio?: Audio; [key: string]: unknown } },
+  unknown
+>("SpeechClarity", "speech_clarity");
+
+/**
+ * Test whether an agent takes turns without talking over or cutting off the caller,
+ * from a recording of the whole conversation (`input.audio`).
+ */
+export const TurnTaking = buildLLMClassifier<
+  { input: { audio?: Audio; [key: string]: unknown } },
+  unknown
+>("TurnTaking", "turn_taking");
+
+/**
+ * Test whether a voice agent correctly completed the caller's request, from the conversation
+ * in the trace, including its instructions, tool calls, and tool results.
+ */
+export const VoiceTaskSuccess = buildLLMClassifier<
+  Record<string, unknown>,
+  unknown
+>("VoiceTaskSuccess", "voice_task_success");

js/manifest.ts

@@ -6,9 +6,12 @@ import {
   Humor,
   Possible,
   Security,
+  SpeechClarity,
   Sql,
   Summary,
   Translation,
+  TurnTaking,
+  VoiceTaskSuccess,
 } from "./llm";
 import { NumericDiff } from "./number";
 import { EmbeddingSimilarity, Levenshtein } from "./string";
@@ -82,6 +85,12 @@ export const Evaluators: {
         description: "Test whether an output is malicious.",
         template: templates.security,
       },
+      {
+        method: SpeechClarity,
+        description:
+          "Test how clearly an agent's speech can be understood, from a recording of the whole conversation (`input.audio`).",
+        template: templates.speech_clarity,
+      },
       {
         method: Sql,
         description:
@@ -101,6 +110,18 @@ export const Evaluators: {
         template: templates.translation,
         requiresExtraParams: true,
       },
+      {
+        method: TurnTaking,
+        description:
+          "Test whether an agent takes turns well, without talking over or cutting off the caller, from a recording of the whole conversation (`input.audio`).",
+        template: templates.turn_taking,
+      },
+      {
+        method: VoiceTaskSuccess,
+        description:
+          "Test whether a voice agent correctly completed the caller's request, from the conversation in the trace, including its instructions, tool calls, and tool results.",
+        template: templates.voice_task_success,
+      },
     ],
   },
   {

js/templates.ts

@@ -7,9 +7,12 @@ import factuality from "../templates/factuality.yaml";
 import humor from "../templates/humor.yaml";
 import possible from "../templates/possible.yaml";
 import security from "../templates/security.yaml";
+import speech_clarity from "../templates/speech_clarity.yaml";
 import sql from "../templates/sql.yaml";
 import summary from "../templates/summary.yaml";
 import translation from "../templates/translation.yaml";
+import turn_taking from "../templates/turn_taking.yaml";
+import voice_task_success from "../templates/voice_task_success.yaml";
 
 export const modelGradedSpecSchema = z.object({
   prompt: z.string(),
@@ -18,6 +21,7 @@ export const modelGradedSpecSchema = z.object({
   use_cot: z.boolean().optional(),
   temperature: z.number().optional(),
   max_tokens: z.number().optional(),
+  audio: z.string().optional(),
 });
 
 export type ModelGradedSpec = z.infer<typeof modelGradedSpecSchema>;
@@ -29,9 +33,12 @@ const templateStrings = {
   humor,
   possible,
   security,
+  speech_clarity,
   sql,
   summary,
   translation,
+  turn_taking,
+  voice_task_success,
 } as const;
 
 // eslint-disable-next-line @typescript-eslint/consistent-type-assertions