Add support for classifier "scorers" (#1553)

Co-authored-by: Luca Forstner <luca.forstner@gmail.com>

Author: aswink

js/dev/server.ts

@@ -8,6 +8,7 @@ import {
   EvalScorer,
   EvaluatorDef,
   OneOrMoreScores,
+  classifierName,
   scorerName,
 } from "../src/framework";
 import { errorHandler } from "./errorHandler";
@@ -117,9 +118,12 @@ export function runDevServer(
 
         evalDefs[name] = {
           parameters,
-          scores: evaluator.scores.map((score, idx) => ({
+          scores: (evaluator.scores ?? []).map((score, idx) => ({
             name: scorerName(score, idx),
           })),
+          classifiers: (evaluator.classifiers ?? []).map((classifier, idx) => ({
+            name: classifierName(classifier, idx),
+          })),
         };
       }
 
@@ -209,7 +213,7 @@ export function runDevServer(
           {
             ...evaluator,
             data: evalData.data,
-            scores: evaluator.scores.concat(
+            scores: (evaluator.scores ?? []).concat(
               scores?.map((score) =>
                 makeScorer(
                   state,

js/dev/types.ts

@@ -111,6 +111,7 @@ export type SerializedParametersContainer = z.infer<
 export const evaluatorDefinitionSchema = z.object({
   parameters: serializedParametersContainerSchema.optional(),
   scores: z.array(z.object({ name: z.string() })).optional(),
+  classifiers: z.array(z.object({ name: z.string() })).optional(),
 });
 export type EvaluatorDefinition = z.infer<typeof evaluatorDefinitionSchema>;
 

js/src/cli/functions/infer-source.ts

@@ -86,7 +86,9 @@ export async function findCodeDefinition({
       fn =
         location.position.type === "task"
           ? evaluator.task
-          : evaluator.scores[location.position.index];
+          : location.position.type === "scorer"
+            ? (evaluator.scores ?? [])[location.position.index]
+            : (evaluator.classifiers ?? [])[location.position.index];
     }
   } else if (location.type === "function") {
     fn = outFileModule.functions[location.index].handler;

js/src/cli/functions/upload.test.ts

@@ -84,4 +84,37 @@ describe("buildBundledFunctionEntry", () => {
 
     expect(entry.tags).toBeUndefined();
   });
+
+  test("preserves classifier experiment locations", async () => {
+    const entry = await buildBundledFunctionEntry({
+      spec: {
+        project_id: "proj-123",
+        name: "test-classifier",
+        slug: "test-classifier",
+        description: "Test classifier",
+        location: {
+          type: "experiment" as const,
+          eval_name: "eval-1",
+          position: {
+            type: "classifier" as const,
+            index: 0,
+          },
+        },
+        function_type: "classifier" as const,
+      },
+      runtime_context: { runtime: "node", version: "22.0.0" },
+      bundleId: "bundle-123",
+      sourceMapContext: undefined,
+    });
+
+    expect(entry.function_type).toBe("classifier");
+    expect(entry.function_data.data.location).toEqual({
+      type: "experiment",
+      eval_name: "eval-1",
+      position: {
+        type: "classifier",
+        index: 0,
+      },
+    });
+  });
 });

js/src/cli/functions/upload.ts

@@ -4,7 +4,7 @@ import {
   type IfExistsType as IfExists,
 } from "../../generated_types";
 import type { BuildSuccess, EvaluatorState, FileHandle } from "../types";
-import { scorerName, warning } from "../../framework";
+import { classifierName, scorerName, warning } from "../../framework";
 import {
   _internalGetGlobalState,
   Experiment,
@@ -181,23 +181,42 @@ export async function uploadHandleBundles({
           function_type: "task",
           origin,
         },
-        ...evaluator.evaluator.scores.map((score, i): BundledFunctionSpec => {
-          const name = scorerName(score, i);
-          return {
-            ...baseInfo,
-            // There is a very small chance that someone names a function with the same convention, but
-            // let's assume it's low enough that it doesn't matter.
-            ...formatNameAndSlug(["eval", namePrefix, "scorer", name]),
-            description: `Score ${name} for eval ${namePrefix}`,
-            location: {
-              type: "experiment",
-              eval_name: evaluator.evaluator.evalName,
-              position: { type: "scorer", index: i },
-            },
-            function_type: "scorer",
-            origin,
-          };
-        }),
+        ...(evaluator.evaluator.scores ?? []).map(
+          (score, i): BundledFunctionSpec => {
+            const name = scorerName(score, i);
+            return {
+              ...baseInfo,
+              // There is a very small chance that someone names a function with the same convention, but
+              // let's assume it's low enough that it doesn't matter.
+              ...formatNameAndSlug(["eval", namePrefix, "scorer", name]),
+              description: `Score ${name} for eval ${namePrefix}`,
+              location: {
+                type: "experiment",
+                eval_name: evaluator.evaluator.evalName,
+                position: { type: "scorer", index: i },
+              },
+              function_type: "scorer",
+              origin,
+            };
+          },
+        ),
+        ...(evaluator.evaluator.classifiers ?? []).map(
+          (classifier, i): BundledFunctionSpec => {
+            const name = classifierName(classifier, i);
+            return {
+              ...baseInfo,
+              ...formatNameAndSlug(["eval", namePrefix, "classifier", name]),
+              description: `Classifier ${name} for eval ${namePrefix}`,
+              location: {
+                type: "experiment",
+                eval_name: evaluator.evaluator.evalName,
+                position: { type: "classifier", index: i },
+              },
+              function_type: "classifier",
+              origin,
+            };
+          },
+        ),
       ];
 
       bundleSpecs.push(...fileSpecs);
@@ -220,9 +239,14 @@ export async function uploadHandleBundles({
                   serializeRemoteEvalParametersContainer(resolvedParameters),
               }
             : {}),
-          scores: evaluator.evaluator.scores.map((score, i) => ({
+          scores: (evaluator.evaluator.scores ?? []).map((score, i) => ({
             name: scorerName(score, i),
           })),
+          classifiers: (evaluator.evaluator.classifiers ?? []).map(
+            (classifier, i) => ({
+              name: classifierName(classifier, i),
+            }),
+          ),
         };
 
         bundleSpecs.push({

js/src/exports.ts

@@ -193,6 +193,7 @@ export type {
   EvalResult,
   EvalScorerArgs,
   EvalScorer,
+  EvalClassifier,
   EvaluatorDef,
   EvaluatorFile,
   ReporterBody,