Add Human Review Client Endpoints

src/client.ts

@@ -1,4 +1,4 @@
-import type { TimeDelta } from './types';
+import type { HumanReviewFieldContentType, TimeDelta } from './types';
 import {
   convertTimeDeltaToMilliSeconds,
   readEnv,
@@ -69,6 +69,59 @@ export enum SystemEventFilterKey {
   LABEL = 'SYSTEM:label',
 }
 
+export interface HumanReviewJob {
+  id: string;
+  name: string;
+  reviewer: { id: string; email: string };
+}
+
+export interface HumanReviewJobWithTestCases extends HumanReviewJob {
+  testCases: { id: string; status: 'Submitted' | 'Pending' }[];
+}
+
+export interface HumanReviewJobTestCaseResult {
+  id: string;
+  reviewer: { id: string; email: string };
+  status: 'Submitted' | 'Pending';
+  grades: { name: string; grade: number }[];
+  automatedEvaluations: {
+    id: string;
+    originalScore: number;
+    overrideScore: number;
+    overrideReason?: string;
+  }[];
+  inputFields: {
+    id: string;
+    name: string;
+    value: string;
+    contentType: HumanReviewFieldContentType;
+  }[];
+  outputFields: {
+    id: string;
+    name: string;
+    value: string;
+    contentType: HumanReviewFieldContentType;
+  }[];
+  fieldComments: {
+    fieldId: string;
+    startIdx?: number;
+    endIdx?: number;
+    value: string;
+    inRelationToGradeName?: string;
+    inRelationToAutomatedEvaluationId?: string;
+  }[];
+  inputComments: {
+    value: string;
+    inRelationToGradeName?: string;
+    inRelationToAutomatedEvaluationId?: string;
+  }[];
+  outputComments: {
+    value: string;
+    inRelationToGradeName?: string;
+    inRelationToAutomatedEvaluationId?: string;
+  }[];
+}
+
 interface ClientArgs {
   apiKey?: string;
   timeout?: TimeDelta;
@@ -151,7 +204,9 @@ export class AutoblocksAPIClient {
     if (args.cursor) {
       params.set('cursor', args.cursor);
     }
-    return this.get(`/views/${args.viewId}/traces?${params.toString()}`);
+    return this.get(
+      `/views/${encodeURIComponent(args.viewId)}/traces?${params.toString()}`,
+    );
   }
 
   public async searchTraces(args: {
@@ -175,4 +230,27 @@ export class AutoblocksAPIClient {
   }): Promise<{ testCases: ManagedTestCase<T>[] }> {
     return this.get(`/test-suites/${args.testSuiteId}/test-cases`);
   }
+
+  public async getHumanReviewJobs(): Promise<{ jobs: HumanReviewJob[] }> {
+    return this.get('/human-review/jobs');
+  }
+
+  public async getTestCasesForHumanReviewJob(args: {
+    jobId: string;
+  }): Promise<HumanReviewJobWithTestCases> {
+    return this.get(
+      `/human-review/jobs/${encodeURIComponent(args.jobId)}/test-cases`,
+    );
+  }
+
+  public async getTestCaseResultForHumanReviewJob(args: {
+    jobId: string;
+    testCaseId: string;
+  }): Promise<HumanReviewJobTestCaseResult> {
+    return this.get(
+      `/human-review/jobs/${encodeURIComponent(args.jobId)}/test-cases/${encodeURIComponent(
+        args.testCaseId,
+      )}`,
+    );
+  }
 }

src/index.ts

@@ -1,5 +1,12 @@
 export { AutoblocksTracer, flush } from './tracer';
 export { AutoblocksAPIClient, SystemEventFilterKey } from './client';
 export { dedent } from './util';
-export type { View, Event, Trace } from './client';
-export type { TimeDelta } from './types';
+export type {
+  View,
+  Event,
+  Trace,
+  HumanReviewJob,
+  HumanReviewJobWithTestCases,
+  HumanReviewJobTestCaseResult,
+} from './client';
+export type { TimeDelta, HumanReviewFieldContentType } from './types';

src/testing/index.ts

@@ -12,8 +12,8 @@ export {
   type EvaluationOverrideField,
   type ScoreChoice,
   type HumanReviewField,
-  HumanReviewFieldContentType,
 } from './models';
+export { HumanReviewFieldContentType } from '../types';
 export { RunManager } from './run-manager';
 export { gridSearchAsyncLocalStorage } from '../asyncLocalStorage';
 export {

src/testing/models.ts

@@ -1,4 +1,5 @@
 import { ArbitraryProperties } from '../types';
+import { HumanReviewFieldContentType } from '../types';
 
 export interface Threshold {
   lt?: number;
@@ -87,13 +88,6 @@ export abstract class BaseEvaluator<TestCaseType, OutputType>
   }): Evaluation | Promise<Evaluation>;
 }
 
-export enum HumanReviewFieldContentType {
-  TEXT = 'text',
-  LINK = 'link',
-  HTML = 'html',
-  MARKDOWN = 'markdown',
-}
-
 export interface HumanReviewField {
   name: string;
   value: string;

src/types.ts

@@ -42,3 +42,10 @@ export const zPromptSchema = z.object({
 });
 
 export type Prompt = z.infer<typeof zPromptSchema>;
+
+export enum HumanReviewFieldContentType {
+  TEXT = 'text',
+  MARKDOWN = 'markdown',
+  HTML = 'html',
+  LINK = 'link',
+}