Add Human Review Client Endpoints

src/client.ts

@@ -1,4 +1,4 @@
-import type { TimeDelta } from './types';
+import type { HumanReviewFieldContentType, TimeDelta } from './types';
 import {
   convertTimeDeltaToMilliSeconds,
   readEnv,
@@ -69,6 +69,59 @@ export enum SystemEventFilterKey {
   LABEL = 'SYSTEM:label',
 }
 
+export interface HumanReviewJob {
+  id: string;
+  name: string;
+  reviewer: { id: string; email: string };
+}
+
+export interface HumanReviewJobWithTestCases extends HumanReviewJob {
+  testCases: { id: string; status: 'Submitted' | 'Pending' }[];
+}
+
+export interface HumanReviewJobTestCaseResult {
+  id: string;
+  reviewerEmail: string;
+  status: 'Submitted' | 'Pending';
+  grades: { name: string; grade: number }[];
+  automatedEvaluations: {
+    id: string;
+    originalScore: number;
+    overrideScore: number;
+    overrideReason?: string;
+  }[];
+  inputFields: {
+    id: string;
+    name: string;
+    value: string;
+    contentType: HumanReviewFieldContentType;
+  }[];
+  outputFields: {
+    id: string;
+    name: string;
+    value: string;
+    contentType: HumanReviewFieldContentType;
+  }[];
+  fieldComments: {
+    fieldId: string;
+    startIdx?: number;
+    endIdx?: number;
+    value: string;
+    inRelationToGradeName?: string;
+    inRelationToAutomatedEvaluationId?: string;
+  }[];
+  inputComments: {
+    value: string;
+    inRelationToGradeName?: string;
+    inRelationToAutomatedEvaluationId?: string;
+  }[];
+  outputComments: {
+    value: string;
+    inRelationToGradeName?: string;
+    inRelationToAutomatedEvaluationId?: string;
+  }[];
+}
+
 interface ClientArgs {
   apiKey?: string;
   timeout?: TimeDelta;
@@ -175,4 +228,21 @@ export class AutoblocksAPIClient {
   }): Promise<{ testCases: ManagedTestCase<T>[] }> {
     return this.get(`/test-suites/${args.testSuiteId}/test-cases`);
   }
+
+  public async getHumanReviewJobs(): Promise<{ jobs: HumanReviewJob[] }> {
+    return this.get('/human-review/jobs');
+  }
+
+  public async getHumanReviewJobTestCases(
+    jobId: string,
+  ): Promise<HumanReviewJobWithTestCases> {
+    return this.get(`/human-review/jobs/${jobId}/test-cases`);
+  }
+
+  public async getHumanReviewJobTestCaseResult(
+    jobId: string,
+    testCaseId: string,
+  ): Promise<HumanReviewJobTestCaseResult> {
+    return this.get(`/human-review/jobs/${jobId}/test-cases/${testCaseId}`);
+  }
 }

src/index.ts

@@ -1,5 +1,12 @@
 export { AutoblocksTracer, flush } from './tracer';
 export { AutoblocksAPIClient, SystemEventFilterKey } from './client';
 export { dedent } from './util';
-export type { View, Event, Trace } from './client';
-export type { TimeDelta } from './types';
+export type {
+  View,
+  Event,
+  Trace,
+  HumanReviewJob,
+  HumanReviewJobWithTestCases,
+  HumanReviewJobTestCaseResult,
+} from './client';
+export type { TimeDelta, HumanReviewFieldContentType } from './types';

src/testing/index.ts

@@ -12,8 +12,8 @@ export {
   type EvaluationOverrideField,
   type ScoreChoice,
   type HumanReviewField,
-  HumanReviewFieldContentType,
 } from './models';
+export { HumanReviewFieldContentType } from '../types';
 export { RunManager } from './run-manager';
 export { gridSearchAsyncLocalStorage } from '../asyncLocalStorage';
 export {

src/testing/models.ts

@@ -1,4 +1,5 @@
 import { ArbitraryProperties } from '../types';
+import { HumanReviewFieldContentType } from '../types';
 
 export interface Threshold {
   lt?: number;
@@ -87,13 +88,6 @@ export abstract class BaseEvaluator<TestCaseType, OutputType>
   }): Evaluation | Promise<Evaluation>;
 }
 
-export enum HumanReviewFieldContentType {
-  TEXT = 'text',
-  LINK = 'link',
-  HTML = 'html',
-  MARKDOWN = 'markdown',
-}
-
 export interface HumanReviewField {
   name: string;
   value: string;

src/types.ts

@@ -42,3 +42,10 @@ export const zPromptSchema = z.object({
 });
 
 export type Prompt = z.infer<typeof zPromptSchema>;
+
+export enum HumanReviewFieldContentType {
+  TEXT = 'text',
+  MARKDOWN = 'markdown',
+  HTML = 'html',
+  LINK = 'link',
+}