feat(weave): playground

weave-js/src/components/FancyPage/useProjectSidebar.ts

@@ -212,6 +212,13 @@ export const useProjectSidebar = (
             isShown: isWeaveOnly,
             iconName: IconNames.CubeContainer,
           },
+          {
+            type: 'button' as const,
+            name: 'Playground',
+            slug: 'weave/playground',
+            isShown: isWeaveOnly,
+            iconName: IconNames.CubeContainer,
+          },
           {
             type: 'menuPlaceholder' as const,
             // name: 'More',
@@ -224,6 +231,7 @@ export const useProjectSidebar = (
               // 'weave/scorers', // Hiding until we want to release
               'weave/operations',
               'weave/objects',
+              'weave/playground',
             ],
           },
         ];

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx

@@ -1,6 +1,7 @@
 import Box from '@mui/material/Box';
 import {useViewerInfo} from '@wandb/weave/common/hooks/useViewerInfo';
 import {Loading} from '@wandb/weave/components/Loading';
+import {urlPrefixed} from '@wandb/weave/config';
 import {useViewTraceEvent} from '@wandb/weave/integrations/analytics/useViewEvents';
 import React, {FC, useCallback, useContext, useEffect, useState} from 'react';
 import {useHistory} from 'react-router-dom';
@@ -75,6 +76,14 @@ const useCallTabs = (call: CallSchema) => {
   const codeURI = call.opVersionRef;
   const {entity, project, callId} = call;
   const weaveRef = makeRefCall(entity, project, callId);
+
+  const handleOpenInPlayground = () => {
+    window.open(
+      urlPrefixed(`/${entity}/${project}/weave/playground/${callId}`),
+      '_blank'
+    );
+  };
+
   return [
     // Disabling Evaluation tab until it's better for single evaluation
     ...(false && isEvaluateOp(call.spanName)
@@ -101,11 +110,20 @@ const useCallTabs = (call: CallSchema) => {
           {
             label: 'Chat',
             content: (
-              <ScrollableTabContent>
-                <Tailwind>
-                  <CallChat call={call.traceCall!} />
-                </Tailwind>
-              </ScrollableTabContent>
+              <>
+                <Button
+                  variant="secondary"
+                  startIcon="sandbox-playground"
+                  className="m-16 mb-8"
+                  onClick={handleOpenInPlayground}>
+                  Open chat in playground
+                </Button>
+                <ScrollableTabContent>
+                  <Tailwind>
+                    <CallChat call={call.traceCall!} />
+                  </Tailwind>
+                </ScrollableTabContent>
+              </>
             ),
           },
         ]

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChatEmptyStateCallout.tsx

@@ -0,0 +1,57 @@
+import {Callout} from '@wandb/weave/components/Callout';
+import {Icon, IconName} from '@wandb/weave/components/Icon';
+import {Tailwind} from '@wandb/weave/components/Tailwind';
+import React from 'react';
+
+import {usePlaygroundContext} from '../PlaygroundPage/PlaygroundContext';
+
+const DEFAULT_EMPTY_STATE_MESSAGE = 'How can I help you today?';
+const DEFAULT_MESSAGES: Array<{message: string; icon: IconName}> = [
+  {
+    message:
+      'Explain the difference between concurrency and parallelism with code examples.',
+    icon: 'settings',
+  },
+  {
+    message:
+      'What are the best practices for optimizing SQL queries in a large database?',
+    icon: 'miller-columns',
+  },
+  {
+    message: 'Generate a regular expression to validate an email address.',
+    icon: 'email-envelope',
+  },
+  {
+    message:
+      'Create a REST API using Node.js and Express that handles user authentication.',
+    icon: 'lock-closed',
+  },
+];
+export const ChatEmptyStateCallout = () => {
+  const {sendMessage} = usePlaygroundContext();
+  return (
+    <Tailwind>
+      <div className="flex justify-center">
+        <div className="mt-32 flex flex-col items-center gap-8 rounded-lg bg-moon-100 p-32">
+          <Callout color="magenta" icon="buzz-bot10" size="large" />
+          <p className="text-gray-500 mb-8 text-2xl font-semibold">
+            {DEFAULT_EMPTY_STATE_MESSAGE}
+          </p>
+          {DEFAULT_MESSAGES.map(message => (
+            <div
+              key={message.message}
+              onClick={() => sendMessage?.('user', message.message)}
+              className="flex min-w-[600px] max-w-[700px] cursor-pointer items-center gap-4 rounded-full border border-moon-300 bg-white p-2">
+              <div className="flex items-center justify-center rounded-full bg-moon-100 p-6">
+                <Icon name={message.icon} />
+              </div>
+              <p className="text-gray-500 flex w-full justify-center">
+                {message.message}
+              </p>
+            </div>
+          ))}
+        </div>
+      </div>
+    </Tailwind>
+  );
+};

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/ChatView.tsx

@@ -1,6 +1,9 @@
 import React, {useEffect, useMemo, useRef} from 'react';
 
 import {useDeepMemo} from '../../../../../../hookUtils';
+import {usePlaygroundContext} from '../PlaygroundPage/PlaygroundContext';
+import {DEFAULT_SYSTEM_MESSAGE_CONTENT} from '../PlaygroundPage/usePlaygroundState';
+import {ChatEmptyStateCallout} from './ChatEmptyStateCallout';
 import {ChoicesView} from './ChoicesView';
 import {MessageList} from './MessageList';
 import {Chat} from './types';
@@ -25,6 +28,14 @@
     }
   }, [chatResult]);
 
+  const {isPlayground} = usePlaygroundContext();
+
+  const showEmptyStateCallout =
+    isPlayground &&
+    chat.request?.messages.length === 1 &&
+    chat.request.messages[0].content === DEFAULT_SYSTEM_MESSAGE_CONTENT &&
+    (chatResult?.choices.length === 0 || chatResult?.choices === undefined);
+
   return (
     <div className="flex flex-col pb-32">
       <span className="mb-[-16px] text-sm font-semibold text-moon-800">
@@ -47,6 +58,7 @@
           </div>
         </>
       )}
+      {/* {showEmptyStateCallout && <ChatEmptyStateCallout />} */}
     </div>
   );
 };

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/MessageList.tsx

@@ -63,6 +63,12 @@ const processToolCallMessages = (messages: Messages): Messages => {
       i++;
     }
 
+    // If the number of tool messages doesn't match the number of tool calls,
+    // log a warning.
+    if (toolMessages.length !== message.tool_calls.length) {
+      // console.warn('Tool call count mismatch');
+    }
+
     const toolCallsWithResponses: ToolCall[] = message.tool_calls.map(
       toolCall => ({
         ...toolCall,

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/ChatView/PlaygroundMessagePanellButtons.tsx

@@ -0,0 +1,92 @@
+import React from 'react';
+import classNames from 'classnames';
+import {Button} from '@wandb/weave/components/Button';
+
+import {usePlaygroundContext} from '../PlaygroundPage/PlaygroundContext';
+
+type PlaygroundMessagePanelButtonsProps = {
+  index: number;
+  isChoice: boolean;
+  isTool: boolean;
+  hasContent: boolean;
+  isNested: boolean;
+  contentRef: React.RefObject<HTMLDivElement>;
+  setEditorHeight: (height: number | null) => void;
+  responseIndexes?: number[];
+};
+
+export const PlaygroundMessagePanelButtons: React.FC<
+  PlaygroundMessagePanelButtonsProps
+> = ({
+  index,
+  isChoice,
+  isTool,
+  hasContent,
+  isNested,
+  contentRef,
+  setEditorHeight,
+  responseIndexes,
+}) => {
+  const {deleteMessage, deleteChoice, retry} = usePlaygroundContext();
+
+  return (
+    <div
+      className={classNames(
+        'absolute right-0 flex w-full items-center justify-end pt-20',
+        isNested ? 'bottom-0' : 'bottom-[-32px]'
+      )}>
+      <div className="z-10 flex gap-4 rounded-lg border border-moon-250 bg-white p-4">
+        <Button
+          variant="quiet"
+          size="small"
+          startIcon="randomize-reset-reload"
+          onClick={() => retry?.(index, isChoice)}
+          tooltip={
+            !hasContent
+              ? 'We currently do not support retrying functions'
+              : 'Retry'
+          }
+          disabled={!hasContent}>
+          Retry
+        </Button>
+        <Button
+          variant="quiet"
+          size="small"
+          startIcon="pencil-edit"
+          onClick={() => {
+            setEditorHeight(
+              contentRef?.current?.clientHeight
+                ? // Accounts for padding and save buttons
+                  contentRef.current.clientHeight - 56
+                : null
+            );
+          }}
+          tooltip={
+            !hasContent
+              ? 'We currently do not support editing functions'
+              : 'Edit'
+          }
+          disabled={!hasContent}>
+          Edit
+        </Button>
+        <Button
+          variant="quiet"
+          size="small"
+          startIcon="delete"
+          onClick={() => {
+            if (isChoice) {
+              deleteChoice?.(index);
+            } else {
+              deleteMessage?.(index, responseIndexes);
+            }
+          }}
+          tooltip={
+            isTool ? 'Tool responses cannot be deleted' : 'Delete message'
+          }
+          disabled={isTool}>
+          Delete
+        </Button>
+      </div>
+    </div>
+  );
+};

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundChat/PlaygroundChat.tsx

@@ -48,6 +48,16 @@
     setChatText
   );
 
+  const {handleRetry, handleSend} = useChatCompletionFunctions(
+    setPlaygroundStates,
+    setIsLoading,
+    chatText,
+    playgroundStates,
+    entity,
+    project,
+    setChatText
+  );
+
   const {deleteMessage, editMessage, deleteChoice, editChoice, addMessage} =
     useChatFunctions(setPlaygroundStateField);
 

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/PlaygroundSettings/PlaygroundSettings.tsx

@@ -137,6 +137,20 @@ export const PlaygroundSettings: React.FC<PlaygroundSettingsProps> = ({
                 label="Presence penalty"
                 value={playgroundState.presencePenalty}
               />
+
+              {/* TODO: Add this back in, N times to run is not supported for all models */}
+              {/* Shawn said to run multiple requests if its not supported */}
+              {/* <PlaygroundSlider
+                min={1}
+                max={100}
+                step={1}
+                setValue={value =>
+                  setPlaygroundStateField(idx, 'nTimes', value)
+                }
+                label="n times to run"
+                value={playgroundState.nTimes}
+              /> */}
+
               <Box
                 sx={{
                   width: '100%',

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/PlaygroundPage/llmMaxTokens.ts

@@ -118,6 +118,66 @@ export const LLM_MAX_TOKENS = {
   'o1-mini': {max_tokens: 65536, supports_function_calling: true},
   'o1-preview-2024-09-12': {max_tokens: 32768, supports_function_calling: true},
   'o1-preview': {max_tokens: 32768, supports_function_calling: true},
+
+  // These were all in our model_providers.json (but dont work)
+  // Leaving them here for now, just in case someone asks why they arent in the list.
+  // This seems like a dupe of claude-3-5-sonnet-20241022.
+  // 'anthropic/claude-3-5-sonnet-20241022': 8192,
+
+  // 422 Unprocessable Entity
+  // 'claude-2.1': 8191,
+  // 'claude-2': 8191,
+  // 'claude-instant-1.2': 8191,
+  // 'claude-instant-1': 8191,
+
+  // error litellm.BadRequestError: OpenAIException - Error code: 400 - {'error': {'message': "[{'type': 'string_type', 'loc': ('body', 'stop', 'str'), 'msg': 'Input should be a valid string', 'input': []}, {'type': 'too_short', 'loc': ('body', 'stop', 'list[str]'), 'msg': 'List should have at least 1 item after validation, not 0', 'input': [], 'ctx': {'field_type': 'List', 'min_length': 1, 'actual_length': 0}}, {'type': 'too_short', 'loc': ('body', 'stop', 'list[list[int]]'), 'msg': 'List should have at least 1 item after validation, not 0', 'input': [], 'ctx': {'field_type': 'List', 'min_length': 1, 'actual_length': 0}}]", 'type': 'invalid_request_error', 'param': None, 'code': None}}
+  // 'chatgpt-4o-latest': 4096,
+  // 'gpt-4o-audio-preview-2024-10-01': 16384,
+  // 'gpt-4o-audio-preview': 16384,
+
+  // error litellm.NotFoundError: OpenAIException - Error code: 404 - {'error': {'message': 'The model `ft:gpt-3.5-turbo-0125` does not exist or you do not have access to it.', 'type': 'invalid_request_error', 'param': None, 'code': 'model_not_found'}}
+  // 'ft:gpt-3.5-turbo-0125': 4096,
+  // 'ft:gpt-3.5-turbo-0613': 4096,
+  // 'ft:gpt-3.5-turbo-1106': 4096,
+  // 'ft:gpt-3.5-turbo': 4096,
+  // 'ft:gpt-4-0613': 4096,
+  // 'ft:gpt-4o-2024-08-06': 16384,
+  // 'ft:gpt-4o-mini-2024-07-18': 16384,
+  // 'gpt-4-32k-0613': 4096,
+  // 'gpt-4-32k': 4096,
+  // 'groq/llama-3.1-405b-reasoning': 8192,
+  // 'groq/llama2-70b-4096': 4096,
+
+  // error litellm.NotFoundError: OpenAIException - Error code: 404 - {'error': {'message': 'The model `gpt-3.5-turbo-0301` has been deprecated, learn more here: https://platform.openai.com/docs/deprecations', 'type': 'invalid_request_error', 'param': None, 'code': 'model_not_found'}}
+  // 'gpt-3.5-turbo-0301': 4096,
+  // 'gpt-3.5-turbo-0613': 4096,
+  // 'gpt-3.5-turbo-16k-0613': 4096,
+  // 'gpt-4-1106-vision-preview': 4096,
+  // 'gpt-4-vision-preview': 4096,
+
+  // error litellm.NotFoundError: VertexAIException - {
+  //   "error": {
+  //     "code": 404,
+  //     "message": "models/gemini-gemma-2-27b-it is not found for API version v1beta, or is not supported for generateContent. Call ListModels to see the list of available models and their supported methods.",
+  //     "status": "NOT_FOUND"
+  //   }
+  // }
+  // 'gemini/gemini-gemma-2-27b-it': 8192,
+  // 'gemini/gemini-gemma-2-9b-it': 8192,
+
+  // error litellm.NotFoundError: VertexAIException - {
+  //   "error": {
+  //     "code": 404,
+  //     "message": "Gemini 1.0 Pro Vision has been deprecated on July 12, 2024. Consider switching to different model, for example gemini-1.5-flash.",
+  //     "status": "NOT_FOUND"
+  //   }
+  // }
+  //   'gemini/gemini-pro-vision': 2048,
+
+  // These are 0 tokens, idk why we would want to use them.
+  // "text-moderation-007": 0,
+  // "text-moderation-latest": 0,
+  // "text-moderation-stable": 0
 };
 
 export type LLMMaxTokensKey = keyof typeof LLM_MAX_TOKENS;