diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx
index 319ee812a3cf3..cefc008eba992 100644
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/evaluation_settings.tsx
@@ -95,6 +95,20 @@ export const EvaluationSettings: React.FC = React.memo(() => {
     },
     [setSelectedDatasetOptions]
   );
+  const onDatasetCreateOption = useCallback(
+    (searchValue: string) => {
+      const normalizedSearchValue = searchValue.trim().toLowerCase();
+      if (!normalizedSearchValue) {
+        return;
+      }
+      const newOption = {
+        label: searchValue,
+      };
+
+      setSelectedDatasetOptions([newOption]);
+    },
+    [setSelectedDatasetOptions]
+  );
 
   // Predictions
   // Connectors / Models
@@ -244,6 +258,7 @@ export const EvaluationSettings: React.FC = React.memo(() => {
             options={datasetOptions}
             selectedOptions={selectedDatasetOptions}
             onChange={onDatasetOptionsChange}
+            onCreateOption={onDatasetCreateOption}
             compressed={true}
           />
         </EuiFormRow>
diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/translations.ts b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/translations.ts
index b1adb6296b2a1..62902d0f14095 100644
--- a/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/translations.ts
+++ b/x-pack/packages/kbn-elastic-assistant/impl/assistant/settings/evaluation_settings/translations.ts
@@ -163,7 +163,8 @@ export const EVALUATOR_DATASET_LABEL = i18n.translate(
 export const LANGSMITH_DATASET_DESCRIPTION = i18n.translate(
   'xpack.elasticAssistant.assistant.settings.evaluationSettings.langsmithDatasetDescription',
   {
-    defaultMessage: 'Name of dataset hosted on LangSmith to evaluate.',
+    defaultMessage:
+      'Name of dataset hosted on LangSmith to evaluate. Must manually enter on cloud environments.',
   }
 );