Merge branch 'main' into create-esql-filter-from-chart

packages/kbn-doc-links/src/get_doc_links.ts

@@ -859,6 +859,7 @@ export const getDocLinks = ({ kibanaBranch, buildFlavor }: GetDocLinkOptions): D
       performancePresets: `${FLEET_DOCS}es-output-settings.html#es-output-settings-performance-tuning-settings`,
       scalingKubernetesResourcesAndLimits: `${FLEET_DOCS}scaling-on-kubernetes.html#_specifying_resources_and_limits_in_agent_manifests`,
       roleAndPrivileges: `${FLEET_DOCS}fleet-roles-and-privileges.html`,
+      proxiesSettings: `${FLEET_DOCS}fleet-agent-proxy-support.html`,
     },
     ecs: {
       guide: `${ELASTIC_WEBSITE_URL}guide/en/ecs/${ECS_VERSION}/index.html`,

packages/kbn-doc-links/src/types.ts

@@ -552,6 +552,7 @@ export interface DocLinks {
     performancePresets: string;
     scalingKubernetesResourcesAndLimits: string;
     roleAndPrivileges: string;
+    proxiesSettings: string;
   }>;
   readonly ecs: {
     readonly guide: string;

packages/kbn-search-connectors/lib/create_connector.test.ts

@@ -0,0 +1,198 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { errors } from '@elastic/elasticsearch';
+
+import { ElasticsearchClient } from '@kbn/core/server';
+import { FeatureName } from '../types';
+
+import { createConnector } from './create_connector';
+
+const notFoundError = new errors.ResponseError({
+  statusCode: 404,
+  body: {
+    error: {
+      type: `document_missing_exception`,
+    },
+  },
+} as any);
+
+describe('createConnector lib', () => {
+  const mockClient = {
+    transport: {
+      request: jest.fn(),
+    },
+  };
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should create connector with _connector API endpoint', async () => {
+    const connectorId = 'connectorId';
+    const mockConnector = {
+      id: connectorId,
+      index_name: 'indexName',
+      language: 'en',
+      is_native: true,
+    };
+    mockClient.transport.request
+      .mockResolvedValueOnce({ id: connectorId })
+      .mockResolvedValueOnce(mockConnector);
+
+    await expect(
+      createConnector(mockClient as unknown as ElasticsearchClient, {
+        isNative: true,
+        indexName: mockConnector.index_name,
+        language: mockConnector.language,
+      })
+    ).resolves.toEqual(mockConnector);
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'POST',
+      path: `/_connector`,
+      body: {
+        index_name: 'indexName',
+        language: 'en',
+        is_native: true,
+        name: '',
+      },
+    });
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'GET',
+      path: `/_connector/${connectorId}`,
+    });
+  });
+
+  it('should update pipeline params if provided', async () => {
+    const connectorId = 'connectorId';
+    const mockConnector = {
+      id: connectorId,
+      index_name: 'indexName',
+      language: 'en',
+      is_native: true,
+    };
+
+    const mockPipeline = {
+      extract_binary_content: true,
+      name: 'test',
+      reduce_whitespace: true,
+      run_ml_inference: true,
+    };
+
+    mockClient.transport.request
+      .mockResolvedValueOnce({ id: connectorId })
+      .mockResolvedValueOnce({ result: 'updated' })
+      .mockResolvedValueOnce(mockConnector);
+
+    await expect(
+      createConnector(mockClient as unknown as ElasticsearchClient, {
+        isNative: true,
+        indexName: 'indexName',
+        language: 'en',
+        pipeline: mockPipeline,
+      })
+    ).resolves.toEqual(mockConnector);
+
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'POST',
+      path: `/_connector`,
+      body: {
+        index_name: 'indexName',
+        language: 'en',
+        is_native: true,
+        name: '',
+      },
+    });
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'PUT',
+      path: `/_connector/${connectorId}/_pipeline`,
+      body: { pipeline: mockPipeline },
+    });
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'GET',
+      path: `/_connector/${connectorId}`,
+    });
+  });
+
+  it('should update connector features if provided', async () => {
+    const connectorId = 'connectorId';
+    const mockConnector = {
+      id: connectorId,
+      index_name: 'indexName',
+      language: 'en',
+      is_native: true,
+    };
+
+    const mockFeatures = {
+      [FeatureName.FILTERING_ADVANCED_CONFIG]: true,
+      [FeatureName.FILTERING_RULES]: true,
+      [FeatureName.SYNC_RULES]: {
+        advanced: { enabled: true },
+        basic: { enabled: true },
+      },
+    };
+
+    mockClient.transport.request
+      .mockResolvedValueOnce({ id: connectorId })
+      .mockResolvedValueOnce({ result: 'updated' })
+      .mockResolvedValueOnce(mockConnector);
+
+    await expect(
+      createConnector(mockClient as unknown as ElasticsearchClient, {
+        isNative: true,
+        indexName: 'indexName',
+        language: 'en',
+        features: mockFeatures,
+      })
+    ).resolves.toEqual(mockConnector);
+
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'POST',
+      path: `/_connector`,
+      body: {
+        index_name: 'indexName',
+        language: 'en',
+        is_native: true,
+        name: '',
+      },
+    });
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'PUT',
+      path: `/_connector/${connectorId}/_features`,
+      body: { features: mockFeatures },
+    });
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'GET',
+      path: `/_connector/${connectorId}`,
+    });
+  });
+
+  it('should throw an error if connector doc is not found', async () => {
+    mockClient.transport.request
+      .mockResolvedValueOnce({ id: 'connectorId' })
+      .mockRejectedValueOnce(notFoundError);
+
+    await expect(
+      createConnector(mockClient as unknown as ElasticsearchClient, {
+        isNative: true,
+        indexName: 'some-index',
+        language: 'somelang',
+      })
+    ).rejects.toEqual(new Error('Could not retrieve the created connector'));
+
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'POST',
+      path: `/_connector`,
+      body: {
+        index_name: 'some-index',
+        is_native: true,
+        language: 'somelang',
+        name: '',
+      },
+    });
+  });
+});

packages/kbn-search-connectors/lib/create_connector.ts

@@ -7,10 +7,10 @@
  */
 
 import { ElasticsearchClient } from '@kbn/core/server';
-import { CURRENT_CONNECTORS_INDEX } from '..';
+import { i18n } from '@kbn/i18n';
+import { fetchConnectorById } from '..';
 
 import { Connector, ConnectorConfiguration, IngestPipelineParams } from '../types/connectors';
-import { createConnectorDocument } from './create_connector_document';
 
 export const createConnector = async (
   client: ElasticsearchClient,
@@ -23,19 +23,46 @@ export const createConnector = async (
     name?: string;
     pipeline?: IngestPipelineParams;
     serviceType?: string | null;
-    instant_response?: boolean;
   }
 ): Promise<Connector> => {
-  const document = createConnectorDocument({
-    ...input,
-    serviceType: input.serviceType || null,
+  const { id: connectorId } = await client.transport.request<{ id: string }>({
+    method: 'POST',
+    path: `/_connector`,
+    body: {
+      ...(input.indexName && { index_name: input.indexName }),
+      is_native: input.isNative,
+      ...(input.language && { language: input.language }),
+      name: input.name || '',
+      ...(input.serviceType && { service_type: input.serviceType }),
+    },
   });
 
-  const result = await client.index({
-    document,
-    index: CURRENT_CONNECTORS_INDEX,
-    refresh: input.instant_response ? false : 'wait_for',
-  });
+  if (input.pipeline) {
+    await client.transport.request({
+      method: 'PUT',
+      path: `/_connector/${connectorId}/_pipeline`,
+      body: { pipeline: input.pipeline },
+    });
+  }
+
+  if (input.features) {
+    await client.transport.request({
+      method: 'PUT',
+      path: `/_connector/${connectorId}/_features`,
+      body: { features: input.features },
+    });
+  }
+
+  // createConnector function expects to return a Connector doc, so we fetch it from the index
+  const connector = await fetchConnectorById(client, connectorId);
+
+  if (!connector) {
+    throw new Error(
+      i18n.translate('searchConnectors.server.connectors.not_found_error', {
+        defaultMessage: 'Could not retrieve the created connector',
+      })
+    );
+  }
 
-  return { ...document, id: result._id };
+  return connector;
 };

src/plugins/esql_datagrid/public/create_datagrid.tsx

@@ -24,6 +24,7 @@ interface ESQLDataGridProps {
   query: AggregateQuery;
   flyoutType?: 'overlay' | 'push';
   isTableView?: boolean;
+  initialColumns?: DatatableColumn[];
 }
 
 const DataGridLazy = withSuspense(lazy(() => import('./data_grid')));

src/plugins/esql_datagrid/public/data_grid.tsx

@@ -30,6 +30,7 @@ interface ESQLDataGridProps {
   query: AggregateQuery;
   flyoutType?: 'overlay' | 'push';
   isTableView?: boolean;
+  initialColumns?: DatatableColumn[];
 }
 type DataTableColumnsMeta = Record<
   string,
@@ -44,7 +45,7 @@ const sortOrder: SortOrder[] = [];
 const DataGrid: React.FC<ESQLDataGridProps> = (props) => {
   const [expandedDoc, setExpandedDoc] = useState<DataTableRecord | undefined>(undefined);
   const [activeColumns, setActiveColumns] = useState<string[]>(
-    props.isTableView ? props.columns.map((c) => c.name) : []
+    (props.initialColumns || (props.isTableView ? props.columns : [])).map((c) => c.name)
   );
   const [rowHeight, setRowHeight] = useState<number>(5);
 

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/utils/connector_status_helpers.ts

@@ -44,9 +44,10 @@ export function connectorStatusToText(connector: Connector): string {
     );
   }
   if (
-    connector.error === SyncStatus.ERROR ||
-    connector.last_sync_error !== null ||
-    connector.last_access_control_sync_error !== null
+    connector.last_sync_status === SyncStatus.ERROR ||
+    connector.last_access_control_sync_status === SyncStatus.ERROR ||
+    connector.last_sync_error != null ||
+    connector.last_access_control_sync_error != null
   ) {
     return i18n.translate(
       'xpack.enterpriseSearch.content.searchIndices.connectorStatus.syncFailure.label',
@@ -87,9 +88,10 @@ export function connectorStatusToColor(connector: Connector): 'warning' | 'dange
   if (
     isLastSeenOld(connector) ||
     connectorStatus === ConnectorStatus.ERROR ||
-    connector.error === SyncStatus.ERROR ||
-    connector.last_sync_error !== null ||
-    connector.last_access_control_sync_error !== null
+    connector.last_sync_status === SyncStatus.ERROR ||
+    connector.last_access_control_sync_status === SyncStatus.ERROR ||
+    connector.last_sync_error != null ||
+    connector.last_access_control_sync_error != null
   ) {
     return 'danger';
   }

x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/download_source_flyout/index.tsx

@@ -24,13 +24,13 @@ import {
   EuiLink,
   EuiSwitch,
   EuiSpacer,
-  EuiBetaBadge,
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 
 import type { DownloadSource, FleetProxy } from '../../../../types';
 import { FLYOUT_MAX_WIDTH } from '../../constants';
 import { useBreadcrumbs, useStartServices } from '../../../../hooks';
+import { ProxyWarning } from '../fleet_proxies_table/proxy_warning';
 
 import { useDowloadSourceFlyoutForm } from './use_download_source_flyout_form';
 
@@ -142,21 +142,7 @@ export const EditDownloadSourceFlyout: React.FunctionComponent<EditDownloadSourc
             label={
               <FormattedMessage
                 id="xpack.fleet.settings.editDownloadSourcesFlyout.proxyIdLabel"
-                defaultMessage="Proxy {badge}"
-                values={{
-                  badge: (
-                    <EuiBetaBadge
-                      size="s"
-                      className="eui-alignTop"
-                      label={i18n.translate(
-                        'xpack.fleet.settings.editDownloadSourcesFlyout.proxyIdBetaBadge',
-                        {
-                          defaultMessage: 'Beta',
-                        }
-                      )}
-                    />
-                  ),
-                }}
+                defaultMessage="Proxy"
               />
             }
             helpText={
@@ -188,6 +174,8 @@ export const EditDownloadSourceFlyout: React.FunctionComponent<EditDownloadSourc
               )}
             />
           </EuiFormRow>
+          <EuiSpacer size="xs" />
+          <ProxyWarning />
           <EuiSpacer size="m" />
           <EuiFormRow fullWidth {...inputs.defaultDownloadSourceInput.formRowProps}>
             <EuiSwitch