Merge branch 'main' into 191206-logs-anomalies--and-categories-access

oas_docs/output/kibana.serverless.staging.yaml

@@ -8080,6 +8080,7 @@ paths:
         - Security Endpoint Management API
   /api/endpoint/metadata/transforms:
     get:
+      deprecated: true
       operationId: GetEndpointMetadataTransform
       responses:
         '200':

oas_docs/output/kibana.serverless.yaml

@@ -8080,6 +8080,7 @@ paths:
         - Security Endpoint Management API
   /api/endpoint/metadata/transforms:
     get:
+      deprecated: true
       operationId: GetEndpointMetadataTransform
       responses:
         '200':

oas_docs/output/kibana.staging.yaml

@@ -11452,6 +11452,7 @@ paths:
         - Security Endpoint Management API
   /api/endpoint/metadata/transforms:
     get:
+      deprecated: true
       operationId: GetEndpointMetadataTransform
       responses:
         '200':

oas_docs/output/kibana.yaml

@@ -11452,6 +11452,7 @@ paths:
         - Security Endpoint Management API
   /api/endpoint/metadata/transforms:
     get:
+      deprecated: true
       operationId: GetEndpointMetadataTransform
       responses:
         '200':

package.json

@@ -1099,7 +1099,7 @@
     "del": "^6.1.0",
     "diff": "^5.1.0",
     "dotenv": "^16.4.5",
-    "elastic-apm-node": "^4.7.3",
+    "elastic-apm-node": "^4.8.0",
     "email-addresses": "^5.0.0",
     "eventsource-parser": "^1.1.1",
     "execa": "^5.1.1",

x-pack/plugins/cloud_security_posture/common/constants.ts

@@ -171,4 +171,4 @@ export const SINGLE_ACCOUNT = 'single-account';
 
 export const CLOUD_SECURITY_PLUGIN_VERSION = '1.9.0';
 // Cloud Credentials Template url was implemented in 1.10.0-preview01. See PR - https://github.com/elastic/integrations/pull/9828
-export const CLOUD_CREDENTIALS_PACKAGE_VERSION = '1.10.0-preview01';
+export const CLOUD_CREDENTIALS_PACKAGE_VERSION = '1.11.0-preview10';

x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_form.tsx

@@ -674,6 +674,7 @@ export const CspPolicyTemplateForm = memo<PackagePolicyReplaceDefineStepExtensio
     const integration = SUPPORTED_POLICY_TEMPLATES.includes(integrationParam)
       ? integrationParam
       : undefined;
+    const isParentSecurityPosture = !integration;
     // Handling validation state
     const [isValid, setIsValid] = useState(true);
     const { cloud } = useKibana().services;
@@ -798,6 +799,12 @@ export const CspPolicyTemplateForm = memo<PackagePolicyReplaceDefineStepExtensio
       // Required for mount only to ensure a single input type is selected
       // This will remove errors in validationResults.vars
       setEnabledPolicyInput(DEFAULT_INPUT_TYPE[input.policy_template]);
+
+      // When the integration is the parent Security Posture (!integration) we need to
+      // reset the setup technology when the integration option changes if it was set to agentless for CSPM
+      if (isParentSecurityPosture && input.policy_template !== 'cspm') {
+        updateSetupTechnology(SetupTechnology.AGENT_BASED);
+      }
       refetch();
       // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [isLoading, input.policy_template, isEditPage]);

x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/policy_template_selectors.tsx

@@ -57,7 +57,11 @@ export const PolicyTemplateSelector = ({
       </EuiText>
       <EuiSpacer size="m" />
       <RadioGroup
-        options={Array.from(policyTemplates, (v) => ({ id: v, label: getPolicyTemplateLabel(v) }))}
+        options={Array.from(policyTemplates, (v) => ({
+          id: v,
+          label: getPolicyTemplateLabel(v),
+          testId: `policy-template-radio-button-${v}`,
+        }))}
         idSelected={selectedTemplate}
         onChange={(id: CloudSecurityPolicyTemplate) => setPolicyTemplate(id)}
         disabled={disabled}

x-pack/plugins/cloud_security_posture/public/components/fleet_extensions/setup_technology_selector/use_setup_technology.ts

@@ -27,7 +27,8 @@ export const useSetupTechnology = ({
   const isAgentlessSupportedForCloudProvider = isCspmAws || isCspmGcp || isCspmAzure;
   const isAgentlessAvailable = isAgentlessSupportedForCloudProvider && isAgentlessEnabled;
   const defaultSetupTechnology =
-    isEditPage && isAgentlessEnabled ? SetupTechnology.AGENTLESS : SetupTechnology.AGENT_BASED;
+    isEditPage && isAgentlessAvailable ? SetupTechnology.AGENTLESS : SetupTechnology.AGENT_BASED;
+
   const [setupTechnology, setSetupTechnology] = useState<SetupTechnology>(defaultSetupTechnology);
 
   const updateSetupTechnology = (value: SetupTechnology) => {

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/single_page_layout/index.tsx

@@ -350,7 +350,7 @@ export const CreatePackagePolicySinglePage: CreatePackagePolicyParams = ({
       "'package-policy-create' and 'package-policy-replace-define-step' cannot both be registered as UI extensions"
     );
   }
-  const { isAgentlessEnabled, isAgentlessIntegration } = useAgentless();
+  const { isAgentlessIntegration } = useAgentless();
   const { handleSetupTechnologyChange, selectedSetupTechnology } = useSetupTechnology({
     newAgentPolicy,
     setNewAgentPolicy,
@@ -374,7 +374,7 @@ export const CreatePackagePolicySinglePage: CreatePackagePolicyParams = ({
             validationResults={validationResults}
             isEditPage={false}
             handleSetupTechnologyChange={handleSetupTechnologyChange}
-            isAgentlessEnabled={isAgentlessEnabled}
+            isAgentlessEnabled={isAgentlessIntegration(packageInfo)}
           />
         </ExtensionWrapper>
       )

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx

@@ -426,7 +426,7 @@ export const PackagePoliciesTable: React.FunctionComponent<Props> = ({
                         ) : (
                           <FormattedMessage
                             id="xpack.fleet.epm.addPackagePolicyButtonPrivilegesRequiredTooltip"
-                            defaultMessage="Elastic Agent Integrations require the All privilege for Fleet and All privilege for Integrations. Contact your administrator."
+                            defaultMessage="Elastic Agent Integrations require the All privilege for Agent policies and All privilege for Integrations. Contact your administrator."
                           />
                         ),
                       }

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx

@@ -103,7 +103,7 @@ export const EditPackagePolicyForm = memo<{
   } = useConfig();
   const { getHref } = useLink();
   const { canUseMultipleAgentPolicies } = useMultipleAgentPolicies();
-  const { isAgentlessAgentPolicy } = useAgentless();
+  const { isAgentlessAgentPolicy, isAgentlessIntegration } = useAgentless();
   const {
     // data
     agentPolicies: existingAgentPolicies,
@@ -130,9 +130,10 @@ export const EditPackagePolicyForm = memo<{
   const hasAgentlessAgentPolicy = useMemo(
     () =>
       existingAgentPolicies.length === 1
-        ? existingAgentPolicies.some((policy) => isAgentlessAgentPolicy(policy))
+        ? existingAgentPolicies.some((policy) => isAgentlessAgentPolicy(policy)) &&
+          isAgentlessIntegration(packageInfo)
         : false,
-    [existingAgentPolicies, isAgentlessAgentPolicy]
+    [existingAgentPolicies, isAgentlessAgentPolicy, packageInfo, isAgentlessIntegration]
   );
 
   const canWriteIntegrationPolicies = useAuthz().integrations.writeIntegrationPolicies;

x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/add_integration_button.tsx

@@ -32,7 +32,7 @@ export function AddIntegrationButton(props: AddIntegrationButtonProps) {
         ) : (
           <FormattedMessage
             id="xpack.fleet.epm.addPackagePolicyButtonPrivilegesRequiredTooltip"
-            defaultMessage="Elastic Agent Integrations require the All privilege for Fleet and All privilege for Integrations. Contact your administrator."
+            defaultMessage="Elastic Agent Integrations require the All privilege for Agent policies and All privilege for Integrations. Contact your administrator."
           />
         ),
       }

x-pack/plugins/integration_assistant/server/graphs/ecs/validate.test.ts

@@ -8,14 +8,14 @@
 import { ECS_RESERVED } from './constants';
 
 import {
+  extractECSMapping,
   findDuplicateFields,
   findInvalidEcsFields,
-  processMapping,
   removeReservedFields,
 } from './validate';
 
 describe('Testing ecs handler', () => {
-  it('processMapping()', async () => {
+  it('extractECSMapping()', async () => {
     const path: string[] = [];
     const value = {
       checkpoint: {
@@ -50,7 +50,7 @@ describe('Testing ecs handler', () => {
       },
     };
     const output: Record<string, string[][]> = {};
-    await processMapping(path, value, output);
+    await extractECSMapping(path, value, output);
     expect(output).toEqual({
       'source.address': [['checkpoint', 'firewall', 'origin']],
       'user.name': [['checkpoint', 'firewall', 'administrator']],
@@ -96,6 +96,110 @@ describe('findInvalidEcsFields', () => {
     const invalid = findInvalidEcsFields(ecsMappingReserved);
     expect(invalid.length).toBe(1);
   });
+
+  it('invalid: date_format fields (natural example)', async () => {
+    const misspelledDateFormatMapping = {
+      ai_postgres_202410050058: {
+        logs: {
+          column1: {
+            target: 'event.created',
+            confidence: 0.9,
+            type: 'date',
+            date_format: ['yyyy-MM-dd HH:mm:ss.SSS z'],
+          },
+          column12: {
+            target: 'log.level',
+            confidence: 0.95,
+            type: 'string',
+            date_format: [],
+          },
+          column11: null,
+          column4: null,
+          column9: {
+            target: 'event.start',
+            confidence: 0.8,
+            type: 'date',
+            date_format: ['yyyy-MM-dd HH:mm:ss z'],
+          },
+          column7: null,
+          column6: null,
+          column14: {
+            target: 'event.reason',
+            confidence: 0.7,
+            type: 'string',
+            date_format: [],
+          },
+          column13: null,
+          column24: {
+            target: 'process.name',
+            confidence: 0.85,
+            type: 'string',
+            date_format: [],
+          },
+          column23: null,
+          column10: null,
+          column5: {
+            target: 'source.address',
+            confidence: 0.9,
+            type: 'string',
+            date_format: [],
+          },
+          column3: {
+            target: 'user.name',
+            confidence: 0.8,
+            type: 'string',
+            date_format: [],
+          },
+          column2: {
+            target: 'destination.user.name',
+            confidence: 0.7,
+            type: 'string',
+            date_format: [],
+          },
+          column8: null,
+        },
+      },
+    };
+
+    const invalid = findInvalidEcsFields(misspelledDateFormatMapping);
+    expect(invalid.length).toBe(1);
+  });
+
+  it('invalid: date_format fields (handcrafted example)', async () => {
+    const mixedMapping = {
+      some_title: {
+        logs: {
+          column1: {
+            target: 'event.created',
+            confidence: 0.9,
+            type: 'date',
+            date_format: ['yyyy-MM-dd HH:mm:ss.SSS z'],
+          },
+          column12: {
+            target: 'log.level',
+            confidence: 0.95,
+            type: 'string',
+            date_formats: [],
+          },
+          column11: null,
+          column4: null,
+          column9: {
+            target: 'event.start',
+            confidence: 0.8,
+            type: 'date',
+            date_format: 'yyyy-MM-dd HH:mm:ss z',
+          },
+          column2: {
+            target: 'destination.user.name',
+            type: 'string',
+            date_format: [],
+          },
+        },
+      },
+    };
+    const invalid = findInvalidEcsFields(mixedMapping);
+    expect(invalid.length).toBe(1);
+  });
 });
 
 describe('findDuplicateFields', () => {