Merge branch '7.16' into docs/7.16.3-rn

docs/developer/contributing/development-functional-tests.asciidoc

@@ -515,3 +515,12 @@ macOS users on a machine with a discrete graphics card may see significant speed
 * Open "Advanced GPU Settings..."
 * Uncheck the "Prefer integrated to discrete GPU" option
 * Restart iTerm
+
+[discrete]
+== Flaky Test Runner
+
+If your functional tests are flaky then the Operations team might skip them and ask that you make them less flaky before enabling them once again. This process usually involves looking at the failures which are logged on the relevant Github issue and finding incorrect assumptions or conditions which need to be awaited at some point in the test. To determine if your changes make the test fail less often you can run your tests in the Flaky Test Runner. This tool runs up to 500 executions of a specific ciGroup. To start a build of the Flaky Test Runner create a PR with your changes and then visit https://ci-stats.kibana.dev/trigger_flaky_test_runner, select your PR, choose the CI Group that your tests are in, and trigger the build.
+
+This will take you to Buildkite where your build will run and tell you if it failed in any execution.
+
+A flaky test may only fail once in 1000 runs, so keep this in mind and make sure you use enough executions to really prove that a test isn't flaky anymore.

docs/user/dashboard/make-dashboards-interactive.asciidoc

@@ -48,7 +48,7 @@ To create *Controls* panels:
 
 . Select the control panel type from the dropdown, then click *Add*.
 
-. Enter the *Control Label*, then select the *{Data-Source}* and *Field*.
+. Enter the *Control Label*, then select the *{data-source-caps}* and *Field*.
 
 . If you are adding a *Range slider*, enter the *Step Size* and *Decimal Places*.
 

docs/user/production-considerations/production.asciidoc

@@ -16,6 +16,11 @@ separate from  your {es} data or master nodes. To distribute {kib}
 traffic across the nodes in your {es} cluster,
 you can configure {kib} to use a list of {es} hosts.
 
+[WARNING]
+====
+{kib} does not support rolling upgrades, and deploying mixed versions of {kib} can result in data loss or upgrade failures. Please shut down all instances of {kib} before performing an upgrade, and ensure all running {kib} instances have matching versions.
+====
+
 [float]
 [[load-balancing-kibana]]
 === Load balancing across multiple {kib} instances
@@ -59,13 +64,18 @@ bin/kibana -c config/instance2.yml
 === Accessing multiple load-balanced {kib} clusters
 
 To access multiple load-balanced {kib} clusters from the same browser,
-set `xpack.security.cookieName` in the configuration.
+explicitly set `xpack.security.cookieName` to the same value in the {kib} configuration
+of each {kib} instance.
+
+Each {kib} cluster must have a different value of `xpack.security.cookieName`.
+
 This avoids conflicts between cookies from the different {kib} instances.
 
-In each cluster, {kib} instances should have the same `cookieName`
-value. This will achieve seamless high availability and keep the session
+This will achieve seamless high availability and keep the session
 active in case of failure from the currently used instance.
 
+
+
 [float]
 [[high-availability]]
 === High availability across multiple {es} nodes

package.json

@@ -668,7 +668,7 @@
     "callsites": "^3.1.0",
     "chai": "3.5.0",
     "chance": "1.0.18",
-    "chromedriver": "^96.0.0",
+    "chromedriver": "^97.0.0",
     "clean-webpack-plugin": "^3.0.0",
     "cmd-shim": "^2.1.0",
     "compression-webpack-plugin": "^4.0.0",
@@ -795,7 +795,7 @@
     "rxjs-marbles": "^5.0.6",
     "sass-loader": "^10.2.0",
     "sass-resources-loader": "^2.0.1",
-    "selenium-webdriver": "^4.1.0",
+    "selenium-webdriver": "^4.1.1",
     "serve-static": "1.14.1",
     "shelljs": "^0.8.4",
     "simple-git": "1.116.0",

test/plugin_functional/test_suites/saved_objects_management/hidden_types.ts

@@ -21,7 +21,8 @@ export default function ({ getService, getPageObjects }: PluginFunctionalProvide
   const esArchiver = getService('esArchiver');
   const testSubjects = getService('testSubjects');
 
-  describe('saved objects management with hidden types', () => {
+  // Failing: See https://github.com/elastic/kibana/issues/116059
+  describe.skip('saved objects management with hidden types', () => {
     before(async () => {
       await esArchiver.load(
         'test/functional/fixtures/es_archiver/saved_objects_management/hidden_types'

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/index.tsx

@@ -145,6 +145,8 @@ export const CreatePackagePolicyPage: React.FunctionComponent = () => {
     inputs: [],
   });
 
+  const [wasNewAgentPolicyCreated, setWasNewAgentPolicyCreated] = useState<boolean>(false);
+
   // Validation state
   const [validationResults, setValidationResults] = useState<PackagePolicyValidationResults>();
   const [hasAgentPolicyError, setHasAgentPolicyError] = useState<boolean>(false);
@@ -274,6 +276,10 @@ export const CreatePackagePolicyPage: React.FunctionComponent = () => {
   }, [packagePolicy, agentCount]);
   const doOnSaveNavigation = useRef<boolean>(true);
 
+  const handleInlineAgentPolicyCreate = useCallback(() => {
+    setWasNewAgentPolicyCreated(true);
+  }, []);
+
   // Detect if user left page
   useEffect(() => {
     return () => {
@@ -293,12 +299,16 @@ export const CreatePackagePolicyPage: React.FunctionComponent = () => {
         return;
       }
 
+      const packagePolicyPath = getPath('policy_details', { policyId: packagePolicy.policy_id });
+
       if (routeState?.onSaveNavigateTo && policy) {
         const [appId, options] = routeState.onSaveNavigateTo;
 
         if (options?.path) {
           const pathWithQueryString = appendOnSaveQueryParamsToPath({
-            path: options.path,
+            // In cases where we created a new agent policy inline, we need to override the initial `path`
+            // value and navigate to the newly-created agent policy instead
+            path: wasNewAgentPolicyCreated ? packagePolicyPath : options.path,
             policy,
             mappingOptions: routeState.onSaveQueryParams,
             paramsToApply,
@@ -308,10 +318,17 @@ export const CreatePackagePolicyPage: React.FunctionComponent = () => {
           handleNavigateTo(routeState.onSaveNavigateTo);
         }
       } else {
-        history.push(getPath('policy_details', { policyId: agentPolicy!.id }));
+        history.push(packagePolicyPath);
       }
     },
-    [agentPolicy, getPath, handleNavigateTo, history, routeState]
+    [
+      packagePolicy.policy_id,
+      getPath,
+      handleNavigateTo,
+      history,
+      routeState,
+      wasNewAgentPolicyCreated,
+    ]
   );
 
   const onSubmit = useCallback(async () => {
@@ -398,9 +415,16 @@ export const CreatePackagePolicyPage: React.FunctionComponent = () => {
         agentPolicy={agentPolicy}
         updateAgentPolicy={updateAgentPolicy}
         setHasAgentPolicyError={setHasAgentPolicyError}
+        onNewAgentPolicyCreate={handleInlineAgentPolicyCreate}
       />
     ),
-    [packageInfo, queryParamsPolicyId, agentPolicy, updateAgentPolicy]
+    [
+      packageInfo,
+      queryParamsPolicyId,
+      agentPolicy,
+      updateAgentPolicy,
+      handleInlineAgentPolicyCreate,
+    ]
   );
 
   const extensionView = useUIExtension(packagePolicy.package?.name ?? '', 'package-policy-create');

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/step_select_agent_policy.tsx

@@ -45,12 +45,14 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{
   agentPolicy: AgentPolicy | undefined;
   updateAgentPolicy: (agentPolicy: AgentPolicy | undefined) => void;
   setHasAgentPolicyError: (hasError: boolean) => void;
+  onNewAgentPolicyCreate: () => void;
 }> = ({
   packageInfo,
   agentPolicy,
   updateAgentPolicy,
   defaultAgentPolicyId,
   setHasAgentPolicyError,
+  onNewAgentPolicyCreate,
 }) => {
   const { isReady: isFleetReady } = useFleetStatus();
 
@@ -203,6 +205,7 @@ export const StepSelectAgentPolicy: React.FunctionComponent<{
             onClose={(newAgentPolicy?: AgentPolicy) => {
               setIsCreateAgentPolicyFlyoutOpen(false);
               if (newAgentPolicy) {
+                onNewAgentPolicyCreate();
                 refreshAgentPolicies();
                 setSelectedPolicyId(newAgentPolicy.id);
               }

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/index.tsx

@@ -65,6 +65,8 @@ import type {
 import type { PackagePolicyEditExtensionComponentProps } from '../../../types';
 import { pkgKeyFromPackageInfo, storedPackagePoliciesToAgentInputs } from '../../../services';
 
+import { hasUpgradeAvailable } from './utils';
+
 export const EditPackagePolicyPage = memo(() => {
   const {
     params: { packagePolicyId },
@@ -84,16 +86,16 @@ export const EditPackagePolicyPage = memo(() => {
       // the edit form in an "upgrade" state regardless of whether the user intended to
       // "edit" their policy or "upgrade" it. This ensures the new policy generated will be
       // set to use the latest version of the package, not its current version.
-      isUpgrade={extensionView?.useLatestPackageVersion}
+      forceUpgrade={extensionView?.useLatestPackageVersion}
     />
   );
 });
 
 export const EditPackagePolicyForm = memo<{
   packagePolicyId: string;
-  isUpgrade?: boolean;
+  forceUpgrade?: boolean;
   from?: EditPackagePolicyFrom;
-}>(({ packagePolicyId, isUpgrade = false, from = 'edit' }) => {
+}>(({ packagePolicyId, forceUpgrade = false, from = 'edit' }) => {
   const { application, notifications } = useStartServices();
   const {
     agents: { enabled: isFleetEnabled },
@@ -119,6 +121,14 @@ export const EditPackagePolicyForm = memo<{
     useState<GetOnePackagePolicyResponse['item']>();
   const [dryRunData, setDryRunData] = useState<UpgradePackagePolicyDryRunResponse>();
 
+  const [isUpgrade, setIsUpgrade] = useState<boolean>(false);
+
+  useEffect(() => {
+    if (forceUpgrade) {
+      setIsUpgrade(true);
+    }
+  }, [forceUpgrade]);
+
   const policyId = agentPolicy?.id ?? '';
 
   // Retrieve agent policy, package, and package policy info
@@ -146,11 +156,24 @@ export const EditPackagePolicyForm = memo<{
           setAgentPolicy(agentPolicyData.item);
         }
 
-        const { data: upgradePackagePolicyDryRunData } = await sendUpgradePackagePolicyDryRun([
-          packagePolicyId,
-        ]);
+        const { data: upgradePackagePolicyDryRunData, error: upgradePackagePolicyDryRunError } =
+          await sendUpgradePackagePolicyDryRun([packagePolicyId]);
 
-        if (upgradePackagePolicyDryRunData) {
+        if (upgradePackagePolicyDryRunError) {
+          throw upgradePackagePolicyDryRunError;
+        }
+
+        const hasUpgrade = upgradePackagePolicyDryRunData
+          ? hasUpgradeAvailable(upgradePackagePolicyDryRunData)
+          : false;
+
+        // If the dry run data doesn't indicate a difference in version numbers, flip the form back
+        // to its non-upgrade state, even if we were initially set to the upgrade view
+        if (!hasUpgrade) {
+          setIsUpgrade(false);
+        }
+
+        if (upgradePackagePolicyDryRunData && hasUpgrade) {
           setDryRunData(upgradePackagePolicyDryRunData);
         }
 
@@ -187,14 +210,32 @@ export const EditPackagePolicyForm = memo<{
             ...restOfPackagePolicy,
             inputs: baseInputs.map((input: any) => {
               // Remove `compiled_input` from all input info, we assign this after saving
-              const { streams, compiled_input: compiledInput, ...restOfInput } = input;
+              const { streams, compiled_input: compiledInput, vars, ...restOfInput } = input;
+              let basePolicyInputVars: any =
+                isUpgrade &&
+                basePolicy.inputs.find(
+                  (i) => i.type === input.type && i.policy_template === input.policy_template
+                )?.vars;
+              let newVars = vars;
+              if (basePolicyInputVars && vars) {
+                // merging vars from dry run with updated ones
+                basePolicyInputVars = Object.keys(vars).reduce(
+                  (acc, curr) => ({ ...acc, [curr]: basePolicyInputVars[curr] }),
+                  {}
+                );
+                newVars = {
+                  ...vars,
+                  ...basePolicyInputVars,
+                };
+              }
               return {
                 ...restOfInput,
                 streams: streams.map((stream: any) => {
                   // eslint-disable-next-line @typescript-eslint/naming-convention
                   const { compiled_stream, ...restOfStream } = stream;
                   return restOfStream;
                 }),
+                vars: newVars,
               };
             }),
             package: basePackage,
@@ -424,7 +465,7 @@ export const EditPackagePolicyForm = memo<{
   const [selectedTab, setSelectedTab] = useState(0);
 
   const layoutProps = {
-    from: extensionView?.useLatestPackageVersion ? 'upgrade-from-extension' : from,
+    from: extensionView?.useLatestPackageVersion && isUpgrade ? 'upgrade-from-extension' : from,
     cancelUrl,
     agentPolicy,
     packageInfo,

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/utils/has_upgrade_available.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import semverGt from 'semver/functions/gt';
+
+import type { UpgradePackagePolicyDryRunResponse } from '../../../../types';
+
+/**
+ * Given a dry run response, determines if a greater version exists in the "proposed"
+ * version of the first package policy in the response.
+ */
+export function hasUpgradeAvailable(dryRunData: UpgradePackagePolicyDryRunResponse) {
+  return (
+    dryRunData &&
+    dryRunData[0].diff &&
+    semverGt(
+      dryRunData[0].diff[1].package?.version ?? '',
+      dryRunData[0].diff[0].package?.version ?? ''
+    )
+  );
+}

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/edit_package_policy_page/utils/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export * from './has_upgrade_available';

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/upgrade_package_policy_page/index.tsx

@@ -30,5 +30,5 @@ export const UpgradePackagePolicyPage = memo(() => {
     from = 'upgrade-from-integrations-policy-list';
   }
 
-  return <EditPackagePolicyForm packagePolicyId={packagePolicyId} from={from} isUpgrade />;
+  return <EditPackagePolicyForm packagePolicyId={packagePolicyId} from={from} forceUpgrade />;
 });