diff --git a/x-pack/plugins/ingest_manager/server/services/agent_policy.ts b/x-pack/plugins/ingest_manager/server/services/agent_policy.ts
index 3b55f7e458141..31bf3e7207774 100644
--- a/x-pack/plugins/ingest_manager/server/services/agent_policy.ts
+++ b/x-pack/plugins/ingest_manager/server/services/agent_policy.ts
@@ -438,7 +438,7 @@ class AgentPolicyService {
   ) {
     // If Agents is not setup skip the creation of POLICY_CHANGE agent actions
     // the action will be created during the fleet setup
-    if (!isAgentsSetup(soClient)) {
+    if (!(await isAgentsSetup(soClient))) {
       return;
     }
     const policy = await agentPolicyService.getFullAgentPolicy(soClient, agentPolicyId);
diff --git a/x-pack/test/ingest_manager_api_integration/apis/settings/update.ts b/x-pack/test/ingest_manager_api_integration/apis/settings/update.ts
index 2c5d154ab0416..4bd7b81b02d57 100644
--- a/x-pack/test/ingest_manager_api_integration/apis/settings/update.ts
+++ b/x-pack/test/ingest_manager_api_integration/apis/settings/update.ts
@@ -81,6 +81,24 @@ export default function (providerContext: FtrProviderContext) {
         });
       createdAgentPolicyIds.push(testPolicyRes.item.id);
 
+      const beforeRes = await esClient.search({
+        index: '.kibana',
+        body: {
+          query: {
+            bool: {
+              must: [
+                {
+                  terms: {
+                    type: ['fleet-agent-actions'],
+                  },
+                },
+                { match: { 'fleet-agent-actions.policy_id': testPolicyRes.item.id } },
+              ],
+            },
+          },
+        },
+      });
+
       await supertest
         .put(`/api/fleet/settings`)
         .set('kbn-xsrf', 'xxxx')
@@ -105,7 +123,7 @@ export default function (providerContext: FtrProviderContext) {
         },
       });
 
-      expect(res.hits.hits.length).equal(2);
+      expect(res.hits.hits.length).equal(beforeRes.hits.hits.length + 1);
     });
   });
 }