From 8d3c7ea7f8844578576c5451a00ca3007cbc7b40 Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Wed, 4 Jan 2023 20:25:30 -0500
Subject: [PATCH 01/14] ci: update cleanup script to reduce throttling

---
 .../src/cleanup-e2e-resources.ts              | 156 ++++++++++++------
 1 file changed, 105 insertions(+), 51 deletions(-)

diff --git a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
index 6247560f626..dbbdf37461e 100644
--- a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
+++ b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
@@ -21,6 +21,11 @@ const AWS_REGIONS_TO_RUN_TESTS = [
   'ap-southeast-2',
 ];
 
+const DELETE_LIMIT_PER_BATCH = {
+  OTHER: 50,
+  CFN_STACK: 100,
+}
+
 const reportPath = path.normalize(path.join(__dirname, '..', 'amplify-e2e-reports', 'stale-resources.json'));
 
 const MULTI_JOB_APP = '<Amplify App reused by multiple apps>';
@@ -88,8 +93,8 @@ type ReportEntry = {
   workflowId?: string;
   lifecycle?: string;
   cciJobDetails?: CircleCIJobDetails;
-  amplifyApps: Record<string, AmplifyAppInfo>;
-  stacks: Record<string, StackInfo>;
+  amplifyApps: AmplifyAppInfo[];
+  stacks: StackInfo[];
   buckets: Record<string, S3BucketInfo>;
   roles: Record<string, IamRoleInfo>;
   pinpointApps: Record<string, PinpointAppInfo>;
@@ -229,29 +234,34 @@ const getAWSConfig = ({ accessKeyId, secretAccessKey, sessionToken }: AWSAccount
  */
 const getAmplifyApps = async (account: AWSAccountInfo, region: string): Promise<AmplifyAppInfo[]> => {
   const amplifyClient = new aws.Amplify(getAWSConfig(account, region));
-  const amplifyApps = await amplifyClient.listApps({ maxResults: 50 }).promise(); // keeping it to 50 as max supported is 50
-  const result: AmplifyAppInfo[] = [];
-  for (const app of amplifyApps.apps) {
-    const backends: Record<string, StackInfo> = {};
-    try {
-      const backendEnvironments = await amplifyClient.listBackendEnvironments({ appId: app.appId, maxResults: 50 }).promise();
-      for (const backendEnv of backendEnvironments.backendEnvironments) {
-        const buildInfo = await getStackDetails(backendEnv.stackName, account, region);
-        if (buildInfo) {
-          backends[backendEnv.environmentName] = buildInfo;
+  try {
+    const amplifyApps = await amplifyClient.listApps({ maxResults: 25 }).promise(); // keeping it to 25 as max supported is 25
+    const result: AmplifyAppInfo[] = [];
+    for (const app of amplifyApps.apps) {
+      const backends: Record<string, StackInfo> = {};
+      try {
+        const backendEnvironments = await amplifyClient.listBackendEnvironments({ appId: app.appId, maxResults: 5 }).promise();
+        for (const backendEnv of backendEnvironments.backendEnvironments) {
+          const buildInfo = await getStackDetails(backendEnv.stackName, account, region);
+          if (buildInfo) {
+            backends[backendEnv.environmentName] = buildInfo;
+          }
         }
+      } catch (e) {
+        // console.log(e);
       }
-    } catch (e) {
-      console.log(e);
+      result.push({
+        appId: app.appId,
+        name: app.name,
+        region,
+        backends,
+      });
     }
-    result.push({
-      appId: app.appId,
-      name: app.name,
-      region,
-      backends,
-    });
+    return result;
+  } catch (e){
+    console.log(e);
+    return [];
   }
-  return result;
 };
 
 /**
@@ -317,7 +327,13 @@ const getStacks = async (account: AWSAccountInfo, region: string): Promise<Stack
     .promise();
 
   // We are interested in only the root stacks that are deployed by amplify-cli
-  const rootStacks = stacks.StackSummaries.filter(stack => !stack.RootId);
+  let rootStacks = stacks.StackSummaries.filter(stack => !stack.RootId);
+  if(rootStacks.length > 20){
+    // we can only delete 100 stacks accross all regions every batch,
+    // so we shouldn't take more than 20 apps from each of 8 regions.
+    // this should at least limit calls to getStackDetails below
+    rootStacks = rootStacks.slice(0, 20);
+  }
   const results: StackInfo[] = [];
   for (const stack of rootStacks) {
     try {
@@ -511,17 +527,20 @@ const mergeResourcesByCCIJob = (
 };
 
 const deleteAmplifyApps = async (account: AWSAccountInfo, accountIndex: number, apps: AmplifyAppInfo[]): Promise<void> => {
-  await Promise.all(apps.map(app => deleteAmplifyApp(account, accountIndex, app)));
+  if(apps.length > 50){
+    // throttle delete calls
+    await Promise.all(apps.slice(0, 50).map(app => deleteAmplifyApp(account, accountIndex, app)));
+  }
 };
 
 const deleteAmplifyApp = async (account: AWSAccountInfo, accountIndex: number, app: AmplifyAppInfo): Promise<void> => {
   const { name, appId, region } = app;
-  console.log(`[ACCOUNT ${accountIndex}] Deleting App ${name}(${appId})`);
+  // console.log(`[ACCOUNT ${accountIndex}] Deleting App ${name}(${appId})`);
   const amplifyClient = new aws.Amplify(getAWSConfig(account, region));
   try {
     await amplifyClient.deleteApp({ appId }).promise();
   } catch (e) {
-    console.log(`[ACCOUNT ${accountIndex}] Deleting Amplify App ${appId} failed with the following error`, e);
+    // console.log(`[ACCOUNT ${accountIndex}] Deleting Amplify App ${appId} failed with the following error`, e);
     if (e.code === 'ExpiredTokenException') {
       handleExpiredTokenException();
     }
@@ -529,20 +548,23 @@ const deleteAmplifyApp = async (account: AWSAccountInfo, accountIndex: number, a
 };
 
 const deleteIamRoles = async (account: AWSAccountInfo, accountIndex: number, roles: IamRoleInfo[]): Promise<void> => {
-  await Promise.all(roles.map(role => deleteIamRole(account, accountIndex, role)));
+  if(roles.length > 50){
+    // throttle delete calls
+    await Promise.all(roles.slice(0, 50).map(role => deleteIamRole(account, accountIndex, role)));
+  }
 };
 
 const deleteIamRole = async (account: AWSAccountInfo, accountIndex: number, role: IamRoleInfo): Promise<void> => {
   const { name: roleName } = role;
   try {
-    console.log(`[ACCOUNT ${accountIndex}] Deleting Iam Role ${roleName}`);
-    console.log(`Role creation time (PST): ${role.createTime.toLocaleTimeString('en-US', { timeZone: 'America/Los_Angeles' })}`);
+    // console.log(`[ACCOUNT ${accountIndex}] Deleting Iam Role ${roleName}`);
+    // console.log(`Role creation time (PST): ${role.createTime.toLocaleTimeString('en-US', { timeZone: 'America/Los_Angeles' })}`);
     const iamClient = new aws.IAM(getAWSConfig(account));
     await deleteAttachedRolePolicies(account, accountIndex, roleName);
     await deleteRolePolicies(account, accountIndex, roleName);
     await iamClient.deleteRole({ RoleName: roleName }).promise();
   } catch (e) {
-    console.log(`[ACCOUNT ${accountIndex}] Deleting iam role ${roleName} failed with error ${e.message}`);
+    // console.log(`[ACCOUNT ${accountIndex}] Deleting iam role ${roleName} failed with error ${e.message}`);
     if (e.code === 'ExpiredTokenException') {
       handleExpiredTokenException();
     }
@@ -566,11 +588,11 @@ const detachIamAttachedRolePolicy = async (
   policy: aws.IAM.AttachedPolicy,
 ): Promise<void> => {
   try {
-    console.log(`[ACCOUNT ${accountIndex}] Detach Iam Attached Role Policy ${policy.PolicyName}`);
+    // console.log(`[ACCOUNT ${accountIndex}] Detach Iam Attached Role Policy ${policy.PolicyName}`);
     const iamClient = new aws.IAM(getAWSConfig(account));
     await iamClient.detachRolePolicy({ RoleName: roleName, PolicyArn: policy.PolicyArn }).promise();
   } catch (e) {
-    console.log(`[ACCOUNT ${accountIndex}] Detach iam role policy ${policy.PolicyName} failed with error ${e.message}`);
+    // console.log(`[ACCOUNT ${accountIndex}] Detach iam role policy ${policy.PolicyName} failed with error ${e.message}`);
     if (e.code === 'ExpiredTokenException') {
       handleExpiredTokenException();
     }
@@ -594,11 +616,11 @@ const deleteIamRolePolicy = async (
   policyName: string,
 ): Promise<void> => {
   try {
-    console.log(`[ACCOUNT ${accountIndex}] Deleting Iam Role Policy ${policyName}`);
+    // console.log(`[ACCOUNT ${accountIndex}] Deleting Iam Role Policy ${policyName}`);
     const iamClient = new aws.IAM(getAWSConfig(account));
     await iamClient.deleteRolePolicy({ RoleName: roleName, PolicyName: policyName }).promise();
   } catch (e) {
-    console.log(`[ACCOUNT ${accountIndex}] Deleting iam role policy ${policyName} failed with error ${e.message}`);
+    // console.log(`[ACCOUNT ${accountIndex}] Deleting iam role policy ${policyName} failed with error ${e.message}`);
     if (e.code === 'ExpiredTokenException') {
       handleExpiredTokenException();
     }
@@ -606,18 +628,21 @@ const deleteIamRolePolicy = async (
 };
 
 const deleteBuckets = async (account: AWSAccountInfo, accountIndex: number, buckets: S3BucketInfo[]): Promise<void> => {
-  await Promise.all(buckets.map(bucket => deleteBucket(account, accountIndex, bucket)));
+  if(buckets.length > 50){
+    // throttle delete calls
+    await Promise.all(buckets.slice(0, 50).map(bucket => deleteBucket(account, accountIndex, bucket)));
+  }
 };
 
 const deleteBucket = async (account: AWSAccountInfo, accountIndex: number, bucket: S3BucketInfo): Promise<void> => {
   const { name } = bucket;
   try {
-    console.log(`[ACCOUNT ${accountIndex}] Deleting S3 Bucket ${name}`);
-    console.log(`Bucket creation time (PST): ${bucket.createTime.toLocaleTimeString('en-US', { timeZone: 'America/Los_Angeles' })}`);
+    // console.log(`[ACCOUNT ${accountIndex}] Deleting S3 Bucket ${name}`);
+    // console.log(`Bucket creation time (PST): ${bucket.createTime.toLocaleTimeString('en-US', { timeZone: 'America/Los_Angeles' })}`);
     const s3 = new aws.S3(getAWSConfig(account));
     await deleteS3Bucket(name, s3);
   } catch (e) {
-    console.log(`[ACCOUNT ${accountIndex}] Deleting bucket ${name} failed with error ${e.message}`);
+    // console.log(`[ACCOUNT ${accountIndex}] Deleting bucket ${name} failed with error ${e.message}`);
     if (e.code === 'ExpiredTokenException') {
       handleExpiredTokenException();
     }
@@ -625,7 +650,10 @@ const deleteBucket = async (account: AWSAccountInfo, accountIndex: number, bucke
 };
 
 const deletePinpointApps = async (account: AWSAccountInfo, accountIndex: number, apps: PinpointAppInfo[]): Promise<void> => {
-  await Promise.all(apps.map(app => deletePinpointApp(account, accountIndex, app)));
+  if(apps.length > 50){
+    // throttle delete calls
+    await Promise.all(apps.slice(0, 50).map(app => deletePinpointApp(account, accountIndex, app)));
+  }
 };
 
 const deletePinpointApp = async (account: AWSAccountInfo, accountIndex: number, app: PinpointAppInfo): Promise<void> => {
@@ -633,17 +661,20 @@ const deletePinpointApp = async (account: AWSAccountInfo, accountIndex: number,
     id, name, region,
   } = app;
   try {
-    console.log(`[ACCOUNT ${accountIndex}] Deleting Pinpoint App ${name}`);
-    console.log(`Pinpoint creation time (PST): ${app.createTime.toLocaleTimeString('en-US', { timeZone: 'America/Los_Angeles' })}`);
+    // console.log(`[ACCOUNT ${accountIndex}] Deleting Pinpoint App ${name}`);
+    // console.log(`Pinpoint creation time (PST): ${app.createTime.toLocaleTimeString('en-US', { timeZone: 'America/Los_Angeles' })}`);
     const pinpoint = new aws.Pinpoint(getAWSConfig(account, region));
     await pinpoint.deleteApp({ ApplicationId: id }).promise();
   } catch (e) {
-    console.log(`[ACCOUNT ${accountIndex}] Deleting pinpoint app ${name} failed with error ${e.message}`);
+    // console.log(`[ACCOUNT ${accountIndex}] Deleting pinpoint app ${name} failed with error ${e.message}`);
   }
 };
 
 const deleteAppSyncApis = async (account: AWSAccountInfo, accountIndex: number, apis: AppSyncApiInfo[]): Promise<void> => {
-  await Promise.all(apis.map(api => deleteAppSyncApi(account, accountIndex, api)));
+  if(apis.length > 50){
+    // throttle delete calls
+    await Promise.all(apis.slice(0, 50).map(api => deleteAppSyncApi(account, accountIndex, api)));
+  }
 };
 
 const deleteAppSyncApi = async (account: AWSAccountInfo, accountIndex: number, api: AppSyncApiInfo): Promise<void> => {
@@ -651,29 +682,32 @@ const deleteAppSyncApi = async (account: AWSAccountInfo, accountIndex: number, a
     apiId, name, region,
   } = api;
   try {
-    console.log(`[ACCOUNT ${accountIndex}] Deleting AppSync Api ${name}`);
+    // console.log(`[ACCOUNT ${accountIndex}] Deleting AppSync Api ${name}`);
     const appSync = new aws.AppSync(getAWSConfig(account, region));
     await appSync.deleteGraphqlApi({ apiId }).promise();
   } catch (e) {
-    console.log(`[ACCOUNT ${accountIndex}] Deleting AppSync Api ${name} failed with error ${e.message}`);
+    // console.log(`[ACCOUNT ${accountIndex}] Deleting AppSync Api ${name} failed with error ${e.message}`);
   }
 };
 
 const deleteCfnStacks = async (account: AWSAccountInfo, accountIndex: number, stacks: StackInfo[]): Promise<void> => {
-  await Promise.all(stacks.map(stack => deleteCfnStack(account, accountIndex, stack)));
+  if(stacks.length > 100){
+    // throttle delete calls, 100 seems to work fine for stacks
+    await Promise.all(stacks.slice(0, 100).map(stack => deleteCfnStack(account, accountIndex, stack)));
+  }
 };
 
 const deleteCfnStack = async (account: AWSAccountInfo, accountIndex: number, stack: StackInfo): Promise<void> => {
   const { stackName, region, resourcesFailedToDelete } = stack;
   const resourceToRetain = resourcesFailedToDelete.length ? resourcesFailedToDelete : undefined;
-  console.log(`[ACCOUNT ${accountIndex}] Deleting CloudFormation stack ${stackName}`);
+  // console.log(`[ACCOUNT ${accountIndex}] Deleting CloudFormation stack ${stackName}`);
   try {
     const cfnClient = new aws.CloudFormation(getAWSConfig(account, region));
     await cfnClient.deleteStack({ StackName: stackName, RetainResources: resourceToRetain }).promise();
-    // we'll only wait up to 10 minutes before moving on
-    await cfnClient.waitFor('stackDeleteComplete', { StackName: stackName, $waiter: { maxAttempts: 20 } }).promise();
+    // we'll only wait up to a minute before moving on
+    // await cfnClient.waitFor('stackDeleteComplete', { StackName: stackName, $waiter: { maxAttempts: 2 } }).promise();
   } catch (e) {
-    console.log(`Deleting CloudFormation stack ${stackName} failed with error ${e.message}`);
+    // console.log(`Deleting CloudFormation stack ${stackName} failed with error ${e.message}`);
     if (e.code === 'ExpiredTokenException') {
       handleExpiredTokenException();
     }
@@ -697,26 +731,32 @@ const deleteResources = async (
   for (const jobId of Object.keys(staleResources)) {
     const resources = staleResources[jobId];
     if (resources.amplifyApps) {
+      console.log(`Deleting up to ${DELETE_LIMIT_PER_BATCH.OTHER} of ${resources.amplifyApps.length} apps on ACCOUNT[${accountIndex}]`);
       await deleteAmplifyApps(account, accountIndex, Object.values(resources.amplifyApps));
     }
 
     if (resources.stacks) {
+      console.log(`Deleting up to ${DELETE_LIMIT_PER_BATCH.CFN_STACK} of ${resources.stacks.length} stacks on ACCOUNT[${accountIndex}]`);
       await deleteCfnStacks(account, accountIndex, Object.values(resources.stacks));
     }
 
     if (resources.buckets) {
+      console.log(`Deleting up to ${DELETE_LIMIT_PER_BATCH.OTHER} of ${resources.buckets.length} buckets on ACCOUNT[${accountIndex}]`);
       await deleteBuckets(account, accountIndex, Object.values(resources.buckets));
     }
 
     if (resources.roles) {
+      console.log(`Deleting up to ${DELETE_LIMIT_PER_BATCH.OTHER} of ${resources.roles.length} roles on ACCOUNT[${accountIndex}]`);
       await deleteIamRoles(account, accountIndex, Object.values(resources.roles));
     }
 
     if (resources.pinpointApps) {
+      console.log(`Deleting up to ${DELETE_LIMIT_PER_BATCH.OTHER} of ${resources.pinpointApps.length} pinpoint apps on ACCOUNT[${accountIndex}]`);
       await deletePinpointApps(account, accountIndex, Object.values(resources.pinpointApps));
     }
 
     if (resources.appSyncApis) {
+      console.log(`Deleting up to ${DELETE_LIMIT_PER_BATCH.OTHER} of ${resources.appSyncApis.length} appSyncApis on ACCOUNT[${accountIndex}]`);
       await deleteAppSyncApis(account, accountIndex, Object.values(resources.appSyncApis));
     }
   }
@@ -829,6 +869,13 @@ const cleanupAccount = async (account: AWSAccountInfo, accountIndex: number, fil
   const allResources = mergeResourcesByCCIJob(
     apps, stacks, buckets, orphanBuckets, orphanIamRoles, orphanPinpointApplications, orphanAppSyncApis
   );
+  // cleanup resources that are <unknown> but that are definitely amplify resources
+  const testapps = (allResources["<unknown>"].amplifyApps as any)?.filter(a => a.name.toLocaleLowerCase().includes('test'));
+  const testStacks = (allResources["<unknown>"].stacks as any)?.filter(s => s.stackName.toLocaleLowerCase().includes('test') && s.stackName.toLocaleLowerCase().includes('amplify'));
+  if(!allResources["<orphan>"].amplifyApps) {allResources["<orphan>"].amplifyApps = []}
+  if(!allResources["<orphan>"].stacks) {allResources["<orphan>"].stacks = []}
+  (allResources["<orphan>"].amplifyApps as any).push(...(testapps ? testapps : []));
+  (allResources["<orphan>"].stacks as any).push(...(testStacks ? testStacks : []));
   const staleResources = _.pickBy(allResources, filterPredicate);
 
   generateReport(staleResources);
@@ -864,9 +911,16 @@ const cleanup = async (): Promise<void> => {
 
   const filterPredicate = getFilterPredicate(args);
   const accounts = await getAccountsToCleanup();
-
-  await Promise.all(accounts.map((account, i) => cleanupAccount(account, i, filterPredicate)));
+  for(let i = 0 ;i < 5; i ++){
+    console.log("CLEANUP ROUND: ", i + 1);
+    await Promise.all(accounts.map((account, i) => cleanupAccount(account, i, filterPredicate)));
+    await sleep(60 * 1000);// run again after 60 seconds
+  }
   console.log('Done cleaning all accounts!');
 };
 
+const sleep = async (ms) => {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
 cleanup();

From 7dadf98fec637971cf3da21260e3905275b80a3b Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Wed, 4 Jan 2023 20:37:56 -0500
Subject: [PATCH 02/14] chore: add extract api file

---
 packages/amplify-opensearch-simulator/API.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/amplify-opensearch-simulator/API.md b/packages/amplify-opensearch-simulator/API.md
index f5f3bbe729b..089b42fa9ef 100644
--- a/packages/amplify-opensearch-simulator/API.md
+++ b/packages/amplify-opensearch-simulator/API.md
@@ -4,8 +4,6 @@
 
 ```ts
 
-/// <reference types="node" />
-
 import { $TSAny } from 'amplify-cli-core';
 import execa from 'execa';
 import { GetPackageAssetPaths } from 'amplify-cli-core';

From 18a6c1d5cdff5f8e8f668952b21d1d93bbed3494 Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Fri, 6 Jan 2023 15:06:04 -0500
Subject: [PATCH 03/14] ci: fix condition

---
 .../src/cleanup-e2e-resources.ts              | 62 ++++++++++++++-----
 1 file changed, 45 insertions(+), 17 deletions(-)

diff --git a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
index dbbdf37461e..1eeff59c9d4 100644
--- a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
+++ b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
@@ -310,29 +310,43 @@ const getStackDetails = async (stackName: string, account: AWSAccountInfo, regio
 
 const getStacks = async (account: AWSAccountInfo, region: string): Promise<StackInfo[]> => {
   const cfnClient = new aws.CloudFormation(getAWSConfig(account, region));
+  const stackStatusFilter = [
+    'CREATE_COMPLETE',
+    'ROLLBACK_FAILED',
+    'ROLLBACK_COMPLETE',
+    'DELETE_FAILED',
+    'UPDATE_COMPLETE',
+    'UPDATE_ROLLBACK_FAILED',
+    'UPDATE_ROLLBACK_COMPLETE',
+    'IMPORT_COMPLETE',
+    'IMPORT_ROLLBACK_FAILED',
+    'IMPORT_ROLLBACK_COMPLETE',
+  ];
   const stacks = await cfnClient
     .listStacks({
-      StackStatusFilter: [
-        'CREATE_COMPLETE',
-        'ROLLBACK_FAILED',
-        'DELETE_FAILED',
-        'UPDATE_COMPLETE',
-        'UPDATE_ROLLBACK_FAILED',
-        'UPDATE_ROLLBACK_COMPLETE',
-        'IMPORT_COMPLETE',
-        'IMPORT_ROLLBACK_FAILED',
-        'IMPORT_ROLLBACK_COMPLETE',
-      ],
+      StackStatusFilter: stackStatusFilter,
     })
     .promise();
+  // loop 
+  let nextToken = stacks.NextToken;
+  while (nextToken && stacks.StackSummaries.length < 50) {
+    const nextPage = await cfnClient
+      .listStacks({
+        StackStatusFilter: stackStatusFilter,
+        NextToken: nextToken,
+      })
+    .promise();
+    stacks.StackSummaries.push(...nextPage.StackSummaries);
+    nextToken = nextPage.NextToken;
+  }
 
   // We are interested in only the root stacks that are deployed by amplify-cli
   let rootStacks = stacks.StackSummaries.filter(stack => !stack.RootId);
-  if(rootStacks.length > 20){
-    // we can only delete 100 stacks accross all regions every batch,
+  if(rootStacks.length > 50){
+    // we can only delete 50 stacks accross all regions every batch,
     // so we shouldn't take more than 20 apps from each of 8 regions.
     // this should at least limit calls to getStackDetails below
-    rootStacks = rootStacks.slice(0, 20);
+    rootStacks = rootStacks.slice(0, 50);
   }
   const results: StackInfo[] = [];
   for (const stack of rootStacks) {
@@ -530,6 +544,8 @@ const deleteAmplifyApps = async (account: AWSAccountInfo, accountIndex: number,
   if(apps.length > 50){
     // throttle delete calls
     await Promise.all(apps.slice(0, 50).map(app => deleteAmplifyApp(account, accountIndex, app)));
+  } else {
+    await Promise.all(apps.map(app => deleteAmplifyApp(account, accountIndex, app)));
   }
 };
 
@@ -551,6 +567,8 @@ const deleteIamRoles = async (account: AWSAccountInfo, accountIndex: number, rol
   if(roles.length > 50){
     // throttle delete calls
     await Promise.all(roles.slice(0, 50).map(role => deleteIamRole(account, accountIndex, role)));
+  } else {
+    await Promise.all(roles.map(role => deleteIamRole(account, accountIndex, role)));
   }
 };
 
@@ -631,6 +649,8 @@ const deleteBuckets = async (account: AWSAccountInfo, accountIndex: number, buck
   if(buckets.length > 50){
     // throttle delete calls
     await Promise.all(buckets.slice(0, 50).map(bucket => deleteBucket(account, accountIndex, bucket)));
+  } else {
+    await Promise.all(buckets.map(bucket => deleteBucket(account, accountIndex, bucket)));
   }
 };
 
@@ -653,6 +673,8 @@ const deletePinpointApps = async (account: AWSAccountInfo, accountIndex: number,
   if(apps.length > 50){
     // throttle delete calls
     await Promise.all(apps.slice(0, 50).map(app => deletePinpointApp(account, accountIndex, app)));
+  } else {
+    await Promise.all(apps.map(app => deletePinpointApp(account, accountIndex, app)));
   }
 };
 
@@ -674,6 +696,8 @@ const deleteAppSyncApis = async (account: AWSAccountInfo, accountIndex: number,
   if(apis.length > 50){
     // throttle delete calls
     await Promise.all(apis.slice(0, 50).map(api => deleteAppSyncApi(account, accountIndex, api)));
+  } else {
+    await Promise.all(apis.map(api => deleteAppSyncApi(account, accountIndex, api)));
   }
 };
 
@@ -694,6 +718,8 @@ const deleteCfnStacks = async (account: AWSAccountInfo, accountIndex: number, st
   if(stacks.length > 100){
     // throttle delete calls, 100 seems to work fine for stacks
     await Promise.all(stacks.slice(0, 100).map(stack => deleteCfnStack(account, accountIndex, stack)));
+  } else {
+    await Promise.all(stacks.map(stack => deleteCfnStack(account, accountIndex, stack)));
   }
 };
 
@@ -705,9 +731,9 @@ const deleteCfnStack = async (account: AWSAccountInfo, accountIndex: number, sta
     const cfnClient = new aws.CloudFormation(getAWSConfig(account, region));
     await cfnClient.deleteStack({ StackName: stackName, RetainResources: resourceToRetain }).promise();
     // we'll only wait up to a minute before moving on
-    // await cfnClient.waitFor('stackDeleteComplete', { StackName: stackName, $waiter: { maxAttempts: 2 } }).promise();
+    // await cfnClient.waitFor('stackDeleteComplete', { StackName: stackName, $waiter: { maxAttempts: 3 } }).promise();
   } catch (e) {
-    // console.log(`Deleting CloudFormation stack ${stackName} failed with error ${e.message}`);
+    console.log(`Deleting CloudFormation stack ${stackName} failed with error ${e.message}`);
     if (e.code === 'ExpiredTokenException') {
       handleExpiredTokenException();
     }
@@ -913,7 +939,9 @@ const cleanup = async (): Promise<void> => {
   const accounts = await getAccountsToCleanup();
   for(let i = 0 ;i < 5; i ++){
     console.log("CLEANUP ROUND: ", i + 1);
-    await Promise.all(accounts.map((account, i) => cleanupAccount(account, i, filterPredicate)));
+    await Promise.all(accounts.map((account, i) => {
+      return cleanupAccount(account, i, filterPredicate);
+    }));
     await sleep(60 * 1000);// run again after 60 seconds
   }
   console.log('Done cleaning all accounts!');

From 197aadafb9591257c238c13f5582b63adb1ac93a Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Fri, 6 Jan 2023 18:34:40 -0500
Subject: [PATCH 04/14] chore: cleanup

---
 .../amplify-e2e-tests/src/cleanup-e2e-resources.ts   | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
index 1eeff59c9d4..918a50c9a1d 100644
--- a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
+++ b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
@@ -115,6 +115,7 @@ type AWSAccountInfo = {
   accessKeyId: string;
   secretAccessKey: string;
   sessionToken: string;
+  parent: boolean;
 };
 
 const PINPOINT_TEST_REGEX = /integtest/;
@@ -233,6 +234,9 @@ const getAWSConfig = ({ accessKeyId, secretAccessKey, sessionToken }: AWSAccount
  * @returns Promise<AmplifyAppInfo[]> a list of Amplify Apps in the region with build info
  */
 const getAmplifyApps = async (account: AWSAccountInfo, region: string): Promise<AmplifyAppInfo[]> => {
+  if(region === 'us-east-1' && account.parent){
+    return []; // temporarily disabled until us-east-1 is re-enabled for this account
+  }
   const amplifyClient = new aws.Amplify(getAWSConfig(account, region));
   try {
     const amplifyApps = await amplifyClient.listApps({ maxResults: 25 }).promise(); // keeping it to 25 as max supported is 25
@@ -341,7 +345,10 @@ const getStacks = async (account: AWSAccountInfo, region: string): Promise<Stack
   }
 
   // We are interested in only the root stacks that are deployed by amplify-cli
-  let rootStacks = stacks.StackSummaries.filter(stack => !stack.RootId);
+  // NOTE: every few months, we should disable the filter , and clean up all stacks (not just root stacks)
+  // this is because some child stacks fail to delete (but we don't let that stop us from deleting root stacks)
+  // eventually, we must clean up those child stacks too.
+  let rootStacks = stacks.StackSummaries.filter(stack => !stack.RootId);; 
   if(rootStacks.length > 50){
     // we can only delete 50 stacks accross all regions every batch,
     // so we shouldn't take more than 20 apps from each of 8 regions.
@@ -844,6 +851,7 @@ const getAccountsToCleanup = async (): Promise<AWSAccountInfo[]> => {
           accessKeyId: process.env.AWS_ACCESS_KEY_ID,
           secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
           sessionToken: process.env.AWS_SESSION_TOKEN,
+          parent: true,
         };
       }
       const randomNumber = Math.floor(Math.random() * 100000);
@@ -859,6 +867,7 @@ const getAccountsToCleanup = async (): Promise<AWSAccountInfo[]> => {
         accessKeyId: assumeRoleRes.Credentials.AccessKeyId,
         secretAccessKey: assumeRoleRes.Credentials.SecretAccessKey,
         sessionToken: assumeRoleRes.Credentials.SessionToken,
+        parent: false,
       };
     });
     return await Promise.all(accountCredentialPromises);
@@ -870,6 +879,7 @@ const getAccountsToCleanup = async (): Promise<AWSAccountInfo[]> => {
         accessKeyId: process.env.AWS_ACCESS_KEY_ID,
         secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
         sessionToken: process.env.AWS_SESSION_TOKEN,
+        parent: true,
       },
     ];
   }

From 4445759e10c2c5e446bf5d972bf7ba3d244b6f00 Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Mon, 9 Jan 2023 15:33:09 -0500
Subject: [PATCH 05/14] chore: add back logs

---
 .../src/cleanup-e2e-resources.ts              | 40 +++++++++----------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
index 918a50c9a1d..b17d4c4591d 100644
--- a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
+++ b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
@@ -348,7 +348,7 @@ const getStacks = async (account: AWSAccountInfo, region: string): Promise<Stack
   // NOTE: every few months, we should disable the filter , and clean up all stacks (not just root stacks)
   // this is because some child stacks fail to delete (but we don't let that stop us from deleting root stacks)
   // eventually, we must clean up those child stacks too.
-  let rootStacks = stacks.StackSummaries.filter(stack => !stack.RootId);; 
+  let rootStacks = stacks.StackSummaries.filter(stack => !stack.RootId);
   if(rootStacks.length > 50){
     // we can only delete 50 stacks accross all regions every batch,
     // so we shouldn't take more than 20 apps from each of 8 regions.
@@ -558,12 +558,12 @@ const deleteAmplifyApps = async (account: AWSAccountInfo, accountIndex: number,
 
 const deleteAmplifyApp = async (account: AWSAccountInfo, accountIndex: number, app: AmplifyAppInfo): Promise<void> => {
   const { name, appId, region } = app;
-  // console.log(`[ACCOUNT ${accountIndex}] Deleting App ${name}(${appId})`);
+  console.log(`[ACCOUNT ${accountIndex}] Deleting App ${name}(${appId})`);
   const amplifyClient = new aws.Amplify(getAWSConfig(account, region));
   try {
     await amplifyClient.deleteApp({ appId }).promise();
   } catch (e) {
-    // console.log(`[ACCOUNT ${accountIndex}] Deleting Amplify App ${appId} failed with the following error`, e);
+    console.log(`[ACCOUNT ${accountIndex}] Deleting Amplify App ${appId} failed with the following error`, e);
     if (e.code === 'ExpiredTokenException') {
       handleExpiredTokenException();
     }
@@ -582,14 +582,14 @@ const deleteIamRoles = async (account: AWSAccountInfo, accountIndex: number, rol
 const deleteIamRole = async (account: AWSAccountInfo, accountIndex: number, role: IamRoleInfo): Promise<void> => {
   const { name: roleName } = role;
   try {
-    // console.log(`[ACCOUNT ${accountIndex}] Deleting Iam Role ${roleName}`);
-    // console.log(`Role creation time (PST): ${role.createTime.toLocaleTimeString('en-US', { timeZone: 'America/Los_Angeles' })}`);
+    console.log(`[ACCOUNT ${accountIndex}] Deleting Iam Role ${roleName}`);
+    console.log(`Role creation time (PST): ${role.createTime.toLocaleTimeString('en-US', { timeZone: 'America/Los_Angeles' })}`);
     const iamClient = new aws.IAM(getAWSConfig(account));
     await deleteAttachedRolePolicies(account, accountIndex, roleName);
     await deleteRolePolicies(account, accountIndex, roleName);
     await iamClient.deleteRole({ RoleName: roleName }).promise();
   } catch (e) {
-    // console.log(`[ACCOUNT ${accountIndex}] Deleting iam role ${roleName} failed with error ${e.message}`);
+    console.log(`[ACCOUNT ${accountIndex}] Deleting iam role ${roleName} failed with error ${e.message}`);
     if (e.code === 'ExpiredTokenException') {
       handleExpiredTokenException();
     }
@@ -613,11 +613,11 @@ const detachIamAttachedRolePolicy = async (
   policy: aws.IAM.AttachedPolicy,
 ): Promise<void> => {
   try {
-    // console.log(`[ACCOUNT ${accountIndex}] Detach Iam Attached Role Policy ${policy.PolicyName}`);
+    console.log(`[ACCOUNT ${accountIndex}] Detach Iam Attached Role Policy ${policy.PolicyName}`);
     const iamClient = new aws.IAM(getAWSConfig(account));
     await iamClient.detachRolePolicy({ RoleName: roleName, PolicyArn: policy.PolicyArn }).promise();
   } catch (e) {
-    // console.log(`[ACCOUNT ${accountIndex}] Detach iam role policy ${policy.PolicyName} failed with error ${e.message}`);
+    console.log(`[ACCOUNT ${accountIndex}] Detach iam role policy ${policy.PolicyName} failed with error ${e.message}`);
     if (e.code === 'ExpiredTokenException') {
       handleExpiredTokenException();
     }
@@ -641,11 +641,11 @@ const deleteIamRolePolicy = async (
   policyName: string,
 ): Promise<void> => {
   try {
-    // console.log(`[ACCOUNT ${accountIndex}] Deleting Iam Role Policy ${policyName}`);
+    console.log(`[ACCOUNT ${accountIndex}] Deleting Iam Role Policy ${policyName}`);
     const iamClient = new aws.IAM(getAWSConfig(account));
     await iamClient.deleteRolePolicy({ RoleName: roleName, PolicyName: policyName }).promise();
   } catch (e) {
-    // console.log(`[ACCOUNT ${accountIndex}] Deleting iam role policy ${policyName} failed with error ${e.message}`);
+    console.log(`[ACCOUNT ${accountIndex}] Deleting iam role policy ${policyName} failed with error ${e.message}`);
     if (e.code === 'ExpiredTokenException') {
       handleExpiredTokenException();
     }
@@ -664,12 +664,12 @@ const deleteBuckets = async (account: AWSAccountInfo, accountIndex: number, buck
 const deleteBucket = async (account: AWSAccountInfo, accountIndex: number, bucket: S3BucketInfo): Promise<void> => {
   const { name } = bucket;
   try {
-    // console.log(`[ACCOUNT ${accountIndex}] Deleting S3 Bucket ${name}`);
-    // console.log(`Bucket creation time (PST): ${bucket.createTime.toLocaleTimeString('en-US', { timeZone: 'America/Los_Angeles' })}`);
+    console.log(`[ACCOUNT ${accountIndex}] Deleting S3 Bucket ${name}`);
+    console.log(`Bucket creation time (PST): ${bucket.createTime.toLocaleTimeString('en-US', { timeZone: 'America/Los_Angeles' })}`);
     const s3 = new aws.S3(getAWSConfig(account));
     await deleteS3Bucket(name, s3);
   } catch (e) {
-    // console.log(`[ACCOUNT ${accountIndex}] Deleting bucket ${name} failed with error ${e.message}`);
+    console.log(`[ACCOUNT ${accountIndex}] Deleting bucket ${name} failed with error ${e.message}`);
     if (e.code === 'ExpiredTokenException') {
       handleExpiredTokenException();
     }
@@ -690,12 +690,12 @@ const deletePinpointApp = async (account: AWSAccountInfo, accountIndex: number,
     id, name, region,
   } = app;
   try {
-    // console.log(`[ACCOUNT ${accountIndex}] Deleting Pinpoint App ${name}`);
-    // console.log(`Pinpoint creation time (PST): ${app.createTime.toLocaleTimeString('en-US', { timeZone: 'America/Los_Angeles' })}`);
+    console.log(`[ACCOUNT ${accountIndex}] Deleting Pinpoint App ${name}`);
+    console.log(`Pinpoint creation time (PST): ${app.createTime.toLocaleTimeString('en-US', { timeZone: 'America/Los_Angeles' })}`);
     const pinpoint = new aws.Pinpoint(getAWSConfig(account, region));
     await pinpoint.deleteApp({ ApplicationId: id }).promise();
   } catch (e) {
-    // console.log(`[ACCOUNT ${accountIndex}] Deleting pinpoint app ${name} failed with error ${e.message}`);
+    console.log(`[ACCOUNT ${accountIndex}] Deleting pinpoint app ${name} failed with error ${e.message}`);
   }
 };
 
@@ -713,11 +713,11 @@ const deleteAppSyncApi = async (account: AWSAccountInfo, accountIndex: number, a
     apiId, name, region,
   } = api;
   try {
-    // console.log(`[ACCOUNT ${accountIndex}] Deleting AppSync Api ${name}`);
+    console.log(`[ACCOUNT ${accountIndex}] Deleting AppSync Api ${name}`);
     const appSync = new aws.AppSync(getAWSConfig(account, region));
     await appSync.deleteGraphqlApi({ apiId }).promise();
   } catch (e) {
-    // console.log(`[ACCOUNT ${accountIndex}] Deleting AppSync Api ${name} failed with error ${e.message}`);
+    console.log(`[ACCOUNT ${accountIndex}] Deleting AppSync Api ${name} failed with error ${e.message}`);
   }
 };
 
@@ -733,12 +733,12 @@ const deleteCfnStacks = async (account: AWSAccountInfo, accountIndex: number, st
 const deleteCfnStack = async (account: AWSAccountInfo, accountIndex: number, stack: StackInfo): Promise<void> => {
   const { stackName, region, resourcesFailedToDelete } = stack;
   const resourceToRetain = resourcesFailedToDelete.length ? resourcesFailedToDelete : undefined;
-  // console.log(`[ACCOUNT ${accountIndex}] Deleting CloudFormation stack ${stackName}`);
+  console.log(`[ACCOUNT ${accountIndex}] Deleting CloudFormation stack ${stackName}`);
   try {
     const cfnClient = new aws.CloudFormation(getAWSConfig(account, region));
     await cfnClient.deleteStack({ StackName: stackName, RetainResources: resourceToRetain }).promise();
     // we'll only wait up to a minute before moving on
-    // await cfnClient.waitFor('stackDeleteComplete', { StackName: stackName, $waiter: { maxAttempts: 3 } }).promise();
+    await cfnClient.waitFor('stackDeleteComplete', { StackName: stackName, $waiter: { maxAttempts: 2 } }).promise();
   } catch (e) {
     console.log(`Deleting CloudFormation stack ${stackName} failed with error ${e.message}`);
     if (e.code === 'ExpiredTokenException') {

From f0d57b5973d12dbfbbee291386f1864038f1725b Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Mon, 9 Jan 2023 15:38:38 -0500
Subject: [PATCH 06/14] chore: move cleanup to start to avoid deleting apps
 while running

---
 .circleci/config.base.yml | 2 --
 .circleci/config.yml      | 2 --
 2 files changed, 4 deletions(-)

diff --git a/.circleci/config.base.yml b/.circleci/config.base.yml
index d0841ca6a25..2297ef5b83d 100644
--- a/.circleci/config.base.yml
+++ b/.circleci/config.base.yml
@@ -1027,8 +1027,6 @@ workflows:
           context:
               - cleanup-resources
               - e2e-test-context
-          requires:
-            - build
           filters:
             branches:
               only:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index c8547cd46a1..74430909230 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -255,5 +255,3 @@ workflows:
           context:
             - cleanup-resources
             - e2e-test-context
-          requires:
-            - build

From dc1ef7584e1c0056b72d59d3904bed1a4fffd306 Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Mon, 9 Jan 2023 15:43:42 -0500
Subject: [PATCH 07/14] chore: update cleanup script

---
 .circleci/config.base.yml | 5 +----
 .circleci/config.yml      | 6 +-----
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/.circleci/config.base.yml b/.circleci/config.base.yml
index 2297ef5b83d..82208660064 100644
--- a/.circleci/config.base.yml
+++ b/.circleci/config.base.yml
@@ -843,10 +843,7 @@ jobs:
   cleanup_resources:
     <<: *linux-e2e-executor-large
     steps:
-      - restore_cache:
-          key: amplify-cli-repo-{{ .Branch }}-{{ .Revision }}
-      - restore_cache:
-          key: amplify-cli-yarn-deps-{{ .Branch }}-{{ checksum "yarn.lock" }}
+      - checkout
       - run:
           name: 'Run cleanup script'
           command: |
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 74430909230..c4161666573 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -185,11 +185,7 @@ jobs:
       AMPLIFY_DIR: /home/circleci/repo/out
       AMPLIFY_PATH: /home/circleci/repo/out/amplify-pkg-linux-x64
     steps:
-      - attach_workspace:
-          at: ./
-      - restore_cache:
-          key: >-
-            amplify-cli-yarn-deps-{{ .Branch }}-{{ checksum "yarn.lock" }}
+      - checkout
       - when: 
           condition:
             equal: [true, << pipeline.parameters.e2e_workflow_cleanup >>]

From a5866693d35a0e95370e3be919457a3003daad4f Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Mon, 9 Jan 2023 16:06:27 -0500
Subject: [PATCH 08/14] chore: revert build change

---
 .circleci/config.base.yml | 7 ++++++-
 .circleci/config.yml      | 8 +++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.base.yml b/.circleci/config.base.yml
index 82208660064..d0841ca6a25 100644
--- a/.circleci/config.base.yml
+++ b/.circleci/config.base.yml
@@ -843,7 +843,10 @@ jobs:
   cleanup_resources:
     <<: *linux-e2e-executor-large
     steps:
-      - checkout
+      - restore_cache:
+          key: amplify-cli-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_cache:
+          key: amplify-cli-yarn-deps-{{ .Branch }}-{{ checksum "yarn.lock" }}
       - run:
           name: 'Run cleanup script'
           command: |
@@ -1024,6 +1027,8 @@ workflows:
           context:
               - cleanup-resources
               - e2e-test-context
+          requires:
+            - build
           filters:
             branches:
               only:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index c4161666573..c8547cd46a1 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -185,7 +185,11 @@ jobs:
       AMPLIFY_DIR: /home/circleci/repo/out
       AMPLIFY_PATH: /home/circleci/repo/out/amplify-pkg-linux-x64
     steps:
-      - checkout
+      - attach_workspace:
+          at: ./
+      - restore_cache:
+          key: >-
+            amplify-cli-yarn-deps-{{ .Branch }}-{{ checksum "yarn.lock" }}
       - when: 
           condition:
             equal: [true, << pipeline.parameters.e2e_workflow_cleanup >>]
@@ -251,3 +255,5 @@ workflows:
           context:
             - cleanup-resources
             - e2e-test-context
+          requires:
+            - build

From 763f39a1aac6853290845b2e140ff3d682dd3bf2 Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Mon, 9 Jan 2023 17:28:32 -0500
Subject: [PATCH 09/14] chore: fix md file

---
 packages/amplify-cli-core/API.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/amplify-cli-core/API.md b/packages/amplify-cli-core/API.md
index 135ec2e8047..3d2e5dfe0de 100644
--- a/packages/amplify-cli-core/API.md
+++ b/packages/amplify-cli-core/API.md
@@ -1489,7 +1489,7 @@ export function validateExportDirectoryPath(directoryPath: any, defaultPath: str
 export class ViewResourceTableParams {
     constructor(cliParams: CLIParams);
     // (undocumented)
-    get categoryList(): string[] | [];
+    get categoryList(): [] | string[];
     // (undocumented)
     get command(): string;
     // (undocumented)

From 33aa02e7a90418e0b69f5a002e533d4ba3cce3c4 Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Mon, 9 Jan 2023 18:04:53 -0500
Subject: [PATCH 10/14] chore: cleanup

---
 .../src/cleanup-e2e-resources.ts              | 60 +++++++++++--------
 1 file changed, 34 insertions(+), 26 deletions(-)

diff --git a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
index 1231988810c..01f111fc766 100644
--- a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
+++ b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
@@ -21,9 +21,17 @@ const AWS_REGIONS_TO_RUN_TESTS = [
   'ap-southeast-2',
 ];
 
-const DELETE_LIMIT_PER_BATCH = {
-  OTHER: 50,
-  CFN_STACK: 100,
+// Limits are efforced per region
+// we collect resources from each region & then delete as an entire batch
+const DELETE_LIMITS = {
+  PER_REGION: {
+    OTHER: 25,
+    CFN_STACK: 50,
+  },
+  PER_BATCH: {
+    OTHER: 50,
+    CFN_STACK: 100,
+  }
 }
 
 const reportPath = path.normalize(path.join(__dirname, '..', 'amplify-e2e-reports', 'stale-resources.json'));
@@ -341,7 +349,7 @@ const getStacks = async (account: AWSAccountInfo, region: string): Promise<Stack
     .promise();
   // loop 
   let nextToken = stacks.NextToken;
-  while (nextToken && stacks.StackSummaries.length < 50) {
+  while (nextToken && stacks.StackSummaries.length < DELETE_LIMITS.PER_REGION.CFN_STACK) {
     const nextPage = await cfnClient
       .listStacks({
         StackStatusFilter: stackStatusFilter,
@@ -357,11 +365,11 @@ const getStacks = async (account: AWSAccountInfo, region: string): Promise<Stack
   // this is because some child stacks fail to delete (but we don't let that stop us from deleting root stacks)
   // eventually, we must clean up those child stacks too.
   let rootStacks = stacks.StackSummaries.filter(stack => !stack.RootId);
-  if(rootStacks.length > 50){
-    // we can only delete 50 stacks accross all regions every batch,
-    // so we shouldn't take more than 20 apps from each of 8 regions.
+  if(rootStacks.length > DELETE_LIMITS.PER_REGION.CFN_STACK){
+    // we can only delete 100 stacks accross all regions every batch,
+    // so we shouldn't take more than 50 stacks from each of those 8 regions.
     // this should at least limit calls to getStackDetails below
-    rootStacks = rootStacks.slice(0, 50);
+    rootStacks = rootStacks.slice(0, DELETE_LIMITS.PER_REGION.CFN_STACK);
   }
   const results: StackInfo[] = [];
   for (const stack of rootStacks) {
@@ -556,9 +564,9 @@ const mergeResourcesByCCIJob = (
 };
 
 const deleteAmplifyApps = async (account: AWSAccountInfo, accountIndex: number, apps: AmplifyAppInfo[]): Promise<void> => {
-  if(apps.length > 50){
+  if(apps.length > DELETE_LIMITS.PER_BATCH.OTHER){
     // throttle delete calls
-    await Promise.all(apps.slice(0, 50).map(app => deleteAmplifyApp(account, accountIndex, app)));
+    await Promise.all(apps.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(app => deleteAmplifyApp(account, accountIndex, app)));
   } else {
     await Promise.all(apps.map(app => deleteAmplifyApp(account, accountIndex, app)));
   }
@@ -579,9 +587,9 @@ const deleteAmplifyApp = async (account: AWSAccountInfo, accountIndex: number, a
 };
 
 const deleteIamRoles = async (account: AWSAccountInfo, accountIndex: number, roles: IamRoleInfo[]): Promise<void> => {
-  if(roles.length > 50){
+  if(roles.length > DELETE_LIMITS.PER_BATCH.OTHER){
     // throttle delete calls
-    await Promise.all(roles.slice(0, 50).map(role => deleteIamRole(account, accountIndex, role)));
+    await Promise.all(roles.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(role => deleteIamRole(account, accountIndex, role)));
   } else {
     await Promise.all(roles.map(role => deleteIamRole(account, accountIndex, role)));
   }
@@ -648,9 +656,9 @@ const deleteIamRolePolicy = async (account: AWSAccountInfo, accountIndex: number
 };
 
 const deleteBuckets = async (account: AWSAccountInfo, accountIndex: number, buckets: S3BucketInfo[]): Promise<void> => {
-  if(buckets.length > 50){
+  if(buckets.length > DELETE_LIMITS.PER_BATCH.OTHER){
     // throttle delete calls
-    await Promise.all(buckets.slice(0, 50).map(bucket => deleteBucket(account, accountIndex, bucket)));
+    await Promise.all(buckets.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(bucket => deleteBucket(account, accountIndex, bucket)));
   } else {
     await Promise.all(buckets.map(bucket => deleteBucket(account, accountIndex, bucket)));
   }
@@ -672,9 +680,9 @@ const deleteBucket = async (account: AWSAccountInfo, accountIndex: number, bucke
 };
 
 const deletePinpointApps = async (account: AWSAccountInfo, accountIndex: number, apps: PinpointAppInfo[]): Promise<void> => {
-  if(apps.length > 50){
+  if(apps.length > DELETE_LIMITS.PER_BATCH.OTHER){
     // throttle delete calls
-    await Promise.all(apps.slice(0, 50).map(app => deletePinpointApp(account, accountIndex, app)));
+    await Promise.all(apps.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(app => deletePinpointApp(account, accountIndex, app)));
   } else {
     await Promise.all(apps.map(app => deletePinpointApp(account, accountIndex, app)));
   }
@@ -693,9 +701,9 @@ const deletePinpointApp = async (account: AWSAccountInfo, accountIndex: number,
 };
 
 const deleteAppSyncApis = async (account: AWSAccountInfo, accountIndex: number, apis: AppSyncApiInfo[]): Promise<void> => {
-  if(apis.length > 50){
+  if(apis.length > DELETE_LIMITS.PER_BATCH.OTHER){
     // throttle delete calls
-    await Promise.all(apis.slice(0, 50).map(api => deleteAppSyncApi(account, accountIndex, api)));
+    await Promise.all(apis.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(api => deleteAppSyncApi(account, accountIndex, api)));
   } else {
     await Promise.all(apis.map(api => deleteAppSyncApi(account, accountIndex, api)));
   }
@@ -715,9 +723,9 @@ const deleteAppSyncApi = async (account: AWSAccountInfo, accountIndex: number, a
 };
 
 const deleteCfnStacks = async (account: AWSAccountInfo, accountIndex: number, stacks: StackInfo[]): Promise<void> => {
-  if(stacks.length > 100){
+  if(stacks.length > DELETE_LIMITS.PER_BATCH.CFN_STACK){
     // throttle delete calls, 100 seems to work fine for stacks
-    await Promise.all(stacks.slice(0, 100).map(stack => deleteCfnStack(account, accountIndex, stack)));
+    await Promise.all(stacks.slice(0, DELETE_LIMITS.PER_BATCH.CFN_STACK).map(stack => deleteCfnStack(account, accountIndex, stack)));
   } else {
     await Promise.all(stacks.map(stack => deleteCfnStack(account, accountIndex, stack)));
   }
@@ -757,32 +765,32 @@ const deleteResources = async (
   for (const jobId of Object.keys(staleResources)) {
     const resources = staleResources[jobId];
     if (resources.amplifyApps) {
-      console.log(`Deleting up to ${DELETE_LIMIT_PER_BATCH.OTHER} of ${resources.amplifyApps.length} apps on ACCOUNT[${accountIndex}]`);
+      console.log(`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.amplifyApps.length} apps on ACCOUNT[${accountIndex}]`);
       await deleteAmplifyApps(account, accountIndex, Object.values(resources.amplifyApps));
     }
 
     if (resources.stacks) {
-      console.log(`Deleting up to ${DELETE_LIMIT_PER_BATCH.CFN_STACK} of ${resources.stacks.length} stacks on ACCOUNT[${accountIndex}]`);
+      console.log(`Deleting up to ${DELETE_LIMITS.PER_BATCH.CFN_STACK} of ${resources.stacks.length} stacks on ACCOUNT[${accountIndex}]`);
       await deleteCfnStacks(account, accountIndex, Object.values(resources.stacks));
     }
 
     if (resources.buckets) {
-      console.log(`Deleting up to ${DELETE_LIMIT_PER_BATCH.OTHER} of ${resources.buckets.length} buckets on ACCOUNT[${accountIndex}]`);
+      console.log(`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.buckets.length} buckets on ACCOUNT[${accountIndex}]`);
       await deleteBuckets(account, accountIndex, Object.values(resources.buckets));
     }
 
     if (resources.roles) {
-      console.log(`Deleting up to ${DELETE_LIMIT_PER_BATCH.OTHER} of ${resources.roles.length} roles on ACCOUNT[${accountIndex}]`);
+      console.log(`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.roles.length} roles on ACCOUNT[${accountIndex}]`);
       await deleteIamRoles(account, accountIndex, Object.values(resources.roles));
     }
 
     if (resources.pinpointApps) {
-      console.log(`Deleting up to ${DELETE_LIMIT_PER_BATCH.OTHER} of ${resources.pinpointApps.length} pinpoint apps on ACCOUNT[${accountIndex}]`);
+      console.log(`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.pinpointApps.length} pinpoint apps on ACCOUNT[${accountIndex}]`);
       await deletePinpointApps(account, accountIndex, Object.values(resources.pinpointApps));
     }
 
     if (resources.appSyncApis) {
-      console.log(`Deleting up to ${DELETE_LIMIT_PER_BATCH.OTHER} of ${resources.appSyncApis.length} appSyncApis on ACCOUNT[${accountIndex}]`);
+      console.log(`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.appSyncApis.length} appSyncApis on ACCOUNT[${accountIndex}]`);
       await deleteAppSyncApis(account, accountIndex, Object.values(resources.appSyncApis));
     }
   }

From d73be1a3f22c768acb30d8b004e8a28b5b5fc96e Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Mon, 9 Jan 2023 18:14:04 -0500
Subject: [PATCH 11/14] chore: address pr feedback

---
 .../src/cleanup-e2e-resources.ts              | 48 +++----------------
 1 file changed, 7 insertions(+), 41 deletions(-)

diff --git a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
index 01f111fc766..7ca880e783a 100644
--- a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
+++ b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
@@ -7,7 +7,7 @@ import * as aws from 'aws-sdk';
 import _ from 'lodash';
 import fs from 'fs-extra';
 import path from 'path';
-import { deleteS3Bucket } from '@aws-amplify/amplify-e2e-core';
+import { deleteS3Bucket, sleep } from '@aws-amplify/amplify-e2e-core';
 
 // Ensure to update scripts/split-e2e-tests.ts is also updated this gets updated
 const AWS_REGIONS_TO_RUN_TESTS = [
@@ -564,12 +564,7 @@ const mergeResourcesByCCIJob = (
 };
 
 const deleteAmplifyApps = async (account: AWSAccountInfo, accountIndex: number, apps: AmplifyAppInfo[]): Promise<void> => {
-  if(apps.length > DELETE_LIMITS.PER_BATCH.OTHER){
-    // throttle delete calls
-    await Promise.all(apps.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(app => deleteAmplifyApp(account, accountIndex, app)));
-  } else {
-    await Promise.all(apps.map(app => deleteAmplifyApp(account, accountIndex, app)));
-  }
+  await Promise.all(apps.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(app => deleteAmplifyApp(account, accountIndex, app)));
 };
 
 const deleteAmplifyApp = async (account: AWSAccountInfo, accountIndex: number, app: AmplifyAppInfo): Promise<void> => {
@@ -587,12 +582,7 @@ const deleteAmplifyApp = async (account: AWSAccountInfo, accountIndex: number, a
 };
 
 const deleteIamRoles = async (account: AWSAccountInfo, accountIndex: number, roles: IamRoleInfo[]): Promise<void> => {
-  if(roles.length > DELETE_LIMITS.PER_BATCH.OTHER){
-    // throttle delete calls
-    await Promise.all(roles.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(role => deleteIamRole(account, accountIndex, role)));
-  } else {
-    await Promise.all(roles.map(role => deleteIamRole(account, accountIndex, role)));
-  }
+  await Promise.all(roles.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(role => deleteIamRole(account, accountIndex, role)));
 };
 
 const deleteIamRole = async (account: AWSAccountInfo, accountIndex: number, role: IamRoleInfo): Promise<void> => {
@@ -656,12 +646,7 @@ const deleteIamRolePolicy = async (account: AWSAccountInfo, accountIndex: number
 };
 
 const deleteBuckets = async (account: AWSAccountInfo, accountIndex: number, buckets: S3BucketInfo[]): Promise<void> => {
-  if(buckets.length > DELETE_LIMITS.PER_BATCH.OTHER){
-    // throttle delete calls
-    await Promise.all(buckets.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(bucket => deleteBucket(account, accountIndex, bucket)));
-  } else {
-    await Promise.all(buckets.map(bucket => deleteBucket(account, accountIndex, bucket)));
-  }
+  await Promise.all(buckets.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(bucket => deleteBucket(account, accountIndex, bucket)));
 };
 
 const deleteBucket = async (account: AWSAccountInfo, accountIndex: number, bucket: S3BucketInfo): Promise<void> => {
@@ -680,12 +665,7 @@ const deleteBucket = async (account: AWSAccountInfo, accountIndex: number, bucke
 };
 
 const deletePinpointApps = async (account: AWSAccountInfo, accountIndex: number, apps: PinpointAppInfo[]): Promise<void> => {
-  if(apps.length > DELETE_LIMITS.PER_BATCH.OTHER){
-    // throttle delete calls
-    await Promise.all(apps.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(app => deletePinpointApp(account, accountIndex, app)));
-  } else {
-    await Promise.all(apps.map(app => deletePinpointApp(account, accountIndex, app)));
-  }
+  await Promise.all(apps.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(app => deletePinpointApp(account, accountIndex, app)));
 };
 
 const deletePinpointApp = async (account: AWSAccountInfo, accountIndex: number, app: PinpointAppInfo): Promise<void> => {
@@ -701,12 +681,7 @@ const deletePinpointApp = async (account: AWSAccountInfo, accountIndex: number,
 };
 
 const deleteAppSyncApis = async (account: AWSAccountInfo, accountIndex: number, apis: AppSyncApiInfo[]): Promise<void> => {
-  if(apis.length > DELETE_LIMITS.PER_BATCH.OTHER){
-    // throttle delete calls
-    await Promise.all(apis.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(api => deleteAppSyncApi(account, accountIndex, api)));
-  } else {
-    await Promise.all(apis.map(api => deleteAppSyncApi(account, accountIndex, api)));
-  }
+  await Promise.all(apis.slice(0, DELETE_LIMITS.PER_BATCH.OTHER).map(api => deleteAppSyncApi(account, accountIndex, api)));
 };
 
 const deleteAppSyncApi = async (account: AWSAccountInfo, accountIndex: number, api: AppSyncApiInfo): Promise<void> => {
@@ -723,12 +698,7 @@ const deleteAppSyncApi = async (account: AWSAccountInfo, accountIndex: number, a
 };
 
 const deleteCfnStacks = async (account: AWSAccountInfo, accountIndex: number, stacks: StackInfo[]): Promise<void> => {
-  if(stacks.length > DELETE_LIMITS.PER_BATCH.CFN_STACK){
-    // throttle delete calls, 100 seems to work fine for stacks
-    await Promise.all(stacks.slice(0, DELETE_LIMITS.PER_BATCH.CFN_STACK).map(stack => deleteCfnStack(account, accountIndex, stack)));
-  } else {
-    await Promise.all(stacks.map(stack => deleteCfnStack(account, accountIndex, stack)));
-  }
+  await Promise.all(stacks.slice(0, DELETE_LIMITS.PER_BATCH.CFN_STACK).map(stack => deleteCfnStack(account, accountIndex, stack)));
 };
 
 const deleteCfnStack = async (account: AWSAccountInfo, accountIndex: number, stack: StackInfo): Promise<void> => {
@@ -960,8 +930,4 @@ const cleanup = async (): Promise<void> => {
   console.log('Done cleaning all accounts!');
 };
 
-const sleep = async (ms) => {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}
-
 cleanup();

From 42444596d4fe9a72b61300c6295d3ad958c1952b Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Mon, 9 Jan 2023 18:24:28 -0500
Subject: [PATCH 12/14] chore: comments

---
 packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
index 7ca880e783a..18e8d7d07a4 100644
--- a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
+++ b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
@@ -879,6 +879,7 @@ const cleanupAccount = async (account: AWSAccountInfo, accountIndex: number, fil
     apps, stacks, buckets, orphanBuckets, orphanIamRoles, orphanPinpointApplications, orphanAppSyncApis
   );
   // cleanup resources that are <unknown> but that are definitely amplify resources
+  // this includes apps with names that include "test" or stacks that include both "amplify" & "test"
   const testapps = (allResources["<unknown>"].amplifyApps as any)?.filter(a => a.name.toLocaleLowerCase().includes('test'));
   const testStacks = (allResources["<unknown>"].stacks as any)?.filter(s => s.stackName.toLocaleLowerCase().includes('test') && s.stackName.toLocaleLowerCase().includes('amplify'));
   if(!allResources["<orphan>"].amplifyApps) {allResources["<orphan>"].amplifyApps = []}

From a9a4526d2a8718bcb8cdf80f2b95410324622df4 Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Mon, 9 Jan 2023 18:34:35 -0500
Subject: [PATCH 13/14] chore: refactor

---
 .../amplify-e2e-tests/src/cleanup-e2e-resources.ts  | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
index 18e8d7d07a4..ef5d1a7b7d2 100644
--- a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
+++ b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
@@ -880,12 +880,13 @@ const cleanupAccount = async (account: AWSAccountInfo, accountIndex: number, fil
   );
   // cleanup resources that are <unknown> but that are definitely amplify resources
   // this includes apps with names that include "test" or stacks that include both "amplify" & "test"
-  const testapps = (allResources["<unknown>"].amplifyApps as any)?.filter(a => a.name.toLocaleLowerCase().includes('test'));
-  const testStacks = (allResources["<unknown>"].stacks as any)?.filter(s => s.stackName.toLocaleLowerCase().includes('test') && s.stackName.toLocaleLowerCase().includes('amplify'));
-  if(!allResources["<orphan>"].amplifyApps) {allResources["<orphan>"].amplifyApps = []}
-  if(!allResources["<orphan>"].stacks) {allResources["<orphan>"].stacks = []}
-  (allResources["<orphan>"].amplifyApps as any).push(...(testapps ? testapps : []));
-  (allResources["<orphan>"].stacks as any).push(...(testStacks ? testStacks : []));
+  const testApps = allResources["<unknown>"].amplifyApps?.filter(a => a.name.toLocaleLowerCase().includes('test'));
+  const testStacks = allResources["<unknown>"].stacks?.filter(s => s.stackName.toLocaleLowerCase().includes('test') && s.stackName.toLocaleLowerCase().includes('amplify'));
+  const orphanedResources = allResources["<orphan>"];
+  orphanedResources.amplifyApps = orphanedResources.amplifyApps ?? [];
+  orphanedResources.stacks = orphanedResources.stacks ?? [];
+  orphanedResources.amplifyApps.push(...(testApps ? testApps : []));
+  orphanedResources.stacks.push(...(testStacks ? testStacks : []));
   const staleResources = _.pickBy(allResources, filterPredicate);
 
   generateReport(staleResources);

From faff00fe7e782e3deefa7aa5c4ca1c820a143a35 Mon Sep 17 00:00:00 2001
From: Armando Luja <aluja@amazon.com>
Date: Mon, 9 Jan 2023 19:01:21 -0500
Subject: [PATCH 14/14] chore: fix lint issues

---
 packages/amplify-cli-core/API.md             | 2 +-
 packages/amplify-opensearch-simulator/API.md | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/amplify-cli-core/API.md b/packages/amplify-cli-core/API.md
index 3d2e5dfe0de..135ec2e8047 100644
--- a/packages/amplify-cli-core/API.md
+++ b/packages/amplify-cli-core/API.md
@@ -1489,7 +1489,7 @@ export function validateExportDirectoryPath(directoryPath: any, defaultPath: str
 export class ViewResourceTableParams {
     constructor(cliParams: CLIParams);
     // (undocumented)
-    get categoryList(): [] | string[];
+    get categoryList(): string[] | [];
     // (undocumented)
     get command(): string;
     // (undocumented)
diff --git a/packages/amplify-opensearch-simulator/API.md b/packages/amplify-opensearch-simulator/API.md
index 089b42fa9ef..f5f3bbe729b 100644
--- a/packages/amplify-opensearch-simulator/API.md
+++ b/packages/amplify-opensearch-simulator/API.md
@@ -4,6 +4,8 @@
 
 ```ts
 
+/// <reference types="node" />
+
 import { $TSAny } from 'amplify-cli-core';
 import execa from 'execa';
 import { GetPackageAssetPaths } from 'amplify-cli-core';