Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ci: fix staleness filters #11801

Merged
merged 4 commits into from
Jan 18, 2023
Merged

ci: fix staleness filters #11801

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5809c7a
ci: fix staleness filters
awsluja Jan 18, 2023
8557b3e
ci: only delete stale apps & stacks
awsluja Jan 18, 2023
1e38fd5
chore: refactor
awsluja Jan 18, 2023
80494fc
Merge branch 'dev' into fix-staleness-filters
awsluja Jan 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 59 additions & 30 deletions packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ const DELETE_LIMITS = {
PER_BATCH: {
OTHER: 50,
CFN_STACK: 100,
}
}
},
};

const reportPath = path.normalize(path.join(__dirname, '..', 'amplify-e2e-reports', 'stale-resources.json'));

Expand Down Expand Up @@ -140,19 +140,28 @@ const handleExpiredTokenException = (): void => {
process.exit();
};

/**
* Check if a resource is stale based on its created date
* @param created
* @returns
*/
const isStale = (created: Date): boolean => {
const now = new Date().getTime();
const isStale = now - created.getTime() > STALE_DURATION_MS;
return isStale;
};

/**
* We define a resource as viable for deletion if it matches TEST_REGEX in the name, and if it is > STALE_DURATION_MS old.
*/
const testBucketStalenessFilter = (resource: aws.S3.Bucket): boolean => {
const isTestResource = resource.Name.match(BUCKET_TEST_REGEX);
const isStaleResource = new Date().getUTCMilliseconds() - resource.CreationDate.getUTCMilliseconds() > STALE_DURATION_MS;
return isTestResource && isStaleResource;
return isTestResource && isStale(resource.CreationDate);
};

const testRoleStalenessFilter = (resource: aws.IAM.Role): boolean => {
const isTestResource = resource.RoleName.match(IAM_TEST_REGEX);
const isStaleResource = new Date().getUTCMilliseconds() - resource.CreateDate.getUTCMilliseconds() > STALE_DURATION_MS;
return isTestResource && isStaleResource;
return isTestResource && isStale(resource.CreateDate);
};

const testAppSyncApiStalenessFilter = (resource: aws.AppSync.GraphqlApi): boolean => {
Expand All @@ -161,15 +170,14 @@ const testAppSyncApiStalenessFilter = (resource: aws.AppSync.GraphqlApi): boolea
let isStaleResource = true;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should not assume it is stale by default

if (createTimeTagValue) {
const createTime = new Date(createTimeTagValue);
isStaleResource = (new Date().getUTCMilliseconds() - createTime.getUTCMilliseconds()) > STALE_DURATION_MS;
isStaleResource = isStale(createTime);
}
return isTestResource && isStaleResource;
};

const testPinpointAppStalenessFilter = (resource: aws.Pinpoint.ApplicationResponse): boolean => {
const isTestResource = resource.Name.match(PINPOINT_TEST_REGEX);
const isStaleResource = new Date().getUTCMilliseconds() - new Date(resource.CreationDate).getUTCMilliseconds() > STALE_DURATION_MS;
return isTestResource && isStaleResource;
return isTestResource && isStale(new Date(resource.CreationDate));
};

/**
Expand Down Expand Up @@ -250,14 +258,17 @@ const getAWSConfig = ({ accessKeyId, secretAccessKey, sessionToken }: AWSAccount
* @returns Promise<AmplifyAppInfo[]> a list of Amplify Apps in the region with build info
*/
const getAmplifyApps = async (account: AWSAccountInfo, region: string): Promise<AmplifyAppInfo[]> => {
if(region === 'us-east-1' && account.parent){
if (region === 'us-east-1' && account.parent) {
return []; // temporarily disabled until us-east-1 is re-enabled for this account
}
const amplifyClient = new aws.Amplify(getAWSConfig(account, region));
try {
const amplifyApps = await amplifyClient.listApps({ maxResults: 25 }).promise(); // keeping it to 25 as max supported is 25
const result: AmplifyAppInfo[] = [];
for (const app of amplifyApps.apps) {
if (!isStale(app.createTime)) {
continue; // skip
}
const backends: Record<string, StackInfo> = {};
try {
const backendEnvironments = await amplifyClient.listBackendEnvironments({ appId: app.appId, maxResults: 5 }).promise();
Expand All @@ -278,7 +289,7 @@ const getAmplifyApps = async (account: AWSAccountInfo, region: string): Promise<
});
}
return result;
} catch (e){
} catch (e) {
console.log(e);
return [];
}
Expand Down Expand Up @@ -347,15 +358,15 @@ const getStacks = async (account: AWSAccountInfo, region: string): Promise<Stack
StackStatusFilter: stackStatusFilter,
})
.promise();
// loop
// loop
let nextToken = stacks.NextToken;
while (nextToken && stacks.StackSummaries.length < DELETE_LIMITS.PER_REGION.CFN_STACK) {
const nextPage = await cfnClient
.listStacks({
StackStatusFilter: stackStatusFilter,
NextToken: nextToken,
})
.promise();
.promise();
stacks.StackSummaries.push(...nextPage.StackSummaries);
nextToken = nextPage.NextToken;
}
Expand All @@ -364,8 +375,14 @@ const getStacks = async (account: AWSAccountInfo, region: string): Promise<Stack
// NOTE: every few months, we should disable the filter , and clean up all stacks (not just root stacks)
// this is because some child stacks fail to delete (but we don't let that stop us from deleting root stacks)
// eventually, we must clean up those child stacks too.
let rootStacks = stacks.StackSummaries.filter(stack => !stack.RootId);
if(rootStacks.length > DELETE_LIMITS.PER_REGION.CFN_STACK){
let rootStacks = stacks.StackSummaries.filter(stack => {
const isRoot = !stack.RootId;
if (!isStale(stack.CreationTime)) {
console.log('Skipping stack because created date is:', stack.CreationTime);
}
return isRoot && isStale;
});
if (rootStacks.length > DELETE_LIMITS.PER_REGION.CFN_STACK) {
// we can only delete 100 stacks accross all regions every batch,
// so we shouldn't take more than 50 stacks from each of those 8 regions.
// this should at least limit calls to getStackDetails below
Expand Down Expand Up @@ -685,9 +702,7 @@ const deleteAppSyncApis = async (account: AWSAccountInfo, accountIndex: number,
};

const deleteAppSyncApi = async (account: AWSAccountInfo, accountIndex: number, api: AppSyncApiInfo): Promise<void> => {
const {
apiId, name, region,
} = api;
const { apiId, name, region } = api;
try {
console.log(`[ACCOUNT ${accountIndex}] Deleting AppSync Api ${name}`);
const appSync = new aws.AppSync(getAWSConfig(account, region));
Expand Down Expand Up @@ -755,12 +770,16 @@ const deleteResources = async (
}

if (resources.pinpointApps) {
console.log(`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.pinpointApps.length} pinpoint apps on ACCOUNT[${accountIndex}]`);
console.log(
`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.pinpointApps.length} pinpoint apps on ACCOUNT[${accountIndex}]`,
);
await deletePinpointApps(account, accountIndex, Object.values(resources.pinpointApps));
}

if (resources.appSyncApis) {
console.log(`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.appSyncApis.length} appSyncApis on ACCOUNT[${accountIndex}]`);
console.log(
`Deleting up to ${DELETE_LIMITS.PER_BATCH.OTHER} of ${resources.appSyncApis.length} appSyncApis on ACCOUNT[${accountIndex}]`,
);
await deleteAppSyncApis(account, accountIndex, Object.values(resources.appSyncApis));
}
}
Expand Down Expand Up @@ -876,13 +895,21 @@ const cleanupAccount = async (account: AWSAccountInfo, accountIndex: number, fil
const orphanAppSyncApis = (await Promise.all(orphanAppSyncApisPromise)).flat();

const allResources = mergeResourcesByCCIJob(
apps, stacks, buckets, orphanBuckets, orphanIamRoles, orphanPinpointApplications, orphanAppSyncApis
apps,
stacks,
buckets,
orphanBuckets,
orphanIamRoles,
orphanPinpointApplications,
orphanAppSyncApis,
);
// cleanup resources that are <unknown> but that are definitely amplify resources
// this includes apps with names that include "test" or stacks that include both "amplify" & "test"
const testApps = allResources["<unknown>"].amplifyApps?.filter(a => a.name.toLocaleLowerCase().includes('test'));
const testStacks = allResources["<unknown>"].stacks?.filter(s => s.stackName.toLocaleLowerCase().includes('test') && s.stackName.toLocaleLowerCase().includes('amplify'));
const orphanedResources = allResources["<orphan>"];
const testApps = allResources['<unknown>'].amplifyApps?.filter(a => a.name.toLocaleLowerCase().includes('test'));
const testStacks = allResources['<unknown>'].stacks?.filter(
s => s.stackName.toLocaleLowerCase().includes('test') && s.stackName.toLocaleLowerCase().includes('amplify'),
);
const orphanedResources = allResources['<orphan>'];
orphanedResources.amplifyApps = orphanedResources.amplifyApps ?? [];
orphanedResources.stacks = orphanedResources.stacks ?? [];
orphanedResources.amplifyApps.push(...(testApps ? testApps : []));
Expand Down Expand Up @@ -922,12 +949,14 @@ const cleanup = async (): Promise<void> => {

const filterPredicate = getFilterPredicate(args);
const accounts = await getAccountsToCleanup();
for(let i = 0 ;i < 3; i ++){
console.log("CLEANUP ROUND: ", i + 1);
await Promise.all(accounts.map((account, i) => {
return cleanupAccount(account, i, filterPredicate);
}));
await sleep(60 * 1000);// run again after 60 seconds
for (let i = 0; i < 3; i++) {
console.log('CLEANUP ROUND: ', i + 1);
await Promise.all(
accounts.map((account, i) => {
return cleanupAccount(account, i, filterPredicate);
}),
);
await sleep(60 * 1000); // run again after 60 seconds
}
console.log('Done cleaning all accounts!');
};
Expand Down