Skip to content

Commit

Permalink
Change APIKEY name
Browse files Browse the repository at this point in the history
  • Loading branch information
@nathanielrindlaub
nathanielrindlaub committed Apr 20, 2024
1 parent d30601e commit e998727
Show file tree
Hide file tree
Showing 4 changed files with 364 additions and 334 deletions.
262 changes: 138 additions & 124 deletions ingest-delete/task.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,53 +3,59 @@ import SSM from '@aws-sdk/client-ssm';
import CW from '@aws-sdk/client-cloudwatch';
import moment from 'moment';

const APIKEY = process.env.APIKEY;
const API_KEY = process.env.API_KEY;

export async function handler(event) {
if (!event) throw new Error('Event not populated');

console.log(`ok - event: ${JSON.stringify(event)}`);

const STAGE = process.env.STAGE || 'dev';
const params = new Map();

let StackName = null;
let batch = null;
if (event.Records && event.Records[0] && event.Records[0].Sns) {
const alarm = JSON.parse(event.Records[0].Sns.Message).AlarmName;
StackName = alarm.replace('-sqs-empty', '');
batch = `batch-${StackName.replace(/^.*-batch-/, '')}`;
} else if (event.batch) {
StackName = `animl-ingest-${STAGE}-${event.batch}`;
batch = event.batch;
} else if (event.source) {
return await scheduledDelete(STAGE);
} else {
throw new Error('Unknown Event Type');
if (!event) throw new Error('Event not populated');

console.log(`ok - event: ${JSON.stringify(event)}`);

const STAGE = process.env.STAGE || 'dev';
const params = new Map();

let StackName = null;
let batch = null;
if (event.Records && event.Records[0] && event.Records[0].Sns) {
const alarm = JSON.parse(event.Records[0].Sns.Message).AlarmName;
StackName = alarm.replace('-sqs-empty', '');
batch = `batch-${StackName.replace(/^.*-batch-/, '')}`;
} else if (event.batch) {
StackName = `animl-ingest-${STAGE}-${event.batch}`;
batch = event.batch;
} else if (event.source) {
return await scheduledDelete(STAGE);
} else {
throw new Error('Unknown Event Type');
}

if (!StackName) throw new Error('StackName could not be determined');

try {
const cf = new CloudFormation.CloudFormationClient({
region: process.env.AWS_DEFAULT_REGION || 'us-west-2',
});
const ssm = new SSM.SSMClient({ region: process.env.AWS_DEFAULT_REGION || 'us-west-2' });

for (const param of (
await ssm.send(
new SSM.GetParametersCommand({
Names: [`/api/url-${STAGE}`],
WithDecryption: true,
}),
)
).Parameters) {
console.log(`ok - setting ${param.Name}`);
params.set(param.Name, param.Value);
}

if (!StackName) throw new Error('StackName could not be determined');

try {
const cf = new CloudFormation.CloudFormationClient({ region: process.env.AWS_DEFAULT_REGION || 'us-west-2' });
const ssm = new SSM.SSMClient({ region: process.env.AWS_DEFAULT_REGION || 'us-west-2' });

for (const param of (await ssm.send(new SSM.GetParametersCommand({
Names: [`/api/url-${STAGE}`],
WithDecryption: true
}))).Parameters) {
console.log(`ok - setting ${param.Name}`);
params.set(param.Name, param.Value);
}

console.log(`ok - deleting: ${StackName}`);
console.log(`ok - deleting: ${StackName}`);

await cf.send(new CloudFormation.DeleteStackCommand({ StackName }));
await cf.send(new CloudFormation.DeleteStackCommand({ StackName }));

console.log(`ok - batch: ${batch}`);
console.log(`ok - batch: ${batch}`);

await fetcher(params.get(`/api/url-${STAGE}`), {
query: `
await fetcher(params.get(`/api/url-${STAGE}`), {
query: `
mutation UpdateBatch($input: UpdateBatchInput!){
updateBatch(input: $input) {
batch {
Expand All @@ -61,21 +67,21 @@ export async function handler(event) {
}
}
`,
variables: {
input: {
_id: batch,
processingEnd: new Date()
}
}
});
variables: {
input: {
_id: batch,
processingEnd: new Date(),
},
},
});

console.log('ok - stack deletion complete');
} catch (err) {
console.error(err);
console.log('ok - stack deletion complete');
} catch (err) {
console.error(err);

if (params.has(`/api/url-${STAGE}`)) {
await fetcher(params.get(`/api/url-${STAGE}`), {
query: `
if (params.has(`/api/url-${STAGE}`)) {
await fetcher(params.get(`/api/url-${STAGE}`), {
query: `
mutation CreateBatchError($input: CreateBatchErrorInput!) {
createBatchError(input: $input) {
_id
Expand All @@ -85,92 +91,100 @@ export async function handler(event) {
}
}
`,
variables: {
input: {
error: err.message,
batch: batch
}
}
});
} else {
console.error('not ok - Failed to post to CreateBatchError');
}

throw err;
variables: {
input: {
error: err.message,
batch: batch,
},
},
});
} else {
console.error('not ok - Failed to post to CreateBatchError');
}

throw err;
}
}

async function scheduledDelete(stage) {
const cf = new CloudFormation.CloudFormationClient({ region: process.env.AWS_DEFAULT_REGION || 'us-west-2' });

let stacks = [];
let nextToken = undefined;
do {
const res = await cf.send(new CloudFormation.ListStacksCommand({
NextToken: nextToken
}));

stacks.push(...res.StackSummaries);
nextToken = res.NextToken;
} while (nextToken);

stacks = stacks.filter((stack) => {
return stack.StackName.startsWith(`animl-ingest-${stage}-batch-`);
}).filter((stack) => {
return stack.StackStatus !== 'DELETE_COMPLETE';
}).filter((stack) => {
return moment(stack.CreationTime).isSameOrBefore(moment().subtract(24, 'hours'));
const cf = new CloudFormation.CloudFormationClient({
region: process.env.AWS_DEFAULT_REGION || 'us-west-2',
});

let stacks = [];
let nextToken = undefined;
do {
const res = await cf.send(
new CloudFormation.ListStacksCommand({
NextToken: nextToken,
}),
);

stacks.push(...res.StackSummaries);
nextToken = res.NextToken;
} while (nextToken);

stacks = stacks
.filter((stack) => {
return stack.StackName.startsWith(`animl-ingest-${stage}-batch-`);
})
.filter((stack) => {
return stack.StackStatus !== 'DELETE_COMPLETE';
})
.filter((stack) => {
return moment(stack.CreationTime).isSameOrBefore(moment().subtract(24, 'hours'));
});

const cw = new CW.CloudWatchClient({ region: process.env.AWS_DEFAULT_REGION || 'us-west-2' });
console.log(`ok - ${stacks.length} candidate stacks for deletion`);
for (const stack of stacks) {
console.log(`ok - checking alarm state ${stack.StackName}`);
const alarm = await cw.send(new CW.DescribeAlarmsCommand({
AlarmNames: [`${stack.StackName}-sqs-empty`]
}));

if (alarm.MetricAlarms[0].StateValue === 'INSUFFICIENT_DATA') {
console.log(`ok - deleting ${stack.StackName}`);
await cf.send(new CloudFormation.DeleteStackCommand({ StackName: stack.StackName }));
}
const cw = new CW.CloudWatchClient({ region: process.env.AWS_DEFAULT_REGION || 'us-west-2' });
console.log(`ok - ${stacks.length} candidate stacks for deletion`);
for (const stack of stacks) {
console.log(`ok - checking alarm state ${stack.StackName}`);
const alarm = await cw.send(
new CW.DescribeAlarmsCommand({
AlarmNames: [`${stack.StackName}-sqs-empty`],
}),
);

if (alarm.MetricAlarms[0].StateValue === 'INSUFFICIENT_DATA') {
console.log(`ok - deleting ${stack.StackName}`);
await cf.send(new CloudFormation.DeleteStackCommand({ StackName: stack.StackName }));
}
}
}

async function fetcher(url, body) {
console.log('Posting metadata to API');
const res = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-api-key': APIKEY
},
body: JSON.stringify(body)
});

if (!res.ok) {
const texterr = await res.text();
let jsonerr;
try {
jsonerr = JSON.parse(texterr);
} catch (err) {
throw new Error(texterr);
}

if (jsonerr.message) throw new Error(jsonerr.message);
throw new Error(texterr);
console.log('Posting metadata to API');
const res = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'x-api-key': API_KEY,
},
body: JSON.stringify(body),
});

if (!res.ok) {
const texterr = await res.text();
let jsonerr;
try {
jsonerr = JSON.parse(texterr);
} catch (err) {
throw new Error(texterr);
}

const json = await res.json();
if (jsonerr.message) throw new Error(jsonerr.message);
throw new Error(texterr);
}

if (json && Array.isArray(json.errors) && json.errors.length) {
throw new Error(json.errors[0].message);
}
const json = await res.json();

return json;
}
if (json && Array.isArray(json.errors) && json.errors.length) {
throw new Error(json.errors[0].message);
}

return json;
}

if (import.meta.url === `file://${process.argv[1]}`) {
await handler(process.env.EVENT ? JSON.parse(process.env.EVENT) : {});
await handler(process.env.EVENT ? JSON.parse(process.env.EVENT) : {});
}
4 changes: 2 additions & 2 deletions ingest-image/task.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ const region = process.env.AWS_DEFAULT_REGION || 'us-west-2';

const IngestType = new Enum(['NONE', 'IMAGE', 'BATCH'], 'IngestType');

const APIKEY = process.env.APIKEY;
const API_KEY = process.env.API_KEY;

const SHARP_CONFIG = { failOn: 'none' };

Expand Down Expand Up @@ -440,7 +440,7 @@ async function fetcher(url, body) {
signal: controller.signal,
headers: {
'Content-Type': 'application/json',
'x-api-key': APIKEY,
'x-api-key': API_KEY,
},
body: JSON.stringify(body),
});
Expand Down
Loading

0 comments on commit e998727

Please sign in to comment.