Skip to content

Commit

Permalink
Cloud metadata (#748) (#750)
Browse files Browse the repository at this point in the history 
* Convert yargs builder function calls to plain objects

Leave environment variable management to yargs

* Remove dead code

* Manage all the environment variables through yargs (#739)

* Fix comment blunder

* replace null

* tf template check

* add null option

* clean up

* use undefined

* rm redundant debug logging

* update from draft example

* clean up

* Update tests

Additionally,
* Handle falsy values as empty strings
* Simplify template code

Co-authored-by: Helio Machado <[email protected]>

Co-authored-by: Daniel Barnes <[email protected]>
Co-authored-by: davidgortega <[email protected]>
  • Loading branch information
@0x2b3bfa0 @dacbd @DavidGOrtega
3 people authored Oct 15, 2021
1 parent 54c0a18 commit fa99ac9
Show file tree
Hide file tree
Showing 4 changed files with 184 additions and 91 deletions.
16 changes: 16 additions & 0 deletions bin/cml/runner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ const runCloud = async (opts) => {
cloud,
cloudRegion: region,
cloudType: type,
cloudMetadata: metadata,
cloudGpu: gpu,
cloudHddSize: hddSize,
cloudSshPrivate: sshPrivate,
Expand Down Expand Up @@ -151,6 +152,7 @@ const runCloud = async (opts) => {
cloud,
region,
type,
metadata,
gpu: gpu === 'tesla' ? 'v100' : gpu,
hddSize,
sshPrivate,
Expand Down Expand Up @@ -190,6 +192,7 @@ const runCloud = async (opts) => {
instanceLaunchTime: attributes.instance_launch_time,
instanceType: attributes.instance_type,
labels: attributes.labels,
metadata: attributes.metadata,
name: attributes.name,
region: attributes.region,
repo: attributes.repo,
Expand Down Expand Up @@ -467,6 +470,19 @@ exports.builder = (yargs) =>
description:
'Instance type. Choices: [m, l, xl]. Also supports native types like i.e. t2.micro'
},
cloudMetadata: {
type: 'array',
string: true,
coerce: (items) => {
const keyValuePairs = items.map((item) => [
...item.split(/=(.+)/),
null
]);
return Object.fromEntries(keyValuePairs);
},
description:
'Key Value pairs to associate cml-runner instance on the provider i.e. tags/labels "key=value"'
},
cloudGpu: {
type: 'string',
choices: ['nogpu', 'k80', 'v100', 'tesla'],
Expand Down
3 changes: 3 additions & 0 deletions bin/cml/runner.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ Options:
[string] [default: \\"us-west\\"]
--cloud-type Instance type. Choices: [m, l, xl]. Also supports
native types like i.e. t2.micro [string]
--cloud-metadata Key Value pairs to associate cml-runner instance
on the provider i.e. tags/labels \\"key=value\\"
[array]
--cloud-gpu GPU type.
[string] [choices: \\"nogpu\\", \\"k80\\", \\"v100\\", \\"tesla\\"]
--cloud-hdd-size HDD size in GB [number]
Expand Down
12 changes: 11 additions & 1 deletion src/terraform.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ const destroy = async (opts = {}) => {
);
};

const mapCloudMetadata = (metadata) =>
Object.entries(metadata).map(([key, value]) => `${key} = "${value || ''}"`);

const iterativeProviderTpl = () => {
return `
terraform {
Expand All @@ -74,6 +77,7 @@ const iterativeCmlRunnerTpl = (opts = {}) => {
name,
single,
type,
metadata,
gpu,
hddSize,
sshPrivate,
Expand All @@ -83,7 +87,7 @@ const iterativeCmlRunnerTpl = (opts = {}) => {
awsSecurityGroup
} = opts;

return `
const template = `
${iterativeProviderTpl()}
resource "iterative_cml_runner" "runner" {
Expand All @@ -108,8 +112,14 @@ resource "iterative_cml_runner" "runner" {
${spotPrice ? `spot_price = ${spotPrice}` : ''}
${startupScript ? `startup_script = "${startupScript}"` : ''}
${awsSecurityGroup ? `aws_security_group = "${awsSecurityGroup}"` : ''}
${
metadata
? `metadata = {\n ${mapCloudMetadata(metadata).join('\n ')}\n }`
: ''
}
}
`;
return template;
};

const checkMinVersion = async () => {
Expand Down
244 changes: 154 additions & 90 deletions src/terraform.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ describe('Terraform tests', () => {
}
"
`);
Expand All @@ -60,40 +61,41 @@ describe('Terraform tests', () => {
awsSecurityGroup: 'mysg'
});
expect(output).toMatchInlineSnapshot(`
"
"
terraform {
required_providers {
iterative = {
source = \\"iterative/iterative\\"
}
}
}
terraform {
required_providers {
iterative = {
source = \\"iterative/iterative\\"
}
}
}
provider \\"iterative\\" {}
provider \\"iterative\\" {}
resource \\"iterative_cml_runner\\" \\"runner\\" {
repo = \\"https://\\"
token = \\"abc\\"
driver = \\"gitlab\\"
labels = \\"mylabel\\"
idle_timeout = 300
name = \\"myrunner\\"
single = \\"true\\"
cloud = \\"aws\\"
region = \\"west\\"
instance_type = \\"mymachinetype\\"
instance_gpu = \\"mygputype\\"
instance_hdd_size = 50
ssh_private = \\"myprivate\\"
spot = true
spot_price = 0.0001
aws_security_group = \\"mysg\\"
}
"
`);
resource \\"iterative_cml_runner\\" \\"runner\\" {
repo = \\"https://\\"
token = \\"abc\\"
driver = \\"gitlab\\"
labels = \\"mylabel\\"
idle_timeout = 300
name = \\"myrunner\\"
single = \\"true\\"
cloud = \\"aws\\"
region = \\"west\\"
instance_type = \\"mymachinetype\\"
instance_gpu = \\"mygputype\\"
instance_hdd_size = 50
ssh_private = \\"myprivate\\"
spot = true
spot_price = 0.0001
aws_security_group = \\"mysg\\"
}
"
`);
});

test('basic settings with runner forever', async () => {
Expand All @@ -115,40 +117,101 @@ describe('Terraform tests', () => {
spotPrice: '0.0001'
});
expect(output).toMatchInlineSnapshot(`
"
"
terraform {
required_providers {
iterative = {
source = \\"iterative/iterative\\"
}
}
}
terraform {
required_providers {
iterative = {
source = \\"iterative/iterative\\"
}
}
}
provider \\"iterative\\" {}
provider \\"iterative\\" {}
resource \\"iterative_cml_runner\\" \\"runner\\" {
repo = \\"https://\\"
token = \\"abc\\"
driver = \\"gitlab\\"
labels = \\"mylabel\\"
idle_timeout = 0
name = \\"myrunner\\"
single = \\"true\\"
cloud = \\"aws\\"
region = \\"west\\"
instance_type = \\"mymachinetype\\"
instance_gpu = \\"mygputype\\"
instance_hdd_size = 50
ssh_private = \\"myprivate\\"
spot = true
spot_price = 0.0001
}
"
`);
resource \\"iterative_cml_runner\\" \\"runner\\" {
repo = \\"https://\\"
token = \\"abc\\"
driver = \\"gitlab\\"
labels = \\"mylabel\\"
idle_timeout = 0
name = \\"myrunner\\"
single = \\"true\\"
cloud = \\"aws\\"
region = \\"west\\"
instance_type = \\"mymachinetype\\"
instance_gpu = \\"mygputype\\"
instance_hdd_size = 50
ssh_private = \\"myprivate\\"
spot = true
spot_price = 0.0001
}
"
`);
});

test('basic settings with metadata', async () => {
const output = iterativeCmlRunnerTpl({
repo: 'https://',
token: 'abc',
driver: 'gitlab',
labels: 'mylabel',
idleTimeout: 300,
name: 'myrunner',
single: true,
cloud: 'aws',
region: 'west',
type: 'mymachinetype',
gpu: 'mygputype',
hddSize: 50,
sshPrivate: 'myprivate',
spot: true,
spotPrice: '0.0001',
metadata: { one: 'value', two: null }
});
expect(output).toMatchInlineSnapshot(`
"
terraform {
required_providers {
iterative = {
source = \\"iterative/iterative\\"
}
}
}
provider \\"iterative\\" {}
resource \\"iterative_cml_runner\\" \\"runner\\" {
repo = \\"https://\\"
token = \\"abc\\"
driver = \\"gitlab\\"
labels = \\"mylabel\\"
idle_timeout = 300
name = \\"myrunner\\"
single = \\"true\\"
cloud = \\"aws\\"
region = \\"west\\"
instance_type = \\"mymachinetype\\"
instance_gpu = \\"mygputype\\"
instance_hdd_size = 50
ssh_private = \\"myprivate\\"
spot = true
spot_price = 0.0001
metadata = {
one = \\"value\\"
two = \\"\\"
}
}
"
`);
});

test('Startup script', async () => {
Expand All @@ -171,39 +234,40 @@ describe('Terraform tests', () => {
startupScript: 'c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ='
});
expect(output).toMatchInlineSnapshot(`
"
"
terraform {
required_providers {
iterative = {
source = \\"iterative/iterative\\"
}
}
}
terraform {
required_providers {
iterative = {
source = \\"iterative/iterative\\"
}
}
}
provider \\"iterative\\" {}
provider \\"iterative\\" {}
resource \\"iterative_cml_runner\\" \\"runner\\" {
repo = \\"https://\\"
token = \\"abc\\"
driver = \\"gitlab\\"
labels = \\"mylabel\\"
idle_timeout = 300
name = \\"myrunner\\"
single = \\"true\\"
cloud = \\"aws\\"
region = \\"west\\"
instance_type = \\"mymachinetype\\"
instance_gpu = \\"mygputype\\"
instance_hdd_size = 50
ssh_private = \\"myprivate\\"
spot = true
spot_price = 0.0001
startup_script = \\"c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=\\"
}
"
`);
resource \\"iterative_cml_runner\\" \\"runner\\" {
repo = \\"https://\\"
token = \\"abc\\"
driver = \\"gitlab\\"
labels = \\"mylabel\\"
idle_timeout = 300
name = \\"myrunner\\"
single = \\"true\\"
cloud = \\"aws\\"
region = \\"west\\"
instance_type = \\"mymachinetype\\"
instance_gpu = \\"mygputype\\"
instance_hdd_size = 50
ssh_private = \\"myprivate\\"
spot = true
spot_price = 0.0001
startup_script = \\"c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=\\"
}
"
`);
});
});

1 comment on commit fa99ac9

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Test Comment

CML watermark

Please sign in to comment.