Skip to content

Commit

Permalink
[#5919] Platform: change default storage option for AWS provider to 1…
Browse files Browse the repository at this point in the history 
…x250GB

Heading:
[5919][Platform] change default storage option for AWS provider to 1x250GB
Description:
We wanted to show default volume count to 1 on the UI for all the instance types starting with "c5./c4."
Testing:
When we select aws provider while creating universe, volume count was showing more than one for some instances types starting with ""c5./c4.". Now with this fix it will show only one.
Added Unit test as well.
  • Loading branch information
Jitendra Kumar committed Dec 30, 2020
1 parent c11dc55 commit fdf8fea
Show file tree
Hide file tree
Showing 2 changed files with 123 additions and 103 deletions.
222 changes: 119 additions & 103 deletions managed/src/main/java/com/yugabyte/yw/cloud/AWSInitializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import java.util.Map;
import java.util.UUID;

import com.typesafe.config.Config;

import com.google.inject.Singleton;
import com.yugabyte.yw.commissioner.Common;
import com.yugabyte.yw.common.ApiResponse;
Expand All @@ -48,13 +50,21 @@
@Singleton
public class AWSInitializer extends AbstractInitializer {
private static final boolean enableVerboseLogging = false;
// Config names
static final String YB_VOLUME_INFO_VOLUME_COUNT = "yb.volumeInfo.volume_count";

private List<Map<String, String>> ec2AvailableInstances = new ArrayList<>();
private Provider provider;
private final Config config;

@Inject
Environment environment;

@Inject
public AWSInitializer(Config config) {
this.config = config;
}

/**
* Entry point to initialize AWS. This will create the various InstanceTypes and their
* corresponding PriceComponents per Region for AWS as well as the EBS pricing info.
Expand Down Expand Up @@ -118,45 +128,45 @@ public Result initialize(UUID customerUUID, UUID providerUUID) {
/**
* This will store the PriceComponents corresponding to EBS. Example IO1 size json blobs:
* "KA7RG53ZHMXMZFAF" : {
* "KA7RG53ZHMXMZFAF.JRTCKXETXF" : {
* "offerTermCode" : "JRTCKXETXF",
* "sku" : "KA7RG53ZHMXMZFAF",
* "effectiveDate" : "2017-06-01T00:00:00Z",
* "priceDimensions" : {
* "KA7RG53ZHMXMZFAF.JRTCKXETXF.6YS6EN2CT7" : {
* "rateCode" : "KA7RG53ZHMXMZFAF.JRTCKXETXF.6YS6EN2CT7",
* "description" : "$0.145 per GB-month of Provisioned IOPS SSD (io1) provisioned storage - EU (London)",
* "beginRange" : "0",
* "endRange" : "Inf",
* "unit" : "GB-Mo",
* "pricePerUnit" : {
* "USD" : "0.1450000000"
* },
* "appliesTo" : [ ]
* }
* },
* "termAttributes" : { }
* }
* "KA7RG53ZHMXMZFAF.JRTCKXETXF" : {
* "offerTermCode" : "JRTCKXETXF",
* "sku" : "KA7RG53ZHMXMZFAF",
* "effectiveDate" : "2017-06-01T00:00:00Z",
* "priceDimensions" : {
* "KA7RG53ZHMXMZFAF.JRTCKXETXF.6YS6EN2CT7" : {
* "rateCode" : "KA7RG53ZHMXMZFAF.JRTCKXETXF.6YS6EN2CT7",
* "description" : "$0.145 per GB-month of Provisioned IOPS SSD (io1) provisioned storage - EU (London)",
* "beginRange" : "0",
* "endRange" : "Inf",
* "unit" : "GB-Mo",
* "pricePerUnit" : {
* "USD" : "0.1450000000"
* },
* "appliesTo" : [ ]
* }
* },
* "termAttributes" : { }
* }
* },
* "KA7RG53ZHMXMZFAF" : {
* "sku" : "KA7RG53ZHMXMZFAF",
* "productFamily" : "Storage",
* "attributes" : {
* "servicecode" : "AmazonEC2",
* "location" : "EU (London)",
* "locationType" : "AWS Region",
* "storageMedia" : "SSD-backed",
* "volumeType" : "Provisioned IOPS",
* "maxVolumeSize" : "16 TiB",
* "maxIopsvolume" : "20000",
* "maxThroughputvolume" : "320 MB/sec",
* "usagetype" : "EUW2-EBS:VolumeUsage.piops",
* "operation" : ""
* }
* "sku" : "KA7RG53ZHMXMZFAF",
* "productFamily" : "Storage",
* "attributes" : {
* "servicecode" : "AmazonEC2",
* "location" : "EU (London)",
* "locationType" : "AWS Region",
* "storageMedia" : "SSD-backed",
* "volumeType" : "Provisioned IOPS",
* "maxVolumeSize" : "16 TiB",
* "maxIopsvolume" : "20000",
* "maxThroughputvolume" : "320 MB/sec",
* "usagetype" : "EUW2-EBS:VolumeUsage.piops",
* "operation" : ""
* }
* },
*
* @param productDetailsListJson Products sub-document with list of EC2 products along with SKU.
* @param onDemandJson Price details json object.
* @param onDemandJson Price details json object.
*/
private void storeEBSPriceComponents(JsonNode productDetailsListJson, JsonNode onDemandJson) {
LOG.info("Parsing product details list to store pricing info");
Expand All @@ -171,9 +181,9 @@ private void storeEBSPriceComponents(JsonNode productDetailsListJson, JsonNode o
continue;
}
Region region = Region.find.query().where()
.eq("provider_uuid", provider.uuid)
.eq("name", regionJson.textValue())
.findOne();
.eq("provider_uuid", provider.uuid)
.eq("name", regionJson.textValue())
.findOne();
if (region == null) {
if (enableVerboseLogging) {
LOG.error("No region " + regionJson.textValue() + " available");
Expand Down Expand Up @@ -205,10 +215,10 @@ private void storeEBSPriceComponents(JsonNode productDetailsListJson, JsonNode o
/**
* Given info about a single EBS item (size/piops) in a specific region, store its PriceComponent.
*
* @param sku SKU of the EBS item in its region.
* @param sku SKU of the EBS item in its region.
* @param componentCode Code for the EBS item (e.g. io1.size).
* @param region The region the EBS item is in (e.g. us-west2).
* @param onDemandJson Price details json object.
* @param region The region the EBS item is in (e.g. us-west2).
* @param onDemandJson Price details json object.
*/
private void storeEBSPriceComponent(String sku, String componentCode, Region region,
JsonNode onDemandJson) {
Expand Down Expand Up @@ -243,30 +253,30 @@ private void storeEBSPriceComponent(String sku, String componentCode, Region reg
/**
* This will store the PriceComponent corresponding to the InstanceType itself.
* Each price detail object has the format:
* "DQ578CGN99KG6ECF" : {
* "DQ578CGN99KG6ECF.JRTCKXETXF" : {
* "offerTermCode" : "JRTCKXETXF",
* "sku" : "DQ578CGN99KG6ECF",
* "effectiveDate" : "2016-08-01T00:00:00Z",
* "priceDimensions" : {
* "DQ578CGN99KG6ECF.JRTCKXETXF.6YS6EN2CT7" : {
* "rateCode" : "DQ578CGN99KG6ECF.JRTCKXETXF.6YS6EN2CT7",
* "description" : "$4.931 per On Demand Windows hs1.8xlarge Instance Hour",
* "beginRange" : "0",
* "endRange" : "Inf",
* "unit" : "Hrs",
* "pricePerUnit" : {
* "USD" : "4.9310000000"
* },
* "appliesTo" : [ ]
* }
* },
* "termAttributes" : { }
* }
* }
* "DQ578CGN99KG6ECF" : {
* "DQ578CGN99KG6ECF.JRTCKXETXF" : {
* "offerTermCode" : "JRTCKXETXF",
* "sku" : "DQ578CGN99KG6ECF",
* "effectiveDate" : "2016-08-01T00:00:00Z",
* "priceDimensions" : {
* "DQ578CGN99KG6ECF.JRTCKXETXF.6YS6EN2CT7" : {
* "rateCode" : "DQ578CGN99KG6ECF.JRTCKXETXF.6YS6EN2CT7",
* "description" : "$4.931 per On Demand Windows hs1.8xlarge Instance Hour",
* "beginRange" : "0",
* "endRange" : "Inf",
* "unit" : "Hrs",
* "pricePerUnit" : {
* "USD" : "4.9310000000"
* },
* "appliesTo" : [ ]
* }
* },
* "termAttributes" : { }
* }
* }
*
* @param productDetailsListJson Products sub-document with list of EC2 products along with SKU.
* @param onDemandJson Price details json object.
* @param onDemandJson Price details json object.
*/
private void storeInstancePriceComponents(JsonNode productDetailsListJson,
JsonNode onDemandJson) {
Expand All @@ -286,10 +296,10 @@ private void storeInstancePriceComponents(JsonNode productDetailsListJson,
include &= (matches(productAttrs, "operatingSystem", FilterOp.Equals, "Linux"));
// Pick the supported license models.
include &= (matches(productAttrs, "licenseModel", FilterOp.Equals, "No License required") ||
matches(productAttrs, "licenseModel", FilterOp.Equals, "NA"));
matches(productAttrs, "licenseModel", FilterOp.Equals, "NA"));
// Pick the valid disk drive types.
include &= (matches(productAttrs, "storage", FilterOp.Contains, "SSD") ||
matches(productAttrs, "storage", FilterOp.Contains, "EBS"));
matches(productAttrs, "storage", FilterOp.Contains, "EBS"));
// Make sure it is current generation.
include &= matches(productAttrs, "currentGeneration", FilterOp.Equals, "Yes");
// Make sure tenancy is shared.
Expand All @@ -302,20 +312,20 @@ private void storeInstancePriceComponents(JsonNode productDetailsListJson,
if (include) {
JsonNode attributesJson = productDetailsJson.get("attributes");
storeInstancePriceComponent(
productDetailsJson.get("sku").textValue(),
attributesJson.get("instanceType").textValue(),
attributesJson.get("location").textValue(),
onDemandJson);
productDetailsJson.get("sku").textValue(),
attributesJson.get("instanceType").textValue(),
attributesJson.get("location").textValue(),
onDemandJson);
}
}
}

/**
* Given info about a single AWS InstanceType in a specific region, store its PriceComponent.
*
* @param sku SKU of the InstanceType in its region.
* @param sku SKU of the InstanceType in its region.
* @param instanceCode Code for the InstanceType (e.g. m3.medium).
* @param regionName Name for the region the InstanceType is in (e.g. "US West (Oregon)").
* @param regionName Name for the region the InstanceType is in (e.g. "US West (Oregon)").
* @param onDemandJson Price details json object.
*/
private void storeInstancePriceComponent(String sku, String instanceCode, String regionName,
Expand Down Expand Up @@ -366,29 +376,29 @@ private void storeInstancePriceComponent(String sku, String instanceCode, String
* map, which contains information on the various instances available through EC2. Each entry in
* the product details map looks like:
* "DQ578CGN99KG6ECF" : {
* "sku" : "DQ578CGN99KG6ECF",
* "productFamily" : "Compute Instance",
* "attributes" : {
* "servicecode" : "AmazonEC2",
* "location" : "US East (N. Virginia)",
* "locationType" : "AWS Region",
* "instanceType" : "hs1.8xlarge",
* "currentGeneration" : "No",
* "instanceFamily" : "Storage optimized",
* "vcpu" : "17",
* "physicalProcessor" : "Intel Xeon E5-2650",
* "clockSpeed" : "2 GHz",
* "memory" : "117 GiB",
* "storage" : "24 x 2000",
* "networkPerformance" : "10 Gigabit",
* "processorArchitecture" : "64-bit",
* "tenancy" : "Shared",
* "operatingSystem" : "Windows",
* "licenseModel" : "License Included",
* "usagetype" : "BoxUsage:hs1.8xlarge",
* "operation" : "RunInstances:0002",
* "preInstalledSw" : "NA"
* }
* "sku" : "DQ578CGN99KG6ECF",
* "productFamily" : "Compute Instance",
* "attributes" : {
* "servicecode" : "AmazonEC2",
* "location" : "US East (N. Virginia)",
* "locationType" : "AWS Region",
* "instanceType" : "hs1.8xlarge",
* "currentGeneration" : "No",
* "instanceFamily" : "Storage optimized",
* "vcpu" : "17",
* "physicalProcessor" : "Intel Xeon E5-2650",
* "clockSpeed" : "2 GHz",
* "memory" : "117 GiB",
* "storage" : "24 x 2000",
* "networkPerformance" : "10 Gigabit",
* "processorArchitecture" : "64-bit",
* "tenancy" : "Shared",
* "operatingSystem" : "Windows",
* "licenseModel" : "License Included",
* "usagetype" : "BoxUsage:hs1.8xlarge",
* "operation" : "RunInstances:0002",
* "preInstalledSw" : "NA"
* }
* }
*
* @param productDetailsListJson A JSON blob as described above.
Expand All @@ -411,10 +421,10 @@ private void parseProductDetailsList(JsonNode productDetailsListJson) {
include &= (matches(productAttrs, "operatingSystem", FilterOp.Equals, "Linux"));
// Pick the supported license models.
include &= (matches(productAttrs, "licenseModel", FilterOp.Equals, "No License required") ||
matches(productAttrs, "licenseModel", FilterOp.Equals, "NA"));
matches(productAttrs, "licenseModel", FilterOp.Equals, "NA"));
// Pick the valid disk drive types.
include &= (matches(productAttrs, "storage", FilterOp.Contains, "SSD") ||
matches(productAttrs, "storage", FilterOp.Contains, "EBS"));
matches(productAttrs, "storage", FilterOp.Contains, "EBS"));
// Make sure it is current generation.
include &= matches(productAttrs, "currentGeneration", FilterOp.Equals, "Yes");
// Make sure tenancy is shared.
Expand All @@ -433,7 +443,7 @@ private void parseProductDetailsList(JsonNode productDetailsListJson) {

if (enableVerboseLogging) {
LOG.info("Found matching product with sku={}, instanceType={}", productAttrs.get("sku"),
productAttrs.get("instanceType"));
productAttrs.get("instanceType"));
}
ec2AvailableInstances.add(productAttrs);
}
Expand All @@ -448,7 +458,7 @@ private void parseProductDetailsList(JsonNode productDetailsListJson) {
private Map<String, String> extractAllAttributes(JsonNode productDetailsJson) {
Map<String, String> productAttrs = new HashMap<>();
productAttrs.put("sku", productDetailsJson.get("sku").textValue());
productAttrs.put("productFamily", productDetailsJson.get("productFamily") != null ?
productAttrs.put("productFamily", productDetailsJson.get("productFamily") != null ?
productDetailsJson.get("productFamily").textValue() : "");

// Iterate over all the attributes.
Expand Down Expand Up @@ -494,8 +504,8 @@ private void storeInstanceTypeInfoToDB() {

// Parse the memory size.
String memSizeStrGB = productAttrs.get("memory")
.replaceAll("(?i) gib", "")
.replaceAll(",", "");
.replaceAll("(?i) gib", "")
.replaceAll(",", "");
Double memSizeGB = Double.parseDouble(memSizeStrGB);

Integer volumeCount;
Expand All @@ -512,12 +522,11 @@ private void storeInstanceTypeInfoToDB() {
if (parts.length < 4) {
if (!productAttrs.get("storage").equals("EBS only")) {
String msg = "Volume type not specified in product sku=" + productAttrs.get("sku") +
", storage={" + productAttrs.get("storage") + "}";
", storage={" + productAttrs.get("storage") + "}";
LOG.error(msg);
throw new UnsupportedOperationException(msg);
} else {
// TODO: hardcode me not?
volumeCount = 2;
volumeCount = getVolumeCount();
volumeSizeGB = 250;
volumeType = VolumeType.EBS;
}
Expand All @@ -540,7 +549,7 @@ private void storeInstanceTypeInfoToDB() {

if (enableVerboseLogging) {
LOG.info("Instance type entry ({}, {}): {} cores, {} GB RAM, {} x {} GB {}", provider.name(),
instanceTypeCode, numCores, memSizeGB, volumeCount, volumeSizeGB, volumeType);
instanceTypeCode, numCores, memSizeGB, volumeCount, volumeSizeGB, volumeType);
}

// Create the instance type model. If one already exists, overwrite it.
Expand Down Expand Up @@ -588,4 +597,11 @@ private boolean isInstanceTypeSupported(Map<String, String> productAttributes) {
return InstanceType.AWS_INSTANCE_PREFIXES_SUPPORTED.stream().anyMatch(
productAttributes.getOrDefault("instanceType", "")::startsWith);
}

/**
* To get the default volume count for AWS insance types
*/
private int getVolumeCount() {
return config.getInt(YB_VOLUME_INFO_VOLUME_COUNT);
}
}
4 changes: 4 additions & 0 deletions managed/src/main/resources/reference.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,8 @@ yb {
# For how long do we let the task be in database after it has completed
task_retention_duration = 120 days
}
volumeInfo {
# Volume count needed for AWS instance types
volume_count = 1
}
}

0 comments on commit fdf8fea

Please sign in to comment.