Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add storage utilization gib per node for autoscaling #1317

Merged
merged 5 commits into from
Jul 22, 2022
Merged

feat: add storage utilization gib per node for autoscaling #1317

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c61869e
feat: add storage utilization gib per node option for autoscaling
kolea2 Jul 1, 2022
b39a753
add additional assertion
kolea2 Jul 19, 2022
c61ceca
add additional tests
kolea2 Jul 20, 2022
bd72c76
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Jul 20, 2022
033bb6c
review comments
kolea2 Jul 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/Cluster.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,20 @@ public int getAutoscalingCpuPercentageTarget() {
.getAutoscalingTargets()
.getCpuUtilizationPercent();
}
/**
* Get the storage utilization that the Autoscaler should be trying to achieve. This number is
* limited between 2560 (2.5TiB) and 5120 (5TiB) for a SSD cluster and between 8192 (8TiB) and
* 16384 (16TiB) for an HDD cluster; otherwise it will return INVALID_ARGUMENT error. If this
* value is set to 0, it will be treated as if it were set to the default value: 2560 for SSD,
* 8192 for HDD.
*/
public int getStorageUtilizationGibPerNode() {
return stateProto
.getClusterConfig()
.getClusterAutoscalingConfig()
.getAutoscalingTargets()
.getStorageUtilizationGibPerNode();
}

/**
* The type of storage used by this cluster to serve its parent instance's tables, unless
Expand Down
42 changes: 42 additions & 0 deletions ...ble/src/main/java/com/google/cloud/bigtable/admin/v2/models/ClusterAutoscalingConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,21 @@ public ClusterAutoscalingConfig setCpuUtilizationTargetPercent(int cpuUtilizatio
return this;
}

public ClusterAutoscalingConfig setStorageUtilizationGibPerNode(
int storageUtilizationGibPerNode) {
builder.setUpdateMask(
FieldMaskUtil.union(
builder.getUpdateMask(),
FieldMaskUtil.fromString(
Cluster.class,
"cluster_config.cluster_autoscaling_config.autoscaling_targets.storage_utilization_gib_per_node")));
clusterConfigBuilder
.getClusterAutoscalingConfigBuilder()
.getAutoscalingTargetsBuilder()
.setStorageUtilizationGibPerNode(storageUtilizationGibPerNode);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to check if this is a valid value?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The server will give back an INVALID_ARGUMENT error if the value is not within the range specified in the comments.

return this;
}

/** Get the minimum number of nodes to scale down to. */
public int getMinNodes() {
return clusterConfigBuilder
Expand Down Expand Up @@ -131,6 +146,20 @@ public int getCpuUtilizationTargetPercent() {
.getCpuUtilizationPercent();
}

/**
* Get the storage utilization that the Autoscaler should be trying to achieve. This number is
* limited between 2560 (2.5TiB) and 5120 (5TiB) for a SSD cluster and between 8192 (8TiB) and
* 16384 (16TiB) for an HDD cluster; otherwise it will return INVALID_ARGUMENT error. If this
* value is set to 0, it will be treated as if it were set to the default value: 2560 for SSD,
* 8192 for HDD.
*/
public int getStorageUtilizationGibPerNode() {
return clusterConfigBuilder
.getClusterAutoscalingConfig()
.getAutoscalingTargets()
.getStorageUtilizationGibPerNode();
}

/**
* Creates the request protobuf. This method is considered an internal implementation detail and
* not meant to be used by applications.
Expand Down Expand Up @@ -184,6 +213,15 @@ public boolean equals(Object o) {
.getClusterAutoscalingConfig()
.getAutoscalingTargets()
.getCpuUtilizationPercent())
&& Objects.equal(
clusterConfigBuilder
.getClusterAutoscalingConfig()
.getAutoscalingTargets()
.getStorageUtilizationGibPerNode(),
that.clusterConfigBuilder
.getClusterAutoscalingConfig()
.getAutoscalingTargets()
.getStorageUtilizationGibPerNode())
&& Objects.equal(clusterId, that.clusterId)
&& Objects.equal(instanceId, that.instanceId);
}
Expand All @@ -203,6 +241,10 @@ public int hashCode() {
.getClusterAutoscalingConfig()
.getAutoscalingTargets()
.getCpuUtilizationPercent(),
clusterConfigBuilder
.getClusterAutoscalingConfig()
.getAutoscalingTargets()
.getStorageUtilizationGibPerNode(),
clusterId,
instanceId);
}
Expand Down
7 changes: 7 additions & 0 deletions ...igtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/CreateClusterRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ public CreateClusterRequest setScalingMode(@Nonnull ClusterAutoscalingConfig aut
int minNodes = autoscalingConfig.getMinNodes();
int maxNodes = autoscalingConfig.getMaxNodes();
int cpuTargetPercent = autoscalingConfig.getCpuUtilizationTargetPercent();
int storageUtilizationGibPerNode = autoscalingConfig.getStorageUtilizationGibPerNode();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we add in the comment "CPU utilization and storage utilization are set"?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good catch, done!


proto
.getClusterBuilder()
Expand All @@ -135,6 +136,12 @@ public CreateClusterRequest setScalingMode(@Nonnull ClusterAutoscalingConfig aut
.getClusterAutoscalingConfigBuilder()
.getAutoscalingTargetsBuilder()
.setCpuUtilizationPercent(cpuTargetPercent);
proto
.getClusterBuilder()
.getClusterConfigBuilder()
.getClusterAutoscalingConfigBuilder()
.getAutoscalingTargetsBuilder()
.setStorageUtilizationGibPerNode(storageUtilizationGibPerNode);
return this;
}

Expand Down
11 changes: 9 additions & 2 deletions ...le/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableInstanceAdminClientTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,10 @@ public class BigtableInstanceAdminClientTests {
.setMinServeNodes(2)
.build())
.setAutoscalingTargets(
AutoscalingTargets.newBuilder().setCpuUtilizationPercent(22).build()))
AutoscalingTargets.newBuilder()
.setCpuUtilizationPercent(22)
.setStorageUtilizationGibPerNode(3000)
.build()))
.build();

@Mock private BigtableInstanceAdminStub mockStub;
Expand Down Expand Up @@ -575,7 +578,8 @@ public void testCreateClusterAutoscaling() {
ClusterAutoscalingConfig.of(INSTANCE_ID, CLUSTER_ID)
.setMinNodes(2)
.setMaxNodes(10)
.setCpuUtilizationTargetPercent(22))
.setCpuUtilizationTargetPercent(22)
.setStorageUtilizationGibPerNode(3000))
.setStorageType(StorageType.SSD));
// Verify
assertThat(actualResult).isEqualTo(Cluster.fromProto(expectedResponse));
Expand Down Expand Up @@ -759,6 +763,8 @@ public void testPartialUpdateCluster() {
"cluster_config.cluster_autoscaling_config.autoscaling_limits.min_serve_nodes")
.addPaths(
"cluster_config.cluster_autoscaling_config.autoscaling_targets.cpu_utilization_percent")
.addPaths(
"cluster_config.cluster_autoscaling_config.autoscaling_targets.storage_utilization_gib_per_node")
.build())
.build();

Expand All @@ -776,6 +782,7 @@ public void testPartialUpdateCluster() {
ClusterAutoscalingConfig.of(INSTANCE_ID, CLUSTER_ID)
.setMaxNodes(10)
.setMinNodes(2)
.setStorageUtilizationGibPerNode(3000)
.setCpuUtilizationTargetPercent(22));

// Verify
Expand Down
18 changes: 17 additions & 1 deletion ...le/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableInstanceAdminClientIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,14 +307,16 @@ public void createClusterWithAutoscalingTest() {
ClusterAutoscalingConfig.of(newInstanceId, clusterId)
.setMaxNodes(4)
.setMinNodes(1)
.setCpuUtilizationTargetPercent(20));
.setCpuUtilizationTargetPercent(20)
.setStorageUtilizationGibPerNode(9200));

Cluster cluster = client.createCluster(createClusterRequest);
assertThat(cluster.getId()).contains(clusterId);
assertThat(cluster.getServeNodes()).isEqualTo(0);
assertThat(cluster.getAutoscalingMinServeNodes()).isEqualTo(1);
assertThat(cluster.getAutoscalingMaxServeNodes()).isEqualTo(4);
assertThat(cluster.getAutoscalingCpuPercentageTarget()).isEqualTo(20);
assertThat(cluster.getStorageUtilizationGibPerNode()).isEqualTo(9200);
} catch (Exception e) {
Assert.fail("error in the test" + e.getMessage());
} finally {
Expand Down Expand Up @@ -343,6 +345,7 @@ public void createClusterWithAutoscalingAndPartialUpdateTest() {
ClusterAutoscalingConfig.of("ignored", clusterId)
.setMaxNodes(4)
.setMinNodes(1)
.setStorageUtilizationGibPerNode(2561)
kolea2 marked this conversation as resolved.
Show resolved Hide resolved
.setCpuUtilizationTargetPercent(20));

Cluster cluster = client.createCluster(createClusterRequest);
Expand Down Expand Up @@ -382,6 +385,15 @@ public void createClusterWithAutoscalingAndPartialUpdateTest() {
assertThat(updatedCluster.getAutoscalingMinServeNodes()).isEqualTo(2);
assertThat(updatedCluster.getAutoscalingMaxServeNodes()).isEqualTo(5);
assertThat(updatedCluster.getAutoscalingCpuPercentageTarget()).isEqualTo(45);

updatedCluster =
client.updateClusterAutoscalingConfig(
ClusterAutoscalingConfig.of(newInstanceId, clusterId)
.setStorageUtilizationGibPerNode(2777));
assertThat(updatedCluster.getAutoscalingMinServeNodes()).isEqualTo(2);
assertThat(updatedCluster.getAutoscalingMaxServeNodes()).isEqualTo(5);
assertThat(updatedCluster.getAutoscalingCpuPercentageTarget()).isEqualTo(45);
assertThat(updatedCluster.getStorageUtilizationGibPerNode()).isEqualTo(2777);
} catch (Exception e) {
Assert.fail("error in the test: " + e.getMessage());
} finally {
Expand Down Expand Up @@ -414,6 +426,7 @@ public void createClusterWithManualScalingTest() {
assertThat(cluster.getAutoscalingMaxServeNodes()).isEqualTo(0);
assertThat(cluster.getAutoscalingMinServeNodes()).isEqualTo(0);
assertThat(cluster.getAutoscalingCpuPercentageTarget()).isEqualTo(0);
assertThat(cluster.getStorageUtilizationGibPerNode()).isEqualTo(0);
} catch (Exception e) {
Assert.fail("error in the test: " + e.getMessage());
} finally {
Expand Down Expand Up @@ -447,16 +460,19 @@ private void basicClusterOperationTestHelper(String targetInstanceId, String tar
ClusterAutoscalingConfig.of(targetInstanceId, targetClusterId)
.setMinNodes(1)
.setMaxNodes(4)
.setStorageUtilizationGibPerNode(2877)
.setCpuUtilizationTargetPercent(40);
Cluster cluster = client.updateClusterAutoscalingConfig(autoscalingConfig);
assertThat(cluster.getAutoscalingMaxServeNodes()).isEqualTo(4);
assertThat(cluster.getAutoscalingMinServeNodes()).isEqualTo(1);
assertThat(cluster.getAutoscalingCpuPercentageTarget()).isEqualTo(40);
assertThat(cluster.getStorageUtilizationGibPerNode()).isEqualTo(2877);

Cluster updatedCluster = client.disableClusterAutoscaling(targetInstanceId, targetClusterId, 3);
assertThat(updatedCluster.getServeNodes()).isEqualTo(3);
assertThat(updatedCluster.getAutoscalingMaxServeNodes()).isEqualTo(0);
assertThat(updatedCluster.getAutoscalingMinServeNodes()).isEqualTo(0);
assertThat(updatedCluster.getAutoscalingCpuPercentageTarget()).isEqualTo(0);
assertThat(updatedCluster.getStorageUtilizationGibPerNode()).isEqualTo(0);
}
}