Skip to content
This repository has been archived by the owner on Nov 14, 2024. It is now read-only.

Feature/qos service api #2629

Merged
merged 47 commits into from
Dec 2, 2017
Merged
Show file tree
Hide file tree
Changes from 42 commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
f5664a2
Extremely basic QosServiceResource
gsheasby Nov 2, 2017
6beb734
Make resource an interface
gsheasby Nov 2, 2017
45cc7fc
Add client PathParam
gsheasby Nov 2, 2017
f0345e9
Clean up javax.ws.rs dependencies
gsheasby Nov 2, 2017
013582c
Create stub for AtlasDbQosClient
gsheasby Nov 3, 2017
a303b1e
Calls to checkLimit use up a credit; throw when out of credits
gsheasby Nov 3, 2017
139a208
Add QosServiceResourceImpl + test
gsheasby Nov 3, 2017
fb925d3
AutoDelegate for Cassandra.Client
gsheasby Nov 3, 2017
50cc876
Rename QosService stuff
gsheasby Nov 3, 2017
bcbde47
Pass AtlasDbQosClient to CassandraClient
gsheasby Nov 3, 2017
9be8291
Check limit on multiget_slice
gsheasby Nov 3, 2017
e2efe14
Check limit on batch_mutate
gsheasby Nov 3, 2017
47de8d5
Don't test we aren't soft-limited while we can never be soft-limited
gsheasby Nov 3, 2017
c7cfa3d
Check limit on remaining CassandraClient methods
gsheasby Nov 6, 2017
d5fa2db
Scheduled refresh of AtlasDbQosClient.credits
gsheasby Nov 6, 2017
9666ee0
Refresh every second
gsheasby Nov 6, 2017
7717ee4
Mount qos-service on Timelock
hsaraogi Nov 6, 2017
db0037b
Checkstyle
hsaraogi Nov 7, 2017
5ccf451
Update dependency locks
gsheasby Nov 7, 2017
c191cb7
Merge branch 'develop' into feature/qos-service-api
hsaraogi Nov 7, 2017
364ae22
Dont throw limitExceededException
hsaraogi Nov 7, 2017
5515d31
Move client param around
hsaraogi Nov 8, 2017
32ed6bc
Comment
hsaraogi Nov 8, 2017
ffa5e5a
Qos Service config (#2644)
hsaraogi Nov 8, 2017
f8d687f
[QoS] qos ete test (#2652)
nziebart Nov 8, 2017
1613da4
[QoS] rate limiter (#2653)
nziebart Nov 10, 2017
3873586
[QoS] Feature/qos client (#2650)
hsaraogi Nov 10, 2017
103fd97
deps
Nov 10, 2017
22b1e7a
fix tests
Nov 10, 2017
3d1f43f
[QoS] Feature/qos meters (#2640)
hsaraogi Nov 10, 2017
3797765
[QoS] QosClient with ratelimiter (#2667)
hsaraogi Nov 13, 2017
509f3e9
locks
hsaraogi Nov 13, 2017
34dcd28
[QoS] Create a jaxrs-client for the integ tests (#2675)
hsaraogi Nov 14, 2017
b8f8312
Nziebart/merge develop into qos (#2683)
nziebart Nov 15, 2017
4fcef06
qos rate limiting (#2709)
fsamuel-bs Nov 20, 2017
d0b9ae0
[QoS] Estimate the number of read bytes w/ number of rows (#2717)
fsamuel-bs Nov 21, 2017
8998354
weight estimates (#2725)
nziebart Nov 21, 2017
bbd272e
[QoS] Fix exceptions thrown on CqlExecutor (#2696)
fsamuel-bs Nov 22, 2017
d2d7b18
[QoS] Qos ete test (#2708)
hsaraogi Nov 24, 2017
be45487
[QoS] Fix/qos system table rate limiting (#2739)
hsaraogi Nov 24, 2017
bf251e1
Merge develop to the feature branch (#2741)
hsaraogi Nov 27, 2017
11192dd
Nziebart/cell timestamps qos (#2745)
nziebart Nov 28, 2017
d4c2c67
Merge branch 'develop' into feature/qos-service-api
hsaraogi Nov 28, 2017
ae1a3d5
Differentiate between read and write limits when logging (#2751)
hsaraogi Nov 29, 2017
b5caa16
Merge branch 'develop' into feature/qos-service-api
nziebart Dec 1, 2017
da8e513
Use longs in the rate limiter and handle negative adjustments. (#2758)
hsaraogi Dec 1, 2017
92aa59c
Merge branch 'develop' into feature/qos-service-api
hsaraogi Dec 1, 2017
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion atlasdb-api/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ apply from: "../gradle/shared.gradle"
dependencies {
compile project(":atlasdb-commons")
compile project(":timestamp-api")
compile 'javax.ws.rs:javax.ws.rs-api:2.0.1'
compile project(":qos-service-api")
compile group: 'javax.ws.rs', name: 'javax.ws.rs-api'
compile group: 'org.apache.commons', name: 'commons-lang3'
compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations'
compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind'
compile group: 'javax.validation', name: 'validation-api'

compile group: 'com.palantir.remoting-api', name: 'ssl-config'
compile group: 'com.palantir.remoting3', name: 'jaxrs-clients'

processor group: 'org.immutables', name: 'value'

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,18 @@
package com.palantir.atlasdb.keyvalue.api;

public enum ClusterAvailabilityStatus {
TERMINAL,
NO_QUORUM_AVAILABLE,
QUORUM_AVAILABLE,
ALL_AVAILABLE
TERMINAL(false),
NO_QUORUM_AVAILABLE(false),
QUORUM_AVAILABLE(false),
ALL_AVAILABLE(true);

private final boolean isHealthy;

ClusterAvailabilityStatus(boolean isHealthy) {
this.isHealthy = isHealthy;
}

public boolean isHealthy() {
return isHealthy;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import javax.ws.rs.core.MediaType;

import com.google.common.collect.Multimap;
import com.palantir.atlasdb.transaction.api.TransactionManager;
import com.palantir.common.annotation.Idempotent;
import com.palantir.common.annotation.NonIdempotent;
import com.palantir.common.base.ClosableIterator;
Expand Down Expand Up @@ -655,8 +656,16 @@ Multimap<Cell, Long> getAllTimestamps(@QueryParam("tableName") TableReference ta
void compactInternally(TableReference tableRef);

/**
* Checks if the KVS has a quorum available to successfully perform reads/writes.
*
* Provides a {@link ClusterAvailabilityStatus}, indicating the current availability of the key value store.
* This can be used to infer product health - in the usual, conservative case, products can call
* {@link ClusterAvailabilityStatus#isHealthy()}, which returns true only if all KVS nodes are up.
* <p>
* Products that use AtlasDB only for reads and writes (no schema mutations or deletes, including having sweep and
* scrub disabled) can also treat {@link ClusterAvailabilityStatus#QUORUM_AVAILABLE} as healthy.
* <p>
* If you have access to a {@link com.palantir.atlasdb.transaction.api.TransactionManager}, then it is recommended
* to use its availability indicator, {@link TransactionManager#getKeyValueServiceStatus()}, instead of this one.
* <p>
* This call must be implemented so that it completes synchronously.
*/
@POST
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import com.palantir.atlasdb.config.LeaderConfig;
import com.palantir.atlasdb.keyvalue.api.KeyValueService;
import com.palantir.atlasdb.keyvalue.api.TableReference;
import com.palantir.atlasdb.qos.FakeQosClient;
import com.palantir.atlasdb.qos.QosClient;
import com.palantir.timestamp.TimestampService;
import com.palantir.timestamp.TimestampStoreInvalidator;

Expand All @@ -34,7 +36,8 @@ public interface AtlasDbFactory {

default KeyValueService createRawKeyValueService(
KeyValueServiceConfig config, Optional<LeaderConfig> leaderConfig) {
return createRawKeyValueService(config, leaderConfig, Optional.empty(), DEFAULT_INITIALIZE_ASYNC);
return createRawKeyValueService(config, leaderConfig, Optional.empty(), DEFAULT_INITIALIZE_ASYNC,
FakeQosClient.INSTANCE);
}

/**
Expand All @@ -46,13 +49,15 @@ default KeyValueService createRawKeyValueService(
* absent. If both are present, they must match.
* @param initializeAsync If the implementations supports it, and initializeAsync is true, the KVS will initialize
* asynchronously when synchronous initialization fails.
* @param qosClient the client for checking limits from the Quality-of-Service service.
* @return The requested KeyValueService instance
*/
KeyValueService createRawKeyValueService(
KeyValueServiceConfig config,
Optional<LeaderConfig> leaderConfig,
Optional<String> namespace,
boolean initializeAsync);
boolean initializeAsync,
QosClient qosClient);

default TimestampService createTimestampService(KeyValueService rawKvs) {
return createTimestampService(rawKvs, Optional.empty(), DEFAULT_INITIALIZE_ASYNC);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,18 @@
package com.palantir.atlasdb.transaction.api;

public enum KeyValueServiceStatus {
TERMINAL,
UNHEALTHY,
HEALTHY_BUT_NO_SCHEMA_MUTATIONS_OR_DELETES,
HEALTHY_ALL_OPERATIONS
TERMINAL(false),
UNHEALTHY(false),
HEALTHY_BUT_NO_SCHEMA_MUTATIONS_OR_DELETES(false),
HEALTHY_ALL_OPERATIONS(true);

private final boolean isHealthy;

KeyValueServiceStatus(boolean isHealthy) {
this.isHealthy = isHealthy;
}

public boolean isHealthy() {
return isHealthy;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -126,9 +126,15 @@ <T, E extends Exception> T runTaskThrowOnConflict(TransactionTask<T, E> task)
long getImmutableTimestamp();

/**
* Return the {@link KeyValueServiceStatus} depending on the availability of the underlying key-value store.
*
* @return status of the key value service, can be used by the application to decide its own health
* Provides a {@link KeyValueServiceStatus}, indicating the current availability of the key value store.
* This can be used to infer product health - in the usual, conservative case, products can call
* {@link KeyValueServiceStatus#isHealthy()}, which returns true only if all KVS nodes are up.
* <p>
* Products that use AtlasDB only for reads and writes (no schema mutations or deletes, including having sweep and
* scrub disabled) can also treat {@link KeyValueServiceStatus#HEALTHY_BUT_NO_SCHEMA_MUTATIONS_OR_DELETES} as
* healthy.
* <p>
* This call must be implemented so that it completes synchronously.
*/
KeyValueServiceStatus getKeyValueServiceStatus();

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Copyright 2017 Palantir Technologies, Inc. All rights reserved.
*
* Licensed under the BSD-3 License (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://opensource.org/licenses/BSD-3-Clause
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.palantir.atlasdb.keyvalue.api;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import org.junit.Test;

public class ClusterAvailabilityStatusTest {
@Test
public void allAvailableMeansHealthy() {
assertTrue(ClusterAvailabilityStatus.ALL_AVAILABLE.isHealthy());
}

@Test
public void quorumAvailableMeansNotHealthy() {
assertFalse(ClusterAvailabilityStatus.QUORUM_AVAILABLE.isHealthy());
}

@Test
public void noQuorumMeansNotHealthy() {
assertFalse(ClusterAvailabilityStatus.NO_QUORUM_AVAILABLE.isHealthy());
}

@Test
public void terminalMeansNotHealthy() {
assertFalse(ClusterAvailabilityStatus.TERMINAL.isHealthy());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright 2017 Palantir Technologies, Inc. All rights reserved.
*
* Licensed under the BSD-3 License (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://opensource.org/licenses/BSD-3-Clause
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.palantir.atlasdb.transaction.api;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import org.junit.Test;

public class KeyValueServiceStatusTest {
@Test
public void allOperationsMeansHealthy() {
assertTrue(KeyValueServiceStatus.HEALTHY_ALL_OPERATIONS.isHealthy());
}
@Test
public void mostOperationsMeansNotHealthy() {
assertFalse(KeyValueServiceStatus.HEALTHY_BUT_NO_SCHEMA_MUTATIONS_OR_DELETES.isHealthy());
}

@Test
public void unhealthyMeansNotHealthy() { // well, obviously
assertFalse(KeyValueServiceStatus.UNHEALTHY.isHealthy());
}

@Test
public void terminalMeansNotHealthy() {
assertFalse(KeyValueServiceStatus.TERMINAL.isHealthy());
}
}
Loading