Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prevent excessive api calls #4966

Merged
merged 54 commits into from
Dec 23, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
54 commits
Select commit Hold shift + click to select a range
dc6144d
Refactor BtcWalletService to let api override fee rates
ghubstan Dec 4, 2020
900d498
Merge branch 'master' into 02-refactor-completePreparedSendBsqTx
ghubstan Dec 4, 2020
159d4cc
Add optional txFeeRate parameter to api sendbsq
ghubstan Dec 4, 2020
2842070
Merge branch 'master' into 03-add-txFeeRate-param
ghubstan Dec 8, 2020
6c9f0c2
Add new api method 'sendbtc' and test
ghubstan Dec 9, 2020
144c5a8
Merge branch 'master' into 04-add-sendbtc-impl
ghubstan Dec 9, 2020
bd66008
Support tx memo field for btc withdrawals from api
ghubstan Dec 9, 2020
478c8f4
Remove unused imports
ghubstan Dec 9, 2020
259bad6
Merge branch 'master' into 05-use-memo-tx-field
ghubstan Dec 10, 2020
150e2f6
Use Bisq's UserThread.executor in gRPC server
ghubstan Dec 11, 2020
6aa385e
Append nullable withdrawalTxId field to Trade proto message
ghubstan Dec 12, 2020
5522d0c
Add new api method gettransaction
ghubstan Dec 14, 2020
a0f1c22
Merge branch 'master' into 08-scratch
ghubstan Dec 14, 2020
0384642
Adjust create TransferwiseAccount test
ghubstan Dec 14, 2020
4be87a6
Disable method test to avoid repetition
ghubstan Dec 14, 2020
a341173
Merge branch 'master' into 09-scratch
ghubstan Dec 15, 2020
e6c6d3b
Add new CoreApiExceptionHandler to gRPC services
ghubstan Dec 16, 2020
1cd47fd
Merge branch 'master' into 09-refactor-grpc-error-handling
ghubstan Dec 16, 2020
c60605f
Fix class level comment
ghubstan Dec 16, 2020
f7c1103
Rename gRPC exception handler class
ghubstan Dec 16, 2020
2572e86
Create grpc interceptor pkg, move auth interceptor into it
ghubstan Dec 16, 2020
fa9ffa1
Put arguments on separate lines
ghubstan Dec 16, 2020
2148a4d
Prevent excessive api calls
ghubstan Dec 17, 2020
89e2187
Change long to int, tidy up error msg
ghubstan Dec 17, 2020
a5ed17e
Add license comment, stop & toString methods, and make isRunning tran…
ghubstan Dec 19, 2020
b307593
Make CallRateMeteringInterceptor configurable via json
ghubstan Dec 19, 2020
5de910a
Add unit test dependencies to daemon subproject
ghubstan Dec 19, 2020
455ed67
Add GrpcServiceRateMeteringConfigTest
ghubstan Dec 19, 2020
830a5f0
Add license note
ghubstan Dec 19, 2020
9f679de
Add license note and toString method
ghubstan Dec 19, 2020
bb8d2ae
Inject Config into CoreApi
ghubstan Dec 19, 2020
ea97a80
Don't cancel gRPC call if an interceptor does not meter all methods
ghubstan Dec 19, 2020
87f75ee
Configure GrpcVersionService's rate metering interceptor
ghubstan Dec 19, 2020
56a5c79
Add ApiTestConfig option --callRateMeteringConfigPath
ghubstan Dec 19, 2020
d5657e9
Install call rate metering config file before startup
ghubstan Dec 19, 2020
fabd7c8
Refactor testcase superclasses to support rate metering configs
ghubstan Dec 19, 2020
abc3940
Test CallRateMeteringInterceptor
ghubstan Dec 19, 2020
a3eb4ed
Remove unused import
ghubstan Dec 19, 2020
672eb79
Revert "Append nullable withdrawalTxId field to Trade proto message"
ghubstan Dec 21, 2020
bdde24a
Ajust TradeInfo to reverting 6aa385e494eb8fa870257c76e078108607503d03
ghubstan Dec 21, 2020
64c2ac5
Adjust grpc.proto to reverting 6aa385e494eb8fa870257c76e078108607503d03
ghubstan Dec 21, 2020
97dcac2
Adjust TradeFormat to reverting 6aa385e494eb8fa870257c76e078108607503d03
ghubstan Dec 21, 2020
3a770f4
Adjust TakeSellBTCOfferTest to reverting 6aa385e494eb8fa870257c76e078…
ghubstan Dec 21, 2020
4aa4270
Adjust TradeTest to reverting 6aa385e494eb8fa870257c76e078108607503d03
ghubstan Dec 21, 2020
1507a2c
Resolve file conflict w/ master
ghubstan Dec 21, 2020
8ee3a15
Merge branch '08-gettransaction' into 09-refactor-grpc-error-handling
ghubstan Dec 21, 2020
27efc5f
Merge branch '09-refactor-grpc-error-handling' into 10-callrate-inter…
ghubstan Dec 21, 2020
01546ad
Use a simpler, time windowing call rate meter
ghubstan Dec 22, 2020
d615212
Remove unused local var
ghubstan Dec 22, 2020
0d4ed95
Remove redundant callCount field
ghubstan Dec 22, 2020
c8ef414
Fix comment
ghubstan Dec 22, 2020
6356476
Support more fine grained rate metering
ghubstan Dec 23, 2020
b8c5a29
Disable CallRateMeteringInterceptorTest and run it from test suite
ghubstan Dec 23, 2020
10727fc
Fix GrpcCallRateMeter method and variable name
ghubstan Dec 23, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 30 additions & 5 deletions apitest/src/main/java/bisq/apitest/Scaffold.java
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,8 @@ public void tearDown() {
try {
log.info("Shutting down executor service ...");
executor.shutdownNow();
executor.awaitTermination(config.supportingApps.size() * 2000, MILLISECONDS);
//noinspection ResultOfMethodCallIgnored
executor.awaitTermination(config.supportingApps.size() * 2000L, MILLISECONDS);

SetupTask[] orderedTasks = new SetupTask[]{
bobNodeTask, aliceNodeTask, arbNodeTask, seedNodeTask, bitcoindTask};
Expand Down Expand Up @@ -218,20 +219,25 @@ public void installDaoSetupDirectories() {
if (copyBitcoinRegtestDir.run().getExitStatus() != 0)
throw new IllegalStateException("Could not install bitcoin regtest dir");

String aliceDataDir = daoSetupDir + "/" + alicedaemon.appName;
BashCommand copyAliceDataDir = new BashCommand(
"cp -rf " + daoSetupDir + "/" + alicedaemon.appName
+ " " + config.rootAppDataDir);
"cp -rf " + aliceDataDir + " " + config.rootAppDataDir);
if (copyAliceDataDir.run().getExitStatus() != 0)
throw new IllegalStateException("Could not install alice data dir");

String bobDataDir = daoSetupDir + "/" + bobdaemon.appName;
BashCommand copyBobDataDir = new BashCommand(
"cp -rf " + daoSetupDir + "/" + bobdaemon.appName
+ " " + config.rootAppDataDir);
"cp -rf " + bobDataDir + " " + config.rootAppDataDir);
if (copyBobDataDir.run().getExitStatus() != 0)
throw new IllegalStateException("Could not install bob data dir");

log.info("Installed dao-setup files into {}", buildDataDir);

if (!config.callRateMeteringConfigPath.isEmpty()) {
installCallRateMeteringConfiguration(aliceDataDir);
installCallRateMeteringConfiguration(bobDataDir);
}

// Copy the blocknotify script from the src resources dir to the build
// resources dir. Users may want to edit comment out some lines when all
// of the default block notifcation ports being will not be used (to avoid
Expand Down Expand Up @@ -287,6 +293,25 @@ private void installBitcoinBlocknotify() {
}
}

private void installCallRateMeteringConfiguration(String dataDir) throws IOException, InterruptedException {
File testRateMeteringFile = new File(config.callRateMeteringConfigPath);
if (!testRateMeteringFile.exists())
throw new FileNotFoundException(
format("Call rate metering config file '%s' not found", config.callRateMeteringConfigPath));

BashCommand copyRateMeteringConfigFile = new BashCommand(
"cp -rf " + config.callRateMeteringConfigPath + " " + dataDir);
if (copyRateMeteringConfigFile.run().getExitStatus() != 0)
throw new IllegalStateException(
format("Could not install %s file in %s",
testRateMeteringFile.getAbsolutePath(), dataDir));

Path destPath = Paths.get(dataDir, testRateMeteringFile.getName());
String chmod700Perms = "rwx------";
Files.setPosixFilePermissions(destPath, PosixFilePermissions.fromString(chmod700Perms));
log.info("Installed {} with perms {}.", destPath.toString(), chmod700Perms);
}

private void installShutdownHook() {
// Background apps can be left running until the jvm is manually shutdown,
// so we add a shutdown hook for that use case.
Expand Down
9 changes: 9 additions & 0 deletions apitest/src/main/java/bisq/apitest/config/ApiTestConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ public class ApiTestConfig {
static final String SKIP_TESTS = "skipTests";
static final String SHUTDOWN_AFTER_TESTS = "shutdownAfterTests";
static final String SUPPORTING_APPS = "supportingApps";
static final String CALL_RATE_METERING_CONFIG_PATH = "callRateMeteringConfigPath";
static final String ENABLE_BISQ_DEBUGGING = "enableBisqDebugging";

// Default values for certain options
Expand Down Expand Up @@ -102,6 +103,7 @@ public class ApiTestConfig {
public final boolean skipTests;
public final boolean shutdownAfterTests;
public final List<String> supportingApps;
public final String callRateMeteringConfigPath;
public final boolean enableBisqDebugging;

// Immutable system configurations set in the constructor.
Expand Down Expand Up @@ -228,6 +230,12 @@ public ApiTestConfig(String... args) {
.ofType(String.class)
.defaultsTo("bitcoind,seednode,arbdaemon,alicedaemon,bobdaemon");

ArgumentAcceptingOptionSpec<String> callRateMeteringConfigPathOpt =
parser.accepts(CALL_RATE_METERING_CONFIG_PATH,
"Install a ratemeters.json file to configure call rate metering interceptors")
.withRequiredArg()
.defaultsTo(EMPTY);

ArgumentAcceptingOptionSpec<Boolean> enableBisqDebuggingOpt =
parser.accepts(ENABLE_BISQ_DEBUGGING,
"Start Bisq apps with remote debug options")
Expand Down Expand Up @@ -289,6 +297,7 @@ public ApiTestConfig(String... args) {
this.skipTests = options.valueOf(skipTestsOpt);
this.shutdownAfterTests = options.valueOf(shutdownAfterTestsOpt);
this.supportingApps = asList(options.valueOf(supportingAppsOpt).split(","));
this.callRateMeteringConfigPath = options.valueOf(callRateMeteringConfigPathOpt);
this.enableBisqDebugging = options.valueOf(enableBisqDebuggingOpt);

// Assign values to special-case static fields.
Expand Down
11 changes: 11 additions & 0 deletions apitest/src/test/java/bisq/apitest/ApiTestCase.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

import java.net.InetAddress;

import java.io.File;
import java.io.IOException;

import java.util.HashMap;
Expand Down Expand Up @@ -72,6 +73,16 @@ public class ApiTestCase {
// gRPC service stubs are used by method & scenario tests, but not e2e tests.
private static final Map<BisqAppConfig, GrpcStubs> grpcStubsCache = new HashMap<>();

public static void setUpScaffold(File callRateMeteringConfigFile,
Enum<?>... supportingApps)
throws InterruptedException, ExecutionException, IOException {
scaffold = new Scaffold(stream(supportingApps).map(Enum::name)
.collect(Collectors.joining(",")))
.setUp();
config = scaffold.config;
bitcoinCli = new BitcoinCliHelper((config));
}

public static void setUpScaffold(Enum<?>... supportingApps)
throws InterruptedException, ExecutionException, IOException {
scaffold = new Scaffold(stream(supportingApps).map(Enum::name)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
/*
* This file is part of Bisq.
*
* Bisq is free software: you can redistribute it and/or modify it
* under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or (at
* your option) any later version.
*
* Bisq is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
* License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Bisq. If not, see <http://www.gnu.org/licenses/>.
*/

package bisq.apitest.method;

import io.grpc.StatusRuntimeException;

import java.io.File;

import lombok.extern.slf4j.Slf4j;

import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;

import static bisq.apitest.Scaffold.BitcoinCoreApp.bitcoind;
import static bisq.apitest.config.BisqAppConfig.alicedaemon;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.HOURS;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;



import bisq.daemon.grpc.GrpcVersionService;
import bisq.daemon.grpc.interceptor.GrpcServiceRateMeteringConfig;

@Disabled
@Slf4j
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
public class CallRateMeteringInterceptorTest extends MethodTest {

private static final GetVersionTest getVersionTest = new GetVersionTest();

@BeforeAll
public static void setUp() {
File callRateMeteringConfigFile = buildInterceptorConfigFile();
startSupportingApps(callRateMeteringConfigFile,
false,
false,
bitcoind, alicedaemon);
}

@BeforeEach
public void sleep200Milliseconds() {
sleep(200);
}

@Test
@Order(1)
public void testGetVersionCall1IsAllowed() {
getVersionTest.testGetVersion();
}

@Test
@Order(2)
public void testGetVersionCall2ShouldThrowException() {
Throwable exception = assertThrows(StatusRuntimeException.class, getVersionTest::testGetVersion);
assertEquals("PERMISSION_DENIED: the maximum allowed number of getversion calls (1/second) has been exceeded",
exception.getMessage());
}

@Test
@Order(3)
public void testGetVersionCall3ShouldThrowException() {
Throwable exception = assertThrows(StatusRuntimeException.class, getVersionTest::testGetVersion);
assertEquals("PERMISSION_DENIED: the maximum allowed number of getversion calls (1/second) has been exceeded",
exception.getMessage());
}

@Test
@Order(4)
public void testGetVersionCall4IsAllowed() {
sleep(1100); // Let the server's rate meter reset the call count.
getVersionTest.testGetVersion();
}

@AfterAll
public static void tearDown() {
tearDownScaffold();
}

public static File buildInterceptorConfigFile() {
GrpcServiceRateMeteringConfig.Builder builder = new GrpcServiceRateMeteringConfig.Builder();
builder.addCallRateMeter(GrpcVersionService.class.getSimpleName(),
"getVersion",
1,
SECONDS);
builder.addCallRateMeter(GrpcVersionService.class.getSimpleName(),
"shouldNotBreakAnything",
1000,
DAYS);
// Only GrpcVersionService is @VisibleForTesting, so we hardcode the class names.
builder.addCallRateMeter("GrpcOffersService",
"createOffer",
5,
MINUTES);
builder.addCallRateMeter("GrpcOffersService",
"takeOffer",
10,
DAYS);
builder.addCallRateMeter("GrpcTradesService",
"withdrawFunds",
3,
HOURS);
return builder.build();
}
}
Loading