From ae6e79c441d2d85c8ef47f712b38eb8e8dbe0c56 Mon Sep 17 00:00:00 2001 From: Andrej Mitrovic Date: Fri, 7 Aug 2020 18:10:20 +0900 Subject: [PATCH 1/2] ConnectionTask: Fix request failure retrying Previously if the request failed after X number of attempts this would throw an Exception which would not be caught in the ConnectionTask. Instead the request should be continuously retried until the request was successful or until the BanManager bans the node. Fixes #1105 --- source/agora/network/NetworkManager.d | 11 +++++--- source/agora/test/Timeout.d | 37 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 source/agora/test/Timeout.d diff --git a/source/agora/network/NetworkManager.d b/source/agora/network/NetworkManager.d index 8ce8fcf701f..1e1a4e6ea93 100644 --- a/source/agora/network/NetworkManager.d +++ b/source/agora/network/NetworkManager.d @@ -162,11 +162,14 @@ public class NetworkManager key = client.getPublicKey(); break; } - catch (HTTPStatusException ex) + catch (Exception ex) { - // 404 => API not implemented, it's a FullNode - if (ex.status == 404) - break; + if (auto http = cast(HTTPStatusException)ex) + { + // 404 => API not implemented, it's a FullNode + if (http.status == 404) + break; + } if (!this.onFailedRequest(this.address)) return; diff --git a/source/agora/test/Timeout.d b/source/agora/test/Timeout.d new file mode 100644 index 00000000000..b8eae49add3 --- /dev/null +++ b/source/agora/test/Timeout.d @@ -0,0 +1,37 @@ +/******************************************************************************* + + Tests connection timeouts + + Copyright: + Copyright (c) 2020 BOS Platform Foundation Korea + All rights reserved. + + License: + MIT License. See LICENSE for details. + +*******************************************************************************/ + +module agora.test.Timeout; + +version (unittest): + +import agora.api.Validator; +import agora.consensus.data.Transaction; +import agora.test.Base; + +/// +unittest +{ + TestConf conf = { retry_delay : 1.msecs, + max_retries : 2, + timeout : 500.msecs, + max_failed_requests : 1000 }; // never ban + auto network = makeTestNetwork(conf); + auto nodes = network.clients; + + nodes[$-1].ctrl.sleep(2.seconds, false); + network.start(); + scope(exit) network.shutdown(); + scope(failure) network.printLogs(); + network.waitForDiscovery(); +} From 430d1d7b37480135b52943000a36cb7d2d2aa69a Mon Sep 17 00:00:00 2001 From: Andrej Mitrovic Date: Fri, 7 Aug 2020 18:29:27 +0900 Subject: [PATCH 2/2] Decrease per-request timeout and increase max retries The per-request timeout is too high. The waitForDiscovery() call in the integration test only waited 5 seconds, but this doesn't give us enough time to attempt a request again. --- tests/system/node/0/config.yaml | 4 ++-- tests/system/node/2/config.yaml | 4 ++-- tests/system/node/3/config.yaml | 4 ++-- tests/system/node/4/config.yaml | 4 ++-- tests/system/node/5/config.yaml | 4 ++-- tests/system/node/6/config.yaml | 4 ++-- tests/system/node/7/config.yaml | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/system/node/0/config.yaml b/tests/system/node/0/config.yaml index 6f3add63ed0..cc169fd9b13 100644 --- a/tests/system/node/0/config.yaml +++ b/tests/system/node/0/config.yaml @@ -8,8 +8,8 @@ node: address: 0.0.0.0 port: 1826 retry_delay: 1500 - max_retries: 5 - timeout: 5000 + max_retries: 25 + timeout: 1000 # Path to the data directory (if the path doesn't exist it will be created) data_dir: .cache genesis_block: 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000549228136441128130986753D019A3EBE69CB8F4718E4848746D55BB81BF56A80BAA95DEC7E13B7B189D7A3DE78E0302688EC38657729F5D6565B5629D158C78000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006ACA35270C439C31DC5C009DAD44CF57F216E3697F95F5A76067891C3447ABCE9C709B87675E1993FCD01805AACEAEF6D1B1A6EFDCF5FA940D6818477833E88462843B73EACDAC613F9816A79B6CD8258B09EF4EA6DB14F5108B21FFB10A9B349234F777A77C84F14553A29788B187AA54049694741DEE8E81C6E09F5F901820A14227414C27024C8AC13E31159AB2D8B1BC463B7BDB915289EDFF5E35F13898BDCEE0B3BF2010D6109D67226B708A59521C8B4EAE180643DE3D2D271258627AB0CEFE69BF0533530DA5632469962923165E4DE89D98156771851D3C4F07B37396ACF630C5671B1ED94BB8D08DB4FA636A301ABDE46AFDA5B9D7F36092E6D80DE4DC197F29D7E9D71799103E5E0D34A420840EE97F217EDC8671CB0C9C28745B81D468C34825C6050C46E7760D774476F763874A58956DFE4286245B7F9888A34D014015C257522778AC4A2A2D85753583E7CD7331D4B9A161647A33341B0B8D09226AFCDA880E7F899C4EC2C91BD1A5468EC8A23A87B818D466B313DC367B898D40EE07056BEB7F356866F57F38F92DF9E723C31C60F0A2914AB1C6FEE0ACD8ED437F5BCFAE0B769CE037B0F616C2BD8B1D48A678A1FDA3008FA2AF45D47A461158CEBFCBF9035B7E54BC88E5AC71EE18C65B4BACC94447A13A7D759D581E65C78DB2A378BC26E7065AFB5F05D20622D424386DADC3CE6EDEAE33DF5D09D7DC643AF1404130031A6489FFB577AD2A13C43B8B7835E19E44B86C235DBCCB4B653473D7E9DA24D47AB38BE951D9C41FAF10744C22A20F338F646ACDB30C97995484F4709B085E09F3B11826E6279366EF28D73DF4AE163D1AD81FB21126CF42A570C183D2DBAC8A9ACED837D2398340F09C3D7B5E01CE305B6878EBB54CF6698C78E9094BC12815566F8FD421DB1451CAE59C5C20ADFC8A3E55A3FFBC696BD1C562BA12DACEB5063DC80B05829E8BED2016AE537F0439B7FEB3C5254340D2243687E4BA21434663A1D39CF6FE86B79C21F63CA2AAC57D5410776A373EDB98AB718457C78FD6668ECDBFDE5C243111EF6DCCE5C55A7E17C7EFFE7657AD49AEB0FA3EC66450EEB08AE4BFEE5F00DA15985C13921B7FD65E3B6DF5A2702A4D6E0BC9EBB871E63719B7970E5F9CD974A99E848792FF1EA7F67CDAB5EF573F5F371A9BDCFA90DB2B42C51FAB1B56745BB70DC76E39626535DDA75B5EC9735A4A06CFF48E77B988EFFD26DE5B878B79D6AECDE64D820B04F92C780A4EA0AF6296781BEDD602950A014316B74AA2693C0D531692C82C840F9BD83FC39899B00DC697809D3E4AC369AD7CFAA1FA1BE9EF0A5ABF15D0BF474F199C3CA0D7477B026FD199FC475D9E12E05852DFE9F69AB65820CAE78A23CBC3CED9160E54AD66F86E6C38EA34AA7126DD7035AC60083BD69330C7970558EEC4FC5365E0ABE823D5397A0CED287BD3139DBFA7B3B7EBC32922900AAEAEB39F25E2DFC6DA635DCB1579ABA3C2EDC34DF467F417B4E8A7046CD460EBA0ECBACA93C3DE2A2D522544E12EA4622C6D4629C1BDD14E24FA71A72F492A50AC73556DED7EC14CFE36F30B0324D647E4BA28243E7715C62FD69CCB74A31B144567E3F4629758BE27CC5C493B57C89BEF51E95E570000E02010006FF0040E59C30120000DAE1934864C67FCB27BD872D70991330222591CD83B970EB8A28FF51A6556A55FF0040E59C30120000DAE19364B4145DEE6F56FA30436AB831274F84E00437552C1E603AA9FEF21892FF0040E59C30120000DAE1938A993F5850F50AEE830FF52171DFDF97AF79F7FF43DFE4337E8477D967FF0040E59C30120000DAE193B3FCEC9D4B9484A6CFA867A68F038934B970B7E463C1AE80D007DD36F4FF0040E59C30120000DAE193D9BDB4D6B9D0419BAE3024CC5DDD6BCB930E68036CA019884A69AC7F79FF0040E59C30120000DAE193E9E928AFCFC7591DA28E81B6497ADCE977C5C017B5C8159234D0E5A3B4000008FF002050B1CA2A02009D0238E0A171400BC6D68A9D9B316ACD5109649113A05C284F4296D2B30122F5FF002050B1CA2A02009D0238E0A171400BC6D68A9D9B316ACD5109649113A05C284F4296D2B30122F5FF002050B1CA2A02009D0238E0A171400BC6D68A9D9B316ACD5109649113A05C284F4296D2B30122F5FF002050B1CA2A02009D0238E0A171400BC6D68A9D9B316ACD5109649113A05C284F4296D2B30122F5FF002050B1CA2A02009D0238E0A171400BC6D68A9D9B316ACD5109649113A05C284F4296D2B30122F5FF002050B1CA2A02009D0238E0A171400BC6D68A9D9B316ACD5109649113A05C284F4296D2B30122F5FF002050B1CA2A02009D0238E0A171400BC6D68A9D9B316ACD5109649113A05C284F4296D2B30122F5FF002050B1CA2A02009D0238E0A171400BC6D68A9D9B316ACD5109649113A05C284F4296D2B30122F503BC75F96F82A6EBF8C9776AC315FA1DA5E9E1F26AF3BF631385A359E27C12835C7129DB46AEE9C5C1DB3D4E4C713B61D74746823D3A9C30985B7F1AC49BCE14635CF9F0B2D9962D874079EE7C154C7E8C5F99201664459F6A182BCE732AEC0AFE0677F4A407C3B84A30661C4A4DA89ED53DEAE3FFA7B97C37FAAE9C6FB9FE5B7A549228136441128130986753D019A3EBE69CB8F4718E4848746D55BB81BF56A80BAA95DEC7E13B7B189D7A3DE78E0302688EC38657729F5D6565B5629D158C78 diff --git a/tests/system/node/2/config.yaml b/tests/system/node/2/config.yaml index 5b76fc25788..6aeb76cb0e6 100644 --- a/tests/system/node/2/config.yaml +++ b/tests/system/node/2/config.yaml @@ -8,8 +8,8 @@ node: address: 0.0.0.0 port: 2826 retry_delay: 1500 - max_retries: 5 - timeout: 5000 + max_retries: 25 + timeout: 1000 # This is a randomly generated keypair # If this node is not a validator, this will be ignored # diff --git a/tests/system/node/3/config.yaml b/tests/system/node/3/config.yaml index b6a3db4ba63..e3c5c921d24 100644 --- a/tests/system/node/3/config.yaml +++ b/tests/system/node/3/config.yaml @@ -8,8 +8,8 @@ node: address: 0.0.0.0 port: 3826 retry_delay: 1500 - max_retries: 5 - timeout: 5000 + max_retries: 25 + timeout: 1000 # This is a randomly generated keypair # If this node is not a validator, this will be ignored # diff --git a/tests/system/node/4/config.yaml b/tests/system/node/4/config.yaml index b79aaf1eb3a..5afb1729518 100644 --- a/tests/system/node/4/config.yaml +++ b/tests/system/node/4/config.yaml @@ -8,8 +8,8 @@ node: address: 0.0.0.0 port: 4826 retry_delay: 1500 - max_retries: 5 - timeout: 5000 + max_retries: 25 + timeout: 1000 # This is a randomly generated keypair # If this node is not a validator, this will be ignored # diff --git a/tests/system/node/5/config.yaml b/tests/system/node/5/config.yaml index eedf1465514..d2b9584ab70 100644 --- a/tests/system/node/5/config.yaml +++ b/tests/system/node/5/config.yaml @@ -8,8 +8,8 @@ node: address: 0.0.0.0 port: 5826 retry_delay: 1500 - max_retries: 5 - timeout: 5000 + max_retries: 25 + timeout: 1000 # This is a randomly generated keypair # If this node is not a validator, this will be ignored # diff --git a/tests/system/node/6/config.yaml b/tests/system/node/6/config.yaml index 6cb04c2a0df..0e94aaf6081 100644 --- a/tests/system/node/6/config.yaml +++ b/tests/system/node/6/config.yaml @@ -8,8 +8,8 @@ node: address: 0.0.0.0 port: 6826 retry_delay: 1500 - max_retries: 5 - timeout: 5000 + max_retries: 25 + timeout: 1000 # This is a randomly generated keypair # If this node is not a validator, this will be ignored # diff --git a/tests/system/node/7/config.yaml b/tests/system/node/7/config.yaml index 966c39a8d84..2caadc64413 100644 --- a/tests/system/node/7/config.yaml +++ b/tests/system/node/7/config.yaml @@ -8,8 +8,8 @@ node: address: 0.0.0.0 port: 7826 retry_delay: 1500 - max_retries: 5 - timeout: 5000 + max_retries: 25 + timeout: 1000 # This is a randomly generated keypair # If this node is not a validator, this will be ignored #