diff --git a/build/arm-tools.mk b/build/arm-tools.mk index ce92c3baeb..6a2ca4fbcb 100644 --- a/build/arm-tools.mk +++ b/build/arm-tools.mk @@ -112,6 +112,11 @@ endif # We are using newlib-nano for all the platforms CFLAGS += --specs=nano.specs +ifneq ($(LTO_EXTRA_OPTIMIZATIONS),) +CFLAGS += -fmerge-all-constants +LDFLAGS += -fmerge-all-constants +endif + # Check if the compiler version is the minimum required version_to_number=$(shell v=$1; v=($${v//./ }); echo $$((v[0] * 10000 + v[1] * 100 + v[2]))) get_major_version=$(shell v=$1; v=($${v//./ }); echo $${v[0]}) diff --git a/hal/network/lwip/lwiphooks.c b/hal/network/lwip/lwiphooks.c index 517f1e2f79..d9b9858d67 100644 --- a/hal/network/lwip/lwiphooks.c +++ b/hal/network/lwip/lwiphooks.c @@ -71,3 +71,6 @@ __attribute__((weak)) struct netif* lwip_hook_ip6_route(const ip6_addr_t* src, c return NULL; } +__attribute__((weak)) struct netif* lwip_hook_dns_get_netif_for_server_index(int index) { + return NULL; +} diff --git a/hal/network/lwip/resolvapi.cpp b/hal/network/lwip/resolvapi.cpp index f0647da751..64091f8c37 100644 --- a/hal/network/lwip/resolvapi.cpp +++ b/hal/network/lwip/resolvapi.cpp @@ -22,6 +22,7 @@ #include "logging.h" #include "ifapi.h" #include +#include "lwiphooks.h" using namespace particle::net; @@ -221,3 +222,10 @@ int resolv_get_dns_server_priority_for_iface(if_t iface, int priority) { index = index * LWIP_DNS_SERVERS_PER_NETIF + std::min(LWIP_DNS_SERVERS_PER_NETIF, priority); return index; } + +struct netif* lwip_hook_dns_get_netif_for_server_index(int index) { + uint8_t netifIdx = index / LWIP_DNS_SERVERS_PER_NETIF; + if_t iface = nullptr; + if_get_by_index(netifIdx, &iface); + return (netif*)iface; +} diff --git a/hal/src/nRF52840/lwip/lwiphooks.h b/hal/src/nRF52840/lwip/lwiphooks.h index cc47849232..0f0823b1c3 100644 --- a/hal/src/nRF52840/lwip/lwiphooks.h +++ b/hal/src/nRF52840/lwip/lwiphooks.h @@ -28,6 +28,7 @@ int lwip_hook_ip4_input(struct pbuf *p, struct netif *inp); int lwip_hook_ip4_input_post_validation(struct pbuf* p, struct netif* inp); struct netif* lwip_hook_ip4_route_src(const ip4_addr_t* src, const ip4_addr_t* dst); int lwip_hook_ip4_input_pre_upper_layers(struct pbuf* p, const struct ip_hdr* iphdr, struct netif* inp); +struct netif* lwip_hook_dns_get_netif_for_server_index(int index); #endif /* LWIP_IPV4 */ /* IPv6 hooks */ @@ -48,6 +49,8 @@ void lwip_hook_memp_free(memp_t type, unsigned available, unsigned size); } #endif /* __cplusplus */ +#define LWIP_HOOK_DNS_GET_NETIF_FOR_SERVER_INDEX(index) lwip_hook_dns_get_netif_for_server_index(index) + /** * LWIP_HOOK_TCP_ISN: * Hook for generation of the Initial Sequence Number (ISN) for a new TCP diff --git a/hal/src/nRF52840/lwip/lwipopts.h b/hal/src/nRF52840/lwip/lwipopts.h index bd50666ff7..79c8fcefa4 100644 --- a/hal/src/nRF52840/lwip/lwipopts.h +++ b/hal/src/nRF52840/lwip/lwipopts.h @@ -753,6 +753,13 @@ void sys_unlock_tcpip_core(void); * LWIP_NETBUF_RECVINFO==1: append destination addr and port to every netbuf. */ #define LWIP_NETBUF_RECVINFO 0 + + +/** + * LWIP_NETBUF_TIMESTAMP==1: append timestamp to netbufs + */ +#define LWIP_NETBUF_TIMESTAMP 1 + /** * @} */ diff --git a/hal/src/nRF52840/lwip/sys_arch.c b/hal/src/nRF52840/lwip/sys_arch.c index 56fea365af..0ee82bd25d 100644 --- a/hal/src/nRF52840/lwip/sys_arch.c +++ b/hal/src/nRF52840/lwip/sys_arch.c @@ -39,6 +39,7 @@ #include "FreeRTOS.h" #include "semphr.h" #include "task.h" +#include "timer_hal.h" /** Set this to 1 if you want the stack size passed to sys_thread_new() to be * interpreted as number of stack words (FreeRTOS-like). @@ -78,7 +79,7 @@ * Default is 1, where FreeRTOS ticks are used to calculate back to ms. */ #ifndef LWIP_FREERTOS_SYS_NOW_FROM_FREERTOS -#define LWIP_FREERTOS_SYS_NOW_FROM_FREERTOS 1 +#define LWIP_FREERTOS_SYS_NOW_FROM_FREERTOS 0 #endif #if !configSUPPORT_DYNAMIC_ALLOCATION @@ -135,6 +136,12 @@ sys_now(void) { return xTaskGetTickCount() * portTICK_PERIOD_MS; } +#else +u32_t +sys_now(void) +{ + return HAL_Timer_Get_Milli_Seconds(); +} #endif u32_t diff --git a/hal/src/rtl872x/lwip/lwiphooks.h b/hal/src/rtl872x/lwip/lwiphooks.h index cc47849232..0f0823b1c3 100644 --- a/hal/src/rtl872x/lwip/lwiphooks.h +++ b/hal/src/rtl872x/lwip/lwiphooks.h @@ -28,6 +28,7 @@ int lwip_hook_ip4_input(struct pbuf *p, struct netif *inp); int lwip_hook_ip4_input_post_validation(struct pbuf* p, struct netif* inp); struct netif* lwip_hook_ip4_route_src(const ip4_addr_t* src, const ip4_addr_t* dst); int lwip_hook_ip4_input_pre_upper_layers(struct pbuf* p, const struct ip_hdr* iphdr, struct netif* inp); +struct netif* lwip_hook_dns_get_netif_for_server_index(int index); #endif /* LWIP_IPV4 */ /* IPv6 hooks */ @@ -48,6 +49,8 @@ void lwip_hook_memp_free(memp_t type, unsigned available, unsigned size); } #endif /* __cplusplus */ +#define LWIP_HOOK_DNS_GET_NETIF_FOR_SERVER_INDEX(index) lwip_hook_dns_get_netif_for_server_index(index) + /** * LWIP_HOOK_TCP_ISN: * Hook for generation of the Initial Sequence Number (ISN) for a new TCP diff --git a/hal/src/rtl872x/lwip/lwipopts.h b/hal/src/rtl872x/lwip/lwipopts.h index a1f67c0b65..58428f5a74 100644 --- a/hal/src/rtl872x/lwip/lwipopts.h +++ b/hal/src/rtl872x/lwip/lwipopts.h @@ -751,6 +751,11 @@ void sys_unlock_tcpip_core(void); * LWIP_NETBUF_RECVINFO==1: append destination addr and port to every netbuf. */ #define LWIP_NETBUF_RECVINFO 0 + +/** + * LWIP_NETBUF_TIMESTAMP==1: append timestamp to netbufs + */ +#define LWIP_NETBUF_TIMESTAMP 1 /** * @} */ diff --git a/hal/src/rtl872x/lwip/sys_arch.c b/hal/src/rtl872x/lwip/sys_arch.c index 56fea365af..0ee82bd25d 100644 --- a/hal/src/rtl872x/lwip/sys_arch.c +++ b/hal/src/rtl872x/lwip/sys_arch.c @@ -39,6 +39,7 @@ #include "FreeRTOS.h" #include "semphr.h" #include "task.h" +#include "timer_hal.h" /** Set this to 1 if you want the stack size passed to sys_thread_new() to be * interpreted as number of stack words (FreeRTOS-like). @@ -78,7 +79,7 @@ * Default is 1, where FreeRTOS ticks are used to calculate back to ms. */ #ifndef LWIP_FREERTOS_SYS_NOW_FROM_FREERTOS -#define LWIP_FREERTOS_SYS_NOW_FROM_FREERTOS 1 +#define LWIP_FREERTOS_SYS_NOW_FROM_FREERTOS 0 #endif #if !configSUPPORT_DYNAMIC_ALLOCATION @@ -135,6 +136,12 @@ sys_now(void) { return xTaskGetTickCount() * portTICK_PERIOD_MS; } +#else +u32_t +sys_now(void) +{ + return HAL_Timer_Get_Milli_Seconds(); +} #endif u32_t diff --git a/modules/argon/system-part1/makefile b/modules/argon/system-part1/makefile index d928b67ce0..8226203ef1 100644 --- a/modules/argon/system-part1/makefile +++ b/modules/argon/system-part1/makefile @@ -9,6 +9,8 @@ NCP_FIRMWARE_MODULE_VERSION=4 DEPENDENCIES = newlib_nano modules/argon/user-part modules/argon/system-part1 dynalib services hal platform system wiring communication rt-dynalib crypto proto_defs LIB_DEPENDENCIES = services system wiring communication hal platform crypto proto_defs +export LTO_EXTRA_OPTIMIZATIONS=1 + # newlib_nano is special in that it's linked automatically by the system, so no need to add it to the library path here MAKE_DEPENDENCIES = newlib_nano $(LIB_DEPENDENCIES) include ../modular.mk diff --git a/modules/b5som/system-part1/makefile b/modules/b5som/system-part1/makefile index 0a6af8aa62..df62d979c0 100644 --- a/modules/b5som/system-part1/makefile +++ b/modules/b5som/system-part1/makefile @@ -5,6 +5,8 @@ BUILD_PATH_EXT = $(BUILD_TARGET_PLATFORM) HAL_LINK := PLATFORM_DFU = 0x30000 +export LTO_EXTRA_OPTIMIZATIONS=1 + DEPENDENCIES = newlib_nano modules/b5som/user-part modules/b5som/system-part1 dynalib services hal platform system wiring communication rt-dynalib crypto proto_defs wiring_globals LIB_DEPENDENCIES = services system wiring communication hal platform crypto proto_defs wiring_globals # newlib_nano is special in that it's linked automatically by the system, so no need to add it to the library path here diff --git a/modules/boron/system-part1/makefile b/modules/boron/system-part1/makefile index 51cfd534d5..d04117a86a 100644 --- a/modules/boron/system-part1/makefile +++ b/modules/boron/system-part1/makefile @@ -5,6 +5,8 @@ BUILD_PATH_EXT = $(BUILD_TARGET_PLATFORM) HAL_LINK := PLATFORM_DFU = 0x30000 +export LTO_EXTRA_OPTIMIZATIONS=1 + DEPENDENCIES = newlib_nano modules/boron/user-part modules/boron/system-part1 dynalib services hal platform system wiring communication rt-dynalib crypto proto_defs wiring_globals LIB_DEPENDENCIES = services system wiring communication hal platform crypto proto_defs wiring_globals # newlib_nano is special in that it's linked automatically by the system, so no need to add it to the library path here diff --git a/modules/tracker/system-part1/makefile b/modules/tracker/system-part1/makefile index 0e90eabb88..e6c10b031b 100644 --- a/modules/tracker/system-part1/makefile +++ b/modules/tracker/system-part1/makefile @@ -5,6 +5,8 @@ BUILD_PATH_EXT = $(BUILD_TARGET_PLATFORM) HAL_LINK := PLATFORM_DFU = 0x30000 +export LTO_EXTRA_OPTIMIZATIONS=1 + DEPENDENCIES = newlib_nano modules/tracker/user-part modules/tracker/system-part1 dynalib services hal platform system wiring communication rt-dynalib crypto proto_defs wiring_globals LIB_DEPENDENCIES = services system wiring communication hal platform crypto proto_defs wiring_globals # newlib_nano is special in that it's linked automatically by the system, so no need to add it to the library path here diff --git a/system/src/system_cloud_connection.h b/system/src/system_cloud_connection.h index c256fd02d5..97ddac0d16 100644 --- a/system/src/system_cloud_connection.h +++ b/system/src/system_cloud_connection.h @@ -24,6 +24,9 @@ #include "ota_flash_hal.h" #include "socket_hal.h" #include +#if HAL_PLATFORM_IFAPI +#include "netdb_hal.h" +#endif // HAL_PLATFORM_IFAPI #ifdef __cplusplus extern "C" { @@ -33,6 +36,13 @@ typedef enum { SYSTEM_CLOUD_DISCONNECT_GRACEFULLY = 1 } system_cloud_connection_flags_t; +typedef enum CloudServerAddressType { + CLOUD_SERVER_ADDRESS_TYPE_NONE = 0, + CLOUD_SERVER_ADDRESS_TYPE_CACHED = 1, + CLOUD_SERVER_ADDRESS_TYPE_CACHED_ADDRINFO = 2, + CLOUD_SERVER_ADDRESS_TYPE_NEW_ADDRINFO = 3 +} CloudServerAddressType; + int system_cloud_connect(int protocol, const ServerAddress* address, sockaddr* saddrCache); int system_cloud_disconnect(int flags); int system_cloud_send(const uint8_t* buf, size_t buflen, int flags); @@ -44,6 +54,10 @@ int system_cloud_set_inet_family_keepalive(int af, unsigned int value, int flags int system_cloud_get_inet_family_keepalive(int af, unsigned int* value); sock_handle_t system_cloud_get_socket_handle(); +#if HAL_PLATFORM_IFAPI +int system_cloud_resolv_address(int protocol, const ServerAddress* address, sockaddr* saddrCache, addrinfo** info, CloudServerAddressType* type, bool useCachedAddrInfo); +#endif // HAL_PLATFORM_IFAPI + #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/system/src/system_cloud_connection_posix.cpp b/system/src/system_cloud_connection_posix.cpp index c91582f9fb..8fa30aeb05 100644 --- a/system/src/system_cloud_connection_posix.cpp +++ b/system/src/system_cloud_connection_posix.cpp @@ -35,13 +35,6 @@ namespace { -enum CloudServerAddressType { - CLOUD_SERVER_ADDRESS_TYPE_NONE = 0, - CLOUD_SERVER_ADDRESS_TYPE_CACHED = 1, - CLOUD_SERVER_ADDRESS_TYPE_CACHED_ADDRINFO = 2, - CLOUD_SERVER_ADDRESS_TYPE_NEW_ADDRINFO = 3 -}; - struct SystemCloudState { int socket = -1; struct addrinfo* addr = nullptr; @@ -54,11 +47,10 @@ const unsigned CLOUD_SOCKET_HALF_CLOSED_WAIT_TIMEOUT = 5000; } /* anonymous */ -int system_cloud_connect(int protocol, const ServerAddress* address, sockaddr* saddrCache) -{ - struct addrinfo* info = nullptr; - CloudServerAddressType type = CLOUD_SERVER_ADDRESS_TYPE_NONE; - bool clean = true; +int system_cloud_resolv_address(int protocol, const ServerAddress* address, sockaddr* saddrCache, addrinfo** info, CloudServerAddressType* type, bool useCachedAddrInfo) { + CHECK_TRUE(info, SYSTEM_ERROR_INVALID_ARGUMENT); + + *type = CLOUD_SERVER_ADDRESS_TYPE_NONE; if (saddrCache && /* protocol == IPPROTO_UDP && */ saddrCache->sa_family != AF_UNSPEC) { char tmphost[INET6_ADDRSTRLEN] = {}; @@ -73,21 +65,21 @@ int system_cloud_connect(int protocol, const ServerAddress* address, sockaddr* s /* FIXME: */ hints.ai_socktype = hints.ai_protocol == IPPROTO_UDP ? SOCK_DGRAM : SOCK_STREAM; - if (!netdb_getaddrinfo(tmphost, tmpserv, &hints, &info)) { - type = CLOUD_SERVER_ADDRESS_TYPE_CACHED; + if (!netdb_getaddrinfo(tmphost, tmpserv, &hints, info)) { + *type = CLOUD_SERVER_ADDRESS_TYPE_CACHED; } } } - if (type == CLOUD_SERVER_ADDRESS_TYPE_NONE) { + if (*type == CLOUD_SERVER_ADDRESS_TYPE_NONE && useCachedAddrInfo) { /* Check if we have another address to try from the cached addrinfo list */ if (s_state.addr && s_state.next) { - info = s_state.next; - type = CLOUD_SERVER_ADDRESS_TYPE_CACHED_ADDRINFO; + *info = s_state.next; + *type = CLOUD_SERVER_ADDRESS_TYPE_CACHED_ADDRINFO; } } - if ((type == CLOUD_SERVER_ADDRESS_TYPE_NONE) && address) { + if ((*type == CLOUD_SERVER_ADDRESS_TYPE_NONE) && address) { /* Use passed ServerAddress */ switch (address->addr_type) { case IP_ADDRESS: { @@ -107,8 +99,8 @@ int system_cloud_connect(int protocol, const ServerAddress* address, sockaddr* s if (inet_inet_ntop(AF_INET, &in, tmphost, sizeof(tmphost))) { snprintf(tmpserv, sizeof(tmpserv), "%u", address->port); - netdb_getaddrinfo(tmphost, tmpserv, &hints, &info); - type = CLOUD_SERVER_ADDRESS_TYPE_NEW_ADDRINFO; + netdb_getaddrinfo(tmphost, tmpserv, &hints, info); + *type = CLOUD_SERVER_ADDRESS_TYPE_NEW_ADDRINFO; } break; } @@ -126,19 +118,31 @@ int system_cloud_connect(int protocol, const ServerAddress* address, sockaddr* s system_string_interpolate(address->domain, tmphost, sizeof(tmphost), system_interpolate_cloud_server_hostname); snprintf(tmpserv, sizeof(tmpserv), "%u", address->port); LOG(TRACE, "Resolving %s#%s", tmphost, tmpserv); - netdb_getaddrinfo(tmphost, tmpserv, &hints, &info); - type = CLOUD_SERVER_ADDRESS_TYPE_NEW_ADDRINFO; + netdb_getaddrinfo(tmphost, tmpserv, &hints, info); + *type = CLOUD_SERVER_ADDRESS_TYPE_NEW_ADDRINFO; break; } } } - int r = SYSTEM_ERROR_NETWORK; - - if (info == nullptr) { + if (*info == nullptr) { LOG(ERROR, "Failed to determine server address"); + return SYSTEM_ERROR_NOT_FOUND; } + return 0; +} + +int system_cloud_connect(int protocol, const ServerAddress* address, sockaddr* saddrCache) +{ + struct addrinfo* info = nullptr; + CloudServerAddressType type = CLOUD_SERVER_ADDRESS_TYPE_NONE; + bool clean = true; + + system_cloud_resolv_address(protocol, address, saddrCache, &info, &type, true /* useCachedAddrInfo */); + + int r = SYSTEM_ERROR_NETWORK; + LOG(TRACE, "Address type: %d", type); for (struct addrinfo* a = info; a != nullptr; a = a->ai_next) { diff --git a/system/src/system_cloud_internal.cpp b/system/src/system_cloud_internal.cpp index b38f0cefd6..f68f2a22f9 100644 --- a/system/src/system_cloud_internal.cpp +++ b/system/src/system_cloud_internal.cpp @@ -1335,8 +1335,17 @@ void Spark_Process_Events() { if (SPARK_CLOUD_SOCKETED && !Spark_Communication_Loop()) { - WARN("Communication loop error, closing cloud socket"); - cloud_disconnect(HAL_PLATFORM_MAY_LEAK_SOCKETS ? CLOUD_DISCONNECT_DONT_CLOSE : 0, CLOUD_DISCONNECT_REASON_ERROR); + // The error is only handled here if we are already out of handshake (or sesssion resume) + // In case of handshake/session resume errors the error is handled in handle_cloud_connection() + // and cloud_disconnect() will also eventually be called with CLOUD_DISCONNECT_REASON_ERROR. + if (SPARK_CLOUD_CONNECTED) { + WARN("Communication loop error, closing cloud socket"); + cloud_disconnect(HAL_PLATFORM_MAY_LEAK_SOCKETS ? CLOUD_DISCONNECT_DONT_CLOSE : 0, CLOUD_DISCONNECT_REASON_ERROR); + } else if (SPARK_CLOUD_HANDSHAKE_PENDING) { + // FIXME: this is a temporary workaround. communication layer should call the appropriate callback on its own in case of errors (?) + SPARK_CLOUD_HANDSHAKE_PENDING = 0; + SPARK_CLOUD_HANDSHAKE_NOTIFY_DONE = 1; + } } else { diff --git a/system/src/system_connection_manager.cpp b/system/src/system_connection_manager.cpp index 05bcfd55f0..6ac9a0ec74 100644 --- a/system/src/system_connection_manager.cpp +++ b/system/src/system_connection_manager.cpp @@ -14,6 +14,7 @@ * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, see . */ + #include "logging.h" LOG_SOURCE_CATEGORY("system.cm") @@ -38,7 +39,7 @@ LOG_SOURCE_CATEGORY("system.cm") namespace particle { namespace system { -typedef struct { +typedef struct DTLSPlaintext_t { uint8_t type; uint8_t version[2]; uint16_t epoch; @@ -46,6 +47,8 @@ typedef struct { uint16_t length; } __attribute__((__packed__)) DTLSPlaintext_t; +constexpr uint16_t EPOCH_BASE = 0x8000; + static const char* netifToName(uint8_t interfaceNumber) { switch(interfaceNumber) { case NETWORK_INTERFACE_ETHERNET: @@ -56,32 +59,26 @@ static const char* netifToName(uint8_t interfaceNumber) { #endif #if HAL_PLATFORM_WIFI case NETWORK_INTERFACE_WIFI_STA: - return "WiFi "; + return "WiFi"; #endif default: return ""; } } -static int getCloudHostnameAndPort(uint16_t * port, char * hostname, int hostnameLength) { - ServerAddress server_addr = {}; - char tmphost[sizeof(server_addr.domain) + 32] = {}; - if (hostnameLength < (int)(sizeof(tmphost)+1)) { - return SYSTEM_ERROR_TOO_LARGE; - } +bool isValidScore(uint32_t score) { + return std::numeric_limits::max() != score; +} +static int getCloudHostnameAndPort(addrinfo** info, CloudServerAddressType* type, bool allowCached = false) { + ServerAddress server_addr = {}; HAL_FLASH_Read_ServerAddress(&server_addr); if (server_addr.port == 0 || server_addr.port == 0xFFFF) { server_addr.port = spark_cloud_udp_port_get(); } - system_string_interpolate(server_addr.domain, tmphost, sizeof(tmphost), system_interpolate_cloud_server_hostname); - strcpy(hostname, tmphost); - *port = server_addr.port; - - LOG_DEBUG(TRACE, "Cloud hostname#port %s#%d", hostname, *port); - return 0; -}; + return system_cloud_resolv_address(IPPROTO_UDP, &server_addr, allowCached ? (sockaddr*)&g_system_cloud_session_data.address : nullptr, info, type, false /* useCachedAddrInfo */); +} ConnectionManager::ConnectionManager() : preferredNetwork_(NETWORK_INTERFACE_ALL) { @@ -98,10 +95,9 @@ void ConnectionManager::setPreferredNetwork(network_handle_t network, bool prefe if (network != NETWORK_INTERFACE_ALL) { preferredNetwork_ = network; - // If cloud is already connected, and a preferred network is set, and it is up, move cloud connection to it immediately + // If cloud is already connected, and a preferred network is set, and it is up, attempt to move cloud connection to it immediately if (spark_cloud_flag_connected() && network_ready(preferredNetwork_, 0, nullptr)) { - auto options = CloudDisconnectOptions().graceful(true).reconnect(true).toSystemOptions(); - spark_cloud_disconnect(&options, nullptr); + scheduleCloudConnectionNetworkCheck(); } } } else { @@ -130,44 +126,209 @@ network_handle_t ConnectionManager::getCloudConnectionNetwork() { network_handle_t ConnectionManager::selectCloudConnectionNetwork() { network_handle_t bestNetwork = NETWORK_INTERFACE_ALL; - if (preferredNetwork_ != NETWORK_INTERFACE_ALL && network_ready(preferredNetwork_, 0, nullptr)) { - LOG_DEBUG(TRACE, "Using preferred network: %lu", preferredNetwork_); - return preferredNetwork_; - } - + bool canUsePreferred = false; // If no preferred network, use the 'best' network based on criteria // Network is ready: ie configured + connected (see ipv4 routable hook) // Network has best criteria based on network tester results for (auto& i: bestNetworks_) { - if (network_ready(i, 0, nullptr)) { - LOG_DEBUG(TRACE, "Using best network: %lu", i); - return i; + if (network_ready(i.first, 0, nullptr)) { + if (bestNetwork == NETWORK_INTERFACE_ALL) { + bestNetwork = i.first; + } + if (preferredNetwork_ != NETWORK_INTERFACE_ALL && preferredNetwork_ == i.first && isValidScore(i.second) /* score */) { + canUsePreferred = true; + } } } // TODO: Determine a specific interface to bind to, even in the default case. // ie: How should we handle selecting a cloud connection when no interfaces are up/ready? // We should have some historical stats to rely on and then bring that network up? - return bestNetwork; + + if (!canUsePreferred) { + if (preferredNetwork_ != NETWORK_INTERFACE_ALL && network_ready(preferredNetwork_, 0, nullptr)) { + nextPeriodicCheck_ = HAL_Timer_Get_Milli_Seconds() + PERIODIC_CHECK_PERIOD_MS; + LOG_DEBUG(TRACE, "Scheduled a periodic check @ %lu ms", nextPeriodicCheck_); + } + LOG(TRACE, "Using best network: %s", netifToName(bestNetwork)); + return bestNetwork; + } else { + nextPeriodicCheck_ = 0; + LOG(TRACE, "Using preferred network: %s", netifToName(preferredNetwork_)); + return preferredNetwork_; + } } -int ConnectionManager::testConnections() { - ConnectionTester tester; - int r = tester.testConnections(); +int ConnectionManager::testConnections(bool background) { + if (!background) { + LOG_DEBUG(INFO, "testConnections full"); + } + if (!background && testResultsActual_) { + // Skip the test once + testResultsActual_ = false; + LOG_DEBUG(TRACE, "Skipping connection test as there are valid cached results"); + return 0; + } + + int r = SYSTEM_ERROR_NETWORK; + Vector metrics; + + if (!background) { + if (backgroundTestInProgress_) { + LOG_DEBUG(WARN, "Background reachability test aborted"); + } + backgroundTestInProgress_ = false; + backgroundTester_.reset(); + testResultsActual_ = false; + + LOG_DEBUG(INFO, "Full reachability test started"); + ConnectionTester tester; + CHECK(tester.prepare(true /* full test */)); + // Blocking call + r = tester.runTest(); + LOG_DEBUG(INFO, "Full reachability test finished (%d)", r); + metrics = tester.getConnectionMetrics(); + } else { + // Background test + testResultsActual_ = false; + if (!backgroundTestInProgress_) { + LOG_DEBUG(INFO, "Background reachability test started"); + backgroundTester_ = std::make_unique(); + CHECK_TRUE(backgroundTester_, SYSTEM_ERROR_NO_MEMORY); + CHECK(backgroundTester_->prepare(false /* full test*/)); + backgroundTestInProgress_ = true; + } + r = backgroundTester_->runTest(0 /* non blocking */); + if (!r || r != SYSTEM_ERROR_BUSY) { + LOG_DEBUG(INFO, "Background reachability test finished (%d)", r); + backgroundTestInProgress_ = false; + metrics = backgroundTester_->getConnectionMetrics(); + backgroundTester_.reset(); + } + } if (r == 0) { - auto metrics = tester.getConnectionMetrics(); bestNetworks_.clear(); for (auto& i: metrics) { - bestNetworks_.append(i.interface); + bestNetworks_.append(std::make_pair(i.interface, i.resultingScore)); + } + if (background) { + // Disable this for now + // testResultsActual_ = true; } } return r; } +int ConnectionManager::scheduleCloudConnectionNetworkCheck() { + testResultsActual_ = false; + checkScheduled_ = true; + return 0; +} + +void ConnectionManager::handlePeriodicCheck() { + if (nextPeriodicCheck_ != 0 && HAL_Timer_Get_Milli_Seconds() >= nextPeriodicCheck_) { + if (!testIsAllowed()) { + return; + } + if (preferredNetwork_ != NETWORK_INTERFACE_ALL && getCloudConnectionNetwork() != preferredNetwork_) { + LOG_DEBUG(TRACE, "Periodic check because preferred interface was not picked during last run"); + scheduleCloudConnectionNetworkCheck(); + } + nextPeriodicCheck_ = 0; + } +} + +bool ConnectionManager::testIsAllowed() const { + uint8_t resetPending = 0; + system_get_flag(SYSTEM_FLAG_RESET_PENDING, &resetPending, nullptr); + + return spark_cloud_flag_connected() && !resetPending && !SPARK_FLASH_UPDATE; +} + +int ConnectionManager::checkCloudConnectionNetwork() { + bool finishedBackgroundTest = false; + if (backgroundTestInProgress_) { + int r = testConnections(true /* background */); + if (checkScheduled_) { + // Invalidate results + r = SYSTEM_ERROR_ABORTED; + } + if (!r) { + // Finished without errors + finishedBackgroundTest = true; + } else if (r == SYSTEM_ERROR_BUSY) { + // Wait to complete + return 0; + } else if (r != SYSTEM_ERROR_ABORTED) { + // Finished with an error, reschedule a check, if aborted - do nothing + return scheduleCloudConnectionNetworkCheck(); + } + } + + if (!checkScheduled_) { + handlePeriodicCheck(); + } + + if (!checkScheduled_ && !finishedBackgroundTest) { + return 0; + } + + if (!testIsAllowed()) { + // Postpone until cloud connection is established or while in OTA + return SYSTEM_ERROR_INVALID_STATE; + } + + checkScheduled_ = false; + + unsigned countReady = 0; + bool matchesCurrent = false; + network_handle_t best = NETWORK_INTERFACE_ALL; + for (const auto& i: bestNetworks_) { + LOG_DEBUG(TRACE, "%s - ready=%d (getCloudConnectionNetwork()=%s)", netifToName(i.first), network_ready(i.first, 0, nullptr), netifToName(getCloudConnectionNetwork())); + if (network_ready(i.first, 0, nullptr)) { + countReady++; + if (i.first == getCloudConnectionNetwork()) { + matchesCurrent = true; + } + best = i.first; + } + } + if (countReady == 0) { + return SYSTEM_ERROR_INVALID_STATE; + } + // Simple case, just perform a cloud ping + if (matchesCurrent && (countReady == 1 || getCloudConnectionNetwork() == getPreferredNetwork())) { + spark_protocol_command(system_cloud_protocol_instance(), ProtocolCommands::PING, 0, nullptr); + LOG_DEBUG(TRACE, "Still using the same network interface (%s) for the cloud connection - perform a cloud ping", netifToName(getCloudConnectionNetwork())); + return 0; + } + if (countReady > 1) { + if (!finishedBackgroundTest) { + // Re-test connections + backgroundTestInProgress_ = false; + return testConnections(true /* background */); + } else { + best = selectCloudConnectionNetwork(); + // If matches current again just perform a ping + if (best == getCloudConnectionNetwork()) { + spark_protocol_command(system_cloud_protocol_instance(), ProtocolCommands::PING, 0, nullptr); + LOG_DEBUG(TRACE, "Best network interface candidate for the cloud connection is still the same (%s) - perform a cloud ping", netifToName(best)); + return 0; + } + } + } + // If best candidate doesn't match current network interface - reconnect + LOG(TRACE, "Best network interface for cloud connection changed (to %s) - move the cloud session", netifToName(best)); + auto options = CloudDisconnectOptions().reconnect(true); + auto systemOptions = options.toSystemOptions(); + spark_cloud_disconnect(&systemOptions, nullptr); + return 0; +} + ConnectionTester::ConnectionTester() { for (const auto& i: getSupportedInterfaces()) { struct ConnectionMetrics interfaceDiagnostics = {}; - interfaceDiagnostics.interface = i; + interfaceDiagnostics.interface = i.first; interfaceDiagnostics.socketDescriptor = -1; // 0 is a valid socket fd number metrics_.append(interfaceDiagnostics); } @@ -184,7 +345,7 @@ ConnectionTester::~ConnectionTester() { free(i.rxBuffer); } } -}; +} ConnectionMetrics* ConnectionTester::metricsFromSocketDescriptor(int socketDescriptor) { for (auto& i : metrics_) { @@ -193,16 +354,20 @@ ConnectionMetrics* ConnectionTester::metricsFromSocketDescriptor(int socketDescr } } return nullptr; -}; +} bool ConnectionTester::testPacketsOutstanding() { for (auto& i : metrics_) { - if (i.txPacketCount > i.rxPacketCount) { + if (i.txPacketCount != REACHABILITY_TEST_MAX_TX_PACKET_COUNT) { return true; + } else { + if (i.txPacketCount != i.rxPacketCount) { + return true; + } } } return false; -}; +} int ConnectionTester::allocateTestPacketBuffers(ConnectionMetrics* metrics) { int maxMessageLength = REACHABILITY_MAX_PAYLOAD_SIZE + sizeof(DTLSPlaintext_t); @@ -216,61 +381,99 @@ int ConnectionTester::allocateTestPacketBuffers(ConnectionMetrics* metrics) { if (rxBuffer) { free(rxBuffer); } - LOG(ERROR, "%s failed to allocate connection test buffers of size %d", netifToName(metrics->interface), maxMessageLength); + LOG_DEBUG(ERROR, "%s failed to allocate connection test buffers of size %d", netifToName(metrics->interface), maxMessageLength); return SYSTEM_ERROR_NO_MEMORY; } metrics->txBuffer = txBuffer; metrics->rxBuffer = rxBuffer; return 0; -}; +} int ConnectionTester::sendTestPacket(ConnectionMetrics* metrics) { int r = 0; - // Only send a new packet if we have received the previous one, or we timeout waiting for a response - if (metrics->txPacketCount == metrics->rxPacketCount || - (millis() > metrics->txPacketStartMillis + REACHABILITY_TEST_PACKET_TIMEOUT_MS)) { - - generateTestPacket(metrics); - - int r = sock_send(metrics->socketDescriptor, metrics->txBuffer, metrics->testPacketSize, 0); + // Only send a new packet every REACHABILITY_TEST_PACKET_TX_TIMEOUT_MS milliseconds + if (HAL_Timer_Get_Milli_Seconds() >= (metrics->txPacketStartMillis + REACHABILITY_TEST_PACKET_TX_TIMEOUT_MS) && metrics->txPacketCount < REACHABILITY_TEST_MAX_TX_PACKET_COUNT) { + size_t testPacketSize = CHECK(generateTestPacket(metrics)); + + int r = sock_send(metrics->socketDescriptor, metrics->txBuffer, testPacketSize, 0); + // Take TX errors into account too + metrics->txPacketStartMillis = HAL_Timer_Get_Milli_Seconds(); + metrics->txPacketCount++; + metrics->testPacketSequenceNumber++; if (r > 0) { - metrics->txPacketStartMillis = millis(); - metrics->txPacketCount++; - metrics->testPacketSequenceNumber++; - metrics->txBytes += metrics->testPacketSize; + metrics->txBytes += testPacketSize; } else { + metrics->txPacketErrors++; LOG_DEBUG(WARN, "Test sock_send failed %d errno %d interface %d", r, errno, metrics->interface); return SYSTEM_ERROR_NETWORK; } - LOG_DEBUG(TRACE, "Sock %d packet # %d tx > %d", metrics->socketDescriptor, metrics->txPacketCount, r); + // LOG_DEBUG(TRACE, "Sock %d packet # %d tx > %d", metrics->socketDescriptor, metrics->txPacketCount, r); } return r; -}; +} int ConnectionTester::receiveTestPacket(ConnectionMetrics* metrics) { - int r = sock_recv(metrics->socketDescriptor, metrics->rxBuffer, metrics->testPacketSize, MSG_DONTWAIT); - if (r > 0) { - metrics->totalPacketWaitMillis += (millis() - metrics->txPacketStartMillis); - metrics->rxPacketCount++; - metrics->rxBytes += metrics->testPacketSize; - - CHECK_TRUE((uint32_t)r == metrics->testPacketSize, SYSTEM_ERROR_BAD_DATA); - - if (memcmp(metrics->rxBuffer, metrics->txBuffer, r)) { - LOG(WARN, "Socket %d Interface %d did not receive the same echo data: %d", metrics->socketDescriptor, metrics->interface, r); - return SYSTEM_ERROR_BAD_DATA; + msghdr msg = {}; + iovec iov = {}; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + iov.iov_base = metrics->rxBuffer; + iov.iov_len = REACHABILITY_MAX_PAYLOAD_SIZE + sizeof(DTLSPlaintext_t); + char controlBuf[CMSG_SPACE(sizeof(timespec))] = {}; + msg.msg_control = controlBuf; + msg.msg_controllen = sizeof(controlBuf); + + int r = sock_recvmsg(metrics->socketDescriptor, &msg, MSG_DONTWAIT); + if (r >= (int)sizeof(DTLSPlaintext_t)) { + system_tick_t rxTimestamp = 0; + for (cmsghdr* cm = CMSG_FIRSTHDR(&msg); cm != nullptr; cm = CMSG_NXTHDR(&msg, cm)) { + if (cm->cmsg_level == SOL_SOCKET && cm->cmsg_type == SO_TIMESTAMPING) { + auto ts = (timespec*)CMSG_DATA(cm); + rxTimestamp = (ts->tv_sec * 1000 + ts->tv_nsec / 1000000); + } } - + if (!rxTimestamp) { + // Just in case + LOG_DEBUG(WARN, "No RX timestamp from SO_TIMESTAMPING"); + rxTimestamp = HAL_Timer_Get_Milli_Seconds(); + } + // Parse packet + auto header = (DTLSPlaintext_t*)metrics->rxBuffer; + header->epoch = bigEndianToNative(header->epoch); + // LOG(TRACE, "epoch=%04x", header->epoch); + CHECK_TRUE(header->epoch >= EPOCH_BASE, SYSTEM_ERROR_BAD_DATA); + CHECK_TRUE((header->epoch & ~(EPOCH_BASE)) == metrics->interface, SYSTEM_ERROR_BAD_DATA); + header->length = bigEndianToNative(header->length); + // LOG(TRACE, "length=%u (r=%d)", header->length, r); + CHECK_TRUE(header->length == (r - sizeof(DTLSPlaintext_t)), SYSTEM_ERROR_BAD_DATA); + uint32_t sentTimestamp = 0; + uint16_t seqNum = 0; + memcpy(&sentTimestamp, header->sequence_number, sizeof(sentTimestamp)); + memcpy(&seqNum, header->sequence_number + sizeof(uint32_t), sizeof(uint16_t)); + seqNum = bigEndianToNative(seqNum); + sentTimestamp = bigEndianToNative(sentTimestamp); + // LOG(TRACE, "seqNum=%u testSeqNum=%u sentTimestamp=%u now=%u", seqNum, metrics->testPacketSequenceNumber, sentTimestamp, HAL_Timer_Get_Milli_Seconds()); + CHECK_TRUE(seqNum <= metrics->testPacketSequenceNumber, SYSTEM_ERROR_BAD_DATA); + CHECK_TRUE(sentTimestamp < HAL_Timer_Get_Milli_Seconds(), SYSTEM_ERROR_BAD_DATA); + if (metrics->rxPacketMask & (1 << seqNum)) { + LOG(TRACE, "Duplicate packet seq=%u mask=%04x", seqNum, metrics->rxPacketMask); + // Already seen this seq num + return 0; + } + metrics->rxPacketMask |= (1 << seqNum); + metrics->totalPacketWaitMillis += (rxTimestamp - sentTimestamp); + metrics->rxPacketCount++; + metrics->rxBytes += header->length; + // LOG_DEBUG(TRACE, "Sock %d packet # %u rx < %d", metrics->socketDescriptor, seqNum, r); } else { LOG_DEBUG(WARN, "Test sock_recv failed %d errno %d interface %d", r, errno, metrics->interface); return SYSTEM_ERROR_NETWORK; } - LOG_DEBUG(TRACE, "Sock %d packet # %d rx < %d", metrics->socketDescriptor, metrics->rxPacketCount, r); return r; -}; +} int ConnectionTester::generateTestPacket(ConnectionMetrics* metrics) { unsigned packetDataLength = random(1, REACHABILITY_MAX_PAYLOAD_SIZE); @@ -278,7 +481,7 @@ int ConnectionTester::generateTestPacket(ConnectionMetrics* metrics) { DTLSPlaintext_t msg = { REACHABILITY_TEST_MSG, // DTLS Message Type {0xfe, 0xfd}, // DTLS type 1.2 - 0x8000, // Differentiate interfaces by epoch field + EPOCH_BASE, // Differentiate interfaces by epoch field {}, // Sequence number 0 // Payload length }; @@ -288,43 +491,47 @@ int ConnectionTester::generateTestPacket(ConnectionMetrics* metrics) { msg.epoch |= metrics->interface; msg.length = packetDataLength; - uint32_t sequenceNumber = nativeToBigEndian(metrics->testPacketSequenceNumber); + uint16_t sequenceNumber = nativeToBigEndian((uint16_t)metrics->testPacketSequenceNumber); msg.epoch = nativeToBigEndian(msg.epoch); msg.length = nativeToBigEndian(msg.length); - memcpy(&msg.sequence_number, &sequenceNumber, sizeof(sequenceNumber)); + uint32_t ts = nativeToBigEndian(HAL_Timer_Get_Milli_Seconds()); + memcpy(msg.sequence_number, &ts, sizeof(ts)); + memcpy(msg.sequence_number + sizeof(ts), &sequenceNumber, sizeof(sequenceNumber)); Random rand; rand.gen((char*)metrics->txBuffer + sizeof(msg), packetDataLength); memcpy(metrics->txBuffer, &msg, headerLength); - metrics->testPacketSize = totalMessageLength; - return 0; -}; + return totalMessageLength; +} int ConnectionTester::pollSockets(struct pollfd* pfds, int socketCount) { - int pollCount = sock_poll(pfds, socketCount, 0); + int pollCount = sock_poll(pfds, socketCount, 1 /* ms */); if (pollCount < 0) { - LOG(ERROR, "Connection test poll error %d", pollCount); + LOG_DEBUG(ERROR, "Connection test poll error %d", pollCount); return 0; } for (int i = 0; i < socketCount; i++) { ConnectionMetrics* connection = metricsFromSocketDescriptor(pfds[i].fd); if (!connection) { - LOG(ERROR, "No connection associated with socket descriptor %d", pfds[i].fd); + LOG_DEBUG(ERROR, "No connection associated with socket descriptor %d", pfds[i].fd); return SYSTEM_ERROR_BAD_DATA; } + // Ignore errors + sendTestPacket(connection); + if (pfds[i].revents & POLLIN) { - receiveTestPacket(connection); - } - if (pfds[i].revents & POLLOUT) { - CHECK(sendTestPacket(connection)); + int r = receiveTestPacket(connection); + if (r == SYSTEM_ERROR_BAD_DATA) { + LOG_DEBUG(WARN, "Reachability packet failed validation"); + } } } return 0; -}; +} // GOAL: To maintain a list of which network interface is "best" at any given time // 1) Retrieve the server hostname and port. Resolve the hostname to an addrinfo list (ie IP addresses of server) @@ -332,40 +539,26 @@ int ConnectionTester::pollSockets(struct pollfd* pfds, int socketCount) { // 3) Add these created+connected sockets to a pollfd structure. Allocate buffers for the reachability test messages. // 4) Poll all the sockets. Polling sends a reachability test message and waits for the response. The test continues for the test duration // 5) After polling completes, free the allocated buffers, reset diagnostics and calculate updated metrics. -int ConnectionTester::testConnections() { - // Step 1: resolve server hostname to IP address +int ConnectionTester::prepare(bool fullTest) { struct addrinfo* info = nullptr; - struct addrinfo hints = {}; - hints.ai_flags = AI_NUMERICSERV | AI_ADDRCONFIG; - hints.ai_protocol = IPPROTO_UDP; - hints.ai_socktype = SOCK_DGRAM; - - char tmphost[128] = {}; // TODO: better size ie sizeof(address->domain) - char tmpserv[8] = {}; - uint16_t tmpport = 0; - - int r = SYSTEM_ERROR_NETWORK; + CloudServerAddressType type = CLOUD_SERVER_ADDRESS_TYPE_NONE; - getCloudHostnameAndPort(&tmpport, tmphost, sizeof(tmphost)); - snprintf(tmpserv, sizeof(tmpserv), "%u", tmpport); - LOG(TRACE, "Resolving %s#%s", tmphost, tmpserv); - // FIXME: get addrinfo/server IP from DNS lookup using the specific interfaces DNS server - r = netdb_getaddrinfo(tmphost, tmpserv, &hints, &info); - if (r) { - LOG(ERROR, "No addrinfo for %s#%s", tmphost, tmpserv); - return SYSTEM_ERROR_NETWORK; - } + // Step 1: Retrieve the server hostname and port. Resolve the hostname to an addrinfo list (ie IP addresses of server) + CHECK(getCloudHostnameAndPort(&info, &type, !fullTest)); SCOPE_GUARD({ netdb_freeaddrinfo(info); }); int socketCount = 0; - auto pfds = std::make_unique(metrics_.size());; + auto pfds = std::make_unique(metrics_.size()); CHECK_TRUE(pfds, SYSTEM_ERROR_NO_MEMORY); + + int r = SYSTEM_ERROR_NETWORK; // Step 2: Create, bind, and connect sockets for each network interface to test for (struct addrinfo* a = info; a != nullptr; a = a->ai_next) { + bool ok = true; // For each network interface to test, create + open a socket with the retrieved server address // If any of the sockets fail to be created + opened with this server address, return an error for (auto& connectionMetrics: metrics_) { @@ -377,13 +570,14 @@ int ConnectionTester::testConnections() { int s = sock_socket(a->ai_family, a->ai_socktype, a->ai_protocol); NAMED_SCOPE_GUARD(guard, { sock_close(s); + ok = false; }); if (s < 0) { - LOG(ERROR, "test socket failed, family=%d, type=%d, protocol=%d, errno=%d", a->ai_family, a->ai_socktype, a->ai_protocol, errno); + LOG_DEBUG(ERROR, "test socket failed, family=%d, type=%d, protocol=%d, errno=%d", a->ai_family, a->ai_socktype, a->ai_protocol, errno); return SYSTEM_ERROR_NETWORK; } - +#ifdef DEBUG_BUILD char serverHost[INET6_ADDRSTRLEN] = {}; uint16_t serverPort = 0; switch (a->ai_family) { @@ -399,19 +593,27 @@ int ConnectionTester::testConnections() { } } LOG_DEBUG(TRACE, "test socket=%d, connecting to %s#%u", s, serverHost, serverPort); - +#endif struct ifreq ifr = {}; if_index_to_name(connectionMetrics.interface, ifr.ifr_name); r = sock_setsockopt(s, SOL_SOCKET, SO_BINDTODEVICE, &ifr, sizeof(ifr)); if (r) { - LOG(ERROR, "test socket=%d, failed to sock_setsockopt to IF %s, errno=%d", s, ifr.ifr_name, errno); + LOG_DEBUG(ERROR, "test socket=%d, failed to sock_setsockopt to IF %s, errno=%d", s, ifr.ifr_name, errno); return SYSTEM_ERROR_NETWORK; } + // Enable timestamps on recvd packets + int dummy = 1; + r = sock_setsockopt(s, SOL_SOCKET, SO_TIMESTAMPING, &dummy, sizeof(dummy)); + if (r) { + LOG_DEBUG(WARN, "test socket=%d, failed to enable timestamping"); + // Not a critical error + } + connectionMetrics.socketConnAttempts++; r = sock_connect(s, a->ai_addr, a->ai_addrlen); if (r) { - LOG(ERROR, "test socket=%d, failed to connect to %s#%u, errno=%d", s, serverHost, serverPort, errno); + LOG_DEBUG(ERROR, "test socket=%d, failed to connect to %s#%u, errno=%d", s, serverHost, serverPort, errno); connectionMetrics.socketConnFailures++; return SYSTEM_ERROR_NETWORK; } @@ -420,69 +622,103 @@ int ConnectionTester::testConnections() { // Step 3: Use the socket descriptor for the polling structure, allocate our test buffers connectionMetrics.socketDescriptor = s; pfds[socketCount].fd = connectionMetrics.socketDescriptor; - pfds[socketCount].events = (POLLIN | POLLOUT); + pfds[socketCount].events = (POLLIN); socketCount++; guard.dismiss(); CHECK(allocateTestPacketBuffers(&connectionMetrics)); } + if (ok) { + r = SYSTEM_ERROR_NONE; + break; + } } - - // Step 4: Send/Receive data on the sockets for the duration of the test time - auto endTime = HAL_Timer_Get_Milli_Seconds() + REACHABILITY_TEST_DURATION_MS; - while (HAL_Timer_Get_Milli_Seconds() < endTime) { - CHECK(pollSockets(pfds.get(), socketCount)); - SystemISRTaskQueue.process(); + + if (!r) { + socketCount_ = socketCount; + pfds_ = std::move(pfds); } - // Only read from sockets to receive any final outstanding packets - for (int i = 0; i < socketCount; i++) { - pfds[i].events = (POLLIN); + endTime_ = HAL_Timer_Get_Milli_Seconds() + REACHABILITY_TEST_DURATION_MS; + + return r; +} + +int ConnectionTester::runTest(system_tick_t maxBlockTime) { + if (finished_) { + return 0; } - endTime = HAL_Timer_Get_Milli_Seconds() + REACHABILITY_TEST_DURATION_MS; - while(testPacketsOutstanding() && HAL_Timer_Get_Milli_Seconds() < endTime) { - pollSockets(pfds.get(), socketCount); + auto start = HAL_Timer_Get_Milli_Seconds(); + + // Step 4: Send/Receive data on the sockets for the duration of the test time + while(testPacketsOutstanding() && HAL_Timer_Get_Milli_Seconds() < endTime_) { + pollSockets(pfds_.get(), socketCount_); SystemISRTaskQueue.process(); + if (HAL_Timer_Get_Milli_Seconds() - start >= maxBlockTime) { + break; + } } - // Step 5: calculate updated metrics - for (auto& i: metrics_) { - if (i.rxPacketCount > 0) { - i.avgPacketRoundTripTime = (i.totalPacketWaitMillis / i.rxPacketCount); - - LOG(INFO,"%s: %d/%d packets %d/%d bytes received, avg rtt: %d", - netifToName(i.interface), - i.rxPacketCount, - i.txPacketCount, - i.rxBytes, - i.txBytes, - i.avgPacketRoundTripTime); + finished_ = !testPacketsOutstanding() || HAL_Timer_Get_Milli_Seconds() >= endTime_; + if (finished_) { + // Step 5: calculate updated metrics + for (auto& i: metrics_) { + if (i.rxPacketCount > 0) { + i.avgPacketRoundTripTime = (i.totalPacketWaitMillis / i.rxPacketCount); + i.resultingScore = i.totalPacketWaitMillis; + unsigned penalty = 0; + unsigned consecutive = 0; + for (unsigned j = 0; j < i.txPacketCount; j++) { + if (i.rxPacketMask & (1 << j)) { + // Received + penalty = 0; + consecutive = 0; + } else { + penalty = i.avgPacketRoundTripTime * (2 << consecutive++) /* 2^(conscutive++) */; + LOG_DEBUG(TRACE, "%d: total=%u consecutive=%u penalty=%u resultingScore=%u new=%u", i.interface, i.totalPacketWaitMillis, consecutive, penalty, i.resultingScore, i.resultingScore + penalty); + i.resultingScore += penalty; + } + } + i.resultingScore /= i.rxPacketCount; + } else { + i.avgPacketRoundTripTime = 0; + i.resultingScore = std::numeric_limits::max(); + } + LOG(INFO,"%s: %lu/%lu packets (%lu tx errors) %lu/%lu bytes received, avg rtt: %lu, mask=%04x, score=%lu", + netifToName(i.interface), + i.rxPacketCount, + i.txPacketCount, + i.txPacketErrors, + i.rxBytes, + i.txBytes, + i.avgPacketRoundTripTime, + i.rxPacketMask, + i.resultingScore); } + // Sort list by packet latency in ascending order, ie fastest to slowest + std::sort(metrics_.begin(), metrics_.end(), [](const ConnectionMetrics& dg1, const ConnectionMetrics& dg2) { + return (dg1.resultingScore < dg2.resultingScore); + }); + return 0; } - - // Sort list by packet latency in ascending order, ie fastest to slowest - std::sort(metrics_.begin(), metrics_.end(), [](const ConnectionMetrics& dg1, const ConnectionMetrics& dg2) { - return (dg1.avgPacketRoundTripTime < dg2.avgPacketRoundTripTime); - }); - - return 0; + return SYSTEM_ERROR_BUSY; } const Vector ConnectionTester::getConnectionMetrics(){ return metrics_; } -const Vector ConnectionTester::getSupportedInterfaces() { - const Vector interfaceList = { +const Vector> ConnectionTester::getSupportedInterfaces() { + const Vector> interfaceList = { #if HAL_PLATFORM_ETHERNET - NETWORK_INTERFACE_ETHERNET, + {NETWORK_INTERFACE_ETHERNET, 0}, #endif #if HAL_PLATFORM_WIFI - NETWORK_INTERFACE_WIFI_STA, + {NETWORK_INTERFACE_WIFI_STA, 0}, #endif #if HAL_PLATFORM_CELLULAR - NETWORK_INTERFACE_CELLULAR + {NETWORK_INTERFACE_CELLULAR, 0} #endif }; diff --git a/system/src/system_connection_manager.h b/system/src/system_connection_manager.h index 9ed38630bb..94ba584158 100644 --- a/system/src/system_connection_manager.h +++ b/system/src/system_connection_manager.h @@ -24,6 +24,7 @@ #include "system_network.h" #include "spark_wiring_vector.h" +#include namespace particle { namespace system { @@ -32,11 +33,11 @@ struct ConnectionMetrics { int socketDescriptor; uint8_t *txBuffer; uint8_t *rxBuffer; - uint32_t testPacketSize; uint32_t testPacketSequenceNumber; - uint32_t testPacketTxMillis; uint32_t txPacketCount; + uint32_t txPacketErrors; uint32_t rxPacketCount; + uint32_t rxPacketMask; uint32_t txPacketStartMillis; uint32_t totalPacketWaitMillis; @@ -47,8 +48,11 @@ struct ConnectionMetrics { uint32_t txBytes; uint32_t rxBytes; uint32_t avgPacketRoundTripTime; + uint32_t resultingScore; }; +class ConnectionTester; + class ConnectionManager { public: ConnectionManager(); @@ -61,11 +65,23 @@ class ConnectionManager { network_handle_t getCloudConnectionNetwork(); network_handle_t selectCloudConnectionNetwork(); - int testConnections(); + int testConnections(bool background = false); + int scheduleCloudConnectionNetworkCheck(); + int checkCloudConnectionNetwork(); + +private: + void handlePeriodicCheck(); + bool testIsAllowed() const; private: network_handle_t preferredNetwork_; - Vector bestNetworks_; + Vector> bestNetworks_; + volatile bool testResultsActual_ = false; + volatile bool checkScheduled_ = false; + bool backgroundTestInProgress_ = false; + std::unique_ptr backgroundTester_; + static constexpr system_tick_t PERIODIC_CHECK_PERIOD_MS = 5 * 60 * 1000; + system_tick_t nextPeriodicCheck_ = 0; }; class ConnectionTester { @@ -73,10 +89,11 @@ class ConnectionTester { ConnectionTester(); ~ConnectionTester(); - int testConnections(); + int prepare(bool fullTest = true); + int runTest(system_tick_t maxBlockTime = 0xffffffff); const Vector getConnectionMetrics(); - static const Vector getSupportedInterfaces(); + static const Vector> getSupportedInterfaces(); private: int allocateTestPacketBuffers(ConnectionMetrics* metrics); @@ -88,11 +105,16 @@ class ConnectionTester { bool testPacketsOutstanding(); const uint8_t REACHABILITY_TEST_MSG = 252; - const unsigned REACHABILITY_MAX_PAYLOAD_SIZE = 256; - const unsigned REACHABILITY_TEST_DURATION_MS = 2500; - const unsigned REACHABILITY_TEST_PACKET_TIMEOUT_MS = 500; + const system_tick_t REACHABILITY_MAX_PAYLOAD_SIZE = 512; // FIXME: get some constant from cloud layer + const system_tick_t REACHABILITY_TEST_DURATION_MS = 5000; + const system_tick_t REACHABILITY_TEST_PACKET_TX_TIMEOUT_MS = 250; + const unsigned REACHABILITY_TEST_MAX_TX_PACKET_COUNT = 10; Vector metrics_; + std::unique_ptr pfds_; + system_tick_t endTime_ = 0; + bool finished_ = false; + size_t socketCount_ = 0; }; } } /* particle::system */ diff --git a/system/src/system_network_manager.cpp b/system/src/system_network_manager.cpp index 43efb9cb3e..726e4bf996 100644 --- a/system/src/system_network_manager.cpp +++ b/system/src/system_network_manager.cpp @@ -96,18 +96,8 @@ int for_each_iface(F&& f) { return 0; } -void forceCloudPingIfConnected() { - const auto task = new(std::nothrow) ISRTaskQueue::Task(); - if (!task) { - return; - } - task->func = [](ISRTaskQueue::Task* task) { - delete task; - if (spark_cloud_flag_connected()) { - spark_protocol_command(system_cloud_protocol_instance(), ProtocolCommands::PING, 0, nullptr); - } - }; - SystemISRTaskQueue.enqueue(task); +void forceCloudPingOrTest() { + ConnectionManager::instance()->scheduleCloudConnectionNetworkCheck(); } const char NETWORK_CONFIG_FILE[] = "/sys/network.dat"; @@ -538,11 +528,6 @@ void NetworkManager::handleIfState(if_t iface, const struct if_event* ev) { } void NetworkManager::handleIfLink(if_t iface, const struct if_event* ev) { - uint8_t netIfIndex = 0; - if_get_index(iface, &netIfIndex); - bool disconnectCloud = false; - auto options = CloudDisconnectOptions().reconnect(true); - if (ev->ev_if_link->state) { /* Interface link state changed to UP */ NetworkInterfaceConfig conf; @@ -617,14 +602,11 @@ void NetworkManager::handleIfLink(if_t iface, const struct if_event* ev) { refreshIpState(); } - // If the cloud is connected, and the preferred network becomes available, move to that network - if (spark_cloud_flag_connected() && ConnectionManager::instance()->getPreferredNetwork() == netIfIndex && !SPARK_FLASH_UPDATE) { - LOG(INFO, "Preferred network %u available, moving cloud connection", netIfIndex); - options.graceful(true); - disconnectCloud = true; + if (getInterfaceIp4State(iface) == ProtocolState::CONFIGURED || getInterfaceIp6State(iface) == ProtocolState::CONFIGURED) { + forceCloudPingOrTest(); } - } else { + auto state = getInterfaceRuntimeState(iface); // Disable by default if_clear_xflags(iface, IFXF_DHCP); resetInterfaceProtocolState(iface); @@ -638,20 +620,11 @@ void NetworkManager::handleIfLink(if_t iface, const struct if_event* ev) { refreshIpState(); } } - - // If the current cloud connection network interfaces goes down, and there are other configured interfaces, close the cloud and connect to them - if (ConnectionManager::instance()->getCloudConnectionNetwork() == netIfIndex && state_ == State::IP_CONFIGURED) { - LOG(WARN, "Cloud connection interface %d link state down, switching interfaces", netIfIndex); - disconnectCloud = true; - } - } - if (spark_cloud_flag_connected() && disconnectCloud) { - auto systemOptions = options.toSystemOptions(); - spark_cloud_disconnect(&systemOptions, nullptr); + if (state && (state->ip4State == ProtocolState::CONFIGURED || state->ip6State == ProtocolState::CONFIGURED)) { + forceCloudPingOrTest(); + } } - - forceCloudPingIfConnected(); } void NetworkManager::clearDnsConfiguration(if_t iface) { @@ -664,7 +637,7 @@ void NetworkManager::handleIfAddr(if_t iface, const struct if_event* ev) { if (state_ == State::IP_CONFIGURED || state_ == State::IFACE_LINK_UP) { refreshIpState(); } - forceCloudPingIfConnected(); + forceCloudPingOrTest(); } void NetworkManager::handleIfLinkLayerAddr(if_t iface, const struct if_event* ev) { @@ -868,7 +841,7 @@ void NetworkManager::resolvEventHandler(const void* data) { refreshIpState(); // NOTE: we could potentially force a cloud ping on DNS change, but // this seems excessive, and it's better to rely on IP state only instead - // forceCloudPingIfConnected(); + // forceCloudPingOrTest(); } const char* NetworkManager::stateToName(State state) const { diff --git a/system/src/system_task.cpp b/system/src/system_task.cpp index f6700bbc40..3901e9f853 100644 --- a/system/src/system_task.cpp +++ b/system/src/system_task.cpp @@ -57,6 +57,7 @@ #include "spark_wiring_led.h" #if HAL_PLATFORM_IFAPI #include "system_listening_mode.h" +#include "system_connection_manager.h" #endif #if HAL_PLATFORM_BLE_SETUP @@ -427,7 +428,7 @@ void handle_cloud_connection(bool force_events) } const auto diag = CloudDiagnostics::instance(); diag->lastError(err); - cloud_disconnect(); + cloud_disconnect(HAL_PLATFORM_MAY_LEAK_SOCKETS ? CLOUD_DISCONNECT_DONT_CLOSE : 0, CLOUD_DISCONNECT_REASON_ERROR); } else { cfod_count = 0; } @@ -447,6 +448,10 @@ void manage_cloud_connection(bool force_events) } else // cloud connection is wanted { +#if HAL_PLATFORM_IFAPI + ConnectionManager::instance()->checkCloudConnectionNetwork(); +#endif // HAL_PLATFORM_IFAPI + establish_cloud_connection(); handle_cloud_connection(force_events); diff --git a/third_party/lwip/lwip b/third_party/lwip/lwip index e1c27762da..6f7cb46486 160000 --- a/third_party/lwip/lwip +++ b/third_party/lwip/lwip @@ -1 +1 @@ -Subproject commit e1c27762da60693ca954b6aeb64f4f1a5fe3fc16 +Subproject commit 6f7cb46486c294244b2b95be52385ad8cc4bc7bd