From 4fb49be5a89e8d9e8d5e6c0fa653150a2f20fa3a Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 14:32:29 -0500 Subject: [PATCH 01/16] HAACK: install guzzle --- composer.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index f867eac..8e78dd2 100644 --- a/composer.json +++ b/composer.json @@ -17,7 +17,8 @@ "php": ">=5.5", "ext-curl": "*", "ext-json": "*", - "ext-mbstring": "*" + "ext-mbstring": "*", + "guzzlehttp/guzzle": "^7.1" }, "require-dev": { "phpunit/phpunit": "4.8.*" @@ -28,4 +29,4 @@ "autoload": { "files": [ "lib/TraackrApi.php"] } -} \ No newline at end of file +} From df9761d38b18bab29abe9bfa0bdf263a6eaa0515 Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 14:32:54 -0500 Subject: [PATCH 02/16] HAACK: add concurrent GET requests --- lib/TraackrApi/TraackrApiObject.php | 209 ++++++++++++++++++++-------- 1 file changed, 150 insertions(+), 59 deletions(-) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index fe8b36c..51fd72e 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -1,6 +1,10 @@ concurrent = true; + $this->guzzleClient = new Client(); + } else { + // init cURL + $this->curl = curl_init(); + } + } + + private function getGuzzleHeaders() + { + $headers = []; + foreach (self::$curl_headers as $header) { + $parts = explode(':', $header); + $property = $parts[0]; + $value = ''; + if (!empty($parts[1])) { + $value = trim($parts[1]); + } + $headers[$property] = $value; + } + $headers['Accept-Encoding'] = 'gzip;q=1.0, deflate;q=0.5, identity;q=0.1'; + return $headers; + } + + private function getGuzzleOpts() { - // init cURL - $this->curl = curl_init(); + $options = [ + 'connect_timeout' => self::$connectionTimeout, + 'timeout' => self::$timeout, + 'headers' => $this->getGuzzleHeaders() + ]; + return $options; } /** @@ -122,6 +160,62 @@ private function prepareParameters($params) return $params; } + private function handleErrorResponse($logger, $url, $httpCode, $body) + { + if ($httpCode == '400') { + // Let's try to see if it's a bad customer key + if ($body === 'Customer key not found') { + $message = 'Invalid Customer Key (HTTP 400)'; + + $logger->error($message); + + throw new InvalidCustomerKeyException( + $message . ': ' . $body, + $httpCode + ); + } + + $message = 'Missing or Invalid argument/parameter (HTTP 400)'; + + $logger->error($message); + + throw new MissingParameterException( + $message . ': ' . $body, + $httpCode + ); + } + + if ($httpCode == '403') { + $message = 'Invalid API key (HTTP 403)'; + $logger->error($message); + + throw new InvalidApiKeyException( + $message . ': ' . $body, + $httpCode + ); + } + + if ($httpCode == '404') { + $message = 'API resource not found (HTTP 404)'; + + $logger->error($message); + + throw new NotFoundException( + $message . ': ' . $url, + $httpCode + ); + } + + $message = 'API HTTP Error (HTTP ' . $httpCode . ')'; + + $logger->error($message); + + throw new TraackrApiException( + $message . ': ' . $body, + $httpCode + ); + } + private function call($decode, $contentTypeHeader) { // Prep headers @@ -153,63 +247,11 @@ private function call($decode, $contentTypeHeader) throw new TraackrApiException($message); } - $httpcode = curl_getinfo($this->curl, CURLINFO_HTTP_CODE); + $httpCode = curl_getinfo($this->curl, CURLINFO_HTTP_CODE); - if ($httpcode != '200') { + if ($httpCode != '200') { $info = curl_getinfo($this->curl); - - if ($httpcode == '400') { - // Let's try to see if it's a bad customer key - if ($curl_exec === 'Customer key not found') { - $message = 'Invalid Customer Key (HTTP 400)'; - - $logger->error($message); - - throw new InvalidCustomerKeyException( - $message . ': ' . $curl_exec, - $httpcode - ); - } - - $message = 'Missing or Invalid argument/parameter (HTTP 400)'; - - $logger->error($message); - - throw new MissingParameterException( - $message . ': ' . $curl_exec, - $httpcode - ); - } - - if ($httpcode == '403') { - $message = 'Invalid API key (HTTP 403)'; - $logger->error($message); - - throw new InvalidApiKeyException( - $message . ': ' . $curl_exec, - $httpcode - ); - } - - if ($httpcode == '404') { - $message = 'API resource not found (HTTP 404)'; - - $logger->error($message); - - throw new NotFoundException( - $message . ': ' . $info['url'], - $httpcode - ); - } - - $message = 'API HTTP Error (HTTP ' . $httpcode . ')'; - - $logger->error($message); - - throw new TraackrApiException( - $message . ': ' . $curl_exec, - $httpcode - ); + $this->handleErrorResponse($logger, $info['url'], $httpCode, $curl_exec); } // API MUST return UTF8 @@ -247,6 +289,55 @@ public function get($url, $params = []) return $this->call(!TraackrAPI::isJsonOutput(), 'Content-Type: application/json;charset=utf-8'); } + public function getConcurrent(array $requests) + { + // build requests + $guzzleOptions = $this->getGuzzleOpts(); + $guzzleRequests = function ($requests) use ($guzzleOptions) { + foreach ($requests as $request) { + $options = $guzzleOptions; + $options['query'] = $request['params']; + $options['headers']['Content-Type'] = 'application/json;charset=utf-8'; + yield new Request('GET', $request['url'], $options); + } + }; + + $results = []; + $logger = TraackrAPI::getLogger(); + // queue up requests + $pool = new Pool($this->guzzleClient, $guzzleRequests($requests), [ + 'concurrency' => $this->maxConcurrentRequests, + 'fulfilled' => function (Response $response, $index) use ($logger, $results, $requests) { + $httpCode = $response->getStatusCode(); + if ($httpCode !== 200) { + $this->handleErrorResponse($logger, $requests[$index]['url'], $httpCode, $response->getBody()); + } + if (!TraackrAPI::isJsonOutput()) { + $rez = json_decode($response->getBody(), true); + } else { + $rez = $response->getBody(); + } + $results[] = null === $rez ? false : $rez; + }, + 'rejected' => function (RequestException $e) use ($logger) { + // TODO: give consumer an option to + // continue with some failed requests? + $url = $e->getRequest()->getUri(); + $message = 'API call failed (' . $url . '): ' . $e->getMessage(); + $logger->error($message); + throw new TraackrApiException($message); + }, + ]); + + // Initiate the transfers and create a promise + $promise = $pool->promise(); + + // Force the pool of requests to complete. + $promise->wait(); + + return $results; + } + public function post($url, $params = [], $isJson = false) { $this->initCurlOpts(); From 347c972ce9e6f8342f0555923b85244c9b8fd57a Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 14:33:40 -0500 Subject: [PATCH 03/16] HAACK: use concurrent GET requests for influencer/show calls with audience for multiple infs --- lib/TraackrApi/Influencers.php | 38 ++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/lib/TraackrApi/Influencers.php b/lib/TraackrApi/Influencers.php index b5453ac..7c105cf 100644 --- a/lib/TraackrApi/Influencers.php +++ b/lib/TraackrApi/Influencers.php @@ -7,7 +7,7 @@ class Influencers extends TraackrApiObject /** * Get an influencer's data * - * @param string $uid + * @param string|array $uid * @param array $p * @return bool|mixed * @throws MissingParameterException @@ -18,8 +18,21 @@ public static function show($uid, $p = array('with_channels' => false)) throw new MissingParameterException("Missing Influencer UID parameter"); } + $concurrent = false; + if ( + !empty($p['with_audience']) + && $p['with_audience'] === true + && is_array($uid) + && count($uid) > 1 + ) { + // we need to use concurrent requests + // to fetch result since `show` with audience + // only supports 1 influencer UID + $concurrent = true; + } + // API Object - $inf = new Influencers(); + $inf = new Influencers($concurrent); //Sanitize default values $p['with_channels'] = $inf->convertBool($p, 'with_channels'); @@ -27,6 +40,27 @@ public static function show($uid, $p = array('with_channels' => false)) // Add customer key + check required params $p = $inf->addCustomerKey($p); $inf->checkRequiredParams($p, array('with_channels')); + + if ($concurrent) { + $requests = array_map(function($item) use ($p) { + return [ + 'url' => TraackrApi::$apiBaseUrl . 'influencers/show/' . $item, + 'params' => $p + ]; + }, $uid); + $results = $inf->getConcurrent($requests); + // merge influencer results + $mergedInfResults = []; + foreach ($results as $result) { + foreach ($result['influencer'] as $key => $inf) { + $mergedInfResults[$key] = $inf; + } + } + return [ + 'influencer' => $mergedInfResults + ]; + } + // support for multi params $uid = is_array($uid) ? implode(',', $uid) : $uid; From f79ba35ae56648e675ad72beb38f903e36c89392 Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 14:41:02 -0500 Subject: [PATCH 04/16] HAACK: change to guzzle version 6 to resolve conflict --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 8e78dd2..e3b9b53 100644 --- a/composer.json +++ b/composer.json @@ -18,7 +18,7 @@ "ext-curl": "*", "ext-json": "*", "ext-mbstring": "*", - "guzzlehttp/guzzle": "^7.1" + "guzzlehttp/guzzle": "~6.0" }, "require-dev": { "phpunit/phpunit": "4.8.*" From 5fe596c1afe426a69570a5cf27d64c5f37383071 Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 14:54:52 -0500 Subject: [PATCH 05/16] HAACK: fix typo reference to curl_headers --- lib/TraackrApi/TraackrApiObject.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index 51fd72e..bcd160c 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -48,7 +48,7 @@ public function __construct($concurrent = false) private function getGuzzleHeaders() { $headers = []; - foreach (self::$curl_headers as $header) { + foreach ($this->curl_headers as $header) { $parts = explode(':', $header); $property = $parts[0]; $value = ''; From 788d617bda8b7c69f92628bfe6983248971ec1c6 Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 14:58:58 -0500 Subject: [PATCH 06/16] HAACK: use sslVerifyPeer in guzzle client init --- lib/TraackrApi/TraackrApiObject.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index bcd160c..c4e0ce3 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -38,7 +38,9 @@ public function __construct($concurrent = false) { if ($concurrent) { $this->concurrent = true; - $this->guzzleClient = new Client(); + $this->guzzleClient = new Client([ + 'verify' => self::$sslVerifyPeer + ]); } else { // init cURL $this->curl = curl_init(); From 77ecbd9071bfeb462e9b0676f272c401ec23634f Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 15:08:25 -0500 Subject: [PATCH 07/16] HAACK: log current GET requests --- lib/TraackrApi/TraackrApiObject.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index c4e0ce3..0a15e40 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -293,19 +293,20 @@ public function get($url, $params = []) public function getConcurrent(array $requests) { + $logger = TraackrAPI::getLogger(); // build requests $guzzleOptions = $this->getGuzzleOpts(); - $guzzleRequests = function ($requests) use ($guzzleOptions) { + $guzzleRequests = function ($requests) use ($guzzleOptions, $logger) { foreach ($requests as $request) { $options = $guzzleOptions; $options['query'] = $request['params']; $options['headers']['Content-Type'] = 'application/json;charset=utf-8'; + $logger->debug('Calling (GET)[concurrent]: ' . $request['url'] . ' with options: ' . print_r($options, true)); yield new Request('GET', $request['url'], $options); } }; $results = []; - $logger = TraackrAPI::getLogger(); // queue up requests $pool = new Pool($this->guzzleClient, $guzzleRequests($requests), [ 'concurrency' => $this->maxConcurrentRequests, From 8506edebf0e2ea6ad630e96f1f52f1d5f550ce39 Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 15:11:08 -0500 Subject: [PATCH 08/16] HAACK: use prepareParameters in concurrent GET --- lib/TraackrApi/TraackrApiObject.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index 0a15e40..cce0144 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -299,7 +299,7 @@ public function getConcurrent(array $requests) $guzzleRequests = function ($requests) use ($guzzleOptions, $logger) { foreach ($requests as $request) { $options = $guzzleOptions; - $options['query'] = $request['params']; + $options['query'] = $this->prepareParameters($request['params']); $options['headers']['Content-Type'] = 'application/json;charset=utf-8'; $logger->debug('Calling (GET)[concurrent]: ' . $request['url'] . ' with options: ' . print_r($options, true)); yield new Request('GET', $request['url'], $options); From 2e9cb6d308d5815ce655dc4835801040b6749fde Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 15:13:41 -0500 Subject: [PATCH 09/16] HAACK: ensure api key is set in concurrent GET --- lib/TraackrApi/TraackrApiObject.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index cce0144..21e7f0b 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -299,7 +299,14 @@ public function getConcurrent(array $requests) $guzzleRequests = function ($requests) use ($guzzleOptions, $logger) { foreach ($requests as $request) { $options = $guzzleOptions; + // Add API key parameter if not present + $api_key = TraackrApi::getApiKey(); + if (!isset($request['params'][PARAM_API_KEY]) && !empty($api_key)) { + $request['params'][PARAM_API_KEY] = $api_key; + } + // set query string based on parameters $options['query'] = $this->prepareParameters($request['params']); + // set content-type header $options['headers']['Content-Type'] = 'application/json;charset=utf-8'; $logger->debug('Calling (GET)[concurrent]: ' . $request['url'] . ' with options: ' . print_r($options, true)); yield new Request('GET', $request['url'], $options); From 9830b32d4efc892a9964211067148929db401734 Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 15:22:02 -0500 Subject: [PATCH 10/16] HAACK: fix pass by value bug in concurrent GET --- lib/TraackrApi/TraackrApiObject.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index 21e7f0b..b38c182 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -317,7 +317,7 @@ public function getConcurrent(array $requests) // queue up requests $pool = new Pool($this->guzzleClient, $guzzleRequests($requests), [ 'concurrency' => $this->maxConcurrentRequests, - 'fulfilled' => function (Response $response, $index) use ($logger, $results, $requests) { + 'fulfilled' => function (Response $response, $index) use ($logger, &$results, $requests) { $httpCode = $response->getStatusCode(); if ($httpCode !== 200) { $this->handleErrorResponse($logger, $requests[$index]['url'], $httpCode, $response->getBody()); From daeefd37e89722e48219fbf29388476818176e3c Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 16:11:14 -0500 Subject: [PATCH 11/16] HAACK: fix headers, query string, guzzle options --- lib/TraackrApi/TraackrApiObject.php | 35 ++++++++++------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index b38c182..2f31f8b 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -12,7 +12,6 @@ abstract class TraackrApiObject public static $timeout = 10; public static $sslVerifyPeer = true; - private $concurrent = false; private $maxConcurrentRequests = 10; private $curl; private $guzzleClient; @@ -37,8 +36,9 @@ abstract class TraackrApiObject public function __construct($concurrent = false) { if ($concurrent) { - $this->concurrent = true; $this->guzzleClient = new Client([ + 'connect_timeout' => self::$connectionTimeout, + 'timeout' => self::$timeout, 'verify' => self::$sslVerifyPeer ]); } else { @@ -63,16 +63,6 @@ private function getGuzzleHeaders() return $headers; } - private function getGuzzleOpts() - { - $options = [ - 'connect_timeout' => self::$connectionTimeout, - 'timeout' => self::$timeout, - 'headers' => $this->getGuzzleHeaders() - ]; - return $options; - } - /** * Initialize self::$curl with the base settings all request types use. */ @@ -295,21 +285,20 @@ public function getConcurrent(array $requests) { $logger = TraackrAPI::getLogger(); // build requests - $guzzleOptions = $this->getGuzzleOpts(); - $guzzleRequests = function ($requests) use ($guzzleOptions, $logger) { + $headers = $this->getGuzzleHeaders(); + $headers['Content-Type'] = 'application/json;charset=utf-8'; + $guzzleRequests = function ($requests) use ($headers, $logger) { foreach ($requests as $request) { - $options = $guzzleOptions; + $params = $this->prepareParameters($request['params']); // Add API key parameter if not present $api_key = TraackrApi::getApiKey(); - if (!isset($request['params'][PARAM_API_KEY]) && !empty($api_key)) { - $request['params'][PARAM_API_KEY] = $api_key; + if (!isset($params[PARAM_API_KEY]) && !empty($api_key)) { + $params[PARAM_API_KEY] = $api_key; } - // set query string based on parameters - $options['query'] = $this->prepareParameters($request['params']); - // set content-type header - $options['headers']['Content-Type'] = 'application/json;charset=utf-8'; - $logger->debug('Calling (GET)[concurrent]: ' . $request['url'] . ' with options: ' . print_r($options, true)); - yield new Request('GET', $request['url'], $options); + $queryString = http_build_query($params); + $url = $request['url'] . '?' . $queryString; + $logger->debug('Calling (GET)[concurrent]: ' . $url); + yield new Request('GET', $url, $headers); } }; From 162ade53f59796a6de72040c91a549c977afccdc Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 16:58:14 -0500 Subject: [PATCH 12/16] HAACK: reduce concurrency to 5 --- lib/TraackrApi/TraackrApiObject.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index 2f31f8b..7b46f1d 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -12,7 +12,7 @@ abstract class TraackrApiObject public static $timeout = 10; public static $sslVerifyPeer = true; - private $maxConcurrentRequests = 10; + private $maxConcurrentRequests = 5; private $curl; private $guzzleClient; From 9d703fde7df4e7cca21495cde0a271db28898cb5 Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Mon, 5 Oct 2020 17:13:04 -0500 Subject: [PATCH 13/16] HAACK: revert concurrency to 10 --- lib/TraackrApi/TraackrApiObject.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index 7b46f1d..2f31f8b 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -12,7 +12,7 @@ abstract class TraackrApiObject public static $timeout = 10; public static $sslVerifyPeer = true; - private $maxConcurrentRequests = 5; + private $maxConcurrentRequests = 10; private $curl; private $guzzleClient; From 47c475644ff3496912e1869a026612058c7fd40b Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Tue, 6 Oct 2020 08:29:33 -0500 Subject: [PATCH 14/16] HAACK: reduce concurrency to 5 --- lib/TraackrApi/TraackrApiObject.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index 2f31f8b..7b46f1d 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -12,7 +12,7 @@ abstract class TraackrApiObject public static $timeout = 10; public static $sslVerifyPeer = true; - private $maxConcurrentRequests = 10; + private $maxConcurrentRequests = 5; private $curl; private $guzzleClient; From 55c235bb4c6567bed101e272a8b43d3fedf58166 Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Tue, 6 Oct 2020 08:31:50 -0500 Subject: [PATCH 15/16] HAACK: expose maxConcurrentRequests publicly --- lib/TraackrApi/TraackrApiObject.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index 7b46f1d..d45aec3 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -11,8 +11,8 @@ abstract class TraackrApiObject public static $connectionTimeout = 10; public static $timeout = 10; public static $sslVerifyPeer = true; + public static $maxConcurrentRequests = 5; - private $maxConcurrentRequests = 5; private $curl; private $guzzleClient; @@ -305,7 +305,7 @@ public function getConcurrent(array $requests) $results = []; // queue up requests $pool = new Pool($this->guzzleClient, $guzzleRequests($requests), [ - 'concurrency' => $this->maxConcurrentRequests, + 'concurrency' => self::$maxConcurrentRequests, 'fulfilled' => function (Response $response, $index) use ($logger, &$results, $requests) { $httpCode = $response->getStatusCode(); if ($httpCode !== 200) { From 217f51a7600e6832cc868d97837c827e18e20e89 Mon Sep 17 00:00:00 2001 From: John Czaplicki Date: Tue, 6 Oct 2020 08:56:27 -0500 Subject: [PATCH 16/16] HAACK: try adding retry to concurrent workflow --- lib/TraackrApi/TraackrApiObject.php | 80 ++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 25 deletions(-) diff --git a/lib/TraackrApi/TraackrApiObject.php b/lib/TraackrApi/TraackrApiObject.php index d45aec3..d35649e 100644 --- a/lib/TraackrApi/TraackrApiObject.php +++ b/lib/TraackrApi/TraackrApiObject.php @@ -1,9 +1,11 @@ logger = TraackrAPI::getLogger(); if ($concurrent) { + $stack = HandlerStack::create(new CurlHandler()); + $stack->push(\GuzzleHttp\Middleware::retry($this->createGuzzleRetryHandler())); $this->guzzleClient = new Client([ 'connect_timeout' => self::$connectionTimeout, 'timeout' => self::$timeout, - 'verify' => self::$sslVerifyPeer + 'verify' => self::$sslVerifyPeer, + 'handler' => $stack ]); } else { // init cURL @@ -47,6 +55,32 @@ public function __construct($concurrent = false) } } + private function createGuzzleRetryHandler() + { + return function ( + $retries, + Request $request, + Response $response = null, + RequestException $exception = null + ) { + if ($retries >= self::$maxRetries) { + return false; + } + if ($response->getStatusCode() != 500) { + return false; + } + $this->logger->debug(sprintf( + 'Retrying %s %s %s/%s, %s', + $request->getMethod(), + $request->getUri(), + $retries + 1, + self::$maxRetries, + $response ? 'status code: ' . $response->getStatusCode() : $exception->getMessage() + ), [$request->getHeader('Host')[0]]); + return true; + }; + } + private function getGuzzleHeaders() { $headers = []; @@ -152,14 +186,14 @@ private function prepareParameters($params) return $params; } - private function handleErrorResponse($logger, $url, $httpCode, $body) + private function handleErrorResponse($url, $httpCode, $body) { if ($httpCode == '400') { // Let's try to see if it's a bad customer key if ($body === 'Customer key not found') { $message = 'Invalid Customer Key (HTTP 400)'; - $logger->error($message); + $this->logger->error($message); throw new InvalidCustomerKeyException( $message . ': ' . $body, @@ -169,7 +203,7 @@ private function handleErrorResponse($logger, $url, $httpCode, $body) $message = 'Missing or Invalid argument/parameter (HTTP 400)'; - $logger->error($message); + $this->logger->error($message); throw new MissingParameterException( $message . ': ' . $body, @@ -179,7 +213,7 @@ private function handleErrorResponse($logger, $url, $httpCode, $body) if ($httpCode == '403') { $message = 'Invalid API key (HTTP 403)'; - $logger->error($message); + $this->logger->error($message); throw new InvalidApiKeyException( $message . ': ' . $body, @@ -190,7 +224,7 @@ private function handleErrorResponse($logger, $url, $httpCode, $body) if ($httpCode == '404') { $message = 'API resource not found (HTTP 404)'; - $logger->error($message); + $this->logger->error($message); throw new NotFoundException( $message . ': ' . $url, @@ -200,7 +234,7 @@ private function handleErrorResponse($logger, $url, $httpCode, $body) $message = 'API HTTP Error (HTTP ' . $httpCode . ')'; - $logger->error($message); + $this->logger->error($message); throw new TraackrApiException( $message . ': ' . $body, @@ -226,7 +260,7 @@ private function call($decode, $contentTypeHeader) $info = curl_getinfo($this->curl); $message = 'API call failed (' . $info['url'] . '): ' . curl_error($this->curl); - $logger->error($message); + $this->logger->error($message); throw new TraackrApiException($message); } @@ -234,7 +268,7 @@ private function call($decode, $contentTypeHeader) if (null === $curl_exec) { $message = 'API call failed. Response was null.'; - $logger->error($message); + $this->logger->error($message); throw new TraackrApiException($message); } @@ -243,7 +277,7 @@ private function call($decode, $contentTypeHeader) if ($httpCode != '200') { $info = curl_getinfo($this->curl); - $this->handleErrorResponse($logger, $info['url'], $httpCode, $curl_exec); + $this->handleErrorResponse($info['url'], $httpCode, $curl_exec); } // API MUST return UTF8 @@ -275,19 +309,17 @@ public function get($url, $params = []) // Sets URL curl_setopt($this->curl, CURLOPT_URL, $url); // Make call - $logger = TraackrAPI::getLogger(); - $logger->debug('Calling (GET): ' . $url); + $this->logger->debug('Calling (GET): ' . $url); return $this->call(!TraackrAPI::isJsonOutput(), 'Content-Type: application/json;charset=utf-8'); } public function getConcurrent(array $requests) { - $logger = TraackrAPI::getLogger(); // build requests $headers = $this->getGuzzleHeaders(); $headers['Content-Type'] = 'application/json;charset=utf-8'; - $guzzleRequests = function ($requests) use ($headers, $logger) { + $guzzleRequests = function ($requests) use ($headers) { foreach ($requests as $request) { $params = $this->prepareParameters($request['params']); // Add API key parameter if not present @@ -297,7 +329,7 @@ public function getConcurrent(array $requests) } $queryString = http_build_query($params); $url = $request['url'] . '?' . $queryString; - $logger->debug('Calling (GET)[concurrent]: ' . $url); + $this->logger->debug('Calling (GET)[concurrent]: ' . $url); yield new Request('GET', $url, $headers); } }; @@ -306,10 +338,10 @@ public function getConcurrent(array $requests) // queue up requests $pool = new Pool($this->guzzleClient, $guzzleRequests($requests), [ 'concurrency' => self::$maxConcurrentRequests, - 'fulfilled' => function (Response $response, $index) use ($logger, &$results, $requests) { + 'fulfilled' => function (Response $response, $index) use (&$results, $requests) { $httpCode = $response->getStatusCode(); if ($httpCode !== 200) { - $this->handleErrorResponse($logger, $requests[$index]['url'], $httpCode, $response->getBody()); + $this->handleErrorResponse($requests[$index]['url'], $httpCode, $response->getBody()); } if (!TraackrAPI::isJsonOutput()) { $rez = json_decode($response->getBody(), true); @@ -318,12 +350,12 @@ public function getConcurrent(array $requests) } $results[] = null === $rez ? false : $rez; }, - 'rejected' => function (RequestException $e) use ($logger) { + 'rejected' => function (RequestException $e) { // TODO: give consumer an option to // continue with some failed requests? $url = $e->getRequest()->getUri(); $message = 'API call failed (' . $url . '): ' . $e->getMessage(); - $logger->error($message); + $this->logger->error($message); throw new TraackrApiException($message); }, ]); @@ -365,8 +397,7 @@ public function post($url, $params = [], $isJson = false) curl_setopt($this->curl, CURLOPT_POSTFIELDS, $http_param_query); // Make call - $logger = TraackrAPI::getLogger(); - $logger->debug('Calling (POST): ' . $url . ' [' . $http_param_query . ']'); + $this->logger->debug('Calling (POST): ' . $url . ' [' . $http_param_query . ']'); return $this->call(!TraackrAPI::isJsonOutput(), $isJson ? 'Content-Type: application/json;charset=utf-8' : 'Content-Type: application/x-www-form-urlencoded;charset=utf-8'); } @@ -395,8 +426,7 @@ public function delete($url, $params = []) // Set Custom Request for DELETE curl_setopt($this->curl, CURLOPT_CUSTOMREQUEST, 'DELETE'); // Make call - $logger = TraackrAPI::getLogger(); - $logger->debug('Calling (DELETE): ' . $url); + $this->logger->debug('Calling (DELETE): ' . $url); return $this->call(!TraackrAPI::isJsonOutput(), 'Content-Type: application/json;charset=utf-8'); }