diff --git a/examples/multiple.php b/examples/multiple.php new file mode 100755 index 000000000..69757c3b1 --- /dev/null +++ b/examples/multiple.php @@ -0,0 +1,45 @@ + 'http://httpbin.org/get', + 'headers' => array('Accept' => 'application/javascript'), + ), + 'post' => array( + 'url' => 'http://httpbin.org/post', + 'data' => array('mydata' => 'something'), + ), + 'delayed' => array( + 'url' => 'http://httpbin.org/delay/10', + 'options' => array( + 'timeout' => 20, + ), + ), +); + +// Setup a callback +function my_callback(&$request, $id) { + var_dump($id, $request); +} + +// Tell Requests to use the callback +$options = array( + 'complete' => 'my_callback', +); + +// Send the request! +$responses = Requests::request_multiple($requests, $options); + +// Note: the response from the above call will be an associative array matching +// $requests with the response data, however we've already handled it in +// my_callback() anyway! +// +// If you don't believe me, uncomment this: +# var_dump($responses); \ No newline at end of file diff --git a/library/Requests.php b/library/Requests.php index 18ff74c94..da178f412 100755 --- a/library/Requests.php +++ b/library/Requests.php @@ -287,6 +287,150 @@ public static function request($url, $headers = array(), $data = array(), $type if (!preg_match('/^http(s)?:\/\//i', $url)) { throw new Requests_Exception('Only HTTP requests are handled.', 'nonhttp', $url); } + if (empty($options['type'])) { + $options['type'] = $type; + } + $options = array_merge(self::get_default_options(), $options); + + self::set_defaults($url, $headers, $data, $type, $options); + + $options['hooks']->dispatch('requests.before_request', array(&$url, &$headers, &$data, &$type, &$options)); + + if (!empty($options['transport'])) { + $transport = $options['transport']; + + if (is_string($options['transport'])) { + $transport = new $transport(); + } + } + else { + $transport = self::get_transport(); + } + $response = $transport->request($url, $headers, $data, $options); + + $options['hooks']->dispatch('requests.before_parse', array(&$response, $url, $headers, $data, $type, $options)); + + return self::parse_response($response, $url, $headers, $data, $options); + } + + /** + * Send multiple HTTP requests simultaneously + * + * The `$requests` parameter takes an associative or indexed array of + * request fields. The key of each request can be used to match up the + * request with the returned data, or with the request passed into your + * `multiple.request.complete` callback. + * + * The request fields value is an associative array with the following keys: + * + * - `url`: Request URL Same as the `$url` parameter to + * {@see Requests::request} + * (string, required) + * - `headers`: Associative array of header fields. Same as the `$headers` + * parameter to {@see Requests::request} + * (array, default: `array()`) + * - `data`: Associative array of data fields or a string. Same as the + * `$data` parameter to {@see Requests::request} + * (array|string, default: `array()`) + * - `type`: HTTP request type (use Requests constants). Same as the `$type` + * parameter to {@see Requests::request} + * (string, default: `Requests::GET`) + * - `data`: Associative array of options. Same as the `$options` parameter + * to {@see Requests::request} + * (array, default: see {@see Requests::request}) + * + * If the `$options` parameter is specified, individual requests will + * inherit options from it. This can be used to use a single hooking system, + * or set all the types to `Requests::POST`, for example. + * + * In addition, the `$options` parameter takes the following global options: + * + * - `complete`: A callback for when a request is complete. Takes two + * parameters, a Requests_Response/Requests_Exception reference, and the + * ID from the request array (Note: this can also be overriden on a + * per-request basis, although that's a little silly) + * (callback) + * + * @param array $requests Requests data (see description for more information) + * @param array $options Global and default options (see {@see Requests::request}) + * @return array Responses (either Requests_Response or a Requests_Exception object) + */ + public static function request_multiple($requests, $options = array()) { + $options = array_merge(self::get_default_options(true), $options); + + if (!empty($options['hooks'])) { + $options['hooks']->register('transport.internal.parse_response', array('Requests', 'parse_multiple')); + if (!empty($options['complete'])) { + $options['hooks']->register('multiple.request.complete', $options['complete']); + } + } + + foreach ($requests as $id => &$request) { + if (!isset($request['headers'])) { + $request['headers'] = array(); + } + if (!isset($request['data'])) { + $request['data'] = array(); + } + if (!isset($request['type'])) { + $request['type'] = self::GET; + } + if (!isset($request['options'])) { + $request['options'] = $options; + $request['options']['type'] = $request['type']; + } + else { + if (empty($request['options']['type'])) { + $request['options']['type'] = $request['type']; + } + $request['options'] = array_merge($options, $request['options']); + } + + self::set_defaults($request['url'], $request['headers'], $request['data'], $request['type'], $request['options']); + + // Ensure we only hook in once + if ($request['options']['hooks'] !== $options['hooks']) { + $request['options']['hooks']->register('transport.internal.parse_response', array('Requests', 'parse_multiple')); + if (!empty($request['options']['complete'])) { + $request['options']['hooks']->register('multiple.request.complete', $request['options']['complete']); + } + } + } + unset($request); + + if (!empty($options['transport'])) { + $transport = $options['transport']; + + if (is_string($options['transport'])) { + $transport = new $transport(); + } + } + else { + $transport = self::get_transport(); + } + $responses = $transport->request_multiple($requests, $options); + + foreach ($responses as $id => &$response) { + // If our hook got messed with somehow, ensure we end up with the + // correct response + if (is_string($response)) { + $request = $requests[$id]; + $response = self::parse_multiple($response, $request); + $request['options']['hooks']->dispatch('multiple.request.complete', array(&$response, $id)); + } + } + + return $responses; + } + + /** + * Get the default options + * + * @see Requests::request() for values returned by this method + * @param boolean $multirequest Is this a multirequest? + * @return array Default option values + */ + protected static function get_default_options($multirequest = false) { $defaults = array( 'timeout' => 10, 'useragent' => 'php-requests/' . self::VERSION, @@ -294,20 +438,34 @@ public static function request($url, $headers = array(), $data = array(), $type 'redirects' => 10, 'follow_redirects' => true, 'blocking' => true, - 'type' => $type, + 'type' => self::GET, 'filename' => false, 'auth' => false, 'idn' => true, 'hooks' => null, 'transport' => null, ); - $options = array_merge($defaults, $options); + if ($multirequest !== false) { + $defaults['complete'] = null; + } + return $defaults; + } + /** + * Set the default values + * + * @param string $url URL to request + * @param array $headers Extra headers to send with the request + * @param array $data Data to send either as a query string for GET/HEAD requests, or in the body for POST requests + * @param string $type HTTP request type + * @param array $options Options for the request + * @return array $options + */ + protected static function set_defaults(&$url, &$headers, &$data, &$type, &$options) { if (empty($options['hooks'])) { $options['hooks'] = new Requests_Hooks(); } - // Special case for simple basic auth if (is_array($options['auth'])) { $options['auth'] = new Requests_Auth_Basic($options['auth']); } @@ -315,29 +473,11 @@ public static function request($url, $headers = array(), $data = array(), $type $options['auth']->register($options['hooks']); } - $options['hooks']->dispatch('requests.before_request', array(&$url, &$headers, &$data, &$type, &$options)); - if ($options['idn'] !== false) { $iri = new Requests_IRI($url); $iri->host = Requests_IDNAEncoder::encode($iri->ihost); $url = $iri->uri; } - - if (!empty($options['transport'])) { - $transport = $options['transport']; - - if (is_string($options['transport'])) { - $transport = new $transport(); - } - } - else { - $transport = self::get_transport(); - } - $response = $transport->request($url, $headers, $data, $options); - - $options['hooks']->dispatch('requests.before_parse', array(&$response, $url, $headers, $data, $type, $options)); - - return self::parse_response($response, $url, $headers, $data, $options); } /** @@ -435,6 +575,25 @@ protected static function parse_response($headers, $url, $req_headers, $req_data return $return; } + /** + * Callback for `transport.internal.parse_response` + * + * Internal use only. Converts a raw HTTP response to a Requests_Response + * while still executing a multiple request. + * + * @param string $headers Full response text including headers and body + * @param array $request Request data as passed into {@see Requests::request_multiple()} + * @return null `$response` is either set to a Requests_Response instance, or a Requests_Exception object + */ + public static function parse_multiple(&$response, $request) { + try { + $response = self::parse_response($response, $request['url'], $request['headers'], $request['data'], $request['options']); + } + catch (Requests_Exception $e) { + $response = $e; + } + } + /** * Decoded a chunked body as per RFC 2616 * diff --git a/library/Requests/Transport.php b/library/Requests/Transport.php index 9948cb809..7e4a26293 100755 --- a/library/Requests/Transport.php +++ b/library/Requests/Transport.php @@ -24,6 +24,15 @@ interface Requests_Transport { */ public function request($url, $headers = array(), $data = array(), $options = array()); + /** + * Send multiple requests simultaneously + * + * @param array $requests Request data (array of 'url', 'headers', 'data', 'options') as per {@see Requests_Transport::request} + * @param array $options Global options, see {@see Requests::response()} for documentation + * @return array Array of Requests_Response objects (may contain Requests_Exception or string responses as well) + */ + public function request_multiple($requests, $options); + /** * Self-test whether the transport can be used * @return bool diff --git a/library/Requests/Transport/cURL.php b/library/Requests/Transport/cURL.php index ede7c0564..21b031e08 100755 --- a/library/Requests/Transport/cURL.php +++ b/library/Requests/Transport/cURL.php @@ -48,6 +48,13 @@ class Requests_Transport_cURL implements Requests_Transport { */ protected $done_headers = false; + /** + * If streaming to a file, keep the file pointer + * + * @var resource + */ + protected $stream_handle; + /** * Constructor */ @@ -77,6 +84,125 @@ public function __construct() { * @return string Raw HTTP result */ public function request($url, $headers = array(), $data = array(), $options = array()) { + $this->setup_handle($url, $headers, $data, $options); + + $options['hooks']->dispatch('curl.before_send', array(&$this->fp)); + + if ($options['filename'] !== false) { + $this->stream_handle = fopen($options['filename'], 'wb'); + curl_setopt($this->fp, CURLOPT_FILE, $this->stream_handle); + } + + $response = curl_exec($this->fp); + + $options['hooks']->dispatch('curl.after_send', array(&$fake_headers)); + + if (curl_errno($this->fp) === 23 || curl_errno($this->fp) === 61) { + curl_setopt($this->fp, CURLOPT_ENCODING, 'none'); + $response = curl_exec($this->fp); + } + + $this->process_response($response, $options); + return $this->headers; + } + + /** + * Send multiple requests simultaneously + * + * @param array $requests Request data + * @param array $options Global options + * @return array Array of Requests_Response objects (may contain Requests_Exception or string responses as well) + */ + public function request_multiple($requests, $options) { + $multihandle = curl_multi_init(); + $subrequests = array(); + $subhandles = array(); + + $class = get_class($this); + foreach ($requests as $id => $request) { + $subrequests[$id] = new $class(); + $subhandles[$id] = $subrequests[$id]->get_subrequest_handle($request['url'], $request['headers'], $request['data'], $request['options']); + $request['options']['hooks']->dispatch('curl.before_multi_add', array(&$subhandles[$id])); + curl_multi_add_handle($multihandle, $subhandles[$id]); + } + + $completed = 0; + $responses = array(); + + $request['options']['hooks']->dispatch('curl.before_multi_exec', array(&$multihandle)); + + do { + $active = false; + + do { + $status = curl_multi_exec($multihandle, $active); + } + while ($status === CURLM_CALL_MULTI_PERFORM); + + $to_process = array(); + + // Read the information as needed + while ($done = curl_multi_info_read($multihandle)) { + $key = array_search($done['handle'], $subhandles, true); + if (!isset($to_process[$key])) { + $to_process[$key] = $done; + } + } + + // Parse the finished requests before we start getting the new ones + foreach ($to_process as $key => $done) { + $options = $requests[$key]['options']; + $responses[$key] = $subrequests[$key]->process_response(curl_multi_getcontent($done['handle']), $options); + + $options['hooks']->dispatch('transport.internal.parse_response', array(&$responses[$key], $requests[$key])); + + curl_multi_remove_handle($multihandle, $done['handle']); + curl_close($done['handle']); + + if (!is_string($responses[$key])) { + $options['hooks']->dispatch('multiple.request.complete', array(&$responses[$key], $key)); + } + $completed++; + } + } + while ($active || $completed < count($subrequests)); + + $request['options']['hooks']->dispatch('curl.after_multi_exec', array(&$multihandle)); + + curl_multi_close($multihandle); + + return $responses; + } + + /** + * Get the cURL handle for use in a multi-request + * + * @param string $url URL to request + * @param array $headers Associative array of request headers + * @param string|array $data Data to send either as the POST body, or as parameters in the URL for a GET/HEAD + * @param array $options Request options, see {@see Requests::response()} for documentation + * @return resource Subrequest's cURL handle + */ + public function &get_subrequest_handle($url, $headers, $data, $options) { + $this->setup_handle($url, $headers, $data, $options); + + if ($options['filename'] !== false) { + $this->stream_handle = fopen($options['filename'], 'wb'); + curl_setopt($this->fp, CURLOPT_FILE, $this->stream_handle); + } + + return $this->fp; + } + + /** + * Setup the cURL handle for the given data + * + * @param string $url URL to request + * @param array $headers Associative array of request headers + * @param string|array $data Data to send either as the POST body, or as parameters in the URL for a GET/HEAD + * @param array $options Request options, see {@see Requests::response()} for documentation + */ + protected function setup_handle($url, $headers, $data, $options) { $options['hooks']->dispatch('curl.before_request', array(&$this->fp)); $headers = Requests::flattern($headers); @@ -112,18 +238,9 @@ public function request($url, $headers = array(), $data = array(), $options = ar if (true === $options['blocking']) { curl_setopt($this->fp, CURLOPT_HEADERFUNCTION, array(&$this, 'stream_headers')); } + } - $options['hooks']->dispatch('curl.before_send', array(&$this->fp)); - - if ($options['filename'] !== false) { - $stream_handle = fopen($options['filename'], 'wb'); - curl_setopt($this->fp, CURLOPT_FILE, $stream_handle); - } - - $response = curl_exec($this->fp); - - $options['hooks']->dispatch('curl.after_send', array(&$fake_headers)); - + public function process_response($response, $options) { if ($options['blocking'] === false) { curl_close($this->fp); $fake_headers = ''; @@ -131,23 +248,18 @@ public function request($url, $headers = array(), $data = array(), $options = ar return false; } if ($options['filename'] !== false) { - fclose($stream_handle); + fclose($this->stream_handle); $this->headers = trim($this->headers); } else { $this->headers .= $response; } - if (curl_errno($this->fp) === 23 || curl_errno($this->fp) === 61) { - curl_setopt($this->fp, CURLOPT_ENCODING, 'none'); - $this->headers = curl_exec($this->fp); - } if (curl_errno($this->fp)) { throw new Requests_Exception('cURL error ' . curl_errno($this->fp) . ': ' . curl_error($this->fp), 'curlerror', $this->fp); return; } $this->info = curl_getinfo($this->fp); - curl_close($this->fp); $options['hooks']->dispatch('curl.after_request', array(&$this->headers)); return $this->headers; diff --git a/library/Requests/Transport/fsockopen.php b/library/Requests/Transport/fsockopen.php index 8a2cd252b..e41e776f7 100755 --- a/library/Requests/Transport/fsockopen.php +++ b/library/Requests/Transport/fsockopen.php @@ -165,6 +165,35 @@ public function request($url, $headers = array(), $data = array(), $options = ar return $this->headers; } + /** + * Send multiple requests simultaneously + * + * @param array $requests Request data (array of 'url', 'headers', 'data', 'options') as per {@see Requests_Transport::request} + * @param array $options Global options, see {@see Requests::response()} for documentation + * @return array Array of Requests_Response objects (may contain Requests_Exception or string responses as well) + */ + public function request_multiple($requests, $options) { + $responses = array(); + $class = get_class($this); + foreach ($requests as $id => $request) { + try { + $handler = new $class(); + $responses[$id] = $handler->request($request['url'], $request['headers'], $request['data'], $request['options']); + + $request['options']['hooks']->dispatch('transport.internal.parse_response', array(&$responses[$id], $request)); + } + catch (Requests_Exception $e) { + $responses[$id] = $e; + } + + if (!is_string($responses[$id])) { + $request['options']['hooks']->dispatch('multiple.request.complete', array(&$responses[$id], $id)); + } + } + + return $responses; + } + /** * Retrieve the encodings we can accept * @@ -202,7 +231,7 @@ protected static function format_get($url_parts, $data) { } if (isset($url_parts['path'])) { if (isset($url_parts['query'])) { - $get = "$url_parts[path]?$url_parts[query]"; + $get = $url_parts['path'] . '?' . $url_parts['query']; } else { $get = $url_parts['path']; diff --git a/tests/Transport/Base.php b/tests/Transport/Base.php index 59b63a847..a96e0134d 100755 --- a/tests/Transport/Base.php +++ b/tests/Transport/Base.php @@ -341,4 +341,157 @@ public function testTimeout() { $request = Requests::get('http://httpbin.org/delay/10', array(), $this->getOptions($options)); var_dump($request); } + + public function testMultiple() { + $requests = array( + 'test1' => array( + 'url' => 'http://httpbin.org/get' + ), + 'test2' => array( + 'url' => 'http://httpbin.org/get' + ), + ); + $responses = Requests::request_multiple($requests, $this->getOptions()); + + // test1 + $this->assertNotEmpty($responses['test1']); + $this->assertInstanceOf('Requests_Response', $responses['test1']); + $this->assertEquals(200, $responses['test1']->status_code); + + $result = json_decode($responses['test1']->body, true); + $this->assertEquals('http://httpbin.org/get', $result['url']); + $this->assertEmpty($result['args']); + + // test2 + $this->assertNotEmpty($responses['test1']); + $this->assertInstanceOf('Requests_Response', $responses['test1']); + $this->assertEquals(200, $responses['test1']->status_code); + + $result = json_decode($responses['test1']->body, true); + $this->assertEquals('http://httpbin.org/get', $result['url']); + $this->assertEmpty($result['args']); + } + + public function testMultipleWithDifferingMethods() { + $requests = array( + 'get' => array( + 'url' => 'http://httpbin.org/get', + ), + 'post' => array( + 'url' => 'http://httpbin.org/post', + 'type' => Requests::POST, + 'data' => 'test', + ), + ); + $responses = Requests::request_multiple($requests, $this->getOptions()); + + // get + $this->assertEquals(200, $responses['get']->status_code); + + // post + $this->assertEquals(200, $responses['post']->status_code); + $result = json_decode($responses['post']->body, true); + $this->assertEquals('test', $result['data']); + } + + /** + * @depends testTimeout + */ + public function testMultipleWithFailure() { + $requests = array( + 'success' => array( + 'url' => 'http://httpbin.org/get', + ), + 'timeout' => array( + 'url' => 'http://httpbin.org/delay/10', + 'options' => array( + 'timeout' => 1, + ), + ), + ); + $responses = Requests::request_multiple($requests, $this->getOptions()); + $this->assertEquals(200, $responses['success']->status_code); + $this->assertInstanceOf('Requests_Exception', $responses['timeout']); + } + + public function testMultipleUsingCallback() { + $requests = array( + 'get' => array( + 'url' => 'http://httpbin.org/get', + ), + 'post' => array( + 'url' => 'http://httpbin.org/post', + 'type' => Requests::POST, + 'data' => 'test', + ), + ); + $this->completed = array(); + $options = array( + 'complete' => array($this, 'completeCallback'), + ); + $responses = Requests::request_multiple($requests, $this->getOptions($options)); + + $this->assertEquals($this->completed, $responses); + $this->completed = array(); + } + + public function testMultipleUsingCallbackAndFailure() { + $requests = array( + 'success' => array( + 'url' => 'http://httpbin.org/get', + ), + 'timeout' => array( + 'url' => 'http://httpbin.org/delay/10', + 'options' => array( + 'timeout' => 1, + ), + ), + ); + $this->completed = array(); + $options = array( + 'complete' => array($this, 'completeCallback'), + ); + $responses = Requests::request_multiple($requests, $this->getOptions($options)); + + $this->assertEquals($this->completed, $responses); + $this->completed = array(); + } + + public function completeCallback($response, $key) { + $this->completed[$key] = $response; + } + + public function testMultipleToFile() { + $requests = array( + 'get' => array( + 'url' => 'http://httpbin.org/get', + 'options' => array( + 'filename' => tempnam(sys_get_temp_dir(), 'RLT') // RequestsLibraryTest + ), + ), + 'post' => array( + 'url' => 'http://httpbin.org/post', + 'type' => Requests::POST, + 'data' => 'test', + 'options' => array( + 'filename' => tempnam(sys_get_temp_dir(), 'RLT') // RequestsLibraryTest + ), + ), + ); + $responses = Requests::request_multiple($requests, $this->getOptions()); + + // GET request + $contents = file_get_contents($requests['get']['options']['filename']); + $result = json_decode($contents, true); + $this->assertEquals('http://httpbin.org/get', $result['url']); + $this->assertEmpty($result['args']); + unlink($requests['get']['options']['filename']); + + // POST request + $contents = file_get_contents($requests['post']['options']['filename']); + $result = json_decode($contents, true); + $this->assertEquals('http://httpbin.org/post', $result['url']); + $this->assertEquals('test', $result['data']); + unlink($requests['post']['options']['filename']); + } } \ No newline at end of file diff --git a/tests/bootstrap.php b/tests/bootstrap.php index 7ba36047c..e89cc136b 100755 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -80,6 +80,25 @@ public function request($url, $headers = array(), $data = array(), $options = ar return $response; } + public function request_multiple($requests, $options) { + $responses = array(); + foreach ($requests as $id => $request) { + $handler = new MockTransport(); + $handler->code = $request['options']['mock.code']; + $handler->chunked = $request['options']['mock.chunked']; + $handler->body = $request['options']['mock.body']; + $handler->raw_headers = $request['options']['mock.raw_headers']; + $responses[$id] = $handler->request($request['url'], $request['headers'], $request['data'], $request['options']); + + if (!empty($options['mock.parse'])) { + $request['options']['hooks']->dispatch('transport.internal.parse_response', array(&$responses[$id], $request)); + $request['options']['hooks']->dispatch('multiple.request.complete', array(&$responses[$id], $id)); + } + } + + return $responses; + } + public static function test() { return true; } @@ -90,6 +109,15 @@ class RawTransport implements Requests_Transport { public function request($url, $headers = array(), $data = array(), $options = array()) { return $this->data; } + public function request_multiple($requests, $options) { + foreach ($requests as $id => &$request) { + $handler = new RawTransport(); + $handler->data = $request['options']['raw.data']; + $request = $handler->request($request['url'], $request['headers'], $request['data'], $request['options']); + } + + return $requests; + } public static function test() { return true; }