From adb021b3f8d36e32f19802422d976856f6e6b528 Mon Sep 17 00:00:00 2001 From: ozh Date: Sun, 4 Oct 2015 17:38:20 +0200 Subject: [PATCH] Sync with current version of Requests Specifically, we're interested in the byte limit response feature --- includes/Requests/Requests.php | 12 ++- includes/Requests/Requests/Cookie.php | 4 +- includes/Requests/Requests/Transport/cURL.php | 90 +++++++++++++++++-- .../Requests/Requests/Transport/fsockopen.php | 78 ++++++++++------ 4 files changed, 150 insertions(+), 34 deletions(-) diff --git a/includes/Requests/Requests.php b/includes/Requests/Requests.php index 31eee40..065227e 100644 --- a/includes/Requests/Requests.php +++ b/includes/Requests/Requests.php @@ -62,6 +62,13 @@ class Requests { */ const PATCH = 'PATCH'; + /** + * Default size of buffer size to read streams + * + * @var integer + */ + const BUFFER_SIZE = 1160; + /** * Current version of Requests * @@ -276,6 +283,8 @@ public static function patch($url, $headers, $data = array(), $options = array() * (Requests_Auth|array|boolean, default: false) * - `proxy`: Proxy details to use for proxy by-passing and authentication * (Requests_Proxy|array|boolean, default: false) + * - `max_bytes`: Limit for the response body size. + * (integer|boolean, default: false) * - `idn`: Enable IDN parsing * (boolean, default: true) * - `transport`: Custom transport. Either a class name, or a @@ -459,6 +468,7 @@ protected static function get_default_options($multirequest = false) { 'auth' => false, 'proxy' => false, 'cookies' => false, + 'max_bytes' => false, 'idn' => true, 'hooks' => null, 'transport' => null, @@ -483,7 +493,7 @@ protected static function get_default_options($multirequest = false) { */ protected static function set_defaults(&$url, &$headers, &$data, &$type, &$options) { if (!preg_match('/^http(s)?:\/\//i', $url, $matches)) { - throw new Requests_Exception('Only HTTP requests are handled.', 'nonhttp', $url); + throw new Requests_Exception('Only HTTP(S) requests are handled.', 'nonhttp', $url); } if (empty($options['hooks'])) { diff --git a/includes/Requests/Requests/Cookie.php b/includes/Requests/Requests/Cookie.php index adfbdbb..d978817 100644 --- a/includes/Requests/Requests/Cookie.php +++ b/includes/Requests/Requests/Cookie.php @@ -113,7 +113,7 @@ public function domainMatches($string) { return false; } - if (strlen($string) <= $domain_string) { + if (strlen($string) <= strlen($domain_string)) { // For obvious reasons, the string cannot be a suffix if the domain // is shorter than the domain string return false; @@ -131,7 +131,7 @@ public function domainMatches($string) { return false; } - if (preg_match('#^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$#', $string)) { + if (preg_match('#^(.+\.)\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$#', $string)) { // The string should be a host name (i.e., not an IP address). return false; } diff --git a/includes/Requests/Requests/Transport/cURL.php b/includes/Requests/Requests/Transport/cURL.php index 678ab73..ee25d36 100644 --- a/includes/Requests/Requests/Transport/cURL.php +++ b/includes/Requests/Requests/Transport/cURL.php @@ -23,6 +23,13 @@ class Requests_Transport_cURL implements Requests_Transport { */ public $headers = ''; + /** + * Raw body data + * + * @var string + */ + public $response_data = ''; + /** * Information on the current request * @@ -58,6 +65,20 @@ class Requests_Transport_cURL implements Requests_Transport { */ protected $stream_handle; + /** + * How many bytes are in the response body? + * + * @var int + */ + protected $response_bytes; + + /** + * What's the maximum number of bytes we should keep? + * + * @var int|bool Byte count, or false if no limit. + */ + protected $response_byte_limit; + /** * Constructor */ @@ -97,7 +118,13 @@ public function request($url, $headers = array(), $data = array(), $options = ar if ($options['filename'] !== false) { $this->stream_handle = fopen($options['filename'], 'wb'); - curl_setopt($this->fp, CURLOPT_FILE, $this->stream_handle); + } + + $this->response_data = ''; + $this->response_bytes = 0; + $this->response_byte_limit = false; + if ($options['max_bytes'] !== false) { + $this->response_byte_limit = $options['max_bytes']; } if (isset($options['verify'])) { @@ -114,13 +141,19 @@ public function request($url, $headers = array(), $data = array(), $options = ar curl_setopt($this->fp, CURLOPT_SSL_VERIFYHOST, 0); } - $response = curl_exec($this->fp); + curl_exec($this->fp); + $response = $this->response_data; $options['hooks']->dispatch('curl.after_send', array(&$fake_headers)); if (curl_errno($this->fp) === 23 || curl_errno($this->fp) === 61) { + // Reset encoding and try again curl_setopt($this->fp, CURLOPT_ENCODING, 'none'); - $response = curl_exec($this->fp); + + $this->response_data = ''; + $this->response_bytes = 0; + curl_exec($this->fp); + $response = $this->response_data; } $this->process_response($response, $options); @@ -174,7 +207,7 @@ public function request_multiple($requests, $options) { // Parse the finished requests before we start getting the new ones foreach ($to_process as $key => $done) { $options = $requests[$key]['options']; - $responses[$key] = $subrequests[$key]->process_response(curl_multi_getcontent($done['handle']), $options); + $responses[$key] = $subrequests[$key]->process_response($subrequests[$key]->response_data, $options); $options['hooks']->dispatch('transport.internal.parse_response', array(&$responses[$key], $requests[$key])); @@ -213,6 +246,13 @@ public function &get_subrequest_handle($url, $headers, $data, $options) { curl_setopt($this->fp, CURLOPT_FILE, $this->stream_handle); } + $this->response_data = ''; + $this->response_bytes = 0; + $this->response_byte_limit = false; + if ($options['max_bytes'] !== false) { + $this->response_byte_limit = $options['max_bytes']; + } + return $this->fp; } @@ -270,6 +310,8 @@ protected function setup_handle($url, $headers, $data, $options) { if (true === $options['blocking']) { curl_setopt($this->fp, CURLOPT_HEADERFUNCTION, array(&$this, 'stream_headers')); + curl_setopt($this->fp, CURLOPT_WRITEFUNCTION, array(&$this, 'stream_body')); + curl_setopt($this->fp, CURLOPT_BUFFERSIZE, Requests::BUFFER_SIZE); } } @@ -289,7 +331,6 @@ public function process_response($response, $options) { if (curl_errno($this->fp)) { throw new Requests_Exception('cURL error ' . curl_errno($this->fp) . ': ' . curl_error($this->fp), 'curlerror', $this->fp); - return; } $this->info = curl_getinfo($this->fp); @@ -304,7 +345,7 @@ public function process_response($response, $options) { * @param string $headers Header string * @return integer Length of provided header */ - protected function stream_headers($handle, $headers) { + public function stream_headers($handle, $headers) { // Why do we do this? cURL will send both the final response and any // interim responses, such as a 100 Continue. We don't need that. // (We may want to keep this somewhere just in case) @@ -320,6 +361,43 @@ protected function stream_headers($handle, $headers) { return strlen($headers); } + /** + * Collect data as it's received + * + * @since 1.6.1 + * + * @param resource $handle cURL resource + * @param string $data Body data + * @return integer Length of provided data + */ + protected function stream_body($handle, $data) { + $data_length = strlen($data); + + // Are we limiting the response size? + if ($this->response_byte_limit) { + if ($this->response_bytes === $this->response_byte_limit) { + // Already at maximum, move on + return $data_length; + } + + if (($this->response_bytes + $data_length) > $this->response_byte_limit) { + // Limit the length + $limited_length = ($this->response_byte_limit - $this->response_bytes); + $data = substr($data, 0, $limited_length); + } + } + + if ($this->stream_handle) { + fwrite($this->stream_handle, $data); + } + else { + $this->response_data .= $data; + } + + $this->response_bytes += strlen($data); + return $data_length; + } + /** * Format a URL given GET data * diff --git a/includes/Requests/Requests/Transport/fsockopen.php b/includes/Requests/Requests/Transport/fsockopen.php index 8975e22..d6391e9 100644 --- a/includes/Requests/Requests/Transport/fsockopen.php +++ b/includes/Requests/Requests/Transport/fsockopen.php @@ -34,6 +34,13 @@ class Requests_Transport_fsockopen implements Requests_Transport { */ public $info; + /** + * What's the maximum number of bytes we should keep? + * + * @var int|bool Byte count, or false if no limit. + */ + protected $max_bytes = false; + protected $connect_error = ''; /** @@ -94,6 +101,8 @@ public function request($url, $headers = array(), $data = array(), $options = ar $remote_socket = 'tcp://' . $host; } + $this->max_bytes = $options['max_bytes']; + $proxy = isset( $options['proxy'] ); $proxy_auth = $proxy && isset( $options['proxy_username'] ) && isset( $options['proxy_password'] ); @@ -212,43 +221,62 @@ public function request($url, $headers = array(), $data = array(), $options = ar $this->info = stream_get_meta_data($fp); - $this->headers = ''; + $response = $body = $headers = ''; $this->info = stream_get_meta_data($fp); - if (!$options['filename']) { - while (!feof($fp)) { - $this->info = stream_get_meta_data($fp); - if ($this->info['timed_out']) { - throw new Requests_Exception('fsocket timed out', 'timeout'); - } + $size = 0; + $doingbody = false; + $download = false; + if ($options['filename']) { + $download = fopen($options['filename'], 'wb'); + } - $this->headers .= fread($fp, 1160); + while (!feof($fp)) { + $this->info = stream_get_meta_data($fp); + if ($this->info['timed_out']) { + throw new Requests_Exception('fsocket timed out', 'timeout'); } - } - else { - $download = fopen($options['filename'], 'wb'); - $doingbody = false; - $response = ''; - while (!feof($fp)) { - $this->info = stream_get_meta_data($fp); - if ($this->info['timed_out']) { - throw new Requests_Exception('fsocket timed out', 'timeout'); + + $block = fread($fp, Requests::BUFFER_SIZE); + if (!$doingbody) { + $response .= $block; + if (strpos($response, "\r\n\r\n")) { + list($headers, $block) = explode("\r\n\r\n", $response, 2); + $doingbody = true; + } + } + + // Are we in body mode now? + if ($doingbody) { + $data_length = strlen($block); + if ($this->max_bytes) { + // Have we already hit a limit? + if ($size === $this->max_bytes) { + continue; + } + if (($size + $data_length) > $this->max_bytes) { + // Limit the length + $limited_length = ($this->max_bytes - $size); + $block = substr($block, 0, $limited_length); + } } - $block = fread($fp, 1160); - if ($doingbody) { + $size += strlen($block); + if ($download) { fwrite($download, $block); } else { - $response .= $block; - if (strpos($response, "\r\n\r\n")) { - list($this->headers, $block) = explode("\r\n\r\n", $response, 2); - $doingbody = true; - fwrite($download, $block); - } + $body .= $block; } } + } + $this->headers = $headers; + + if ($download) { fclose($download); } + else { + $this->headers .= "\r\n\r\n" . $body; + } fclose($fp); $options['hooks']->dispatch('fsockopen.after_request', array(&$this->headers)); -- 2.45.0