7 * Various HTTP related functions
11 static $httpEngine = false;
14 * Perform an HTTP request
16 * @param $method String: HTTP method. Usually GET/POST
17 * @param $url String: full URL to act on. If protocol-relative, will be expanded to an http:// URL
18 * @param $options Array: options to pass to MWHttpRequest object.
19 * Possible keys for the array:
20 * - timeout Timeout length in seconds
21 * - postData An array of key-value pairs or a url-encoded form data
22 * - proxy The proxy to use.
23 * Will use $wgHTTPProxy (if set) otherwise.
24 * - noProxy Override $wgHTTPProxy (if set) and don't use any proxy at all.
25 * - sslVerifyHost (curl only) Verify hostname against certificate
26 * - sslVerifyCert (curl only) Verify SSL certificate
27 * - caInfo (curl only) Provide CA information
28 * - maxRedirects Maximum number of redirects to follow (defaults to 5)
29 * - followRedirects Whether to follow redirects (defaults to false).
30 * Note: this should only be used when the target URL is trusted,
31 * to avoid attacks on intranet services accessible by HTTP.
32 * @return Mixed: (bool)false on failure or a string on success
34 public static function request( $method, $url, $options = array() ) {
35 wfDebug( "HTTP: $method: $url\n" );
36 $options['method'] = strtoupper( $method );
38 if ( !isset( $options['timeout'] ) ) {
39 $options['timeout'] = 'default';
42 $req = MWHttpRequest
::factory( $url, $options );
43 $status = $req->execute();
45 if ( $status->isOK() ) {
46 return $req->getContent();
53 * Simple wrapper for Http::request( 'GET' )
54 * @see Http::request()
57 * @param $timeout string
58 * @param $options array
61 public static function get( $url, $timeout = 'default', $options = array() ) {
62 $options['timeout'] = $timeout;
63 return Http
::request( 'GET', $url, $options );
67 * Simple wrapper for Http::request( 'POST' )
68 * @see Http::request()
71 * @param $options array
74 public static function post( $url, $options = array() ) {
75 return Http
::request( 'POST', $url, $options );
79 * Check if the URL can be served by localhost
81 * @param $url String: full url to check
84 public static function isLocalURL( $url ) {
85 global $wgCommandLineMode, $wgConf;
87 if ( $wgCommandLineMode ) {
93 if ( preg_match( '!^http://([\w.-]+)[/:].*$!', $url, $matches ) ) {
96 $domainParts = explode( '.', $host );
97 // Check if this domain or any superdomain is listed in $wgConf as a local virtual host
98 $domainParts = array_reverse( $domainParts );
101 for ( $i = 0; $i < count( $domainParts ); $i++
) {
102 $domainPart = $domainParts[$i];
104 $domain = $domainPart;
106 $domain = $domainPart . '.' . $domain;
109 if ( $wgConf->isLocalVHost( $domain ) ) {
119 * A standard user-agent we can use for external requests.
122 public static function userAgent() {
124 return "MediaWiki/$wgVersion";
128 * Checks that the given URI is a valid one. Hardcoding the
129 * protocols, because we only want protocols that both cURL
132 * file:// should not be allowed here for security purpose (r67684)
134 * @fixme this is wildly inaccurate and fails to actually check most stuff
136 * @param $uri Mixed: URI to check for validity
139 public static function isValidURI( $uri ) {
141 '/^https?:\/\/[^\/\s]\S*$/D',
148 * This wrapper class will call out to curl (if available) or fallback
149 * to regular PHP if necessary for handling internal HTTP requests.
151 * Renamed from HttpRequest to MWHttpRequest to avoid conflict with
152 * PHP's HTTP extension.
154 class MWHttpRequest
{
155 const SUPPORTS_FILE_POSTS
= false;
158 protected $timeout = 'default';
159 protected $headersOnly = null;
160 protected $postData = null;
161 protected $proxy = null;
162 protected $noProxy = false;
163 protected $sslVerifyHost = true;
164 protected $sslVerifyCert = true;
165 protected $caInfo = null;
166 protected $method = "GET";
167 protected $reqHeaders = array();
169 protected $parsedUrl;
171 protected $maxRedirects = 5;
172 protected $followRedirects = false;
177 protected $cookieJar;
179 protected $headerList = array();
180 protected $respVersion = "0.9";
181 protected $respStatus = "200 Ok";
182 protected $respHeaders = array();
187 * @param $url String: url to use. If protocol-relative, will be expanded to an http:// URL
188 * @param $options Array: (optional) extra params to pass (see Http::request())
190 function __construct( $url, $options = array() ) {
191 global $wgHTTPTimeout;
193 $this->url
= wfExpandUrl( $url, PROTO_HTTP
);
194 $this->parsedUrl
= wfParseUrl( $this->url
);
196 if ( !$this->parsedUrl ||
!Http
::isValidURI( $this->url
) ) {
197 $this->status
= Status
::newFatal( 'http-invalid-url' );
199 $this->status
= Status
::newGood( 100 ); // continue
202 if ( isset( $options['timeout'] ) && $options['timeout'] != 'default' ) {
203 $this->timeout
= $options['timeout'];
205 $this->timeout
= $wgHTTPTimeout;
208 $members = array( "postData", "proxy", "noProxy", "sslVerifyHost", "caInfo",
209 "method", "followRedirects", "maxRedirects", "sslVerifyCert", "callback" );
211 foreach ( $members as $o ) {
212 if ( isset( $options[$o] ) ) {
213 $this->$o = $options[$o];
219 * Simple function to test if we can make any sort of requests at all, using
223 public static function canMakeRequests() {
224 return function_exists( 'curl_init' ) ||
wfIniGetBool( 'allow_url_fopen' );
228 * Generate a new request object
229 * @param $url String: url to use
230 * @param $options Array: (optional) extra params to pass (see Http::request())
231 * @return \CurlHttpRequest|\PhpHttpRequest
232 * @see MWHttpRequest::__construct
234 public static function factory( $url, $options = null ) {
235 if ( !Http
::$httpEngine ) {
236 Http
::$httpEngine = function_exists( 'curl_init' ) ?
'curl' : 'php';
237 } elseif ( Http
::$httpEngine == 'curl' && !function_exists( 'curl_init' ) ) {
238 throw new MWException( __METHOD__
. ': curl (http://php.net/curl) is not installed, but' .
239 ' Http::$httpEngine is set to "curl"' );
242 switch( Http
::$httpEngine ) {
244 return new CurlHttpRequest( $url, $options );
246 if ( !wfIniGetBool( 'allow_url_fopen' ) ) {
247 throw new MWException( __METHOD__
. ': allow_url_fopen needs to be enabled for pure PHP' .
248 ' http requests to work. If possible, curl should be used instead. See http://php.net/curl.' );
250 return new PhpHttpRequest( $url, $options );
252 throw new MWException( __METHOD__
. ': The setting of Http::$httpEngine is not valid.' );
257 * Get the body, or content, of the response to the request
261 public function getContent() {
262 return $this->content
;
266 * Set the parameters of the request
269 * @todo overload the args param
271 public function setData( $args ) {
272 $this->postData
= $args;
276 * Take care of setting up the proxy
277 * (override in subclass)
281 public function proxySetup() {
284 if ( $this->proxy
) {
288 if ( Http
::isLocalURL( $this->url
) ) {
290 } elseif ( $wgHTTPProxy ) {
291 $this->proxy
= $wgHTTPProxy ;
292 } elseif ( getenv( "http_proxy" ) ) {
293 $this->proxy
= getenv( "http_proxy" );
298 * Set the refererer header
300 public function setReferer( $url ) {
301 $this->setHeader( 'Referer', $url );
308 public function setUserAgent( $UA ) {
309 $this->setHeader( 'User-Agent', $UA );
313 * Set an arbitrary header
317 public function setHeader( $name, $value ) {
318 // I feel like I should normalize the case here...
319 $this->reqHeaders
[$name] = $value;
323 * Get an array of the headers
326 public function getHeaderList() {
329 if ( $this->cookieJar
) {
330 $this->reqHeaders
['Cookie'] =
331 $this->cookieJar
->serializeToHttpRequest(
332 $this->parsedUrl
['path'],
333 $this->parsedUrl
['host']
337 foreach ( $this->reqHeaders
as $name => $value ) {
338 $list[] = "$name: $value";
345 * Set a read callback to accept data read from the HTTP request.
346 * By default, data is appended to an internal buffer which can be
347 * retrieved through $req->getContent().
349 * To handle data as it comes in -- especially for large files that
350 * would not fit in memory -- you can instead set your own callback,
351 * in the form function($resource, $buffer) where the first parameter
352 * is the low-level resource being read (implementation specific),
353 * and the second parameter is the data buffer.
355 * You MUST return the number of bytes handled in the buffer; if fewer
356 * bytes are reported handled than were passed to you, the HTTP fetch
359 * @param $callback Callback
361 public function setCallback( $callback ) {
362 if ( !is_callable( $callback ) ) {
363 throw new MWException( 'Invalid MwHttpRequest callback' );
365 $this->callback
= $callback;
369 * A generic callback to read the body of the response from a remote
373 * @param $content String
375 public function read( $fh, $content ) {
376 $this->content
.= $content;
377 return strlen( $content );
381 * Take care of whatever is necessary to perform the URI request.
385 public function execute() {
390 if ( strtoupper( $this->method
) == "HEAD" ) {
391 $this->headersOnly
= true;
394 if ( is_object( $wgTitle ) && !isset( $this->reqHeaders
['Referer'] ) ) {
395 $this->setReferer( wfExpandUrl( $wgTitle->getFullURL(), PROTO_CURRENT
) );
398 if ( !$this->noProxy
) {
402 if ( !$this->callback
) {
403 $this->setCallback( array( $this, 'read' ) );
406 if ( !isset( $this->reqHeaders
['User-Agent'] ) ) {
407 $this->setUserAgent( Http
::userAgent() );
412 * Parses the headers, including the HTTP status code and any
413 * Set-Cookie headers. This function expectes the headers to be
414 * found in an array in the member variable headerList.
418 protected function parseHeader() {
421 foreach ( $this->headerList
as $header ) {
422 if ( preg_match( "#^HTTP/([0-9.]+) (.*)#", $header, $match ) ) {
423 $this->respVersion
= $match[1];
424 $this->respStatus
= $match[2];
425 } elseif ( preg_match( "#^[ \t]#", $header ) ) {
426 $last = count( $this->respHeaders
[$lastname] ) - 1;
427 $this->respHeaders
[$lastname][$last] .= "\r\n$header";
428 } elseif ( preg_match( "#^([^:]*):[\t ]*(.*)#", $header, $match ) ) {
429 $this->respHeaders
[strtolower( $match[1] )][] = $match[2];
430 $lastname = strtolower( $match[1] );
434 $this->parseCookies();
438 * Sets HTTPRequest status member to a fatal value with the error
439 * message if the returned integer value of the status code was
440 * not successful (< 300) or a redirect (>=300 and < 400). (see
441 * RFC2616, section 10,
442 * http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html for a
443 * list of status codes.)
447 protected function setStatus() {
448 if ( !$this->respHeaders
) {
449 $this->parseHeader();
452 if ( (int)$this->respStatus
> 399 ) {
453 list( $code, $message ) = explode( " ", $this->respStatus
, 2 );
454 $this->status
->fatal( "http-bad-status", $code, $message );
459 * Get the integer value of the HTTP status code (e.g. 200 for "200 Ok")
460 * (see RFC2616, section 10, http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
461 * for a list of status codes.)
465 public function getStatus() {
466 if ( !$this->respHeaders
) {
467 $this->parseHeader();
470 return (int)$this->respStatus
;
475 * Returns true if the last status code was a redirect.
479 public function isRedirect() {
480 if ( !$this->respHeaders
) {
481 $this->parseHeader();
484 $status = (int)$this->respStatus
;
486 if ( $status >= 300 && $status <= 303 ) {
494 * Returns an associative array of response headers after the
495 * request has been executed. Because some headers
496 * (e.g. Set-Cookie) can appear more than once the, each value of
497 * the associative array is an array of the values given.
501 public function getResponseHeaders() {
502 if ( !$this->respHeaders
) {
503 $this->parseHeader();
506 return $this->respHeaders
;
510 * Returns the value of the given response header.
512 * @param $header String
515 public function getResponseHeader( $header ) {
516 if ( !$this->respHeaders
) {
517 $this->parseHeader();
520 if ( isset( $this->respHeaders
[strtolower ( $header ) ] ) ) {
521 $v = $this->respHeaders
[strtolower ( $header ) ];
522 return $v[count( $v ) - 1];
529 * Tells the MWHttpRequest object to use this pre-loaded CookieJar.
531 * @param $jar CookieJar
533 public function setCookieJar( $jar ) {
534 $this->cookieJar
= $jar;
538 * Returns the cookie jar in use.
542 public function getCookieJar() {
543 if ( !$this->respHeaders
) {
544 $this->parseHeader();
547 return $this->cookieJar
;
551 * Sets a cookie. Used before a request to set up any individual
552 * cookies. Used internally after a request to parse the
553 * Set-Cookie headers.
559 public function setCookie( $name, $value = null, $attr = null ) {
560 if ( !$this->cookieJar
) {
561 $this->cookieJar
= new CookieJar
;
564 $this->cookieJar
->setCookie( $name, $value, $attr );
568 * Parse the cookies in the response headers and store them in the cookie jar.
570 protected function parseCookies() {
571 if ( !$this->cookieJar
) {
572 $this->cookieJar
= new CookieJar
;
575 if ( isset( $this->respHeaders
['set-cookie'] ) ) {
576 $url = parse_url( $this->getFinalUrl() );
577 foreach ( $this->respHeaders
['set-cookie'] as $cookie ) {
578 $this->cookieJar
->parseCookieResponseHeader( $cookie, $url['host'] );
584 * Returns the final URL after all redirections.
586 * Relative values of the "Location" header are incorrect as stated in RFC, however they do happen and modern browsers support them.
587 * This function loops backwards through all locations in order to build the proper absolute URI - Marooned at wikia-inc.com
591 public function getFinalUrl() {
592 $headers = $this->getResponseHeaders();
594 //return full url (fix for incorrect but handled relative location)
595 if ( isset( $headers[ 'location' ] ) ) {
596 $locations = $headers[ 'location' ];
598 $foundRelativeURI = false;
599 $countLocations = count($locations);
601 for ( $i = $countLocations - 1; $i >= 0; $i-- ) {
602 $url = parse_url( $locations[ $i ] );
604 if ( isset($url[ 'host' ]) ) {
605 $domain = $url[ 'scheme' ] . '://' . $url[ 'host' ];
606 break; //found correct URI (with host)
608 $foundRelativeURI = true;
612 if ( $foundRelativeURI ) {
614 return $domain . $locations[ $countLocations - 1 ];
616 $url = parse_url( $this->url
);
617 if ( isset($url[ 'host' ]) ) {
618 return $url[ 'scheme' ] . '://' . $url[ 'host' ] . $locations[ $countLocations - 1 ];
622 return $locations[ $countLocations - 1 ];
630 * Returns true if the backend can follow redirects. Overridden by the
634 public function canFollowRedirects() {
640 * MWHttpRequest implemented using internal curl compiled into PHP
642 class CurlHttpRequest
extends MWHttpRequest
{
643 const SUPPORTS_FILE_POSTS
= true;
645 static $curlMessageMap = array(
646 6 => 'http-host-unreachable',
647 28 => 'http-timed-out'
650 protected $curlOptions = array();
651 protected $headerText = "";
658 protected function readHeader( $fh, $content ) {
659 $this->headerText
.= $content;
660 return strlen( $content );
663 public function execute() {
666 if ( !$this->status
->isOK() ) {
667 return $this->status
;
670 $this->curlOptions
[CURLOPT_PROXY
] = $this->proxy
;
671 $this->curlOptions
[CURLOPT_TIMEOUT
] = $this->timeout
;
672 $this->curlOptions
[CURLOPT_HTTP_VERSION
] = CURL_HTTP_VERSION_1_0
;
673 $this->curlOptions
[CURLOPT_WRITEFUNCTION
] = $this->callback
;
674 $this->curlOptions
[CURLOPT_HEADERFUNCTION
] = array( $this, "readHeader" );
675 $this->curlOptions
[CURLOPT_MAXREDIRS
] = $this->maxRedirects
;
676 $this->curlOptions
[CURLOPT_ENCODING
] = ""; # Enable compression
678 /* not sure these two are actually necessary */
679 if ( isset( $this->reqHeaders
['Referer'] ) ) {
680 $this->curlOptions
[CURLOPT_REFERER
] = $this->reqHeaders
['Referer'];
682 $this->curlOptions
[CURLOPT_USERAGENT
] = $this->reqHeaders
['User-Agent'];
684 if ( isset( $this->sslVerifyHost
) ) {
685 $this->curlOptions
[CURLOPT_SSL_VERIFYHOST
] = $this->sslVerifyHost
;
688 if ( isset( $this->sslVerifyCert
) ) {
689 $this->curlOptions
[CURLOPT_SSL_VERIFYPEER
] = $this->sslVerifyCert
;
692 if ( $this->caInfo
) {
693 $this->curlOptions
[CURLOPT_CAINFO
] = $this->caInfo
;
696 if ( $this->headersOnly
) {
697 $this->curlOptions
[CURLOPT_NOBODY
] = true;
698 $this->curlOptions
[CURLOPT_HEADER
] = true;
699 } elseif ( $this->method
== 'POST' ) {
700 $this->curlOptions
[CURLOPT_POST
] = true;
701 $this->curlOptions
[CURLOPT_POSTFIELDS
] = $this->postData
;
702 // Suppress 'Expect: 100-continue' header, as some servers
703 // will reject it with a 417 and Curl won't auto retry
704 // with HTTP 1.0 fallback
705 $this->reqHeaders
['Expect'] = '';
707 $this->curlOptions
[CURLOPT_CUSTOMREQUEST
] = $this->method
;
710 $this->curlOptions
[CURLOPT_HTTPHEADER
] = $this->getHeaderList();
712 $curlHandle = curl_init( $this->url
);
714 if ( !curl_setopt_array( $curlHandle, $this->curlOptions
) ) {
715 throw new MWException( "Error setting curl options." );
718 if ( $this->followRedirects
&& $this->canFollowRedirects() ) {
719 wfSuppressWarnings();
720 if ( ! curl_setopt( $curlHandle, CURLOPT_FOLLOWLOCATION
, true ) ) {
721 wfDebug( __METHOD__
. ": Couldn't set CURLOPT_FOLLOWLOCATION. " .
722 "Probably safe_mode or open_basedir is set.\n" );
723 // Continue the processing. If it were in curl_setopt_array,
724 // processing would have halted on its entry
729 if ( false === curl_exec( $curlHandle ) ) {
730 $code = curl_error( $curlHandle );
732 if ( isset( self
::$curlMessageMap[$code] ) ) {
733 $this->status
->fatal( self
::$curlMessageMap[$code] );
735 $this->status
->fatal( 'http-curl-error', curl_error( $curlHandle ) );
738 $this->headerList
= explode( "\r\n", $this->headerText
);
741 curl_close( $curlHandle );
743 $this->parseHeader();
746 return $this->status
;
752 public function canFollowRedirects() {
753 if ( strval( ini_get( 'open_basedir' ) ) !== '' ||
wfIniGetBool( 'safe_mode' ) ) {
754 wfDebug( "Cannot follow redirects in safe mode\n" );
758 if ( !defined( 'CURLOPT_REDIR_PROTOCOLS' ) ) {
759 wfDebug( "Cannot follow redirects with libcurl < 7.19.4 due to CVE-2009-0037\n" );
767 class PhpHttpRequest
extends MWHttpRequest
{
773 protected function urlToTcp( $url ) {
774 $parsedUrl = parse_url( $url );
776 return 'tcp://' . $parsedUrl['host'] . ':' . $parsedUrl['port'];
779 public function execute() {
782 if ( is_array( $this->postData
) ) {
783 $this->postData
= wfArrayToCGI( $this->postData
);
786 if ( $this->parsedUrl
['scheme'] != 'http' &&
787 $this->parsedUrl
['scheme'] != 'https' ) {
788 $this->status
->fatal( 'http-invalid-scheme', $this->parsedUrl
['scheme'] );
791 $this->reqHeaders
['Accept'] = "*/*";
792 if ( $this->method
== 'POST' ) {
793 // Required for HTTP 1.0 POSTs
794 $this->reqHeaders
['Content-Length'] = strlen( $this->postData
);
795 $this->reqHeaders
['Content-type'] = "application/x-www-form-urlencoded";
799 if ( $this->proxy
&& !$this->noProxy
) {
800 $options['proxy'] = $this->urlToTCP( $this->proxy
);
801 $options['request_fulluri'] = true;
804 if ( !$this->followRedirects
) {
805 $options['max_redirects'] = 0;
807 $options['max_redirects'] = $this->maxRedirects
;
810 $options['method'] = $this->method
;
811 $options['header'] = implode( "\r\n", $this->getHeaderList() );
812 // Note that at some future point we may want to support
813 // HTTP/1.1, but we'd have to write support for chunking
814 // in version of PHP < 5.3.1
815 $options['protocol_version'] = "1.0";
817 // This is how we tell PHP we want to deal with 404s (for example) ourselves.
818 // Only works on 5.2.10+
819 $options['ignore_errors'] = true;
821 if ( $this->postData
) {
822 $options['content'] = $this->postData
;
825 $options['timeout'] = $this->timeout
;
827 $context = stream_context_create( array( 'http' => $options ) );
829 $this->headerList
= array();
837 wfSuppressWarnings();
838 $fh = fopen( $url, "r", false, $context );
845 $result = stream_get_meta_data( $fh );
846 $this->headerList
= $result['wrapper_data'];
847 $this->parseHeader();
849 if ( !$this->followRedirects
) {
853 # Handle manual redirection
854 if ( !$this->isRedirect() ||
$reqCount > $this->maxRedirects
) {
857 # Check security of URL
858 $url = $this->getResponseHeader( "Location" );
860 if ( !Http
::isValidURI( $url ) ) {
861 wfDebug( __METHOD__
. ": insecure redirection\n" );
868 if ( $fh === false ) {
869 $this->status
->fatal( 'http-request-error' );
870 return $this->status
;
873 if ( $result['timed_out'] ) {
874 $this->status
->fatal( 'http-timed-out', $this->url
);
875 return $this->status
;
878 // If everything went OK, or we recieved some error code
879 // get the response body content.
880 if ( $this->status
->isOK()
881 ||
(int)$this->respStatus
>= 300) {
882 while ( !feof( $fh ) ) {
883 $buf = fread( $fh, 8192 );
885 if ( $buf === false ) {
886 $this->status
->fatal( 'http-read-error' );
890 if ( strlen( $buf ) ) {
891 call_user_func( $this->callback
, $fh, $buf );
897 return $this->status
;