7 * Various HTTP related functions
12 * Perform an HTTP request
13 * @param $method string HTTP method. Usually GET/POST
14 * @param $url string Full URL to act on
15 * @param $opts options to pass to HttpRequest object
16 * @returns mixed (bool)false on failure or a string on success
18 public static function request( $method, $url, $opts = array() ) {
19 $opts['method'] = strtoupper( $method );
20 if ( !array_key_exists( 'timeout', $opts ) ) {
21 $opts['timeout'] = 'default';
23 $req = HttpRequest
::factory( $url, $opts );
24 $status = $req->doRequest();
25 if ( $status->isOK() ) {
26 return $req->getContent();
33 * Simple wrapper for Http::request( 'GET' )
34 * @see Http::request()
36 public static function get( $url, $timeout = 'default', $opts = array() ) {
37 $opts['timeout'] = $timeout;
38 return Http
::request( 'GET', $url, $opts );
42 * Simple wrapper for Http::request( 'POST' )
43 * @see Http::request()
45 public static function post( $url, $opts = array() ) {
46 return Http
::request( 'POST', $url, $opts );
50 * Check if the URL can be served by localhost
51 * @param $url string Full url to check
54 public static function isLocalURL( $url ) {
55 global $wgCommandLineMode, $wgConf;
56 if ( $wgCommandLineMode ) {
62 if ( preg_match( '!^http://([\w.-]+)[/:].*$!', $url, $matches ) ) {
65 $domainParts = explode( '.', $host );
66 // Check if this domain or any superdomain is listed in $wgConf as a local virtual host
67 $domainParts = array_reverse( $domainParts );
68 for ( $i = 0; $i < count( $domainParts ); $i++
) {
69 $domainPart = $domainParts[$i];
71 $domain = $domainPart;
73 $domain = $domainPart . '.' . $domain;
75 if ( $wgConf->isLocalVHost( $domain ) ) {
84 * A standard user-agent we can use for external requests.
87 public static function userAgent() {
89 return "MediaWiki/$wgVersion";
93 * Checks that the given URI is a valid one
94 * @param $uri Mixed: URI to check for validity
97 public static function isValidURI( $uri ) {
99 '/(ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/',
105 * Fetch a URL, write the result to a file.
106 * @params $url string url to fetch
107 * @params $targetFilePath string full path (including filename) to write the file to
108 * @params $async bool whether the download should be asynchronous (defaults to false)
109 * @params $redirectCount int used internally to keep track of the number of redirects
111 * @returns Status -- for async requests this will contain the request key
113 public static function doDownload( $url, $targetFilePath, $async = false, $redirectCount = 0 ) {
114 global $wgPhpCli, $wgMaxUploadSize, $wgMaxRedirects;
116 // do a quick check to HEAD to insure the file size is not > $wgMaxUploadSize
117 $headRequest = HttpRequest
::factory( $url, array( 'headersOnly' => true ) );
118 $headResponse = $headRequest->doRequest();
119 if ( !$headResponse->isOK() ) {
120 return $headResponse;
122 $head = $headResponse->value
;
124 // check for redirects:
125 if ( $redirectCount < 0 ) {
128 if ( isset( $head['Location'] ) && strrpos( $head[0], '302' ) !== false ) {
129 if ( $redirectCount < $wgMaxRedirects ) {
130 if ( self
::isValidURI( $head['Location'] ) ) {
131 return self
::doDownload( $head['Location'], $targetFilePath,
132 $async, $redirectCount++
);
134 return Status
::newFatal( 'upload-proto-error' );
137 return Status
::newFatal( 'upload-too-many-redirects' );
140 // we did not get a 200 ok response:
141 if ( strrpos( $head[0], '200 OK' ) === false ) {
142 return Status
::newFatal( 'upload-http-error', htmlspecialchars( $head[0] ) );
145 $contentLength = $head['Content-Length'];
146 if ( $contentLength ) {
147 if ( $contentLength > $wgMaxUploadSize ) {
148 return Status
::newFatal( 'requested file length ' . $contentLength .
149 ' is greater than $wgMaxUploadSize: ' . $wgMaxUploadSize );
153 // check if we can find phpCliPath (for doing a background shell request to
154 // php to do the download:
155 if ( $async && $wgPhpCli && wfShellExecEnabled() ) {
156 wfDebug( __METHOD__
. "\nASYNC_DOWNLOAD\n" );
157 // setup session and shell call:
158 return self
::startBackgroundRequest( $url, $targetFilePath, $contentLength );
160 wfDebug( __METHOD__
. "\nSYNC_DOWNLOAD\n" );
161 // SYNC_DOWNLOAD download as much as we can in the time we have to execute
162 $opts['method'] = 'GET';
163 $opts['targetFilePath'] = $mTargetFilePath;
164 $req = HttpRequest
::factory( $url, $opts );
165 return $req->doRequest();
170 * Start backgrounded (i.e. non blocking) request. The
171 * backgrounded request will provide updates to the user's session
173 * @param $url string the URL to download
174 * @param $targetFilePath string the destination for the downloaded file
175 * @param $contentLength int (optional) the length of the download from the HTTP header
179 private static function startBackgroundRequest( $url, $targetFilePath, $contentLength = null ) {
180 global $IP, $wgPhpCli, $wgServer;
181 $status = Status
::newGood();
183 // generate a session id with all the details for the download (pid, targetFilePath )
184 $requestKey = self
::createRequestKey();
185 $sessionID = session_id();
187 // store the url and target path:
188 $_SESSION['wsBgRequest'][$requestKey]['url'] = $url;
189 $_SESSION['wsBgRequest'][$requestKey]['targetFilePath'] = $targetFilePath;
190 // since we request from the cmd line we lose the original host name pass in the session:
191 $_SESSION['wsBgRequest'][$requestKey]['orgServer'] = $wgServer;
193 if ( $contentLength ) {
194 $_SESSION['wsBgRequest'][$requestKey]['contentLength'] = $contentLength;
197 // set initial loaded bytes:
198 $_SESSION['wsBgRequest'][$requestKey]['loaded'] = 0;
200 // run the background download request:
201 $cmd = $wgPhpCli . ' ' . $IP . "/maintenance/httpSessionDownload.php " .
202 "--sid {$sessionID} --usk {$requestKey} --wiki " . wfWikiId();
203 $pid = wfShellBackgroundExec( $cmd );
204 // the pid is not of much use since we won't be visiting this same apache any-time soon.
206 return Status
::newFatal( 'http-could-not-background' );
208 // update the status value with the $requestKey (for the user to
209 // check on the status of the download)
210 $status->value
= $requestKey;
212 // return good status
217 * Returns a unique, random string that can be used as an request key and
218 * preloads it into the session data.
222 static function createRequestKey() {
223 if ( !array_key_exists( 'wsBgRequest', $_SESSION ) ) {
224 $_SESSION['wsBgRequest'] = array();
227 $key = uniqid( 'bgrequest', true );
229 // This is probably over-defensive.
230 while ( array_key_exists( $key, $_SESSION['wsBgRequest'] ) ) {
231 $key = uniqid( 'bgrequest', true );
233 $_SESSION['wsBgRequest'][$key] = array();
239 * Recover the necessary session and request information
240 * @param $sessionID string
241 * @param $requestKey string the HTTP request key
243 * @returns array request information
245 private static function recoverSession( $sessionID, $requestKey ) {
246 global $wgUser, $wgServer, $wgSessionsInMemcached;
248 // set session to the provided key:
249 session_id( $sessionID );
250 // fire up mediaWiki session system:
254 if ( session_start() === false ) {
255 wfDebug( __METHOD__
. ' could not start session' );
257 // get all the vars we need from session_id
258 if ( !isset( $_SESSION[ 'wsBgRequest' ][ $requestKey ] ) ) {
259 wfDebug( __METHOD__
. ' Error:could not find upload session' );
262 // setup the global user from the session key we just inherited
263 $wgUser = User
::newFromSession();
265 // grab the session data to setup the request:
266 $sd =& $_SESSION['wsBgRequest'][$requestKey];
268 // update the wgServer var ( since cmd line thinks we are localhost
269 // when we are really orgServer)
270 if ( isset( $sd['orgServer'] ) && $sd['orgServer'] ) {
271 $wgServer = $sd['orgServer'];
273 // close down the session so we can other http queries can get session
274 // updates: (if not $wgSessionsInMemcached)
275 if ( !$wgSessionsInMemcached ) {
276 session_write_close();
283 * Update the session with the finished information.
284 * @param $sessionID string
285 * @param $requestKey string the HTTP request key
287 private static function updateSession( $sessionID, $requestKey, $status ) {
289 if ( session_start() === false ) {
290 wfDebug( __METHOD__
. ' ERROR:: Could not start session' );
293 $sd =& $_SESSION['wsBgRequest'][$requestKey];
294 if ( !$status->isOK() ) {
295 $sd['apiUploadResult'] = FormatJson
::encode(
296 array( 'error' => $status->getWikiText() )
299 $sd['apiUploadResult'] = FormatJson
::encode( $status->value
);
302 session_write_close();
306 * Take care of the downloaded file
308 * @param $status Status
312 private static function doFauxRequest( $sd, $status ) {
313 global $wgEnableWriteAPI;
315 if ( $status->isOK() ) {
316 $fauxReqData = $sd['mParams'];
318 // Fix boolean parameters
319 foreach ( $fauxReqData as $k => $v ) {
321 unset( $fauxReqData[$k] );
324 $fauxReqData['action'] = 'upload';
325 $fauxReqData['format'] = 'json';
326 $fauxReqData['internalhttpsession'] = $requestKey;
328 // evil but no other clean way about it:
329 $fauxReq = new FauxRequest( $fauxReqData, true );
330 $processor = new ApiMain( $fauxReq, $wgEnableWriteAPI );
332 // init the mUpload var for the $processor
333 $processor->execute();
334 $processor->getResult()->cleanUpUTF8();
335 $printer = $processor->createPrinterByName( 'json' );
336 $printer->initPrinter( false );
340 // the status updates runner will grab the result form the session:
341 $status->value
= ob_get_clean();
347 * Run a session based download.
349 * @param $sessionID string: the session id with the download details
350 * @param $requestKey string: the key of the given upload session
351 * (a given client could have started a few http uploads at once)
353 public static function doSessionIdDownload( $sessionID, $requestKey ) {
354 global $wgAsyncHTTPTimeout;
356 wfDebug( __METHOD__
. "\n\n doSessionIdDownload :\n\n" );
357 $sd = self
::recoverSession( $sessionID );
358 $req = HttpRequest
::factory( $sd['url'],
360 'targetFilePath' => $sd['targetFilePath'],
361 'requestKey' => $requestKey,
362 'timeout' => $wgAsyncHTTPTimeout,
365 // run the actual request .. (this can take some time)
366 wfDebug( __METHOD__
. 'do Session Download :: ' . $sd['url'] . ' tf: ' .
367 $sd['targetFilePath'] . "\n\n" );
368 $status = $req->doRequest();
370 self
::updateSession( $sessionID, $requestKey,
371 self
::handleFauxResponse( $sd, $status ) );
376 * This wrapper class will call out to curl (if available) or fallback
377 * to regular PHP if necessary for handling internal HTTP requests.
380 private $targetFilePath;
383 protected $timeout = 'default';
384 protected $headersOnly = null;
385 protected $postdata = null;
386 protected $proxy = null;
387 protected $no_proxy = false;
388 protected $sslVerifyHost = true;
389 protected $caInfo = null;
390 protected $method = "GET";
395 * @param $url string url to use
396 * @param $options array (optional) extra params to pass
397 * Possible keys for the array:
409 function __construct( $url = null, $opt ) {
410 global $wgHTTPTimeout;
414 if ( !ini_get( 'allow_url_fopen' ) ) {
415 $this->status
= Status
::newFatal( 'allow_url_fopen needs to be enabled for http copy to work' );
416 } elseif ( !Http
::isValidURI( $this->url
) ) {
417 $this->status
= Status
::newFatal( 'bad-url' );
419 $this->status
= Status
::newGood( 100 ); // continue
422 if ( array_key_exists( 'timeout', $opt ) && $opt['timeout'] != 'default' ) {
423 $this->timeout
= $opt['timeout'];
425 $this->timeout
= $wgHTTPTimeout;
428 $members = array( "targetFilePath", "requestKey", "headersOnly", "postdata",
429 "proxy", "no_proxy", "sslVerifyHost", "caInfo", "method" );
430 foreach ( $members as $o ) {
431 if ( array_key_exists( $o, $opt ) ) {
432 $this->$o = $opt[$o];
436 if ( is_array( $this->postdata
) ) {
437 $this->postdata
= wfArrayToCGI( $this->postdata
);
442 * For backwards compatibility, we provide a __toString method so
443 * that any code that expects a string result from Http::Get()
444 * will see the content of the request.
446 function __toString() {
447 return $this->content
;
451 * Generate a new request object
452 * @see HttpRequest::__construct
454 public static function factory( $url, $opt ) {
455 global $wgForceHTTPEngine;
457 if ( function_exists( 'curl_init' ) && $wgForceHTTPEngine == "curl" ) {
458 return new CurlHttpRequest( $url, $opt );
460 return new PhpHttpRequest( $url, $opt );
464 public function getContent() {
465 return $this->content
;
468 public function handleOutput() {
469 // if we wrote to a target file close up or return error
470 if ( $this->targetFilePath
) {
471 $this->writer
->close();
472 if ( !$this->writer
->status
->isOK() ) {
473 $this->status
= $this->writer
->status
;
474 return $this->status
;
479 public function doRequest() {
482 if ( !$this->status
->isOK() ) {
483 return $this->status
;
486 $this->initRequest();
488 if ( !$this->no_proxy
) {
492 # Set the referer to $wgTitle, even in command-line mode
493 # This is useful for interwiki transclusion, where the foreign
494 # server wants to know what the referring page is.
495 # $_SERVER['REQUEST_URI'] gives a less reliable indication of the
497 if ( is_object( $wgTitle ) ) {
498 $this->set_referer( $wgTitle->getFullURL() );
501 $this->setupOutputHandler();
503 if ( $this->status
->isOK() ) {
504 $this->spinTheWheel();
507 if ( !$this->status
->isOK() ) {
508 return $this->status
;
511 $this->handleOutput();
514 return $this->status
;
517 public function setupOutputHandler() {
518 if ( $this->targetFilePath
) {
519 $this->writer
= new SimpleFileWriter( $this->targetFilePath
,
521 if ( !$this->writer
->status
->isOK() ) {
522 wfDebug( __METHOD__
. "ERROR in setting up SimpleFileWriter\n" );
523 $this->status
= $this->writer
->status
;
524 return $this->status
;
526 $this->setCallback( array( $this, 'readAndSave' ) );
528 $this->setCallback( array( $this, 'readOnly' ) );
534 * HttpRequest implemented using internal curl compiled into PHP
536 class CurlHttpRequest
extends HttpRequest
{
539 public function initRequest() {
540 $this->c
= curl_init( $this->url
);
543 public function proxySetup() {
546 if ( is_string( $this->proxy
) ) {
547 curl_setopt( $this->c
, CURLOPT_PROXY
, $this->proxy
);
548 } else if ( Http
::isLocalURL( $this->url
) ) { /* Not sure this makes any sense. */
549 curl_setopt( $this->c
, CURLOPT_PROXY
, 'localhost:80' );
550 } else if ( $wgHTTPProxy ) {
551 curl_setopt( $this->c
, CURLOPT_PROXY
, $wgHTTPProxy );
555 public function setCallback( $cb ) {
556 curl_setopt( $this->c
, CURLOPT_WRITEFUNCTION
, $cb );
559 public function spinTheWheel() {
560 curl_setopt( $this->c
, CURLOPT_TIMEOUT
, $this->timeout
);
561 curl_setopt( $this->c
, CURLOPT_USERAGENT
, Http
::userAgent() );
562 curl_setopt( $this->c
, CURLOPT_HTTP_VERSION
, CURL_HTTP_VERSION_1_0
);
564 if ( $this->sslVerifyHost
) {
565 curl_setopt( $this->c
, CURLOPT_SSL_VERIFYHOST
, $this->sslVerifyHost
);
568 if ( $this->caInfo
) {
569 curl_setopt( $this->c
, CURLOPT_CAINFO
, $this->caInfo
);
572 if ( $this->headersOnly
) {
573 curl_setopt( $this->c
, CURLOPT_NOBODY
, true );
574 curl_setopt( $this->c
, CURLOPT_HEADER
, true );
575 } elseif ( $this->method
== 'POST' ) {
576 curl_setopt( $this->c
, CURLOPT_POST
, true );
577 curl_setopt( $this->c
, CURLOPT_POSTFIELDS
, $this->postdata
);
578 // Suppress 'Expect: 100-continue' header, as some servers
579 // will reject it with a 417 and Curl won't auto retry
580 // with HTTP 1.0 fallback
581 curl_setopt( $this->c
, CURLOPT_HTTPHEADER
, array( 'Expect:' ) );
583 curl_setopt( $this->c
, CURLOPT_CUSTOMREQUEST
, $this->method
);
587 if ( false === curl_exec( $this->c
) ) {
588 $error_txt = 'Error sending request: #' . curl_errno( $this->c
) . ' ' . curl_error( $this->c
);
589 wfDebug( __METHOD__
. $error_txt . "\n" );
590 $this->status
->fatal( $error_txt ); /* i18n? */
592 } catch ( Exception
$e ) {
593 $errno = curl_errno( $this->c
);
594 if ( $errno != CURLE_OK
) {
595 $errstr = curl_error( $this->c
);
596 wfDebug( __METHOD__
. ": CURL error code $errno: $errstr\n" );
597 $this->status
->fatal( "CURL error code $errno: $errstr\n" ); /* i18n? */
602 public function readOnly( $curlH, $content ) {
603 $this->content
.= $content;
604 return strlen( $content );
607 public function readAndSave( $curlH, $content ) {
608 return $this->writer
->write( $content );
611 public function getCode() {
612 # Don't return truncated output
613 $code = curl_getinfo( $this->c
, CURLINFO_HTTP_CODE
);
615 $this->status
->setResult( true, $code );
617 $this->status
->setResult( false, $code );
621 public function finish() {
622 curl_close( $this->c
);
627 class PhpHttpRequest
extends HttpRequest
{
632 public function initRequest() {
633 $this->reqHeaders
[] = "User-Agent: " . Http
::userAgent();
634 $this->reqHeaders
[] = "Accept: */*";
635 if ( $this->method
== 'POST' ) {
636 // Required for HTTP 1.0 POSTs
637 $this->reqHeaders
[] = "Content-Length: " . strlen( $this->postdata
);
638 $this->reqHeaders
[] = "Content-type: application/x-www-form-urlencoded";
642 public function proxySetup() {
645 if ( $this->proxy
) {
646 $this->proxy
= 'tcp://' . $this->proxy
;
647 } elseif ( Http
::isLocalURL( $this->url
) ) {
648 $this->proxy
= 'tcp://localhost:80';
649 } elseif ( $wgHTTPProxy ) {
650 $this->proxy
= 'tcp://' . $wgHTTPProxy ;
654 public function setReferrer( $url ) {
655 $this->reqHeaders
[] = "Referer: $url";
658 public function setCallback( $cb ) {
659 $this->callback
= $cb;
662 public function readOnly( $contents ) {
663 if ( $this->headersOnly
) {
666 $this->content
.= $contents;
668 return strlen( $contents );
671 public function readAndSave( $contents ) {
672 if ( $this->headersOnly
) {
675 return $this->writer
->write( $content );
678 public function finish() {
682 public function spinTheWheel() {
684 if ( $this->proxy
&& !$this->no_proxy
) {
685 $opts['proxy'] = $this->proxy
;
686 $opts['request_fulluri'] = true;
689 $opts['method'] = $this->method
;
690 $opts['timeout'] = $this->timeout
;
691 $opts['header'] = implode( "\r\n", $this->reqHeaders
);
692 if ( version_compare( "5.3.0", phpversion(), ">" ) ) {
693 $opts['protocol_version'] = "1.0";
695 $opts['protocol_version'] = "1.1";
698 if ( $this->postdata
) {
699 $opts['content'] = $this->postdata
;
702 $context = stream_context_create( array( 'http' => $opts ) );
703 $this->fh
= fopen( $this->url
, "r", false, $context );
704 $result = stream_get_meta_data( $this->fh
);
706 if ( $result['timed_out'] ) {
707 $this->status
->error( __CLASS__
. '::timed-out-in-headers' );
710 $this->headers
= $result['wrapper_data'];
715 $contents = fread( $this->fh
, $size );
716 $size = call_user_func( $this->callback
, $contents );
717 $end = ( $size == 0 ) ||
feof( $this->fh
);
723 * SimpleFileWriter with session id updates
725 class SimpleFileWriter
{
726 private $targetFilePath = null;
727 private $status = null;
728 private $sessionId = null;
729 private $sessionUpdateInterval = 0; // how often to update the session while downloading
730 private $currentFileSize = 0;
731 private $requestKey = null;
732 private $prevTime = 0;
736 * @param $targetFilePath string the path to write the file out to
737 * @param $requestKey string the request to update
739 function __construct__( $targetFilePath, $requestKey ) {
740 $this->targetFilePath
= $targetFilePath;
741 $this->requestKey
= $requestKey;
742 $this->status
= Status
::newGood();
744 $this->fp
= fopen( $this->targetFilePath
, 'w' );
745 if ( $this->fp
=== false ) {
746 $this->status
= Status
::newFatal( 'HTTP::could-not-open-file-for-writing' );
749 $this->prevTime
= time();
752 public function write( $dataPacket ) {
753 global $wgMaxUploadSize, $wgLang;
755 if ( !$this->status
->isOK() ) {
759 // write out the content
760 if ( fwrite( $this->fp
, $dataPacket ) === false ) {
761 wfDebug( __METHOD__
. " ::could-not-write-to-file\n" );
762 $this->status
= Status
::newFatal( 'HTTP::could-not-write-to-file' );
768 $this->currentFileSize
= filesize( $this->targetFilePath
);
770 if ( $this->currentFileSize
> $wgMaxUploadSize ) {
771 wfDebug( __METHOD__
. " ::http-download-too-large\n" );
772 $this->status
= Status
::newFatal( 'HTTP::file-has-grown-beyond-upload-limit-killing: ' . /* i18n? */
773 'downloaded more than ' .
774 $wgLang->formatSize( $wgMaxUploadSize ) . ' ' );
777 // if more than session_update_interval second have passed updateProgress
778 if ( $this->requestKey
&&
779 ( ( time() - $this->prevTime
) > $this->sessionUpdateInterval
) ) {
780 $this->prevTime
= time();
781 $session_status = $this->updateProgress();
782 if ( !$session_status->isOK() ) {
783 $this->status
= $session_status;
784 wfDebug( __METHOD__
. ' update session failed or was canceled' );
788 return strlen( $dataPacket );
791 public function updateProgress() {
792 global $wgSessionsInMemcached;
794 // start the session (if necessary)
795 if ( !$wgSessionsInMemcached ) {
796 wfSuppressWarnings();
797 if ( session_start() === false ) {
798 wfDebug( __METHOD__
. ' could not start session' );
803 $sd =& $_SESSION['wsBgRequest'][ $this->requestKey
];
804 // check if the user canceled the request:
805 if ( $sd['userCancel'] ) {
806 // @@todo kill the download
807 return Status
::newFatal( 'user-canceled-request' );
809 // update the progress bytes download so far:
810 $sd['loaded'] = $this->currentFileSize
;
812 // close down the session so we can other http queries can get session updates:
813 if ( !$wgSessionsInMemcached )
814 session_write_close();
816 return Status
::newGood();
819 public function close() {
820 $this->updateProgress();
822 // close up the file handle:
823 if ( false === fclose( $this->fp
) ) {
824 $this->status
= Status
::newFatal( 'HTTP::could-not-close-file' );