7 * Various HTTP related functions
11 // Syncronous download (in a single request)
12 const SYNC_DOWNLOAD
= 1;
14 // Asynchronous download ( background process with multiple requests )
15 const ASYNC_DOWNLOAD
= 2;
18 * Get the contents of a file by HTTP
19 * @param $method string HTTP method. Usually GET/POST
20 * @param $url string Full URL to act on
21 * @param $timeout int Seconds to timeout. 'default' falls to $wgHTTPTimeout
22 * @param $curlOptions array Optional array of extra params to pass
25 public static function request( $method, $url, $opts = array() ) {
26 $opts['method'] = ( strtoupper( $method ) == 'GET' ||
strtoupper( $method ) == 'POST' )
27 ?
strtoupper( $method ) : null;
28 $req = HttpRequest
::newRequest( $url, $opts );
29 $status = $req->doRequest();
30 if( $status->isOK() ) {
31 return $status->value
;
33 wfDebug( 'http error: ' . $status->getWikiText() );
39 * Simple wrapper for Http::request( 'GET' )
40 * @see Http::request()
42 public static function get( $url, $timeout = false, $opts = array() ) {
43 global $wgSyncHTTPTimeout;
45 $opts['timeout'] = $timeout;
46 return Http
::request( 'GET', $url, $opts );
50 * Simple wrapper for Http::request( 'POST' )
51 * @see Http::request()
53 public static function post( $url, $opts = array() ) {
54 return Http
::request( 'POST', $url, $opts );
57 public static function doDownload( $url, $target_file_path, $dl_mode = self
::SYNC_DOWNLOAD
,
60 global $wgPhpCli, $wgMaxUploadSize, $wgMaxRedirects;
61 // do a quick check to HEAD to insure the file size is not > $wgMaxUploadSize
62 $headRequest = HttpRequest
::newRequest( $url, array( 'headers_only' => true ) );
63 $headResponse = $headRequest->doRequest();
64 if( !$headResponse->isOK() ) {
67 $head = $headResponse->value
;
69 // check for redirects:
70 if( isset( $head['Location'] ) && strrpos( $head[0], '302' ) !== false ) {
71 if( $redirectCount < $wgMaxRedirects ) {
72 if( self
::isValidURI( $head['Location'] ) ) {
73 return self
::doDownload( $head['Location'], $target_file_path,
74 $dl_mode, $redirectCount++
);
76 return Status
::newFatal( 'upload-proto-error' );
79 return Status
::newFatal( 'upload-too-many-redirects' );
82 // we did not get a 200 ok response:
83 if( strrpos( $head[0], '200 OK' ) === false ) {
84 return Status
::newFatal( 'upload-http-error', htmlspecialchars( $head[0] ) );
87 $content_length = ( isset( $head['Content-Length'] ) ) ?
$head['Content-Length'] : null;
88 if( $content_length ) {
89 if( $content_length > $wgMaxUploadSize ) {
90 return Status
::newFatal( 'requested file length ' . $content_length .
91 ' is greater than $wgMaxUploadSize: ' . $wgMaxUploadSize );
95 // check if we can find phpCliPath (for doing a background shell request to
96 // php to do the download:
97 if( $wgPhpCli && wfShellExecEnabled() && $dl_mode == self
::ASYNC_DOWNLOAD
) {
98 wfDebug( __METHOD__
. "\nASYNC_DOWNLOAD\n" );
99 //setup session and shell call:
100 return self
::initBackgroundDownload( $url, $target_file_path, $content_length );
102 wfDebug( __METHOD__
. "\nSYNC_DOWNLOAD\n" );
103 // SYNC_DOWNLOAD download as much as we can in the time we have to execute
104 $opts['method'] = 'GET';
105 $opts['target_file_path'] = $target_file_path;
106 $req = HttpRequest
::newRequest( $url, $opts );
107 return $req->doRequest();
112 * a non blocking request (generally an exit point in the application)
113 * should write to a file location and give updates
116 private static function initBackgroundDownload( $url, $target_file_path,
117 $content_length = null )
119 global $IP, $wgPhpCli, $wgServer;
120 $status = Status
::newGood();
122 // generate a session id with all the details for the download (pid, target_file_path )
123 $upload_session_key = self
::getUploadSessionKey();
124 $session_id = session_id();
126 // store the url and target path:
127 $_SESSION['wsDownload'][$upload_session_key]['url'] = $url;
128 $_SESSION['wsDownload'][$upload_session_key]['target_file_path'] = $target_file_path;
129 // since we request from the cmd line we lose the original host name pass in the session:
130 $_SESSION['wsDownload'][$upload_session_key]['orgServer'] = $wgServer;
132 if( $content_length )
133 $_SESSION['wsDownload'][$upload_session_key]['content_length'] = $content_length;
135 // set initial loaded bytes:
136 $_SESSION['wsDownload'][$upload_session_key]['loaded'] = 0;
138 // run the background download request:
139 $cmd = $wgPhpCli . ' ' . $IP . "/maintenance/http_session_download.php " .
140 "--sid {$session_id} --usk {$upload_session_key} --wiki " . wfWikiId();
141 $pid = wfShellBackgroundExec( $cmd );
142 // the pid is not of much use since we won't be visiting this same apache any-time soon.
144 return Status
::newFatal( 'could not run background shell exec' );
146 // update the status value with the $upload_session_key (for the user to
147 // check on the status of the upload)
148 $status->value
= $upload_session_key;
150 // return good status
154 static function getUploadSessionKey() {
155 $key = mt_rand( 0, 0x7fffffff );
156 $_SESSION['wsUploadData'][$key] = array();
161 * used to run a session based download. Is initiated via the shell.
163 * @param $session_id String: the session id to grab download details from
164 * @param $upload_session_key String: the key of the given upload session
165 * (a given client could have started a few http uploads at once)
167 public static function doSessionIdDownload( $session_id, $upload_session_key ) {
168 global $wgUser, $wgEnableWriteAPI, $wgAsyncHTTPTimeout, $wgServer,
169 $wgSessionsInMemcached, $wgSessionHandler, $wgSessionStarted;
170 wfDebug( __METHOD__
. "\n\n doSessionIdDownload :\n\n" );
171 // set session to the provided key:
172 session_id( $session_id );
173 //fire up mediaWiki session system:
177 if( session_start() === false ) {
178 wfDebug( __METHOD__
. ' could not start session' );
180 // get all the vars we need from session_id
181 if( !isset( $_SESSION[ 'wsDownload' ][$upload_session_key] ) ) {
182 wfDebug( __METHOD__
. ' Error:could not find upload session');
185 // setup the global user from the session key we just inherited
186 $wgUser = User
::newFromSession();
188 // grab the session data to setup the request:
189 $sd =& $_SESSION['wsDownload'][$upload_session_key];
191 // update the wgServer var ( since cmd line thinks we are localhost
192 // when we are really orgServer)
193 if( isset( $sd['orgServer'] ) && $sd['orgServer'] ) {
194 $wgServer = $sd['orgServer'];
196 // close down the session so we can other http queries can get session
197 // updates: (if not $wgSessionsInMemcached)
198 if( !$wgSessionsInMemcached )
199 session_write_close();
201 $req = HttpRequest
::newRequest( $sd['url'], array(
202 'target_file_path' => $sd['target_file_path'],
203 'upload_session_key'=> $upload_session_key,
204 'timeout' => $wgAsyncHTTPTimeout,
205 'do_close_session_update' => true
207 // run the actual request .. (this can take some time)
208 wfDebug( __METHOD__
. 'do Session Download :: ' . $sd['url'] . ' tf: ' .
209 $sd['target_file_path'] . "\n\n");
210 $status = $req->doRequest();
211 //wfDebug("done with req status is: ". $status->isOK(). ' '.$status->getWikiText(). "\n");
213 // start up the session again:
214 if( session_start() === false ) {
215 wfDebug( __METHOD__
. ' ERROR:: Could not start session');
217 // grab the updated session data pointer
218 $sd =& $_SESSION['wsDownload'][$upload_session_key];
219 // if error update status:
220 if( !$status->isOK() ) {
221 $sd['apiUploadResult'] = FormatJson
::encode(
222 array( 'error' => $status->getWikiText() )
225 // if status okay process upload using fauxReq to api:
226 if( $status->isOK() ){
227 // setup the FauxRequest
228 $fauxReqData = $sd['mParams'];
230 // Fix boolean parameters
231 foreach( $fauxReqData as $k => $v ) {
233 unset( $fauxReqData[$k] );
236 $fauxReqData['action'] = 'upload';
237 $fauxReqData['format'] = 'json';
238 $fauxReqData['internalhttpsession'] = $upload_session_key;
239 // evil but no other clean way about it:
240 $faxReq = new FauxRequest( $fauxReqData, true );
241 $processor = new ApiMain( $faxReq, $wgEnableWriteAPI );
243 //init the mUpload var for the $processor
244 $processor->execute();
245 $processor->getResult()->cleanUpUTF8();
246 $printer = $processor->createPrinterByName( 'json' );
247 $printer->initPrinter( false );
250 $apiUploadResult = ob_get_clean();
252 // the status updates runner will grab the result form the session:
253 $sd['apiUploadResult'] = $apiUploadResult;
255 // close the session:
256 session_write_close();
260 * Check if the URL can be served by localhost
261 * @param $url string Full url to check
264 public static function isLocalURL( $url ) {
265 global $wgCommandLineMode, $wgConf;
266 if ( $wgCommandLineMode ) {
272 if ( preg_match( '!^http://([\w.-]+)[/:].*$!', $url, $matches ) ) {
275 $domainParts = explode( '.', $host );
276 // Check if this domain or any superdomain is listed in $wgConf as a local virtual host
277 $domainParts = array_reverse( $domainParts );
278 for ( $i = 0; $i < count( $domainParts ); $i++
) {
279 $domainPart = $domainParts[$i];
281 $domain = $domainPart;
283 $domain = $domainPart . '.' . $domain;
285 if ( $wgConf->isLocalVHost( $domain ) ) {
294 * Return a standard user-agent we can use for external requests.
296 public static function userAgent() {
298 return "MediaWiki/$wgVersion";
302 * Checks that the given URI is a valid one
303 * @param $uri Mixed: URI to check for validity
305 public static function isValidURI( $uri ){
307 '/(ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/',
315 var $target_file_path;
316 var $upload_session_key;
317 function __construct( $url, $opt ){
319 global $wgSyncHTTPTimeout;
321 // set the timeout to default sync timeout (unless the timeout option is provided)
322 $this->timeout
= ( isset( $opt['timeout'] ) ) ?
$opt['timeout'] : $wgSyncHTTPTimeout;
323 //check special key default
324 if($this->timeout
== 'default'){
325 $opts['timeout'] = $wgSyncHTTPTimeout;
328 $this->method
= ( isset( $opt['method'] ) ) ?
$opt['method'] : 'GET';
329 $this->target_file_path
= ( isset( $opt['target_file_path'] ) )
330 ?
$opt['target_file_path'] : false;
331 $this->upload_session_key
= ( isset( $opt['upload_session_key'] ) )
332 ?
$opt['upload_session_key'] : false;
333 $this->headers_only
= ( isset( $opt['headers_only'] ) ) ?
$opt['headers_only'] : false;
334 $this->do_close_session_update
= isset( $opt['do_close_session_update'] );
335 $this->postData
= isset( $opt['postdata'] ) ?
$opt['postdata'] : '';
337 $this->proxy
= isset( $opt['proxy'] )?
$opt['proxy'] : '';
339 $this->ssl_verifyhost
= (isset( $opt['ssl_verifyhost'] ))?
$opt['ssl_verifyhost']: false;
341 $this->cainfo
= (isset( $opt['cainfo'] ))?
$op['cainfo']: false;
345 public static function newRequest($url, $opt){
346 # select the handler (use curl if available)
347 if ( function_exists( 'curl_init' ) ) {
348 return new curlHttpRequest($url, $opt);
350 return new phpHttpRequest($url, $opt);
355 * Get the contents of a file by HTTP
356 * @param $url string Full URL to act on
357 * @param $Opt associative array Optional array of options:
358 * 'method' => 'GET', 'POST' etc.
359 * 'target_file_path' => if curl should output to a target file
360 * 'adapter' => 'curl', 'soket'
362 public function doRequest() {
363 # Make sure we have a valid url
364 if( !Http
::isValidURI( $this->url
) )
365 return Status
::newFatal('bad-url');
366 //do the actual request:
367 return $this->doReq();
370 class curlHttpRequest
extends HttpRequest
{
371 public function doReq(){
372 global $wgHTTPProxy, $wgTitle;
374 $status = Status
::newGood();
375 $c = curl_init( $this->url
);
377 // only do proxy setup if ( not suppressed $this->proxy === false )
378 if( $this->proxy
!== false ){
380 curl_setopt( $c, CURLOPT_PROXY
, $this->proxy
);
381 } else if ( Http
::isLocalURL( $this->url
) ) {
382 curl_setopt( $c, CURLOPT_PROXY
, 'localhost:80' );
383 } else if ( $wgHTTPProxy ) {
384 curl_setopt( $c, CURLOPT_PROXY
, $wgHTTPProxy );
388 curl_setopt( $c, CURLOPT_TIMEOUT
, $this->timeout
);
389 curl_setopt( $c, CURLOPT_USERAGENT
, Http
::userAgent() );
391 if( $this->ssl_verifyhost
)
392 curl_setopt( $c, CURLOPT_SSL_VERIFYHOST
, $this->ssl_verifyhost
);
395 curl_setopt( $c, CURLOPT_CAINFO
, $this->cainfo
);
397 if ( $this->headers_only
) {
398 curl_setopt( $c, CURLOPT_NOBODY
, true );
399 curl_setopt( $c, CURLOPT_HEADER
, true );
400 } elseif ( $this->method
== 'POST' ) {
401 curl_setopt( $c, CURLOPT_POST
, true );
402 curl_setopt( $c, CURLOPT_POSTFIELDS
, $this->postData
);
403 // Suppress 'Expect: 100-continue' header, as some servers
404 // will reject it with a 417 and Curl won't auto retry
405 // with HTTP 1.0 fallback
406 curl_setopt( $c, CURLOPT_HTTPHEADER
, array( 'Expect:' ) );
408 curl_setopt( $c, CURLOPT_CUSTOMREQUEST
, $this->method
);
411 # Set the referer to $wgTitle, even in command-line mode
412 # This is useful for interwiki transclusion, where the foreign
413 # server wants to know what the referring page is.
414 # $_SERVER['REQUEST_URI'] gives a less reliable indication of the
416 if ( is_object( $wgTitle ) ) {
417 curl_setopt( $c, CURLOPT_REFERER
, $wgTitle->getFullURL() );
420 // set the write back function (if we are writing to a file)
421 if( $this->target_file_path
) {
422 $cwrite = new simpleFileWriter( $this->target_file_path
,
423 $this->upload_session_key
,
424 $this->do_close_session_update
426 if( !$cwrite->status
->isOK() ) {
427 wfDebug( __METHOD__
. "ERROR in setting up simpleFileWriter\n" );
428 $status = $cwrite->status
;
431 curl_setopt( $c, CURLOPT_WRITEFUNCTION
, array( $cwrite, 'callbackWriteBody' ) );
434 // start output grabber:
435 if( !$this->target_file_path
)
438 //run the actual curl_exec:
440 if ( false === curl_exec( $c ) ) {
441 $error_txt ='Error sending request: #' . curl_errno( $c ) .' '. curl_error( $c );
442 wfDebug( __METHOD__
. $error_txt . "\n" );
443 $status = Status
::newFatal( $error_txt );
445 } catch ( Exception
$e ) {
446 // do something with curl exec error?
448 // if direct request output the results to the stats value:
449 if( !$this->target_file_path
&& $status->isOK() ) {
450 $status->value
= ob_get_contents();
453 // if we wrote to a target file close up or return error
454 if( $this->target_file_path
) {
456 if( !$cwrite->status
->isOK() ) {
457 return $cwrite->status
;
461 if ( $this->headers_only
) {
462 $headers = explode( "\n", $status->value
);
463 $headerArray = array();
464 foreach ( $headers as $header ) {
465 if ( !strlen( trim( $header ) ) )
467 $headerParts = explode( ':', $header, 2 );
468 if ( count( $headerParts ) == 1 ) {
469 $headerArray[] = trim( $header );
471 list( $key, $val ) = $headerParts;
472 $headerArray[trim( $key )] = trim( $val );
475 $status->value
= $headerArray;
477 # Don't return the text of error messages, return false on error
478 $retcode = curl_getinfo( $c, CURLINFO_HTTP_CODE
);
479 if ( $retcode != 200 ) {
480 wfDebug( __METHOD__
. ": HTTP return code $retcode\n" );
481 $status = Status
::newFatal( "HTTP return code $retcode\n" );
483 # Don't return truncated output
484 $errno = curl_errno( $c );
485 if ( $errno != CURLE_OK
) {
486 $errstr = curl_error( $c );
487 wfDebug( __METHOD__
. ": CURL error code $errno: $errstr\n" );
488 $status = Status
::newFatal( " CURL error code $errno: $errstr\n" );
493 // return the result obj
497 class phpHttpRequest
extends HttpRequest
{
498 public function doReq() {
499 global $wgTitle, $wgHTTPProxy;
500 # Check for php.ini allow_url_fopen
501 if( !ini_get( 'allow_url_fopen' ) ) {
502 return Status
::newFatal( 'allow_url_fopen needs to be enabled for http copy to work' );
505 // start with good status:
506 $status = Status
::newGood();
508 if ( $this->headers_only
) {
509 $status->value
= get_headers( $this->url
, 1 );
514 $headers = array( "User-Agent: " . Http
::userAgent() );
515 if ( is_object( $wgTitle ) ) {
516 $headers[] = "Referer: ". $wgTitle->getFullURL();
519 if( strcasecmp( $this->method
, 'post' ) == 0 ) {
520 // Required for HTTP 1.0 POSTs
521 $headers[] = "Content-Length: 0";
524 $httpContextOptions = array(
525 'method' => $this->method
,
526 'header' => implode( "\r\n", $headers ),
527 'timeout' => $this->timeout
532 $httpContextOptions['proxy'] = 'tcp://' . $this->proxy
;
533 }else if ( Http
::isLocalURL( $this->url
) ) {
534 $httpContextOptions['proxy'] = 'tcp://localhost:80';
535 } elseif ( $wgHTTPProxy ) {
536 $httpContextOptions['proxy'] = 'tcp://' . $wgHTTPProxy ;
539 $fcontext = stream_context_create (
541 'http' => $httpContextOptions
545 $fh = fopen( $this->url
, "r", false, $fcontext);
547 // set the write back function (if we are writing to a file)
548 if( $this->target_file_path
) {
549 $cwrite = new simpleFileWriter( $this->target_file_path
,
550 $this->upload_session_key
, $this->do_close_session_update
);
551 if( !$cwrite->status
->isOK() ) {
552 wfDebug( __METHOD__
. "ERROR in setting up simpleFileWriter\n" );
553 $status = $cwrite->status
;
557 // Read $fh into the simpleFileWriter (grab in 64K chunks since
558 // it's likely a ~large~ media file)
559 while ( !feof( $fh ) ) {
560 $contents = fread( $fh, 65536 );
561 $cwrite->callbackWriteBody( $fh, $contents );
564 // check for simpleFileWriter error:
565 if( !$cwrite->status
->isOK() ) {
566 return $cwrite->status
;
569 // read $fh into status->value
570 $status->value
= @stream_get_contents
( $fh );
572 //close the url file wrapper
576 if( $status->value
=== false ) {
577 $status->error( 'file_get_contents-failed' );
585 * SimpleFileWriter with session id updates
587 class simpleFileWriter
{
588 var $target_file_path;
590 var $session_id = null;
591 var $session_update_interval = 0; // how often to update the session while downloading
593 function simpleFileWriter( $target_file_path, $upload_session_key,
594 $do_close_session_update = false )
596 $this->target_file_path
= $target_file_path;
597 $this->upload_session_key
= $upload_session_key;
598 $this->status
= Status
::newGood();
599 $this->do_close_session_update
= $do_close_session_update;
601 $this->fp
= fopen( $this->target_file_path
, 'w' );
602 if( $this->fp
=== false ) {
603 $this->status
= Status
::newFatal( 'HTTP::could-not-open-file-for-writing' );
606 $this->prevTime
= time();
609 public function callbackWriteBody( $ch, $data_packet ) {
610 global $wgMaxUploadSize, $wgLang;
612 // write out the content
613 if( fwrite( $this->fp
, $data_packet ) === false ) {
614 wfDebug( __METHOD__
." ::could-not-write-to-file\n" );
615 $this->status
= Status
::newFatal( 'HTTP::could-not-write-to-file' );
621 $this->current_fsize
= filesize( $this->target_file_path
);
623 if( $this->current_fsize
> $wgMaxUploadSize ) {
624 wfDebug( __METHOD__
. " ::http download too large\n" );
625 $this->status
= Status
::newFatal( 'HTTP::file-has-grown-beyond-upload-limit-killing: ' .
626 'downloaded more than ' .
627 $wgLang->formatSize( $wgMaxUploadSize ) . ' ' );
630 // if more than session_update_interval second have passed update_session_progress
631 if( $this->do_close_session_update
&& $this->upload_session_key
&&
632 ( ( time() - $this->prevTime
) > $this->session_update_interval
) ) {
633 $this->prevTime
= time();
634 $session_status = $this->update_session_progress();
635 if( !$session_status->isOK() ) {
636 $this->status
= $session_status;
637 wfDebug( __METHOD__
. ' update session failed or was canceled');
641 return strlen( $data_packet );
644 public function update_session_progress() {
645 global $wgSessionsInMemcached;
646 $status = Status
::newGood();
647 // start the session (if necessary)
648 if( !$wgSessionsInMemcached ) {
649 wfSuppressWarnings();
650 if( session_start() === false ) {
651 wfDebug( __METHOD__
. ' could not start session' );
656 $sd =& $_SESSION['wsDownload'][ $this->upload_session_key
];
657 // check if the user canceled the request:
658 if( isset( $sd['user_cancel'] ) && $sd['user_cancel'] == true ) {
659 //@@todo kill the download
660 return Status
::newFatal( 'user-canceled-request' );
662 // update the progress bytes download so far:
663 $sd['loaded'] = $this->current_fsize
;
665 // close down the session so we can other http queries can get session updates:
666 if( !$wgSessionsInMemcached )
667 session_write_close();
672 public function close() {
673 // do a final session update:
674 if( $this->do_close_session_update
) {
675 $this->update_session_progress();
677 // close up the file handle:
678 if( false === fclose( $this->fp
) ) {
679 $this->status
= Status
::newFatal( 'HTTP::could-not-close-file' );