includes/HttpFunctions.php

   1 <?php
   2 /**
   3  * HTTP handling class
   4  * @defgroup HTTP HTTP
   5  * @file
   6  * @ingroup HTTP
   7  */
   8
   9 class Http {
  10         const SYNC_DOWNLOAD = 1;  // syncronys upload (in a single request)
  11         const ASYNC_DOWNLOAD = 2; // asynchronous upload we should spawn out another process and monitor progress if possible)
  12
  13         var $body = '';
  14
  15         public static function request( $method, $url, $opts = array() ){
  16                 $opts['method'] = ( strtoupper( $method ) == 'GET' || strtoupper( $method ) == 'POST' ) ? strtoupper( $method ) : null;
  17                 $req = new HttpRequest( $url, $opts );
  18                 $status = $req->doRequest();
  19                 if( $status->isOK() ){
  20                         return $status->value;
  21                 } else {
  22                         wfDebug( 'http error: ' . $status->getWikiText() );
  23                         return false;
  24                 }
  25         }
  26
  27         /**
  28          * Simple wrapper for Http::request( 'GET' )
  29          */
  30         public static function get( $url, $timeout = false ) {
  31                 $opts = array();
  32                 if( $timeout )
  33                         $opts['timeout'] = $timeout;
  34                 return Http::request( 'GET', $url, $opts );
  35         }
  36
  37         /**
  38          * Simple wrapper for Http::request( 'POST' )
  39          */
  40         public static function post( $url, $opts = array() ) {
  41                 return Http::request( 'POST', $url, $opts );
  42         }
  43
  44         public static function doDownload( $url, $target_file_path, $dl_mode = self::SYNC_DOWNLOAD, $redirectCount = 0 ){
  45                 global $wgPhpCli, $wgMaxUploadSize, $wgMaxRedirects;
  46                 // do a quick check to HEAD to insure the file size is not > $wgMaxUploadSize
  47                 $headRequest = new HttpRequest( $url, array( 'headers_only' => true ) );
  48                 $headResponse = $headRequest->doRequest();
  49                 if( !$headResponse->isOK() ){
  50                         return $headResponse;
  51                 }
  52                 $head = $headResponse->value;
  53
  54                 // check for redirects:
  55                 if( isset( $head['Location'] ) && strrpos( $head[0], '302' ) !== false ){
  56                         if( $redirectCount < $wgMaxRedirects ){
  57                                 if( UploadFromUrl::isValidURI( $head['Location'] ) ){
  58                                         return self::doDownload( $head['Location'], $target_file_path, $dl_mode, $redirectCount++ );
  59                                 } else {
  60                                         return Status::newFatal( 'upload-proto-error' );
  61                                 }
  62                         } else {
  63                                 return Status::newFatal( 'upload-too-many-redirects' );
  64                         }
  65                 }
  66                 // we did not get a 200 ok response:
  67                 if( strrpos( $head[0], '200 OK' ) === false ){
  68                         return Status::newFatal( 'upload-http-error', htmlspecialchars( $head[0] ) );
  69                 }
  70
  71                 $content_length = ( isset( $head['Content-Length'] ) ) ? $head['Content-Length'] : null;
  72                 if( $content_length ){
  73                         if( $content_length > $wgMaxUploadSize ){
  74                                 return Status::newFatal( 'requested file length ' . $content_length . ' is greater than $wgMaxUploadSize: ' . $wgMaxUploadSize );
  75                         }
  76                 }
  77
  78                 // check if we can find phpCliPath (for doing a background shell request to php to do the download:
  79                 if( $wgPhpCli && wfShellExecEnabled() && $dl_mode == self::ASYNC_DOWNLOAD ){
  80                         wfDebug( __METHOD__ . "\ASYNC_DOWNLOAD\n" );
  81                         // setup session and shell call:
  82                         return self::initBackgroundDownload( $url, $target_file_path, $content_length );
  83                 } else {
  84                         wfDebug( __METHOD__ . "\nSYNC_DOWNLOAD\n" );
  85                         // SYNC_DOWNLOAD download as much as we can in the time we have to execute
  86                         $opts['method'] = 'GET';
  87                         $opts['target_file_path'] = $target_file_path;
  88                         $req = new HttpRequest( $url, $opts );
  89                         return $req->doRequest();
  90                 }
  91         }
  92
  93         /**
  94          * a non blocking request (generally an exit point in the application)
  95          * should write to a file location and give updates
  96          *
  97          */
  98         private static function initBackgroundDownload( $url, $target_file_path, $content_length = null ){
  99                 global $wgMaxUploadSize, $IP, $wgPhpCli, $wgServer;
 100                 $status = Status::newGood();
 101
 102                 // generate a session id with all the details for the download (pid, target_file_path )
 103                 $upload_session_key = self::getUploadSessionKey();
 104                 $session_id = session_id();
 105
 106                 // store the url and target path:
 107                 $_SESSION['wsDownload'][$upload_session_key]['url'] = $url;
 108                 $_SESSION['wsDownload'][$upload_session_key]['target_file_path'] = $target_file_path;
 109                 // since we request from the cmd line we lose the original host name pass in the session:
 110                 $_SESSION['wsDownload'][$upload_session_key]['orgServer'] = $wgServer;
 111
 112                 if( $content_length )
 113                         $_SESSION['wsDownload'][$upload_session_key]['content_length'] = $content_length;
 114
 115                 // set initial loaded bytes:
 116                 $_SESSION['wsDownload'][$upload_session_key]['loaded'] = 0;
 117
 118                 // run the background download request:
 119                 $cmd = $wgPhpCli . ' ' . $IP . "/maintenance/http_session_download.php --sid {$session_id} --usk {$upload_session_key}";
 120                 $pid = wfShellBackgroundExec( $cmd );
 121                 // the pid is not of much use since we won't be visiting this same apache any-time soon.
 122                 if( !$pid )
 123                         return Status::newFatal( 'could not run background shell exec' );
 124
 125                 // update the status value with the $upload_session_key (for the user to check on the status of the upload)
 126                 $status->value = $upload_session_key;
 127
 128                 // return good status
 129                 return $status;
 130         }
 131
 132         static function getUploadSessionKey(){
 133                 $key = mt_rand( 0, 0x7fffffff );
 134                 $_SESSION['wsUploadData'][$key] = array();
 135                 return $key;
 136         }
 137
 138         /**
 139          * used to run a session based download. Is initiated via the shell.
 140          *
 141          * @param $session_id String: the session id to grab download details from
 142          * @param $upload_session_key String: the key of the given upload session
 143          *  (a given client could have started a few http uploads at once)
 144          */
 145         public static function doSessionIdDownload( $session_id, $upload_session_key ){
 146                 global $wgUser, $wgEnableWriteAPI, $wgAsyncHTTPTimeout, $wgServer;
 147                 wfDebug( __METHOD__ . "\n\ndoSessionIdDownload:\n\n" );
 148
 149                 // set session to the provided key:
 150                 session_id( $session_id );
 151                 // start the session
 152                 if( session_start() === false ){
 153                         wfDebug( __METHOD__ . ' could not start session' );
 154                 }
 155                 // get all the vars we need from session_id
 156                 if( !isset( $_SESSION[ 'wsDownload' ][$upload_session_key] ) ){
 157                         wfDebug(  __METHOD__ . ' Error:could not find upload session');
 158                         exit();
 159                 }
 160                 // setup the global user from the session key we just inherited
 161                 $wgUser = User::newFromSession();
 162
 163                 // grab the session data to setup the request:
 164                 $sd =& $_SESSION['wsDownload'][$upload_session_key];
 165
 166                 // update the wgServer var ( since cmd line thinks we are localhost when we are really orgServer)
 167                 if( isset( $sd['orgServer'] ) && $sd['orgServer'] ){
 168                         $wgServer = $sd['orgServer'];
 169                 }
 170                 // close down the session so we can other http queries can get session updates:
 171                 session_write_close();
 172
 173                 $req = new HttpRequest( $sd['url'], array(
 174                         'target_file_path'  => $sd['target_file_path'],
 175                         'upload_session_key'=> $upload_session_key,
 176                         'timeout'           => $wgAsyncHTTPTimeout,
 177                         'do_close_session_update' => true
 178                 ) );
 179                 // run the actual request .. (this can take some time)
 180                 wfDebug( __METHOD__ . 'do Request: ' . $sd['url'] . ' tf: ' . $sd['target_file_path'] );
 181                 $status = $req->doRequest();
 182                 //wfDebug("done with req status is: ". $status->isOK(). ' '.$status->getWikiText(). "\n");
 183
 184                 // start up the session again:
 185                 if( session_start() === false ){
 186                         wfDebug( __METHOD__ . ' ERROR:: Could not start session');
 187                 }
 188                 // grab the updated session data pointer
 189                 $sd =& $_SESSION['wsDownload'][$upload_session_key];
 190                 // if error update status:
 191                 if( !$status->isOK() ){
 192                         $sd['apiUploadResult'] = ApiFormatJson::getJsonEncode(
 193                                 array( 'error' => $status->getWikiText() )
 194                         );
 195                 }
 196                 // if status okay process upload using fauxReq to api:
 197                 if( $status->isOK() ){
 198                         // setup the FauxRequest
 199                         $fauxReqData = $sd['mParams'];
 200
 201                         // Fix boolean parameters
 202                         foreach( $fauxReqData as $k => $v ) {
 203                                 if( $v === false )
 204                                         unset( $fauxReqData[$k] );
 205                         }
 206
 207                         $fauxReqData['action'] = 'upload';
 208                         $fauxReqData['format'] = 'json';
 209                         $fauxReqData['internalhttpsession'] = $upload_session_key;
 210                         // evil but no other clean way about it:
 211                         $faxReq = new FauxRequest( $fauxReqData, true );
 212                         $processor = new ApiMain( $faxReq, $wgEnableWriteAPI );
 213
 214                         //init the mUpload var for the $processor
 215                         $processor->execute();
 216                         $processor->getResult()->cleanUpUTF8();
 217                         $printer = $processor->createPrinterByName( 'json' );
 218                         $printer->initPrinter( false );
 219                         ob_start();
 220                         $printer->execute();
 221                         $apiUploadResult = ob_get_clean();
 222
 223                         wfDebug( __METHOD__ . "\n\n got api result:: $apiUploadResult \n" );
 224                         // the status updates runner will grab the result form the session:
 225                         $sd['apiUploadResult'] = $apiUploadResult;
 226                 }
 227                 // close the session:
 228                 session_write_close();
 229         }
 230
 231         /**
 232          * Check if the URL can be served by localhost
 233          * @param $url string Full url to check
 234          * @return bool
 235          */
 236         public static function isLocalURL( $url ) {
 237                 global $wgCommandLineMode, $wgConf;
 238                 if ( $wgCommandLineMode ) {
 239                         return false;
 240                 }
 241
 242                 // Extract host part
 243                 $matches = array();
 244                 if ( preg_match( '!^http://([\w.-]+)[/:].*$!', $url, $matches ) ) {
 245                         $host = $matches[1];
 246                         // Split up dotwise
 247                         $domainParts = explode( '.', $host );
 248                         // Check if this domain or any superdomain is listed in $wgConf as a local virtual host
 249                         $domainParts = array_reverse( $domainParts );
 250                         for ( $i = 0; $i < count( $domainParts ); $i++ ) {
 251                                 $domainPart = $domainParts[$i];
 252                                 if ( $i == 0 ) {
 253                                         $domain = $domainPart;
 254                                 } else {
 255                                         $domain = $domainPart . '.' . $domain;
 256                                 }
 257                                 if ( $wgConf->isLocalVHost( $domain ) ) {
 258                                         return true;
 259                                 }
 260                         }
 261                 }
 262                 return false;
 263         }
 264
 265         /**
 266          * Return a standard user-agent we can use for external requests.
 267          */
 268         public static function userAgent() {
 269                 global $wgVersion;
 270                 return "MediaWiki/$wgVersion";
 271         }
 272 }
 273
 274 class HttpRequest {
 275         var $target_file_path;
 276         var $upload_session_key;
 277
 278         function __construct( $url, $opt ){
 279                 global $wgSyncHTTPTimeout;
 280                 // double check that it's a valid url:
 281                 $this->url = $url;
 282
 283                 // set the timeout to default sync timeout (unless the timeout option is provided)
 284                 $this->timeout = ( isset( $opt['timeout'] ) ) ? $opt['timeout'] : $wgSyncHTTPTimeout;
 285                 $this->method = ( isset( $opt['method'] ) ) ? $opt['method'] : 'GET';
 286                 $this->target_file_path = ( isset( $opt['target_file_path'] ) ) ? $opt['target_file_path'] : false;
 287                 $this->upload_session_key = ( isset( $opt['upload_session_key'] ) ) ? $opt['upload_session_key'] : false;
 288                 $this->headers_only = ( isset( $opt['headers_only'] ) ) ? $opt['headers_only'] : false;
 289                 $this->do_close_session_update = isset( $opt['do_close_session_update'] );
 290                 $this->postData = isset( $opt['postdata'] ) ? $opt['postdata'] : '';
 291         }
 292
 293         /**
 294          * Get the contents of a file by HTTP
 295          * @param $url string Full URL to act on
 296          * @param $Opt associative array Optional array of options:
 297          *     'method'           => 'GET', 'POST' etc.
 298          *     'target_file_path' => if curl should output to a target file
 299          *     'adapter'          => 'curl', 'soket'
 300          */
 301         public function doRequest() {
 302                 # Make sure we have a valid url
 303                 if( !UploadFromUrl::isValidURI( $this->url ) )
 304                         return Status::newFatal('bad-url');
 305
 306                 # Use curl if available
 307                 if ( function_exists( 'curl_init' ) ) {
 308                         return $this->doCurlReq();
 309                 } else {
 310                         return $this->doPhpReq();
 311                 }
 312         }
 313
 314         private function doCurlReq(){
 315                 global $wgHTTPProxy, $wgTitle;
 316
 317                 $status = Status::newGood();
 318                 $c = curl_init( $this->url );
 319
 320                 // proxy setup:
 321                 if ( Http::isLocalURL( $this->url ) ) {
 322                         curl_setopt( $c, CURLOPT_PROXY, 'localhost:80' );
 323                 } else if ( $wgHTTPProxy ) {
 324                         curl_setopt( $c, CURLOPT_PROXY, $wgHTTPProxy );
 325                 }
 326
 327                 curl_setopt( $c, CURLOPT_TIMEOUT, $this->timeout );
 328                 curl_setopt( $c, CURLOPT_USERAGENT, Http::userAgent() );
 329
 330                 if ( $this->headers_only ) {
 331                         curl_setopt( $c, CURLOPT_NOBODY, true );
 332                         curl_setopt( $c, CURLOPT_HEADER, true );
 333                 } elseif ( $this->method == 'POST' ) {
 334                         curl_setopt( $c, CURLOPT_POST, true );
 335                         curl_setopt( $c, CURLOPT_POSTFIELDS, $this->postData );
 336                 } else {
 337                         curl_setopt( $c, CURLOPT_CUSTOMREQUEST, $this->method );
 338                 }
 339
 340                 # Set the referer to $wgTitle, even in command-line mode
 341                 # This is useful for interwiki transclusion, where the foreign
 342                 # server wants to know what the referring page is.
 343                 # $_SERVER['REQUEST_URI'] gives a less reliable indication of the
 344                 # referring page.
 345                 if ( is_object( $wgTitle ) ) {
 346                         curl_setopt( $c, CURLOPT_REFERER, $wgTitle->getFullURL() );
 347                 }
 348
 349                 // set the write back function (if we are writing to a file)
 350                 if( $this->target_file_path ){
 351                         $cwrite = new simpleFileWriter( $this->target_file_path,
 352                                 $this->upload_session_key,
 353                                 $this->do_close_session_update
 354                         );
 355                         if( !$cwrite->status->isOK() ){
 356                                 wfDebug( __METHOD__ . "ERROR in setting up simpleFileWriter\n" );
 357                                 $status = $cwrite->status;
 358                                 return $status;
 359                         }
 360                         curl_setopt( $c, CURLOPT_WRITEFUNCTION, array( $cwrite, 'callbackWriteBody' ) );
 361                 }
 362
 363                 // start output grabber:
 364                 if( !$this->target_file_path )
 365                         ob_start();
 366
 367                 //run the actual curl_exec:
 368                 try {
 369                         if ( false === curl_exec( $c ) ) {
 370                                 $error_txt ='Error sending request: #' . curl_errno( $c ) .' '. curl_error( $c );
 371                                 wfDebug( __METHOD__ . $error_txt . "\n" );
 372                                 $status = Status::newFatal( $error_txt );
 373                         }
 374                 } catch ( Exception $e ) {
 375                         // do something with curl exec error?
 376                 }
 377                 // if direct request output the results to the stats value:
 378                 if( !$this->target_file_path && $status->isOK() ){
 379                         $status->value = ob_get_contents();
 380                         ob_end_clean();
 381                 }
 382                 // if we wrote to a target file close up or return error
 383                 if( $this->target_file_path ){
 384                         $cwrite->close();
 385                         if( !$cwrite->status->isOK() ){
 386                                 return $cwrite->status;
 387                         }
 388                 }
 389
 390                 if ( $this->headers_only ) {
 391                         $headers = explode( "\n", $status->value );
 392                         $headerArray = array();
 393                         foreach ( $headers as $header ) {
 394                                 if ( !strlen( trim( $header ) ) )
 395                                         continue;
 396                                 $headerParts = explode( ':', $header, 2 );
 397                                 if ( count( $headerParts ) == 1 ) {
 398                                         $headerArray[] = trim( $header );
 399                                 } else {
 400                                         list( $key, $val ) = $headerParts;
 401                                         $headerArray[trim( $key )] = trim( $val );
 402                                 }
 403                         }
 404                         $status->value = $headerArray;
 405                 } else {
 406                         # Don't return the text of error messages, return false on error
 407                         $retcode = curl_getinfo( $c, CURLINFO_HTTP_CODE );
 408                         if ( $retcode != 200 ) {
 409                                 wfDebug( __METHOD__ . ": HTTP return code $retcode\n" );
 410                                 $status = Status::newFatal( "HTTP return code $retcode\n" );
 411                         }
 412                         # Don't return truncated output
 413                         $errno = curl_errno( $c );
 414                         if ( $errno != CURLE_OK ) {
 415                                 $errstr = curl_error( $c );
 416                                 wfDebug( __METHOD__ . ": CURL error code $errno: $errstr\n" );
 417                                 $status = Status::newFatal( " CURL error code $errno: $errstr\n" );
 418                         }
 419                 }
 420
 421                 curl_close( $c );
 422
 423                 // return the result obj
 424                 return $status;
 425         }
 426
 427         public function doPhpReq(){
 428                 global $wgTitle, $wgHTTPProxy;
 429
 430                 # Check for php.ini allow_url_fopen
 431                 if( !ini_get( 'allow_url_fopen' ) ){
 432                         return Status::newFatal( 'allow_url_fopen needs to be enabled for http copy to work' );
 433                 }
 434
 435                 // start with good status:
 436                 $status = Status::newGood();
 437
 438                 if ( $this->headers_only ) {
 439                         $status->value = get_headers( $this->url, 1 );
 440                         return $status;
 441                 }
 442
 443                 // setup the headers
 444                 $headers = array( "User-Agent: " . Http::userAgent() );
 445                 if ( is_object( $wgTitle ) ) {
 446                         $headers[] = "Referer: ". $wgTitle->getFullURL();
 447                 }
 448
 449                 if( strcasecmp( $this->method, 'post' ) == 0 ) {
 450                         // Required for HTTP 1.0 POSTs
 451                         $headers[] = "Content-Length: 0";
 452                 }
 453                 $fcontext = stream_context_create ( array(
 454                         'http' => array(
 455                                 'method' => $this->method,
 456                                 'header' => implode( "\r\n", $headers ),
 457                                 'timeout' => $this->timeout )
 458                         )
 459                 );
 460
 461                 $fh = fopen( $this->url, "r", false, $fcontext);
 462
 463                 // set the write back function (if we are writing to a file)
 464                 if( $this->target_file_path ){
 465                         $cwrite = new simpleFileWriter( $this->target_file_path, $this->upload_session_key, $this->do_close_session_update );
 466                         if( !$cwrite->status->isOK() ){
 467                                 wfDebug( __METHOD__ . "ERROR in setting up simpleFileWriter\n" );
 468                                 $status = $cwrite->status;
 469                                 return $status;
 470                         }
 471
 472                         // read $fh into the simpleFileWriter (grab in 64K chunks since its likely a media file)
 473                         while ( !feof( $fh ) ) {
 474                                 $contents = fread( $fh, 65536 );
 475                                 $cwrite->callbackWriteBody( $fh, $contents );
 476                         }
 477
 478                         $cwrite->close();
 479                         // check for simpleFileWriter error:
 480                         if( !$cwrite->status->isOK() ){
 481                                 return $cwrite->status;
 482                         }
 483                 } else {
 484                         // read $fh into status->value
 485                         $status->value = @stream_get_contents( $fh );
 486                 }
 487                 //close the url file wrapper
 488                 fclose( $fh );
 489
 490                 // check for "false"
 491                 if( $status->value === false ){
 492                         $status->error( 'file_get_contents-failed' );
 493                 }
 494                 return $status;
 495         }
 496
 497 }
 498
 499 /**
 500  * a simpleFileWriter with session id updates
 501  */
 502 class simpleFileWriter {
 503         var $target_file_path;
 504         var $status = null;
 505         var $session_id = null;
 506         var $session_update_interval = 0; // how often to update the session while downloading
 507
 508         function simpleFileWriter( $target_file_path, $upload_session_key, $do_close_session_update = false ){
 509                 $this->target_file_path = $target_file_path;
 510                 $this->upload_session_key = $upload_session_key;
 511                 $this->status = Status::newGood();
 512                 $this->do_close_session_update = $do_close_session_update;
 513                 // open the file:
 514                 $this->fp = fopen( $this->target_file_path, 'w' );
 515                 if( $this->fp === false ){
 516                         $this->status = Status::newFatal( 'HTTP::could-not-open-file-for-writing' );
 517                 }
 518                 // true start time
 519                 $this->prevTime = time();
 520         }
 521
 522         public function callbackWriteBody( $ch, $data_packet ){
 523                 global $wgMaxUploadSize, $wgLang;
 524
 525                 // write out the content
 526                 if( fwrite( $this->fp, $data_packet ) === false ){
 527                         wfDebug( __METHOD__ ." ::could-not-write-to-file\n" );
 528                         $this->status = Status::newFatal( 'HTTP::could-not-write-to-file' );
 529                         return 0;
 530                 }
 531
 532                 // check file size:
 533                 clearstatcache();
 534                 $this->current_fsize = filesize( $this->target_file_path );
 535
 536                 if( $this->current_fsize > $wgMaxUploadSize ){
 537                         wfDebug( __METHOD__ . " ::http download too large\n" );
 538                         $this->status = Status::newFatal( 'HTTP::file-has-grown-beyond-upload-limit-killing: downloaded more than ' .
 539                                 $wgLang->formatSize( $wgMaxUploadSize ) . ' ' );
 540                         return 0;
 541                 }
 542
 543                 // if more than session_update_interval second have passed update_session_progress
 544                 if( $this->do_close_session_update && $this->upload_session_key &&
 545                         ( ( time() - $this->prevTime ) > $this->session_update_interval ) ) {
 546                                 $this->prevTime = time();
 547                                 $session_status = $this->update_session_progress();
 548                                 if( !$session_status->isOK() ){
 549                                         $this->status = $session_status;
 550                                         wfDebug( __METHOD__ . ' update session failed or was canceled');
 551                                         return 0;
 552                                 }
 553                 }
 554                 return strlen( $data_packet );
 555         }
 556
 557         public function update_session_progress(){
 558                 $status = Status::newGood();
 559                 // start the session (if nessesary)
 560                 if(  @session_start() === false){
 561                         wfDebug( __METHOD__ . ' could not start session' );
 562                         exit( 0 );
 563                 }
 564                 $sd =& $_SESSION['wsDownload'][$this->upload_session_key];
 565                 // check if the user canceled the request:
 566                 if( isset( $sd['user_cancel'] ) && $sd['user_cancel'] == true ){
 567                         //@@todo kill the download
 568                         return Status::newFatal( 'user-canceled-request' );
 569                 }
 570                 // update the progress bytes download so far:
 571                 $sd['loaded'] = $this->current_fsize;
 572                 // close down the session so we can other http queries can get session updates:
 573
 574                 session_write_close();
 575
 576                 return $status;
 577         }
 578
 579         public function close(){
 580                 // do a final session update:
 581                 if( $this->do_close_session_update ){
 582                         $this->update_session_progress();
 583                 }
 584                 // close up the file handle:
 585                 if( false === fclose( $this->fp ) ){
 586                         $this->status = Status::newFatal( 'HTTP::could-not-close-file' );
 587                 }
 588         }
 589
 590 }