includes/HttpFunctions.php

   1 <?php
   2 /**
   3  * HTTP handling class
   4  *
   5  */
   6
   7
   8 class Http {
   9         const SYNC_DOWNLOAD = 1;  //syncronys upload (in a single request)
  10         const ASYNC_DOWNLOAD = 2; //asynchronous upload we should spawn out another process and monitor progress if possible)
  11
  12         var $body = '';
  13         public static function request( $url, $opts = array() ) {
  14        $req = new HttpRequest( $url, $opts );
  15        $status = $req->doRequest();
  16        if( $status->isOK() ){
  17            return $status->value;
  18        }else{
  19            return false;
  20        }
  21     }
  22         /**
  23          * Simple wrapper for Http::request( 'GET' )
  24          */
  25         public static function get( $url, $opts = array() ) {
  26                 $opt['method'] = 'GET';
  27                 return Http::request($url, $opts);
  28         }
  29         /**
  30          * Simple wrapper for Http::request( 'POST' )
  31          */
  32         public static function post( $url, $opts = array() ) {
  33                 $opts['method']='POST';
  34                 return Http::request($url, $opts);
  35         }
  36
  37         public static function doDownload( $url, $target_file_path , $dl_mode = self::SYNC_DOWNLOAD , $redirectCount=0){
  38                 global $wgPhpCliPath, $wgMaxUploadSize, $wgMaxRedirects;
  39                 //do a quick check to HEAD to insure the file size is not > $wgMaxUploadSize
  40                 $head = get_headers($url, 1);
  41
  42                 //check for redirects:
  43                 if( isset( $head['Location'] ) && strrpos($head[0], '302')!==false ){
  44                         if($redirectCount < $wgMaxRedirects){
  45                                 if( UploadFromUrl::isValidURI( $head['Location'] )){
  46                                         return self::doDownload ( $head['Location'], $target_file_path , $dl_mode, $redirectCount++);
  47                                 }else{
  48                                         return Status::newFatal('upload-proto-error');
  49                                 }
  50                         }else{
  51                                 return Status::newFatal('upload-too-many-redirects');
  52                         }
  53                 }
  54                 //we did not get a 200 ok response:
  55                 if( strrpos($head[0], '200 OK') === false){
  56                         return Status::newFatal( 'upload-http-error', htmlspecialchars($head[0]) );
  57                 }
  58
  59
  60                 $content_length = (isset($head['Content-Length']))?$head['Content-Length']:null;
  61                 if($content_length){
  62                         if($content_length > $wgMaxUploadSize){
  63                                 return Status::newFatal('requested file length ' . $content_length . ' is greater than $wgMaxUploadSize: ' . $wgMaxUploadSize);
  64                         }
  65                 }
  66
  67                 //check if we can find phpCliPath (for doing a background shell request to php to do the download:
  68                 if( $wgPhpCliPath && wfShellExecEnabled() && $dl_mode == self::ASYNC_DOWNLOAD){
  69                         wfDebug("\ASYNC_DOWNLOAD\n");
  70                         //setup session and shell call:
  71                         return self::initBackgroundDownload( $url, $target_file_path, $content_length );
  72                 }else if( $dl_mode== self::SYNC_DOWNLOAD ){
  73                         wfDebug("\nSYNC_DOWNLOAD\n");
  74                         //SYNC_DOWNLOAD download as much as we can in the time we have to execute
  75                         $opts['method']='GET';
  76                         $opts['target_file_path'] = $target_file_path;
  77                         $req = new HttpRequest($url, $opts );
  78                         return $req->doRequest();
  79                 }
  80         }
  81         /**
  82          * a non blocking request (generally an exit point in the application)
  83          * should write to a file location and give updates
  84          *
  85          */
  86         private function initBackgroundDownload( $url, $target_file_path, $content_length = null ){
  87                 global $wgMaxUploadSize, $IP, $wgPhpCliPath;
  88                 $status = Status::newGood();
  89
  90                 //generate a session id with all the details for the download (pid, target_file_path )
  91                 $upload_session_key = self::getUploadSessionKey();
  92                 $session_id = session_id();
  93
  94                 //store the url and target path:
  95                 $_SESSION[ 'wsDownload' ][$upload_session_key]['url'] = $url;
  96                 $_SESSION[ 'wsDownload' ][$upload_session_key]['target_file_path'] = $target_file_path;
  97
  98                 if($content_length)
  99                         $_SESSION[ 'wsDownload' ][$upload_session_key]['content_length'] = $content_length;
 100
 101                 //set initial loaded bytes:
 102                 $_SESSION[ 'wsDownload' ][$upload_session_key]['loaded'] = 0;
 103
 104
 105                 //run the background download request:
 106                 $cmd = $wgPhpCliPath . ' ' . $IP . "/maintenance/http_session_download.php --sid {$session_id} --usk {$upload_session_key}";
 107                 $pid = wfShellBackgroundExec($cmd , $retval);
 108                 //the pid is not of much use since we won't be visiting this same apache any-time soon.
 109                 if(!$pid)
 110                         return Status::newFatal('could not run background shell exec');
 111
 112                 //update the status value with the $upload_session_key (for the user to check on the status of the upload)
 113                 $status->value = $upload_session_key;
 114
 115                 //return good status
 116                 return $status;
 117         }
 118         function getUploadSessionKey(){
 119                 $key = mt_rand( 0, 0x7fffffff );
 120                 $_SESSION['wsUploadData'][$key] = array();
 121                 return $key;
 122         }
 123         /**
 124          * used to run a session based download. Is initiated via the shell.
 125          *
 126          * @param string $session_id  // the session id to grab download details from
 127          * @param string $upload_session_key //the key of the given upload session
 128          *                      (a given client could have started a few http uploads at once)
 129          */
 130         public static function doSessionIdDownload( $session_id, $upload_session_key ){
 131                 global $wgUser, $wgEnableWriteAPI, $wgAsyncHTTPTimeout;
 132                 wfDebug("\n\ndoSessionIdDownload\n\n");
 133                 //set session to the provided key:
 134                 session_id($session_id);
 135                 //start the session
 136                 if( session_start() === false){
 137                         wfDebug( __METHOD__ . ' could not start session');
 138                 }
 139                 //get all the vars we need from session_id
 140                 if(!isset($_SESSION[ 'wsDownload' ][$upload_session_key])){
 141                         wfDebug(  __METHOD__ .' Error:could not find upload session');
 142                         exit();
 143                 }
 144                 //setup the global user from the session key we just inherited
 145                 $wgUser = User::newFromSession();
 146
 147                 //grab the session data to setup the request:
 148                 $sd =& $_SESSION[ 'wsDownload' ][$upload_session_key];
 149                 //close down the session so we can other http queries can get session updates:
 150                 session_write_close();
 151
 152                 $req = new HttpRequest( $sd['url'], array(
 153                         'target_file_path'      => $sd['target_file_path'],
 154                         'upload_session_key'=> $upload_session_key,
 155                         'timeout'                       => $wgAsyncHTTPTimeout
 156                 ) );
 157                 //run the actual request .. (this can take some time)
 158                 wfDebug("do Request: " . $sd['url'] . ' tf: ' . $sd['target_file_path'] );
 159                 $status = $req->doRequest();
 160                 //wfDebug("done with req status is: ". $status->isOK(). ' '.$status->getWikiText(). "\n");
 161
 162                 //start up the session again:
 163                 if( session_start() === false){
 164                         wfDebug( __METHOD__ . ' ERROR:: Could not start session');
 165                 }
 166                 //grab the updated session data pointer
 167                 $sd =& $_SESSION[ 'wsDownload' ][$upload_session_key];
 168                 //if error update status:
 169                 if( !$status->isOK() ){
 170                         $sd['apiUploadResult']= ApiFormatJson::getJsonEncode(
 171                                                                                 array( 'error' => $status->getWikiText() )
 172                                                                         );
 173                 }
 174                 //if status oky process upload using fauxReq to api:
 175                 if( $status->isOK() ){
 176                         //setup the faxRequest
 177                         $fauxReqData = $sd['mParams'];
 178                         $fauxReqData['action'] = 'upload';
 179                         $fauxReqData['format'] = 'json';
 180                         $fauxReqData['internalhttpsession'] = $upload_session_key;
 181                         //evil but no other clean way about it:
 182
 183                         $faxReq = new FauxRequest($fauxReqData, true);
 184                         $processor = new ApiMain($faxReq, $wgEnableWriteAPI);
 185
 186                         //init the mUpload var for the $processor
 187                         $processor->execute();
 188                         $processor->getResult()->cleanUpUTF8();
 189                         $printer = $processor->createPrinterByName('json');
 190                         $printer->initPrinter(false);
 191                         ob_start();
 192                         $printer->execute();
 193                         $apiUploadResult = ob_get_clean();
 194
 195                         wfDebug("\n\n got api result:: $apiUploadResult \n" );
 196                         //the status updates runner will grab the result form the session:
 197                         $sd['apiUploadResult'] = $apiUploadResult;
 198                 }
 199                 //close the session:
 200                 session_write_close();
 201         }
 202
 203         /**
 204          * Check if the URL can be served by localhost
 205          * @param $url string Full url to check
 206          * @return bool
 207          */
 208         public static function isLocalURL( $url ) {
 209                 global $wgCommandLineMode, $wgConf;
 210                 if ( $wgCommandLineMode ) {
 211                         return false;
 212                 }
 213
 214                 // Extract host part
 215                 $matches = array();
 216                 if ( preg_match( '!^http://([\w.-]+)[/:].*$!', $url, $matches ) ) {
 217                         $host = $matches[1];
 218                         // Split up dotwise
 219                         $domainParts = explode( '.', $host );
 220                         // Check if this domain or any superdomain is listed in $wgConf as a local virtual host
 221                         $domainParts = array_reverse( $domainParts );
 222                         for ( $i = 0; $i < count( $domainParts ); $i++ ) {
 223                                 $domainPart = $domainParts[$i];
 224                                 if ( $i == 0 ) {
 225                                         $domain = $domainPart;
 226                                 } else {
 227                                         $domain = $domainPart . '.' . $domain;
 228                                 }
 229                                 if ( $wgConf->isLocalVHost( $domain ) ) {
 230                                         return true;
 231                                 }
 232                         }
 233                 }
 234                 return false;
 235         }
 236
 237         /**
 238          * Return a standard user-agent we can use for external requests.
 239          */
 240         public static function userAgent() {
 241                 global $wgVersion;
 242                 return "MediaWiki/$wgVersion";
 243         }
 244 }
 245 class HttpRequest{
 246         var $target_file_path;
 247         var $upload_session_key;
 248         function __construct($url, $opt){
 249                 global $wgSyncHTTPTimeout;
 250                 $this->url = $url;
 251                 //set the timeout to default sync timeout (unless the timeout option is provided)
 252                 $this->timeout = (isset($opt['timeout']))?$opt['timeout']:$wgSyncHTTPTimeout;
 253                 $this->method = (isset($opt['method']))?$opt['method']:'GET';
 254                 $this->target_file_path = (isset($opt['target_file_path']))?$opt['target_file_path']:false;
 255                 $this->upload_session_key = (isset($opt['upload_session_key']))?$opt['upload_session_key']:false;
 256         }
 257 /**
 258          * Get the contents of a file by HTTP
 259          * @param $url string Full URL to act on
 260          * @param $Opt associative array Optional array of options:
 261          *              'method'          => 'GET', 'POST' etc.
 262          *              'target_file_path' => if curl should output to a target file
 263          *              'adapter'         => 'curl', 'soket'
 264          */
 265          public function doRequest() {
 266                 # Use curl if available
 267                 if ( function_exists( 'curl_init' ) ) {
 268                         return $this->doCurlReq();
 269                 }else{
 270                         return $this->doPhpReq();
 271                 }
 272          }
 273          private function doCurlReq(){
 274                 global $wgHTTPProxy, $wgTitle;
 275
 276                 $status = Status::newGood();
 277                 $c = curl_init( $this->url );
 278
 279                 //proxy setup:
 280                 if ( Http::isLocalURL( $this->url ) ) {
 281                         curl_setopt( $c, CURLOPT_PROXY, 'localhost:80' );
 282                 } else if ($wgHTTPProxy) {
 283                         curl_setopt($c, CURLOPT_PROXY, $wgHTTPProxy);
 284                 }
 285
 286                 curl_setopt( $c, CURLOPT_TIMEOUT, $this->timeout );
 287
 288
 289                 curl_setopt( $c, CURLOPT_USERAGENT, Http::userAgent() );
 290
 291                 if ( $this->method == 'POST' ) {
 292                         curl_setopt( $c, CURLOPT_POST, true );
 293                         curl_setopt( $c, CURLOPT_POSTFIELDS, '' );
 294                 }else{
 295                         curl_setopt( $c, CURLOPT_CUSTOMREQUEST, $this->method );
 296                 }
 297
 298                 # Set the referer to $wgTitle, even in command-line mode
 299                 # This is useful for interwiki transclusion, where the foreign
 300                 # server wants to know what the referring page is.
 301                 # $_SERVER['REQUEST_URI'] gives a less reliable indication of the
 302                 # referring page.
 303                 if ( is_object( $wgTitle ) ) {
 304                         curl_setopt( $c, CURLOPT_REFERER, $wgTitle->getFullURL() );
 305                 }
 306
 307                 //set the write back function (if we are writing to a file)
 308                 if( $this->target_file_path ){
 309                         $cwrite = new simpleFileWriter( $this->target_file_path, $this->upload_session_key );
 310                         if(!$cwrite->status->isOK()){
 311                                 wfDebug("ERROR in setting up simpleFileWriter\n");
 312                                 $status = $cwrite->status;
 313                         }
 314                         curl_setopt( $c, CURLOPT_WRITEFUNCTION, array($cwrite, 'callbackWriteBody') );
 315                 }
 316
 317                 //start output grabber:
 318                 if(!$this->target_file_path)
 319                         ob_start();
 320
 321                 //run the actual curl_exec:
 322                 try {
 323             if (false === curl_exec($c)) {
 324                 $error_txt ='Error sending request: #' . curl_errno($c) .' '. curl_error($c);
 325                 wfDebug($error_txt . "\n");
 326                 $status = Status::newFatal( $error_txt);
 327             }
 328         } catch (Exception $e) {
 329                 //do something with curl exec error?
 330         }
 331                 //if direct request output the results to the stats value:
 332                 if( !$this->target_file_path && $status->isOK() ){
 333                 $status->value = ob_get_contents();
 334                         ob_end_clean();
 335                 }
 336                 //if we wrote to a target file close up or return error
 337                 if( $this->target_file_path ){
 338                         $cwrite->close();
 339                         if( ! $cwrite->status->isOK() ){
 340                                 return $cwrite->status;
 341                         }
 342                 }
 343
 344                 # Don't return the text of error messages, return false on error
 345                 $retcode = curl_getinfo( $c, CURLINFO_HTTP_CODE );
 346                 if ( $retcode != 200 ) {
 347                         wfDebug( __METHOD__ . ": HTTP return code $retcode\n" );
 348                         $status = Status::newFatal( "HTTP return code $retcode\n" );
 349                 }
 350                 # Don't return truncated output
 351                 $errno = curl_errno( $c );
 352                 if ( $errno != CURLE_OK ) {
 353                         $errstr = curl_error( $c );
 354                         wfDebug( __METHOD__ . ": CURL error code $errno: $errstr\n" );
 355                                 $status = Status::newFatal( " CURL error code $errno: $errstr\n" );
 356                 }
 357                 curl_close( $c );
 358
 359                 //return the result obj
 360                 return $status;
 361         }
 362         public function doPhpReq(){
 363                 #$use file_get_contents...
 364                 # This doesn't have local fetch capabilities...
 365
 366                 $headers = array( "User-Agent: " . self :: userAgent() );
 367                 if( strcasecmp( $method, 'post' ) == 0 ) {
 368                         // Required for HTTP 1.0 POSTs
 369                         $headers[] = "Content-Length: 0";
 370                 }
 371                 $opts = array(
 372                         'http' => array(
 373                                 'method' => $method,
 374                                 'header' => implode( "\r\n", $headers ),
 375                                 'timeout' => $timeout ) );
 376                 $ctx = stream_context_create( $opts );
 377
 378                 $status->value = file_get_contents( $url, false, $ctx );
 379                 if(!$status->value){
 380                         $status->error('file_get_contents-failed');
 381                 }
 382                 return $status;
 383         }
 384 }
 385 /**
 386  * a simpleFileWriter with session id updates
 387  *
 388  */
 389 class simpleFileWriter{
 390         var $target_file_path;
 391         var $status = null;
 392         var $session_id = null;
 393         var $session_update_interval = 0; //how offten to update the session while downloading
 394
 395         function simpleFileWriter($target_file_path, $upload_session_key){
 396                 $this->target_file_path = $target_file_path;
 397                 $this->upload_session_key = $upload_session_key;
 398                 $this->status = Status::newGood();
 399                 //open the file:
 400                 $this->fp = fopen( $this->target_file_path, 'w');
 401                 if( $this->fp === false ){
 402                         $this->status = Status::newFatal('HTTP::could-not-open-file-for-writing');
 403                 }
 404                 //true start time
 405                 $this->prevTime = time();
 406         }
 407         public function callbackWriteBody($ch, $data_packet){
 408                 global $wgMaxUploadSize;
 409
 410                 //write out the content
 411                 if( fwrite($this->fp, $data_packet) === false){
 412                         wfDebug(__METHOD__ ." ::could-not-write-to-file\n");
 413                         $this->status = Status::newFatal('HTTP::could-not-write-to-file');
 414                         return 0;
 415                 }
 416
 417                 //check file size:
 418                 clearstatcache();
 419                 $this->current_fsize = filesize( $this->target_file_path);
 420
 421                 if( $this->current_fsize > $wgMaxUploadSize){
 422                         wfDebug( __METHOD__ . " ::http download too large\n");
 423                         $this->status = Status::newFatal('HTTP::file-has-grown-beyond-upload-limit-killing: downloaded more than ' .
 424                                 Language::formatSize($wgMaxUploadSize) . ' ');
 425                         return 0;
 426                 }
 427
 428                 //if more than session_update_interval second have passed update_session_progress
 429                 if($this->upload_session_key && ( (time() - $this->prevTime) > $this->session_update_interval )) {
 430                         $this->prevTime = time();
 431                         $session_status = $this->update_session_progress();
 432                         if( !$session_status->isOK() ){
 433                                 $this->status = $session_status;
 434                                 wfDebug( __METHOD__ . ' update session failed or was canceled');
 435                                 return 0;
 436                         }
 437                 }
 438                 return strlen( $data_packet );
 439         }
 440         public function update_session_progress(){
 441                 $status = Status::newGood();
 442                 //start the session
 443                 if( session_start() === false){
 444                         wfDebug( __METHOD__ . ' could not start session');
 445                         exit(0);
 446                 }
 447                 $sd =& $_SESSION[ 'wsDownload' ][ $this->upload_session_key ];
 448                 //check if the user canceled the request:
 449                 if( $sd['user_cancel'] == true ){
 450                         //kill the download
 451                         return Status::newFatal('user-canceled-request');
 452                 }
 453                 //update the progress bytes download so far:
 454                 $sd['loaded'] = $this->current_fsize;
 455                 wfDebug('set session loaded amount to: ' . $sd['loaded'] . "\n");
 456                 //close down the session so we can other http queries can get session updates:
 457                 session_write_close();
 458                 return $status;
 459         }
 460         public function close(){
 461                 //do a final session update:
 462                 $this->update_session_progress();
 463                 //close up the file handle:
 464                 if(false === fclose( $this->fp )){
 465                         $this->status = Status::newFatal('HTTP::could-not-close-file');
 466                 }
 467         }
 468 }