* added curl option support for get requests
[lhc/web/wiklou.git] / includes / HttpFunctions.php
1 <?php
2 /**
3 * HTTP handling class
4 * @defgroup HTTP HTTP
5 * @file
6 * @ingroup HTTP
7 */
8
9 class Http {
10 const SYNC_DOWNLOAD = 1; // syncronys upload (in a single request)
11 const ASYNC_DOWNLOAD = 2; // asynchronous upload we should spawn out another process and monitor progress if possible)
12
13 var $body = '';
14
15 public static function request( $method, $url, $opts = array() ){
16 $opts['method'] = ( strtoupper( $method ) == 'GET' || strtoupper( $method ) == 'POST' ) ? strtoupper( $method ) : null;
17 $req = new HttpRequest( $url, $opts );
18 $status = $req->doRequest();
19 if( $status->isOK() ){
20 return $status->value;
21 } else {
22 wfDebug( 'http error: ' . $status->getWikiText() );
23 return false;
24 }
25 }
26
27 /**
28 * Simple wrapper for Http::request( 'GET' )
29 */
30 public static function get( $url, $timeout = false, $opts = array() ) {
31 global $wgSyncHTTPTimeout;
32 $opts = array();
33 if( $timeout )
34 $opts['timeout'] = $timeout;
35 return Http::request( 'GET', $url, $opts );
36 }
37
38 /**
39 * Simple wrapper for Http::request( 'POST' )
40 */
41 public static function post( $url, $opts = array() ) {
42 return Http::request( 'POST', $url, $opts );
43 }
44
45 public static function doDownload( $url, $target_file_path, $dl_mode = self::SYNC_DOWNLOAD, $redirectCount = 0 ){
46 global $wgPhpCli, $wgMaxUploadSize, $wgMaxRedirects;
47 // do a quick check to HEAD to insure the file size is not > $wgMaxUploadSize
48 $headRequest = new HttpRequest( $url, array( 'headers_only' => true ) );
49 $headResponse = $headRequest->doRequest();
50 if( !$headResponse->isOK() ){
51 return $headResponse;
52 }
53 $head = $headResponse->value;
54
55 // check for redirects:
56 if( isset( $head['Location'] ) && strrpos( $head[0], '302' ) !== false ){
57 if( $redirectCount < $wgMaxRedirects ){
58 if( UploadFromUrl::isValidURI( $head['Location'] ) ){
59 return self::doDownload( $head['Location'], $target_file_path, $dl_mode, $redirectCount++ );
60 } else {
61 return Status::newFatal( 'upload-proto-error' );
62 }
63 } else {
64 return Status::newFatal( 'upload-too-many-redirects' );
65 }
66 }
67 // we did not get a 200 ok response:
68 if( strrpos( $head[0], '200 OK' ) === false ){
69 return Status::newFatal( 'upload-http-error', htmlspecialchars( $head[0] ) );
70 }
71
72 $content_length = ( isset( $head['Content-Length'] ) ) ? $head['Content-Length'] : null;
73 if( $content_length ){
74 if( $content_length > $wgMaxUploadSize ){
75 return Status::newFatal( 'requested file length ' . $content_length . ' is greater than $wgMaxUploadSize: ' . $wgMaxUploadSize );
76 }
77 }
78
79 // check if we can find phpCliPath (for doing a background shell request to php to do the download:
80 if( $wgPhpCli && wfShellExecEnabled() && $dl_mode == self::ASYNC_DOWNLOAD ){
81 wfDebug( __METHOD__ . "\ASYNC_DOWNLOAD\n" );
82 //setup session and shell call:
83 return self::initBackgroundDownload( $url, $target_file_path, $content_length );
84 } else {
85 wfDebug( __METHOD__ . "\nSYNC_DOWNLOAD\n" );
86 // SYNC_DOWNLOAD download as much as we can in the time we have to execute
87 $opts['method'] = 'GET';
88 $opts['target_file_path'] = $target_file_path;
89 $req = new HttpRequest( $url, $opts );
90 return $req->doRequest();
91 }
92 }
93
94 /**
95 * a non blocking request (generally an exit point in the application)
96 * should write to a file location and give updates
97 *
98 */
99 private static function initBackgroundDownload( $url, $target_file_path, $content_length = null ){
100 global $wgMaxUploadSize, $IP, $wgPhpCli, $wgServer;
101 $status = Status::newGood();
102
103 // generate a session id with all the details for the download (pid, target_file_path )
104 $upload_session_key = self::getUploadSessionKey();
105 $session_id = session_id();
106
107 // store the url and target path:
108 $_SESSION['wsDownload'][$upload_session_key]['url'] = $url;
109 $_SESSION['wsDownload'][$upload_session_key]['target_file_path'] = $target_file_path;
110 // since we request from the cmd line we lose the original host name pass in the session:
111 $_SESSION['wsDownload'][$upload_session_key]['orgServer'] = $wgServer;
112
113 if( $content_length )
114 $_SESSION['wsDownload'][$upload_session_key]['content_length'] = $content_length;
115
116 // set initial loaded bytes:
117 $_SESSION['wsDownload'][$upload_session_key]['loaded'] = 0;
118
119 // run the background download request:
120 $cmd = $wgPhpCli . ' ' . $IP . "/maintenance/http_session_download.php --sid {$session_id} --usk {$upload_session_key}";
121 $pid = wfShellBackgroundExec( $cmd );
122 // the pid is not of much use since we won't be visiting this same apache any-time soon.
123 if( !$pid )
124 return Status::newFatal( 'could not run background shell exec' );
125
126 // update the status value with the $upload_session_key (for the user to check on the status of the upload)
127 $status->value = $upload_session_key;
128
129 // return good status
130 return $status;
131 }
132
133 static function getUploadSessionKey(){
134 $key = mt_rand( 0, 0x7fffffff );
135 $_SESSION['wsUploadData'][$key] = array();
136 return $key;
137 }
138
139 /**
140 * used to run a session based download. Is initiated via the shell.
141 *
142 * @param $session_id String: the session id to grab download details from
143 * @param $upload_session_key String: the key of the given upload session
144 * (a given client could have started a few http uploads at once)
145 */
146 public static function doSessionIdDownload( $session_id, $upload_session_key ){
147 global $wgUser, $wgEnableWriteAPI, $wgAsyncHTTPTimeout, $wgServer,
148 $wgSessionsInMemcached, $wgSessionHandler, $wgSessionStarted;
149 wfDebug( __METHOD__ . "\n\n doSessionIdDownload :\n\n" );
150 // set session to the provided key:
151 session_id( $session_id );
152 //fire up mediaWiki session system:
153 wfSetupSession();
154
155 // start the session
156 if( session_start() === false ){
157 wfDebug( __METHOD__ . ' could not start session' );
158 }
159 // get all the vars we need from session_id
160 if( !isset( $_SESSION[ 'wsDownload' ][$upload_session_key] ) ){
161 wfDebug( __METHOD__ . ' Error:could not find upload session');
162 exit();
163 }
164 // setup the global user from the session key we just inherited
165 $wgUser = User::newFromSession();
166
167 // grab the session data to setup the request:
168 $sd =& $_SESSION['wsDownload'][$upload_session_key];
169
170 // update the wgServer var ( since cmd line thinks we are localhost when we are really orgServer)
171 if( isset( $sd['orgServer'] ) && $sd['orgServer'] ){
172 $wgServer = $sd['orgServer'];
173 }
174 // close down the session so we can other http queries can get session updates: (if not $wgSessionsInMemcached)
175 if( !$wgSessionsInMemcached )
176 session_write_close();
177
178 $req = new HttpRequest( $sd['url'], array(
179 'target_file_path' => $sd['target_file_path'],
180 'upload_session_key'=> $upload_session_key,
181 'timeout' => $wgAsyncHTTPTimeout,
182 'do_close_session_update' => true
183 ) );
184 // run the actual request .. (this can take some time)
185 wfDebug( __METHOD__ . 'do Session Download :: ' . $sd['url'] . ' tf: ' . $sd['target_file_path'] . "\n\n");
186 $status = $req->doRequest();
187 //wfDebug("done with req status is: ". $status->isOK(). ' '.$status->getWikiText(). "\n");
188
189 // start up the session again:
190 if( session_start() === false ){
191 wfDebug( __METHOD__ . ' ERROR:: Could not start session');
192 }
193 // grab the updated session data pointer
194 $sd =& $_SESSION['wsDownload'][$upload_session_key];
195 // if error update status:
196 if( !$status->isOK() ){
197 $sd['apiUploadResult'] = ApiFormatJson::getJsonEncode(
198 array( 'error' => $status->getWikiText() )
199 );
200 }
201 // if status okay process upload using fauxReq to api:
202 if( $status->isOK() ){
203 // setup the FauxRequest
204 $fauxReqData = $sd['mParams'];
205
206 // Fix boolean parameters
207 foreach( $fauxReqData as $k => $v ) {
208 if( $v === false )
209 unset( $fauxReqData[$k] );
210 }
211
212 $fauxReqData['action'] = 'upload';
213 $fauxReqData['format'] = 'json';
214 $fauxReqData['internalhttpsession'] = $upload_session_key;
215 // evil but no other clean way about it:
216 $faxReq = new FauxRequest( $fauxReqData, true );
217 $processor = new ApiMain( $faxReq, $wgEnableWriteAPI );
218
219 //init the mUpload var for the $processor
220 $processor->execute();
221 $processor->getResult()->cleanUpUTF8();
222 $printer = $processor->createPrinterByName( 'json' );
223 $printer->initPrinter( false );
224 ob_start();
225 $printer->execute();
226 $apiUploadResult = ob_get_clean();
227
228 // the status updates runner will grab the result form the session:
229 $sd['apiUploadResult'] = $apiUploadResult;
230 }
231 // close the session:
232 session_write_close();
233 }
234
235 /**
236 * Check if the URL can be served by localhost
237 * @param $url string Full url to check
238 * @return bool
239 */
240 public static function isLocalURL( $url ) {
241 global $wgCommandLineMode, $wgConf;
242 if ( $wgCommandLineMode ) {
243 return false;
244 }
245
246 // Extract host part
247 $matches = array();
248 if ( preg_match( '!^http://([\w.-]+)[/:].*$!', $url, $matches ) ) {
249 $host = $matches[1];
250 // Split up dotwise
251 $domainParts = explode( '.', $host );
252 // Check if this domain or any superdomain is listed in $wgConf as a local virtual host
253 $domainParts = array_reverse( $domainParts );
254 for ( $i = 0; $i < count( $domainParts ); $i++ ) {
255 $domainPart = $domainParts[$i];
256 if ( $i == 0 ) {
257 $domain = $domainPart;
258 } else {
259 $domain = $domainPart . '.' . $domain;
260 }
261 if ( $wgConf->isLocalVHost( $domain ) ) {
262 return true;
263 }
264 }
265 }
266 return false;
267 }
268
269 /**
270 * Return a standard user-agent we can use for external requests.
271 */
272 public static function userAgent() {
273 global $wgVersion;
274 return "MediaWiki/$wgVersion";
275 }
276 }
277
278 class HttpRequest {
279 var $target_file_path;
280 var $upload_session_key;
281 var $supportedCurlOpts = array(
282 'CURLOPT_SSL_VERIFYHOST',
283 'CURLOPT_CAINFO',
284 'CURLOPT_COOKIE',
285 'CURLOPT_FOLLOWLOCATION',
286 'CURLOPT_FAILONERROR'
287 );
288 function __construct( $url, $opt ){
289 global $wgSyncHTTPTimeout;
290 // double check that it's a valid url:
291 $this->url = $url;
292
293 // set the timeout to default sync timeout (unless the timeout option is provided)
294 $this->timeout = ( isset( $opt['timeout'] ) ) ? $opt['timeout'] : $wgSyncHTTPTimeout;
295 //check special key default
296 if($timeout == 'default'){
297 $opts['timeout'] = $wgSyncHTTPTimeout;
298 }
299
300 $this->method = ( isset( $opt['method'] ) ) ? $opt['method'] : 'GET';
301 $this->target_file_path = ( isset( $opt['target_file_path'] ) ) ? $opt['target_file_path'] : false;
302 $this->upload_session_key = ( isset( $opt['upload_session_key'] ) ) ? $opt['upload_session_key'] : false;
303 $this->headers_only = ( isset( $opt['headers_only'] ) ) ? $opt['headers_only'] : false;
304 $this->do_close_session_update = isset( $opt['do_close_session_update'] );
305 $this->postData = isset( $opt['postdata'] ) ? $opt['postdata'] : '';
306
307 $this->curlOpt = array();
308 //check for some curl options:
309 foreach($this->supportedCurlOpts as $curlOpt){
310 if(isset($opt[ $curlOpt ])){
311 $this->curlOpt[$curlOpt] = $opt[ $curlOpt ];
312 }
313 }
314 }
315
316 /**
317 * Get the contents of a file by HTTP
318 * @param $url string Full URL to act on
319 * @param $Opt associative array Optional array of options:
320 * 'method' => 'GET', 'POST' etc.
321 * 'target_file_path' => if curl should output to a target file
322 * 'adapter' => 'curl', 'soket'
323 */
324 public function doRequest() {
325 # Make sure we have a valid url
326 if( !UploadFromUrl::isValidURI( $this->url ) )
327 return Status::newFatal('bad-url');
328
329 # Use curl if available
330 if ( function_exists( 'curl_init' ) ) {
331 return $this->doCurlReq();
332 } else {
333 return $this->doPhpReq();
334 }
335 }
336
337 private function doCurlReq(){
338 global $wgHTTPProxy, $wgTitle;
339
340 $status = Status::newGood();
341 $c = curl_init( $this->url );
342
343 // proxy setup:
344 if ( Http::isLocalURL( $this->url ) ) {
345 curl_setopt( $c, CURLOPT_PROXY, 'localhost:80' );
346 } else if ( $wgHTTPProxy ) {
347 curl_setopt( $c, CURLOPT_PROXY, $wgHTTPProxy );
348 }
349
350 curl_setopt( $c, CURLOPT_TIMEOUT, $this->timeout );
351 curl_setopt( $c, CURLOPT_USERAGENT, Http::userAgent() );
352
353 //set any curl specific opts:
354 foreach($this->curlOpt as $optKey => $optVal){
355 curl_setopt($c, constant( $optKey ), $optVal);
356 }
357
358 if ( $this->headers_only ) {
359 curl_setopt( $c, CURLOPT_NOBODY, true );
360 curl_setopt( $c, CURLOPT_HEADER, true );
361 } elseif ( $this->method == 'POST' ) {
362 curl_setopt( $c, CURLOPT_POST, true );
363 curl_setopt( $c, CURLOPT_POSTFIELDS, $this->postData );
364 // Suppress 'Expect: 100-continue' header, as some servers
365 // will reject it with a 417 and Curl won't auto retry
366 // with HTTP 1.0 fallback
367 curl_setopt( $c, CURLOPT_HTTPHEADER, array( 'Expect:' ) );
368 } else {
369 curl_setopt( $c, CURLOPT_CUSTOMREQUEST, $this->method );
370 }
371
372 # Set the referer to $wgTitle, even in command-line mode
373 # This is useful for interwiki transclusion, where the foreign
374 # server wants to know what the referring page is.
375 # $_SERVER['REQUEST_URI'] gives a less reliable indication of the
376 # referring page.
377 if ( is_object( $wgTitle ) ) {
378 curl_setopt( $c, CURLOPT_REFERER, $wgTitle->getFullURL() );
379 }
380
381 // set the write back function (if we are writing to a file)
382 if( $this->target_file_path ){
383 $cwrite = new simpleFileWriter( $this->target_file_path,
384 $this->upload_session_key,
385 $this->do_close_session_update
386 );
387 if( !$cwrite->status->isOK() ){
388 wfDebug( __METHOD__ . "ERROR in setting up simpleFileWriter\n" );
389 $status = $cwrite->status;
390 return $status;
391 }
392 curl_setopt( $c, CURLOPT_WRITEFUNCTION, array( $cwrite, 'callbackWriteBody' ) );
393 }
394
395 // start output grabber:
396 if( !$this->target_file_path )
397 ob_start();
398
399 //run the actual curl_exec:
400 try {
401 if ( false === curl_exec( $c ) ) {
402 $error_txt ='Error sending request: #' . curl_errno( $c ) .' '. curl_error( $c );
403 wfDebug( __METHOD__ . $error_txt . "\n" );
404 $status = Status::newFatal( $error_txt );
405 }
406 } catch ( Exception $e ) {
407 // do something with curl exec error?
408 }
409 // if direct request output the results to the stats value:
410 if( !$this->target_file_path && $status->isOK() ){
411 $status->value = ob_get_contents();
412 ob_end_clean();
413 }
414 // if we wrote to a target file close up or return error
415 if( $this->target_file_path ){
416 $cwrite->close();
417 if( !$cwrite->status->isOK() ){
418 return $cwrite->status;
419 }
420 }
421
422 if ( $this->headers_only ) {
423 $headers = explode( "\n", $status->value );
424 $headerArray = array();
425 foreach ( $headers as $header ) {
426 if ( !strlen( trim( $header ) ) )
427 continue;
428 $headerParts = explode( ':', $header, 2 );
429 if ( count( $headerParts ) == 1 ) {
430 $headerArray[] = trim( $header );
431 } else {
432 list( $key, $val ) = $headerParts;
433 $headerArray[trim( $key )] = trim( $val );
434 }
435 }
436 $status->value = $headerArray;
437 } else {
438 # Don't return the text of error messages, return false on error
439 $retcode = curl_getinfo( $c, CURLINFO_HTTP_CODE );
440 if ( $retcode != 200 ) {
441 wfDebug( __METHOD__ . ": HTTP return code $retcode\n" );
442 $status = Status::newFatal( "HTTP return code $retcode\n" );
443 }
444 # Don't return truncated output
445 $errno = curl_errno( $c );
446 if ( $errno != CURLE_OK ) {
447 $errstr = curl_error( $c );
448 wfDebug( __METHOD__ . ": CURL error code $errno: $errstr\n" );
449 $status = Status::newFatal( " CURL error code $errno: $errstr\n" );
450 }
451 }
452
453 curl_close( $c );
454
455 // return the result obj
456 return $status;
457 }
458
459 public function doPhpReq(){
460 global $wgTitle, $wgHTTPProxy;
461 # Check for php.ini allow_url_fopen
462 if( !ini_get( 'allow_url_fopen' ) ){
463 return Status::newFatal( 'allow_url_fopen needs to be enabled for http copy to work' );
464 }
465
466 // start with good status:
467 $status = Status::newGood();
468
469 if ( $this->headers_only ) {
470 $status->value = get_headers( $this->url, 1 );
471 return $status;
472 }
473
474 // setup the headers
475 $headers = array( "User-Agent: " . Http::userAgent() );
476 if ( is_object( $wgTitle ) ) {
477 $headers[] = "Referer: ". $wgTitle->getFullURL();
478 }
479
480 if( strcasecmp( $this->method, 'post' ) == 0 ) {
481 // Required for HTTP 1.0 POSTs
482 $headers[] = "Content-Length: 0";
483 }
484 $fcontext = stream_context_create ( array(
485 'http' => array(
486 'method' => $this->method,
487 'header' => implode( "\r\n", $headers ),
488 'timeout' => $this->timeout )
489 )
490 );
491 $fh = fopen( $this->url, "r", false, $fcontext);
492
493 // set the write back function (if we are writing to a file)
494 if( $this->target_file_path ){
495 $cwrite = new simpleFileWriter( $this->target_file_path, $this->upload_session_key, $this->do_close_session_update );
496 if( !$cwrite->status->isOK() ){
497 wfDebug( __METHOD__ . "ERROR in setting up simpleFileWriter\n" );
498 $status = $cwrite->status;
499 return $status;
500 }
501
502 // read $fh into the simpleFileWriter (grab in 64K chunks since its likely a ~large~ media file)
503 while ( !feof( $fh ) ) {
504 $contents = fread( $fh, 65536 );
505 $cwrite->callbackWriteBody( $fh, $contents );
506 }
507 $cwrite->close();
508 // check for simpleFileWriter error:
509 if( !$cwrite->status->isOK() ){
510 return $cwrite->status;
511 }
512 } else {
513 // read $fh into status->value
514 $status->value = @stream_get_contents( $fh );
515 }
516 //close the url file wrapper
517 fclose( $fh );
518
519 // check for "false"
520 if( $status->value === false ){
521 $status->error( 'file_get_contents-failed' );
522 }
523 return $status;
524 }
525
526 }
527
528 /**
529 * a simpleFileWriter with session id updates
530 */
531 class simpleFileWriter {
532 var $target_file_path;
533 var $status = null;
534 var $session_id = null;
535 var $session_update_interval = 0; // how often to update the session while downloading
536
537 function simpleFileWriter( $target_file_path, $upload_session_key, $do_close_session_update = false ){
538 $this->target_file_path = $target_file_path;
539 $this->upload_session_key = $upload_session_key;
540 $this->status = Status::newGood();
541 $this->do_close_session_update = $do_close_session_update;
542 // open the file:
543 $this->fp = fopen( $this->target_file_path, 'w' );
544 if( $this->fp === false ){
545 $this->status = Status::newFatal( 'HTTP::could-not-open-file-for-writing' );
546 }
547 // true start time
548 $this->prevTime = time();
549 }
550
551 public function callbackWriteBody( $ch, $data_packet ){
552 global $wgMaxUploadSize, $wgLang;
553
554 // write out the content
555 if( fwrite( $this->fp, $data_packet ) === false ){
556 wfDebug( __METHOD__ ." ::could-not-write-to-file\n" );
557 $this->status = Status::newFatal( 'HTTP::could-not-write-to-file' );
558 return 0;
559 }
560
561 // check file size:
562 clearstatcache();
563 $this->current_fsize = filesize( $this->target_file_path );
564
565 if( $this->current_fsize > $wgMaxUploadSize ){
566 wfDebug( __METHOD__ . " ::http download too large\n" );
567 $this->status = Status::newFatal( 'HTTP::file-has-grown-beyond-upload-limit-killing: downloaded more than ' .
568 $wgLang->formatSize( $wgMaxUploadSize ) . ' ' );
569 return 0;
570 }
571 // if more than session_update_interval second have passed update_session_progress
572 if( $this->do_close_session_update && $this->upload_session_key &&
573 ( ( time() - $this->prevTime ) > $this->session_update_interval ) ) {
574 $this->prevTime = time();
575 $session_status = $this->update_session_progress();
576 if( !$session_status->isOK() ){
577 $this->status = $session_status;
578 wfDebug( __METHOD__ . ' update session failed or was canceled');
579 return 0;
580 }
581 }
582 return strlen( $data_packet );
583 }
584
585 public function update_session_progress(){
586 global $wgSessionsInMemcached;
587 $status = Status::newGood();
588 // start the session (if necessary)
589 if( !$wgSessionsInMemcached ){
590 if( @session_start() === false){
591 wfDebug( __METHOD__ . ' could not start session' );
592 exit( 0 );
593 }
594 }
595 $sd =& $_SESSION['wsDownload'][ $this->upload_session_key ];
596 // check if the user canceled the request:
597 if( isset( $sd['user_cancel'] ) && $sd['user_cancel'] == true ){
598 //@@todo kill the download
599 return Status::newFatal( 'user-canceled-request' );
600 }
601 // update the progress bytes download so far:
602 $sd['loaded'] = $this->current_fsize;
603
604 // close down the session so we can other http queries can get session updates:
605 if( !$wgSessionsInMemcached )
606 session_write_close();
607
608 return $status;
609 }
610
611 public function close(){
612 // do a final session update:
613 if( $this->do_close_session_update ){
614 $this->update_session_progress();
615 }
616 // close up the file handle:
617 if( false === fclose( $this->fp ) ){
618 $this->status = Status::newFatal( 'HTTP::could-not-close-file' );
619 }
620 }
621
622 }