includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Russ Nelson
   6  * @author Aaron Schulz
   7  */
   8
   9 /**
  10  * Class for an OpenStack Swift based file backend.
  11  *
  12  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  13  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  14  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  15  *
  16  * Status messages should avoid mentioning the Swift account name.
  17  * Likewise, error suppression should be used to avoid path disclosure.
  18  *
  19  * @ingroup FileBackend
  20  * @since 1.19
  21  */
  22 class SwiftFileBackend extends FileBackend {
  23         /** @var CF_Authentication */
  24         protected $auth; // Swift authentication handler
  25         protected $authTTL; // integer seconds
  26         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  27         protected $maxContCacheSize = 20; // integer; max containers with entries
  28
  29         /** @var CF_Connection */
  30         protected $conn; // Swift connection handle
  31         protected $connStarted = 0; // integer UNIX timestamp
  32         protected $connContainers = array(); // container object cache
  33
  34         /**
  35          * @see FileBackend::__construct()
  36          * Additional $config params include:
  37          *    swiftAuthUrl       : Swift authentication server URL
  38          *    swiftUser          : Swift user used by MediaWiki (account:username)
  39          *    swiftKey           : Swift authentication key for the above user
  40          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  41          *    swiftAnonUser      : Swift user used for end-user requests (account:username)
  42          *    shardViaHashLevels : Map of container names to the number of hash levels
  43          */
  44         public function __construct( array $config ) {
  45                 parent::__construct( $config );
  46                 // Required settings
  47                 $this->auth = new CF_Authentication(
  48                         $config['swiftUser'],
  49                         $config['swiftKey'],
  50                         null, // account; unused
  51                         $config['swiftAuthUrl']
  52                 );
  53                 // Optional settings
  54                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  55                         ? $config['swiftAuthTTL']
  56                         : 120; // some sane number
  57                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  58                         ? $config['swiftAnonUser']
  59                         : '';
  60                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  61                         ? $config['shardViaHashLevels']
  62                         : '';
  63         }
  64
  65         /**
  66          * @see FileBackend::resolveContainerPath()
  67          */
  68         protected function resolveContainerPath( $container, $relStoragePath ) {
  69                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
  70                         return null; // too long for Swift
  71                 }
  72                 return $relStoragePath;
  73         }
  74
  75         /**
  76          * @see FileBackend::isPathUsableInternal()
  77          */
  78         public function isPathUsableInternal( $storagePath ) {
  79                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
  80                 if ( $rel === null ) {
  81                         return false; // invalid
  82                 }
  83
  84                 try {
  85                         $this->getContainer( $container );
  86                         return true; // container exists
  87                 } catch ( NoSuchContainerException $e ) {
  88                 } catch ( InvalidResponseException $e ) {
  89                 } catch ( Exception $e ) { // some other exception?
  90                         $this->logException( $e, __METHOD__, array( 'path' => $storagePath ) );
  91                 }
  92
  93                 return false;
  94         }
  95
  96         /**
  97          * @see FileBackend::doCopyInternal()
  98          */
  99         protected function doCreateInternal( array $params ) {
 100                 $status = Status::newGood();
 101
 102                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 103                 if ( $dstRel === null ) {
 104                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 105                         return $status;
 106                 }
 107
 108                 // (a) Check the destination container and object
 109                 try {
 110                         $dContObj = $this->getContainer( $dstCont );
 111                         if ( empty( $params['overwrite'] ) &&
 112                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 113                         {
 114                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 115                                 return $status;
 116                         }
 117                 } catch ( NoSuchContainerException $e ) {
 118                         $status->fatal( 'backend-fail-create', $params['dst'] );
 119                         return $status;
 120                 } catch ( InvalidResponseException $e ) {
 121                         $status->fatal( 'backend-fail-connect', $this->name );
 122                         return $status;
 123                 } catch ( Exception $e ) { // some other exception?
 124                         $status->fatal( 'backend-fail-internal', $this->name );
 125                         $this->logException( $e, __METHOD__, $params );
 126                         return $status;
 127                 }
 128
 129                 // (b) Get a SHA-1 hash of the object
 130                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 131
 132                 // (c) Actually create the object
 133                 try {
 134                         // Create a fresh CF_Object with no fields preloaded.
 135                         // We don't want to preserve headers, metadata, and such.
 136                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 137                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 138                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 139                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 140                         // The MD5 here will be checked within Swift against its own MD5.
 141                         $obj->set_etag( md5( $params['content'] ) );
 142                         // Use the same content type as StreamFile for security
 143                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 144                         // Actually write the object in Swift
 145                         $obj->write( $params['content'] );
 146                 } catch ( BadContentTypeException $e ) {
 147                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 148                 } catch ( InvalidResponseException $e ) {
 149                         $status->fatal( 'backend-fail-connect', $this->name );
 150                 } catch ( Exception $e ) { // some other exception?
 151                         $status->fatal( 'backend-fail-internal', $this->name );
 152                         $this->logException( $e, __METHOD__, $params );
 153                 }
 154
 155                 return $status;
 156         }
 157
 158         /**
 159          * @see FileBackend::doStoreInternal()
 160          */
 161         protected function doStoreInternal( array $params ) {
 162                 $status = Status::newGood();
 163
 164                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 165                 if ( $dstRel === null ) {
 166                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 167                         return $status;
 168                 }
 169
 170                 // (a) Check the destination container and object
 171                 try {
 172                         $dContObj = $this->getContainer( $dstCont );
 173                         if ( empty( $params['overwrite'] ) &&
 174                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 175                         {
 176                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 177                                 return $status;
 178                         }
 179                 } catch ( NoSuchContainerException $e ) {
 180                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 181                         return $status;
 182                 } catch ( InvalidResponseException $e ) {
 183                         $status->fatal( 'backend-fail-connect', $this->name );
 184                         return $status;
 185                 } catch ( Exception $e ) { // some other exception?
 186                         $status->fatal( 'backend-fail-internal', $this->name );
 187                         $this->logException( $e, __METHOD__, $params );
 188                         return $status;
 189                 }
 190
 191                 // (b) Get a SHA-1 hash of the object
 192                 $sha1Hash = sha1_file( $params['src'] );
 193                 if ( $sha1Hash === false ) { // source doesn't exist?
 194                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 195                         return $status;
 196                 }
 197                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 198
 199                 // (c) Actually store the object
 200                 try {
 201                         // Create a fresh CF_Object with no fields preloaded.
 202                         // We don't want to preserve headers, metadata, and such.
 203                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 204                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 205                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 206                         // The MD5 here will be checked within Swift against its own MD5.
 207                         $obj->set_etag( md5_file( $params['src'] ) );
 208                         // Use the same content type as StreamFile for security
 209                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 210                         // Actually write the object in Swift
 211                         $obj->load_from_filename( $params['src'], True ); // calls $obj->write()
 212                 } catch ( BadContentTypeException $e ) {
 213                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 214                 } catch ( IOException $e ) {
 215                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 216                 } catch ( InvalidResponseException $e ) {
 217                         $status->fatal( 'backend-fail-connect', $this->name );
 218                 } catch ( Exception $e ) { // some other exception?
 219                         $status->fatal( 'backend-fail-internal', $this->name );
 220                         $this->logException( $e, __METHOD__, $params );
 221                 }
 222
 223                 return $status;
 224         }
 225
 226         /**
 227          * @see FileBackend::doCopyInternal()
 228          */
 229         protected function doCopyInternal( array $params ) {
 230                 $status = Status::newGood();
 231
 232                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 233                 if ( $srcRel === null ) {
 234                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 235                         return $status;
 236                 }
 237
 238                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 239                 if ( $dstRel === null ) {
 240                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 241                         return $status;
 242                 }
 243
 244                 // (a) Check the source/destination containers and destination object
 245                 try {
 246                         $sContObj = $this->getContainer( $srcCont );
 247                         $dContObj = $this->getContainer( $dstCont );
 248                         if ( empty( $params['overwrite'] ) &&
 249                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 250                         {
 251                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 252                                 return $status;
 253                         }
 254                 } catch ( NoSuchContainerException $e ) {
 255                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 256                         return $status;
 257                 } catch ( InvalidResponseException $e ) {
 258                         $status->fatal( 'backend-fail-connect', $this->name );
 259                         return $status;
 260                 } catch ( Exception $e ) { // some other exception?
 261                         $status->fatal( 'backend-fail-internal', $this->name );
 262                         $this->logException( $e, __METHOD__, $params );
 263                         return $status;
 264                 }
 265
 266                 // (b) Actually copy the file to the destination
 267                 try {
 268                         $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 269                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 270                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 271                 } catch ( InvalidResponseException $e ) {
 272                         $status->fatal( 'backend-fail-connect', $this->name );
 273                 } catch ( Exception $e ) { // some other exception?
 274                         $status->fatal( 'backend-fail-internal', $this->name );
 275                         $this->logException( $e, __METHOD__, $params );
 276                 }
 277
 278                 return $status;
 279         }
 280
 281         /**
 282          * @see FileBackend::doDeleteInternal()
 283          */
 284         protected function doDeleteInternal( array $params ) {
 285                 $status = Status::newGood();
 286
 287                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 288                 if ( $srcRel === null ) {
 289                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 290                         return $status;
 291                 }
 292
 293                 try {
 294                         $sContObj = $this->getContainer( $srcCont );
 295                         $sContObj->delete_object( $srcRel );
 296                 } catch ( NoSuchContainerException $e ) {
 297                         $status->fatal( 'backend-fail-delete', $params['src'] );
 298                 } catch ( NoSuchObjectException $e ) {
 299                         if ( empty( $params['ignoreMissingSource'] ) ) {
 300                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 301                         }
 302                 } catch ( InvalidResponseException $e ) {
 303                         $status->fatal( 'backend-fail-connect', $this->name );
 304                 } catch ( Exception $e ) { // some other exception?
 305                         $status->fatal( 'backend-fail-internal', $this->name );
 306                         $this->logException( $e, __METHOD__, $params );
 307                 }
 308
 309                 return $status;
 310         }
 311
 312         /**
 313          * @see FileBackend::doPrepareInternal()
 314          */
 315         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 316                 $status = Status::newGood();
 317
 318                 // (a) Check if container already exists
 319                 try {
 320                         $contObj = $this->getContainer( $fullCont );
 321                         // NoSuchContainerException not thrown: container must exist
 322                         return $status; // already exists
 323                 } catch ( NoSuchContainerException $e ) {
 324                         // NoSuchContainerException thrown: container does not exist
 325                 } catch ( InvalidResponseException $e ) {
 326                         $status->fatal( 'backend-fail-connect', $this->name );
 327                         return $status;
 328                 } catch ( Exception $e ) { // some other exception?
 329                         $status->fatal( 'backend-fail-internal', $this->name );
 330                         $this->logException( $e, __METHOD__, $params );
 331                         return $status;
 332                 }
 333
 334                 // (b) Create container as needed
 335                 try {
 336                         $contObj = $this->createContainer( $fullCont );
 337                         if ( $this->swiftAnonUser != '' ) {
 338                                 // Make container public to end-users...
 339                                 $status->merge( $this->setContainerAccess(
 340                                         $contObj,
 341                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 342                                         array( $this->auth->username ) // write
 343                                 ) );
 344                         }
 345                 } catch ( InvalidResponseException $e ) {
 346                         $status->fatal( 'backend-fail-connect', $this->name );
 347                         return $status;
 348                 } catch ( Exception $e ) { // some other exception?
 349                         $status->fatal( 'backend-fail-internal', $this->name );
 350                         $this->logException( $e, __METHOD__, $params );
 351                         return $status;
 352                 }
 353
 354                 return $status;
 355         }
 356
 357         /**
 358          * @see FileBackend::doSecureInternal()
 359          */
 360         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 361                 $status = Status::newGood();
 362
 363                 if ( $this->swiftAnonUser != '' ) {
 364                         // Restrict container from end-users...
 365                         try {
 366                                 // doPrepareInternal() should have been called,
 367                                 // so the Swift container should already exist...
 368                                 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 369                                 // NoSuchContainerException not thrown: container must exist
 370                                 if ( !isset( $contObj->mw_wasSecured ) ) {
 371                                         $status->merge( $this->setContainerAccess(
 372                                                 $contObj,
 373                                                 array( $this->auth->username ), // read
 374                                                 array( $this->auth->username ) // write
 375                                         ) );
 376                                         // @TODO: when php-cloudfiles supports container
 377                                         // metadata, we can make use of that to avoid RTTs
 378                                         $contObj->mw_wasSecured = true; // avoid useless RTTs
 379                                 }
 380                         } catch ( InvalidResponseException $e ) {
 381                                 $status->fatal( 'backend-fail-connect', $this->name );
 382                         } catch ( Exception $e ) { // some other exception?
 383                                 $status->fatal( 'backend-fail-internal', $this->name );
 384                                 $this->logException( $e, __METHOD__, $params );
 385                         }
 386                 }
 387
 388                 return $status;
 389         }
 390
 391         /**
 392          * @see FileBackend::doCleanInternal()
 393          */
 394         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 395                 $status = Status::newGood();
 396
 397                 // Only containers themselves can be removed, all else is virtual
 398                 if ( $dir != '' ) {
 399                         return $status; // nothing to do
 400                 }
 401
 402                 // (a) Check the container
 403                 try {
 404                         $contObj = $this->getContainer( $fullCont, true );
 405                 } catch ( NoSuchContainerException $e ) {
 406                         return $status; // ok, nothing to do
 407                 } catch ( InvalidResponseException $e ) {
 408                         $status->fatal( 'backend-fail-connect', $this->name );
 409                         return $status;
 410                 } catch ( Exception $e ) { // some other exception?
 411                         $status->fatal( 'backend-fail-internal', $this->name );
 412                         $this->logException( $e, __METHOD__, $params );
 413                         return $status;
 414                 }
 415
 416                 // (b) Delete the container if empty
 417                 if ( $contObj->object_count == 0 ) {
 418                         try {
 419                                 $this->deleteContainer( $fullCont );
 420                         } catch ( NoSuchContainerException $e ) {
 421                                 return $status; // race?
 422                         } catch ( InvalidResponseException $e ) {
 423                                 $status->fatal( 'backend-fail-connect', $this->name );
 424                                 return $status;
 425                         } catch ( Exception $e ) { // some other exception?
 426                                 $status->fatal( 'backend-fail-internal', $this->name );
 427                                 $this->logException( $e, __METHOD__, $params );
 428                                 return $status;
 429                         }
 430                 }
 431
 432                 return $status;
 433         }
 434
 435         /**
 436          * @see FileBackend::doFileExists()
 437          */
 438         protected function doGetFileStat( array $params ) {
 439                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 440                 if ( $srcRel === null ) {
 441                         return false; // invalid storage path
 442                 }
 443
 444                 $stat = false;
 445                 try {
 446                         $contObj = $this->getContainer( $srcCont );
 447                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 448                         $this->addMissingMetadata( $srcObj, $params['src'] );
 449                         $stat = array(
 450                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 451                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 452                                 'size'  => $srcObj->content_length,
 453                                 'sha1'  => $srcObj->metadata['Sha1base36']
 454                         );
 455                 } catch ( NoSuchContainerException $e ) {
 456                 } catch ( NoSuchObjectException $e ) {
 457                 } catch ( InvalidResponseException $e ) {
 458                         $stat = null;
 459                 } catch ( Exception $e ) { // some other exception?
 460                         $stat = null;
 461                         $this->logException( $e, __METHOD__, $params );
 462                 }
 463
 464                 return $stat;
 465         }
 466
 467         /**
 468          * Fill in any missing object metadata and save it to Swift
 469          *
 470          * @param $obj CF_Object
 471          * @param $path string Storage path to object
 472          * @return bool Success
 473          * @throws Exception cloudfiles exceptions
 474          */
 475         protected function addMissingMetadata( CF_Object $obj, $path ) {
 476                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 477                         return true; // nothing to do
 478                 }
 479                 $status = Status::newGood();
 480                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 481                 if ( $status->isOK() ) {
 482                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
 483                         if ( $tmpFile ) {
 484                                 $hash = $tmpFile->getSha1Base36();
 485                                 if ( $hash !== false ) {
 486                                         $obj->metadata['Sha1base36'] = $hash;
 487                                         $obj->sync_metadata(); // save to Swift
 488                                         return true; // success
 489                                 }
 490                         }
 491                 }
 492                 $obj->metadata['Sha1base36'] = false;
 493                 return false; // failed
 494         }
 495
 496         /**
 497          * @see FileBackendBase::getFileContents()
 498          */
 499         public function getFileContents( array $params ) {
 500                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 501                 if ( $srcRel === null ) {
 502                         return false; // invalid storage path
 503                 }
 504
 505                 if ( !$this->fileExists( $params ) ) {
 506                         return null;
 507                 }
 508
 509                 $data = false;
 510                 try {
 511                         $sContObj = $this->getContainer( $srcCont );
 512                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request
 513                         $data = $obj->read( $this->headersFromParams( $params ) );
 514                 } catch ( NoSuchContainerException $e ) {
 515                 } catch ( InvalidResponseException $e ) {
 516                 } catch ( Exception $e ) { // some other exception?
 517                         $this->logException( $e, __METHOD__, $params );
 518                 }
 519
 520                 return $data;
 521         }
 522
 523         /**
 524          * @see FileBackend::getFileListInternal()
 525          */
 526         public function getFileListInternal( $fullCont, $dir, array $params ) {
 527                 return new SwiftFileBackendFileList( $this, $fullCont, $dir );
 528         }
 529
 530         /**
 531          * Do not call this function outside of SwiftFileBackendFileList
 532          *
 533          * @param $fullCont string Resolved container name
 534          * @param $dir string Resolved storage directory with no trailing slash
 535          * @param $after string Storage path of file to list items after
 536          * @param $limit integer Max number of items to list
 537          * @return Array
 538          */
 539         public function getFileListPageInternal( $fullCont, $dir, $after, $limit ) {
 540                 $files = array();
 541
 542                 try {
 543                         $container = $this->getContainer( $fullCont );
 544                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 545                         $files = $container->list_objects( $limit, $after, $prefix );
 546                 } catch ( NoSuchContainerException $e ) {
 547                 } catch ( NoSuchObjectException $e ) {
 548                 } catch ( InvalidResponseException $e ) {
 549                 } catch ( Exception $e ) { // some other exception?
 550                         $this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
 551                 }
 552
 553                 return $files;
 554         }
 555
 556         /**
 557          * @see FileBackend::doGetFileSha1base36()
 558          */
 559         public function doGetFileSha1base36( array $params ) {
 560                 $stat = $this->getFileStat( $params );
 561                 if ( $stat ) {
 562                         return $stat['sha1'];
 563                 } else {
 564                         return false;
 565                 }
 566         }
 567
 568         /**
 569          * @see FileBackend::doStreamFile()
 570          */
 571         protected function doStreamFile( array $params ) {
 572                 $status = Status::newGood();
 573
 574                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 575                 if ( $srcRel === null ) {
 576                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 577                 }
 578
 579                 try {
 580                         $cont = $this->getContainer( $srcCont );
 581                 } catch ( NoSuchContainerException $e ) {
 582                         $status->fatal( 'backend-fail-stream', $params['src'] );
 583                         return $status;
 584                 } catch ( InvalidResponseException $e ) {
 585                         $status->fatal( 'backend-fail-connect', $this->name );
 586                         return $status;
 587                 } catch ( Exception $e ) { // some other exception?
 588                         $status->fatal( 'backend-fail-stream', $params['src'] );
 589                         $this->logException( $e, __METHOD__, $params );
 590                         return $status;
 591                 }
 592
 593                 try {
 594                         $output = fopen( 'php://output', 'w' );
 595                         // FileBackend::streamFile() already checks existence
 596                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
 597                         $obj->stream( $output, $this->headersFromParams( $params ) );
 598                 } catch ( InvalidResponseException $e ) { // 404? connection problem?
 599                         $status->fatal( 'backend-fail-stream', $params['src'] );
 600                 } catch ( Exception $e ) { // some other exception?
 601                         $status->fatal( 'backend-fail-stream', $params['src'] );
 602                         $this->logException( $e, __METHOD__, $params );
 603                 }
 604
 605                 return $status;
 606         }
 607
 608         /**
 609          * @see FileBackend::getLocalCopy()
 610          */
 611         public function getLocalCopy( array $params ) {
 612                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 613                 if ( $srcRel === null ) {
 614                         return null;
 615                 }
 616
 617                 if ( !$this->fileExists( $params ) ) {
 618                         return null;
 619                 }
 620
 621                 $tmpFile = null;
 622                 try {
 623                         $sContObj = $this->getContainer( $srcCont );
 624                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 625                         // Get source file extension
 626                         $ext = FileBackend::extensionFromPath( $srcRel );
 627                         // Create a new temporary file...
 628                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 629                         if ( $tmpFile ) {
 630                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 631                                 if ( $handle ) {
 632                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 633                                         fclose( $handle );
 634                                 } else {
 635                                         $tmpFile = null; // couldn't open temp file
 636                                 }
 637                         }
 638                 } catch ( NoSuchContainerException $e ) {
 639                         $tmpFile = null;
 640                 } catch ( InvalidResponseException $e ) {
 641                         $tmpFile = null;
 642                 } catch ( Exception $e ) { // some other exception?
 643                         $tmpFile = null;
 644                         $this->logException( $e, __METHOD__, $params );
 645                 }
 646
 647                 return $tmpFile;
 648         }
 649
 650         /**
 651          * Get headers to send to Swift when reading a file based
 652          * on a FileBackend params array, e.g. that of getLocalCopy().
 653          * $params is currently only checked for a 'latest' flag.
 654          *
 655          * @param $params Array
 656          * @return Array
 657          */
 658         protected function headersFromParams( array $params ) {
 659                 $hdrs = array();
 660                 if ( !empty( $params['latest'] ) ) {
 661                         $hdrs[] = 'X-Newest: true';
 662                 }
 663                 return $hdrs;
 664         }
 665
 666         /**
 667          * Set read/write permissions for a Swift container
 668          *
 669          * @param $contObj CF_Container Swift container
 670          * @param $readGrps Array Swift users who can read (account:user)
 671          * @param $writeGrps Array Swift users who can write (account:user)
 672          * @return Status
 673          */
 674         protected function setContainerAccess(
 675                 CF_Container $contObj, array $readGrps, array $writeGrps
 676         ) {
 677                 $creds = $contObj->cfs_auth->export_credentials();
 678
 679                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
 680
 681                 // Note: 10 second timeout consistent with php-cloudfiles
 682                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
 683                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
 684                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
 685                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
 686
 687                 return $req->execute(); // should return 204
 688         }
 689
 690         /**
 691          * Get a connection to the Swift proxy
 692          *
 693          * @return CF_Connection|false
 694          * @throws InvalidResponseException
 695          */
 696         protected function getConnection() {
 697                 if ( $this->conn === false ) {
 698                         throw new InvalidResponseException; // failed last attempt
 699                 }
 700                 // Session keys expire after a while, so we renew them periodically
 701                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
 702                         $this->conn->close(); // close active cURL connections
 703                         $this->conn = null;
 704                 }
 705                 // Authenticate with proxy and get a session key...
 706                 if ( $this->conn === null ) {
 707                         $this->connContainers = array();
 708                         try {
 709                                 $this->auth->authenticate();
 710                                 $this->conn = new CF_Connection( $this->auth );
 711                                 $this->connStarted = time();
 712                         } catch ( AuthenticationException $e ) {
 713                                 $this->conn = false; // don't keep re-trying
 714                         } catch ( InvalidResponseException $e ) {
 715                                 $this->conn = false; // don't keep re-trying
 716                         }
 717                 }
 718                 if ( !$this->conn ) {
 719                         throw new InvalidResponseException; // auth/connection problem
 720                 }
 721                 return $this->conn;
 722         }
 723
 724         /**
 725          * @see FileBackend::doClearCache()
 726          */
 727         protected function doClearCache( array $paths = null ) {
 728                 $this->connContainers = array(); // clear container object cache
 729         }
 730
 731         /**
 732          * Get a Swift container object, possibly from process cache.
 733          * Use $reCache if the file count or byte count is needed.
 734          *
 735          * @param $container string Container name
 736          * @param $reCache bool Refresh the process cache
 737          * @return CF_Container
 738          */
 739         protected function getContainer( $container, $reCache = false ) {
 740                 $conn = $this->getConnection(); // Swift proxy connection
 741                 if ( $reCache ) {
 742                         unset( $this->connContainers[$container] ); // purge cache
 743                 }
 744                 if ( !isset( $this->connContainers[$container] ) ) {
 745                         $contObj = $conn->get_container( $container );
 746                         // NoSuchContainerException not thrown: container must exist
 747                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
 748                                 reset( $this->connContainers );
 749                                 $key = key( $this->connContainers );
 750                                 unset( $this->connContainers[$key] );
 751                         }
 752                         $this->connContainers[$container] = $contObj; // cache it
 753                 }
 754                 return $this->connContainers[$container];
 755         }
 756
 757         /**
 758          * Create a Swift container
 759          *
 760          * @param $container string Container name
 761          * @return CF_Container
 762          */
 763         protected function createContainer( $container ) {
 764                 $conn = $this->getConnection(); // Swift proxy connection
 765                 $contObj = $conn->create_container( $container );
 766                 $this->connContainers[$container] = $contObj; // cache it
 767                 return $contObj;
 768         }
 769
 770         /**
 771          * Delete a Swift container
 772          *
 773          * @param $container string Container name
 774          * @return void
 775          */
 776         protected function deleteContainer( $container ) {
 777                 $conn = $this->getConnection(); // Swift proxy connection
 778                 $conn->delete_container( $container );
 779                 unset( $this->connContainers[$container] ); // purge cache
 780         }
 781
 782         /**
 783          * Log an unexpected exception for this backend
 784          *
 785          * @param $e Exception
 786          * @param $func string
 787          * @param $params Array
 788          * @return void
 789          */
 790         protected function logException( Exception $e, $func, array $params ) {
 791                 wfDebugLog( 'SwiftBackend',
 792                         get_class( $e ) . " in '{$this->name}': '{$func}' with " . serialize( $params )
 793                 );
 794         }
 795 }
 796
 797 /**
 798  * SwiftFileBackend helper class to page through object listings.
 799  * Swift also has a listing limit of 10,000 objects for sanity.
 800  * Do not use this class from places outside SwiftFileBackend.
 801  *
 802  * @ingroup FileBackend
 803  */
 804 class SwiftFileBackendFileList implements Iterator {
 805         /** @var Array */
 806         protected $bufferIter = array();
 807         protected $bufferAfter = null; // string; list items *after* this path
 808         protected $pos = 0; // integer
 809
 810         /** @var SwiftFileBackend */
 811         protected $backend;
 812         protected $container; //
 813         protected $dir; // string storage directory
 814         protected $suffixStart; // integer
 815
 816         const PAGE_SIZE = 5000; // file listing buffer size
 817
 818         /**
 819          * @param $backend SwiftFileBackend
 820          * @param $fullCont string Resolved container name
 821          * @param $dir string Resolved directory relative to container
 822          */
 823         public function __construct( SwiftFileBackend $backend, $fullCont, $dir ) {
 824                 $this->backend = $backend;
 825                 $this->container = $fullCont;
 826                 $this->dir = $dir;
 827                 if ( substr( $this->dir, -1 ) === '/' ) {
 828                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
 829                 }
 830                 if ( $this->dir == '' ) { // whole container
 831                         $this->suffixStart = 0;
 832                 } else { // dir within container
 833                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
 834                 }
 835         }
 836
 837         public function current() {
 838                 return substr( current( $this->bufferIter ), $this->suffixStart );
 839         }
 840
 841         public function key() {
 842                 return $this->pos;
 843         }
 844
 845         public function next() {
 846                 // Advance to the next file in the page
 847                 next( $this->bufferIter );
 848                 ++$this->pos;
 849                 // Check if there are no files left in this page and
 850                 // advance to the next page if this page was not empty.
 851                 if ( !$this->valid() && count( $this->bufferIter ) ) {
 852                         $this->bufferAfter = end( $this->bufferIter );
 853                         $this->bufferIter = $this->backend->getFileListPageInternal(
 854                                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE
 855                         );
 856                 }
 857         }
 858
 859         public function rewind() {
 860                 $this->pos = 0;
 861                 $this->bufferAfter = null;
 862                 $this->bufferIter = $this->backend->getFileListPageInternal(
 863                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE
 864                 );
 865         }
 866
 867         public function valid() {
 868                 return ( current( $this->bufferIter ) !== false ); // no paths can have this value
 869         }
 870 }