includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Russ Nelson
   6  * @author Aaron Schulz
   7  */
   8
   9 /**
  10  * Class for an OpenStack Swift based file backend.
  11  *
  12  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  13  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  14  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  15  *
  16  * Status messages should avoid mentioning the Swift account name.
  17  * Likewise, error suppression should be used to avoid path disclosure.
  18  *
  19  * @ingroup FileBackend
  20  * @since 1.19
  21  */
  22 class SwiftFileBackend extends FileBackendStore {
  23         /** @var CF_Authentication */
  24         protected $auth; // Swift authentication handler
  25         protected $authTTL; // integer seconds
  26         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  27         protected $maxContCacheSize = 100; // integer; max containers with entries
  28
  29         /** @var CF_Connection */
  30         protected $conn; // Swift connection handle
  31         protected $connStarted = 0; // integer UNIX timestamp
  32         protected $connContainers = array(); // container object cache
  33
  34         /**
  35          * @see FileBackendStore::__construct()
  36          * Additional $config params include:
  37          *    swiftAuthUrl       : Swift authentication server URL
  38          *    swiftUser          : Swift user used by MediaWiki (account:username)
  39          *    swiftKey           : Swift authentication key for the above user
  40          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  41          *    swiftAnonUser      : Swift user used for end-user requests (account:username)
  42          *    shardViaHashLevels : Map of container names to sharding config with:
  43          *                         'base'   : base of hash characters, 16 or 36
  44          *                         'levels' : the number of hash levels (and digits)
  45          *                         'repeat' : hash subdirectories are prefixed with all the
  46          *                                    parent hash directory names (e.g. "a/ab/abc")
  47          */
  48         public function __construct( array $config ) {
  49                 parent::__construct( $config );
  50                 // Required settings
  51                 $this->auth = new CF_Authentication(
  52                         $config['swiftUser'],
  53                         $config['swiftKey'],
  54                         null, // account; unused
  55                         $config['swiftAuthUrl']
  56                 );
  57                 // Optional settings
  58                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  59                         ? $config['swiftAuthTTL']
  60                         : 120; // some sane number
  61                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  62                         ? $config['swiftAnonUser']
  63                         : '';
  64                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  65                         ? $config['shardViaHashLevels']
  66                         : '';
  67         }
  68
  69         /**
  70          * @see FileBackendStore::resolveContainerPath()
  71          */
  72         protected function resolveContainerPath( $container, $relStoragePath ) {
  73                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
  74                         return null; // too long for Swift
  75                 }
  76                 return $relStoragePath;
  77         }
  78
  79         /**
  80          * @see FileBackendStore::isPathUsableInternal()
  81          */
  82         public function isPathUsableInternal( $storagePath ) {
  83                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
  84                 if ( $rel === null ) {
  85                         return false; // invalid
  86                 }
  87
  88                 try {
  89                         $this->getContainer( $container );
  90                         return true; // container exists
  91                 } catch ( NoSuchContainerException $e ) {
  92                 } catch ( InvalidResponseException $e ) {
  93                 } catch ( Exception $e ) { // some other exception?
  94                         $this->logException( $e, __METHOD__, array( 'path' => $storagePath ) );
  95                 }
  96
  97                 return false;
  98         }
  99
 100         /**
 101          * @see FileBackendStore::doCreateInternal()
 102          */
 103         protected function doCreateInternal( array $params ) {
 104                 $status = Status::newGood();
 105
 106                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 107                 if ( $dstRel === null ) {
 108                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 109                         return $status;
 110                 }
 111
 112                 // (a) Check the destination container and object
 113                 try {
 114                         $dContObj = $this->getContainer( $dstCont );
 115                         if ( empty( $params['overwrite'] ) &&
 116                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 117                         {
 118                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 119                                 return $status;
 120                         }
 121                 } catch ( NoSuchContainerException $e ) {
 122                         $status->fatal( 'backend-fail-create', $params['dst'] );
 123                         return $status;
 124                 } catch ( InvalidResponseException $e ) {
 125                         $status->fatal( 'backend-fail-connect', $this->name );
 126                         return $status;
 127                 } catch ( Exception $e ) { // some other exception?
 128                         $status->fatal( 'backend-fail-internal', $this->name );
 129                         $this->logException( $e, __METHOD__, $params );
 130                         return $status;
 131                 }
 132
 133                 // (b) Get a SHA-1 hash of the object
 134                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 135
 136                 // (c) Actually create the object
 137                 try {
 138                         // Create a fresh CF_Object with no fields preloaded.
 139                         // We don't want to preserve headers, metadata, and such.
 140                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 141                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 142                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 143                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 144                         // The MD5 here will be checked within Swift against its own MD5.
 145                         $obj->set_etag( md5( $params['content'] ) );
 146                         // Use the same content type as StreamFile for security
 147                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 148                         // Actually write the object in Swift
 149                         $obj->write( $params['content'] );
 150                 } catch ( BadContentTypeException $e ) {
 151                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 152                 } catch ( InvalidResponseException $e ) {
 153                         $status->fatal( 'backend-fail-connect', $this->name );
 154                 } catch ( Exception $e ) { // some other exception?
 155                         $status->fatal( 'backend-fail-internal', $this->name );
 156                         $this->logException( $e, __METHOD__, $params );
 157                 }
 158
 159                 return $status;
 160         }
 161
 162         /**
 163          * @see FileBackendStore::doStoreInternal()
 164          */
 165         protected function doStoreInternal( array $params ) {
 166                 $status = Status::newGood();
 167
 168                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 169                 if ( $dstRel === null ) {
 170                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 171                         return $status;
 172                 }
 173
 174                 // (a) Check the destination container and object
 175                 try {
 176                         $dContObj = $this->getContainer( $dstCont );
 177                         if ( empty( $params['overwrite'] ) &&
 178                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 179                         {
 180                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 181                                 return $status;
 182                         }
 183                 } catch ( NoSuchContainerException $e ) {
 184                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 185                         return $status;
 186                 } catch ( InvalidResponseException $e ) {
 187                         $status->fatal( 'backend-fail-connect', $this->name );
 188                         return $status;
 189                 } catch ( Exception $e ) { // some other exception?
 190                         $status->fatal( 'backend-fail-internal', $this->name );
 191                         $this->logException( $e, __METHOD__, $params );
 192                         return $status;
 193                 }
 194
 195                 // (b) Get a SHA-1 hash of the object
 196                 $sha1Hash = sha1_file( $params['src'] );
 197                 if ( $sha1Hash === false ) { // source doesn't exist?
 198                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 199                         return $status;
 200                 }
 201                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 202
 203                 // (c) Actually store the object
 204                 try {
 205                         // Create a fresh CF_Object with no fields preloaded.
 206                         // We don't want to preserve headers, metadata, and such.
 207                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 208                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 209                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 210                         // The MD5 here will be checked within Swift against its own MD5.
 211                         $obj->set_etag( md5_file( $params['src'] ) );
 212                         // Use the same content type as StreamFile for security
 213                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 214                         // Actually write the object in Swift
 215                         $obj->load_from_filename( $params['src'], True ); // calls $obj->write()
 216                 } catch ( BadContentTypeException $e ) {
 217                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 218                 } catch ( IOException $e ) {
 219                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 220                 } catch ( InvalidResponseException $e ) {
 221                         $status->fatal( 'backend-fail-connect', $this->name );
 222                 } catch ( Exception $e ) { // some other exception?
 223                         $status->fatal( 'backend-fail-internal', $this->name );
 224                         $this->logException( $e, __METHOD__, $params );
 225                 }
 226
 227                 return $status;
 228         }
 229
 230         /**
 231          * @see FileBackendStore::doCopyInternal()
 232          */
 233         protected function doCopyInternal( array $params ) {
 234                 $status = Status::newGood();
 235
 236                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 237                 if ( $srcRel === null ) {
 238                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 239                         return $status;
 240                 }
 241
 242                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 243                 if ( $dstRel === null ) {
 244                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 245                         return $status;
 246                 }
 247
 248                 // (a) Check the source/destination containers and destination object
 249                 try {
 250                         $sContObj = $this->getContainer( $srcCont );
 251                         $dContObj = $this->getContainer( $dstCont );
 252                         if ( empty( $params['overwrite'] ) &&
 253                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 254                         {
 255                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 256                                 return $status;
 257                         }
 258                 } catch ( NoSuchContainerException $e ) {
 259                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 260                         return $status;
 261                 } catch ( InvalidResponseException $e ) {
 262                         $status->fatal( 'backend-fail-connect', $this->name );
 263                         return $status;
 264                 } catch ( Exception $e ) { // some other exception?
 265                         $status->fatal( 'backend-fail-internal', $this->name );
 266                         $this->logException( $e, __METHOD__, $params );
 267                         return $status;
 268                 }
 269
 270                 // (b) Actually copy the file to the destination
 271                 try {
 272                         $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 273                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 274                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 275                 } catch ( InvalidResponseException $e ) {
 276                         $status->fatal( 'backend-fail-connect', $this->name );
 277                 } catch ( Exception $e ) { // some other exception?
 278                         $status->fatal( 'backend-fail-internal', $this->name );
 279                         $this->logException( $e, __METHOD__, $params );
 280                 }
 281
 282                 return $status;
 283         }
 284
 285         /**
 286          * @see FileBackendStore::doDeleteInternal()
 287          */
 288         protected function doDeleteInternal( array $params ) {
 289                 $status = Status::newGood();
 290
 291                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 292                 if ( $srcRel === null ) {
 293                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 294                         return $status;
 295                 }
 296
 297                 try {
 298                         $sContObj = $this->getContainer( $srcCont );
 299                         $sContObj->delete_object( $srcRel );
 300                 } catch ( NoSuchContainerException $e ) {
 301                         $status->fatal( 'backend-fail-delete', $params['src'] );
 302                 } catch ( NoSuchObjectException $e ) {
 303                         if ( empty( $params['ignoreMissingSource'] ) ) {
 304                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 305                         }
 306                 } catch ( InvalidResponseException $e ) {
 307                         $status->fatal( 'backend-fail-connect', $this->name );
 308                 } catch ( Exception $e ) { // some other exception?
 309                         $status->fatal( 'backend-fail-internal', $this->name );
 310                         $this->logException( $e, __METHOD__, $params );
 311                 }
 312
 313                 return $status;
 314         }
 315
 316         /**
 317          * @see FileBackendStore::doPrepareInternal()
 318          */
 319         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 320                 $status = Status::newGood();
 321
 322                 // (a) Check if container already exists
 323                 try {
 324                         $contObj = $this->getContainer( $fullCont );
 325                         // NoSuchContainerException not thrown: container must exist
 326                         return $status; // already exists
 327                 } catch ( NoSuchContainerException $e ) {
 328                         // NoSuchContainerException thrown: container does not exist
 329                 } catch ( InvalidResponseException $e ) {
 330                         $status->fatal( 'backend-fail-connect', $this->name );
 331                         return $status;
 332                 } catch ( Exception $e ) { // some other exception?
 333                         $status->fatal( 'backend-fail-internal', $this->name );
 334                         $this->logException( $e, __METHOD__, $params );
 335                         return $status;
 336                 }
 337
 338                 // (b) Create container as needed
 339                 try {
 340                         $contObj = $this->createContainer( $fullCont );
 341                         if ( $this->swiftAnonUser != '' ) {
 342                                 // Make container public to end-users...
 343                                 $status->merge( $this->setContainerAccess(
 344                                         $contObj,
 345                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 346                                         array( $this->auth->username ) // write
 347                                 ) );
 348                         }
 349                 } catch ( InvalidResponseException $e ) {
 350                         $status->fatal( 'backend-fail-connect', $this->name );
 351                         return $status;
 352                 } catch ( Exception $e ) { // some other exception?
 353                         $status->fatal( 'backend-fail-internal', $this->name );
 354                         $this->logException( $e, __METHOD__, $params );
 355                         return $status;
 356                 }
 357
 358                 return $status;
 359         }
 360
 361         /**
 362          * @see FileBackendStore::doSecureInternal()
 363          */
 364         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 365                 $status = Status::newGood();
 366
 367                 if ( $this->swiftAnonUser != '' ) {
 368                         // Restrict container from end-users...
 369                         try {
 370                                 // doPrepareInternal() should have been called,
 371                                 // so the Swift container should already exist...
 372                                 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 373                                 // NoSuchContainerException not thrown: container must exist
 374                                 if ( !isset( $contObj->mw_wasSecured ) ) {
 375                                         $status->merge( $this->setContainerAccess(
 376                                                 $contObj,
 377                                                 array( $this->auth->username ), // read
 378                                                 array( $this->auth->username ) // write
 379                                         ) );
 380                                         // @TODO: when php-cloudfiles supports container
 381                                         // metadata, we can make use of that to avoid RTTs
 382                                         $contObj->mw_wasSecured = true; // avoid useless RTTs
 383                                 }
 384                         } catch ( InvalidResponseException $e ) {
 385                                 $status->fatal( 'backend-fail-connect', $this->name );
 386                         } catch ( Exception $e ) { // some other exception?
 387                                 $status->fatal( 'backend-fail-internal', $this->name );
 388                                 $this->logException( $e, __METHOD__, $params );
 389                         }
 390                 }
 391
 392                 return $status;
 393         }
 394
 395         /**
 396          * @see FileBackendStore::doCleanInternal()
 397          */
 398         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 399                 $status = Status::newGood();
 400
 401                 // Only containers themselves can be removed, all else is virtual
 402                 if ( $dir != '' ) {
 403                         return $status; // nothing to do
 404                 }
 405
 406                 // (a) Check the container
 407                 try {
 408                         $contObj = $this->getContainer( $fullCont, true );
 409                 } catch ( NoSuchContainerException $e ) {
 410                         return $status; // ok, nothing to do
 411                 } catch ( InvalidResponseException $e ) {
 412                         $status->fatal( 'backend-fail-connect', $this->name );
 413                         return $status;
 414                 } catch ( Exception $e ) { // some other exception?
 415                         $status->fatal( 'backend-fail-internal', $this->name );
 416                         $this->logException( $e, __METHOD__, $params );
 417                         return $status;
 418                 }
 419
 420                 // (b) Delete the container if empty
 421                 if ( $contObj->object_count == 0 ) {
 422                         try {
 423                                 $this->deleteContainer( $fullCont );
 424                         } catch ( NoSuchContainerException $e ) {
 425                                 return $status; // race?
 426                         } catch ( InvalidResponseException $e ) {
 427                                 $status->fatal( 'backend-fail-connect', $this->name );
 428                                 return $status;
 429                         } catch ( Exception $e ) { // some other exception?
 430                                 $status->fatal( 'backend-fail-internal', $this->name );
 431                                 $this->logException( $e, __METHOD__, $params );
 432                                 return $status;
 433                         }
 434                 }
 435
 436                 return $status;
 437         }
 438
 439         /**
 440          * @see FileBackendStore::doFileExists()
 441          */
 442         protected function doGetFileStat( array $params ) {
 443                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 444                 if ( $srcRel === null ) {
 445                         return false; // invalid storage path
 446                 }
 447
 448                 $stat = false;
 449                 try {
 450                         $contObj = $this->getContainer( $srcCont );
 451                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 452                         $this->addMissingMetadata( $srcObj, $params['src'] );
 453                         $stat = array(
 454                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 455                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 456                                 'size'  => $srcObj->content_length,
 457                                 'sha1'  => $srcObj->metadata['Sha1base36']
 458                         );
 459                 } catch ( NoSuchContainerException $e ) {
 460                 } catch ( NoSuchObjectException $e ) {
 461                 } catch ( InvalidResponseException $e ) {
 462                         $stat = null;
 463                 } catch ( Exception $e ) { // some other exception?
 464                         $stat = null;
 465                         $this->logException( $e, __METHOD__, $params );
 466                 }
 467
 468                 return $stat;
 469         }
 470
 471         /**
 472          * Fill in any missing object metadata and save it to Swift
 473          *
 474          * @param $obj CF_Object
 475          * @param $path string Storage path to object
 476          * @return bool Success
 477          * @throws Exception cloudfiles exceptions
 478          */
 479         protected function addMissingMetadata( CF_Object $obj, $path ) {
 480                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 481                         return true; // nothing to do
 482                 }
 483                 $status = Status::newGood();
 484                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 485                 if ( $status->isOK() ) {
 486                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
 487                         if ( $tmpFile ) {
 488                                 $hash = $tmpFile->getSha1Base36();
 489                                 if ( $hash !== false ) {
 490                                         $obj->metadata['Sha1base36'] = $hash;
 491                                         $obj->sync_metadata(); // save to Swift
 492                                         return true; // success
 493                                 }
 494                         }
 495                 }
 496                 $obj->metadata['Sha1base36'] = false;
 497                 return false; // failed
 498         }
 499
 500         /**
 501          * @see FileBackend::getFileContents()
 502          */
 503         public function getFileContents( array $params ) {
 504                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 505                 if ( $srcRel === null ) {
 506                         return false; // invalid storage path
 507                 }
 508
 509                 if ( !$this->fileExists( $params ) ) {
 510                         return null;
 511                 }
 512
 513                 $data = false;
 514                 try {
 515                         $sContObj = $this->getContainer( $srcCont );
 516                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request
 517                         $data = $obj->read( $this->headersFromParams( $params ) );
 518                 } catch ( NoSuchContainerException $e ) {
 519                 } catch ( InvalidResponseException $e ) {
 520                 } catch ( Exception $e ) { // some other exception?
 521                         $this->logException( $e, __METHOD__, $params );
 522                 }
 523
 524                 return $data;
 525         }
 526
 527         /**
 528          * @see FileBackendStore::getFileListInternal()
 529          */
 530         public function getFileListInternal( $fullCont, $dir, array $params ) {
 531                 return new SwiftFileBackendFileList( $this, $fullCont, $dir );
 532         }
 533
 534         /**
 535          * Do not call this function outside of SwiftFileBackendFileList
 536          *
 537          * @param $fullCont string Resolved container name
 538          * @param $dir string Resolved storage directory with no trailing slash
 539          * @param $after string Storage path of file to list items after
 540          * @param $limit integer Max number of items to list
 541          * @return Array
 542          */
 543         public function getFileListPageInternal( $fullCont, $dir, $after, $limit ) {
 544                 $files = array();
 545
 546                 try {
 547                         $container = $this->getContainer( $fullCont );
 548                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 549                         $files = $container->list_objects( $limit, $after, $prefix );
 550                 } catch ( NoSuchContainerException $e ) {
 551                 } catch ( NoSuchObjectException $e ) {
 552                 } catch ( InvalidResponseException $e ) {
 553                 } catch ( Exception $e ) { // some other exception?
 554                         $this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) );
 555                 }
 556
 557                 return $files;
 558         }
 559
 560         /**
 561          * @see FileBackendStore::doGetFileSha1base36()
 562          */
 563         public function doGetFileSha1base36( array $params ) {
 564                 $stat = $this->getFileStat( $params );
 565                 if ( $stat ) {
 566                         return $stat['sha1'];
 567                 } else {
 568                         return false;
 569                 }
 570         }
 571
 572         /**
 573          * @see FileBackendStore::doStreamFile()
 574          */
 575         protected function doStreamFile( array $params ) {
 576                 $status = Status::newGood();
 577
 578                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 579                 if ( $srcRel === null ) {
 580                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 581                 }
 582
 583                 try {
 584                         $cont = $this->getContainer( $srcCont );
 585                 } catch ( NoSuchContainerException $e ) {
 586                         $status->fatal( 'backend-fail-stream', $params['src'] );
 587                         return $status;
 588                 } catch ( InvalidResponseException $e ) {
 589                         $status->fatal( 'backend-fail-connect', $this->name );
 590                         return $status;
 591                 } catch ( Exception $e ) { // some other exception?
 592                         $status->fatal( 'backend-fail-stream', $params['src'] );
 593                         $this->logException( $e, __METHOD__, $params );
 594                         return $status;
 595                 }
 596
 597                 try {
 598                         $output = fopen( 'php://output', 'wb' );
 599                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
 600                         $obj->stream( $output, $this->headersFromParams( $params ) );
 601                 } catch ( InvalidResponseException $e ) { // 404? connection problem?
 602                         $status->fatal( 'backend-fail-stream', $params['src'] );
 603                 } catch ( Exception $e ) { // some other exception?
 604                         $status->fatal( 'backend-fail-stream', $params['src'] );
 605                         $this->logException( $e, __METHOD__, $params );
 606                 }
 607
 608                 return $status;
 609         }
 610
 611         /**
 612          * @see FileBackendStore::getLocalCopy()
 613          */
 614         public function getLocalCopy( array $params ) {
 615                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 616                 if ( $srcRel === null ) {
 617                         return null;
 618                 }
 619
 620                 if ( !$this->fileExists( $params ) ) {
 621                         return null;
 622                 }
 623
 624                 $tmpFile = null;
 625                 try {
 626                         $sContObj = $this->getContainer( $srcCont );
 627                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 628                         // Get source file extension
 629                         $ext = FileBackend::extensionFromPath( $srcRel );
 630                         // Create a new temporary file...
 631                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 632                         if ( $tmpFile ) {
 633                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 634                                 if ( $handle ) {
 635                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 636                                         fclose( $handle );
 637                                 } else {
 638                                         $tmpFile = null; // couldn't open temp file
 639                                 }
 640                         }
 641                 } catch ( NoSuchContainerException $e ) {
 642                         $tmpFile = null;
 643                 } catch ( InvalidResponseException $e ) {
 644                         $tmpFile = null;
 645                 } catch ( Exception $e ) { // some other exception?
 646                         $tmpFile = null;
 647                         $this->logException( $e, __METHOD__, $params );
 648                 }
 649
 650                 return $tmpFile;
 651         }
 652
 653         /**
 654          * Get headers to send to Swift when reading a file based
 655          * on a FileBackend params array, e.g. that of getLocalCopy().
 656          * $params is currently only checked for a 'latest' flag.
 657          *
 658          * @param $params Array
 659          * @return Array
 660          */
 661         protected function headersFromParams( array $params ) {
 662                 $hdrs = array();
 663                 if ( !empty( $params['latest'] ) ) {
 664                         $hdrs[] = 'X-Newest: true';
 665                 }
 666                 return $hdrs;
 667         }
 668
 669         /**
 670          * Set read/write permissions for a Swift container
 671          *
 672          * @param $contObj CF_Container Swift container
 673          * @param $readGrps Array Swift users who can read (account:user)
 674          * @param $writeGrps Array Swift users who can write (account:user)
 675          * @return Status
 676          */
 677         protected function setContainerAccess(
 678                 CF_Container $contObj, array $readGrps, array $writeGrps
 679         ) {
 680                 $creds = $contObj->cfs_auth->export_credentials();
 681
 682                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
 683
 684                 // Note: 10 second timeout consistent with php-cloudfiles
 685                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
 686                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
 687                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
 688                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
 689
 690                 return $req->execute(); // should return 204
 691         }
 692
 693         /**
 694          * Get a connection to the Swift proxy
 695          *
 696          * @return CF_Connection|bool
 697          * @throws InvalidResponseException
 698          */
 699         protected function getConnection() {
 700                 if ( $this->conn === false ) {
 701                         throw new InvalidResponseException; // failed last attempt
 702                 }
 703                 // Session keys expire after a while, so we renew them periodically
 704                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
 705                         $this->conn->close(); // close active cURL connections
 706                         $this->conn = null;
 707                 }
 708                 // Authenticate with proxy and get a session key...
 709                 if ( $this->conn === null ) {
 710                         $this->connContainers = array();
 711                         try {
 712                                 $this->auth->authenticate();
 713                                 $this->conn = new CF_Connection( $this->auth );
 714                                 $this->connStarted = time();
 715                         } catch ( AuthenticationException $e ) {
 716                                 $this->conn = false; // don't keep re-trying
 717                         } catch ( InvalidResponseException $e ) {
 718                                 $this->conn = false; // don't keep re-trying
 719                         }
 720                 }
 721                 if ( !$this->conn ) {
 722                         throw new InvalidResponseException; // auth/connection problem
 723                 }
 724                 return $this->conn;
 725         }
 726
 727         /**
 728          * @see FileBackendStore::doClearCache()
 729          */
 730         protected function doClearCache( array $paths = null ) {
 731                 $this->connContainers = array(); // clear container object cache
 732         }
 733
 734         /**
 735          * Get a Swift container object, possibly from process cache.
 736          * Use $reCache if the file count or byte count is needed.
 737          *
 738          * @param $container string Container name
 739          * @param $reCache bool Refresh the process cache
 740          * @return CF_Container
 741          */
 742         protected function getContainer( $container, $reCache = false ) {
 743                 $conn = $this->getConnection(); // Swift proxy connection
 744                 if ( $reCache ) {
 745                         unset( $this->connContainers[$container] ); // purge cache
 746                 }
 747                 if ( !isset( $this->connContainers[$container] ) ) {
 748                         $contObj = $conn->get_container( $container );
 749                         // NoSuchContainerException not thrown: container must exist
 750                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
 751                                 reset( $this->connContainers );
 752                                 $key = key( $this->connContainers );
 753                                 unset( $this->connContainers[$key] );
 754                         }
 755                         $this->connContainers[$container] = $contObj; // cache it
 756                 }
 757                 return $this->connContainers[$container];
 758         }
 759
 760         /**
 761          * Create a Swift container
 762          *
 763          * @param $container string Container name
 764          * @return CF_Container
 765          */
 766         protected function createContainer( $container ) {
 767                 $conn = $this->getConnection(); // Swift proxy connection
 768                 $contObj = $conn->create_container( $container );
 769                 $this->connContainers[$container] = $contObj; // cache it
 770                 return $contObj;
 771         }
 772
 773         /**
 774          * Delete a Swift container
 775          *
 776          * @param $container string Container name
 777          * @return void
 778          */
 779         protected function deleteContainer( $container ) {
 780                 $conn = $this->getConnection(); // Swift proxy connection
 781                 $conn->delete_container( $container );
 782                 unset( $this->connContainers[$container] ); // purge cache
 783         }
 784
 785         /**
 786          * Log an unexpected exception for this backend
 787          *
 788          * @param $e Exception
 789          * @param $func string
 790          * @param $params Array
 791          * @return void
 792          */
 793         protected function logException( Exception $e, $func, array $params ) {
 794                 wfDebugLog( 'SwiftBackend',
 795                         get_class( $e ) . " in '{$this->name}': '{$func}' with " . serialize( $params )
 796                 );
 797         }
 798 }
 799
 800 /**
 801  * SwiftFileBackend helper class to page through object listings.
 802  * Swift also has a listing limit of 10,000 objects for sanity.
 803  * Do not use this class from places outside SwiftFileBackend.
 804  *
 805  * @ingroup FileBackend
 806  */
 807 class SwiftFileBackendFileList implements Iterator {
 808         /** @var Array */
 809         protected $bufferIter = array();
 810         protected $bufferAfter = null; // string; list items *after* this path
 811         protected $pos = 0; // integer
 812
 813         /** @var SwiftFileBackend */
 814         protected $backend;
 815         protected $container; //
 816         protected $dir; // string storage directory
 817         protected $suffixStart; // integer
 818
 819         const PAGE_SIZE = 5000; // file listing buffer size
 820
 821         /**
 822          * @param $backend SwiftFileBackend
 823          * @param $fullCont string Resolved container name
 824          * @param $dir string Resolved directory relative to container
 825          */
 826         public function __construct( SwiftFileBackend $backend, $fullCont, $dir ) {
 827                 $this->backend = $backend;
 828                 $this->container = $fullCont;
 829                 $this->dir = $dir;
 830                 if ( substr( $this->dir, -1 ) === '/' ) {
 831                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
 832                 }
 833                 if ( $this->dir == '' ) { // whole container
 834                         $this->suffixStart = 0;
 835                 } else { // dir within container
 836                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
 837                 }
 838         }
 839
 840         public function current() {
 841                 return substr( current( $this->bufferIter ), $this->suffixStart );
 842         }
 843
 844         public function key() {
 845                 return $this->pos;
 846         }
 847
 848         public function next() {
 849                 // Advance to the next file in the page
 850                 next( $this->bufferIter );
 851                 ++$this->pos;
 852                 // Check if there are no files left in this page and
 853                 // advance to the next page if this page was not empty.
 854                 if ( !$this->valid() && count( $this->bufferIter ) ) {
 855                         $this->bufferAfter = end( $this->bufferIter );
 856                         $this->bufferIter = $this->backend->getFileListPageInternal(
 857                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE
 858                         );
 859                 }
 860         }
 861
 862         public function rewind() {
 863                 $this->pos = 0;
 864                 $this->bufferAfter = null;
 865                 $this->bufferIter = $this->backend->getFileListPageInternal(
 866                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE
 867                 );
 868         }
 869
 870         public function valid() {
 871                 return ( current( $this->bufferIter ) !== false ); // no paths can have this value
 872         }
 873 }