includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  45         protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
  46         protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
  47
  48         protected $maxContCacheSize = 300; // integer; max containers with entries
  49
  50         /** @var CF_Connection */
  51         protected $conn; // Swift connection handle
  52         protected $connStarted = 0; // integer UNIX timestamp
  53         protected $connContainers = array(); // container object cache
  54         protected $connException; // CloudFiles exception
  55
  56         /**
  57          * @see FileBackendStore::__construct()
  58          * Additional $config params include:
  59          *    swiftAuthUrl       : Swift authentication server URL
  60          *    swiftUser          : Swift user used by MediaWiki (account:username)
  61          *    swiftKey           : Swift authentication key for the above user
  62          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  63          *    swiftAnonUser      : Swift user used for end-user requests (account:username)
  64          *    swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  65          *    swiftCDNExpiry     : How long (in seconds) to store content in the CDN.
  66          *                         If files may likely change, this should probably not exceed
  67          *                         a few days. For example, deletions may take this long to apply.
  68          *                         If object purging is enabled, however, this is not an issue.
  69          *    swiftCDNPurgable   : Whether object purge requests are allowed by the CDN.
  70          *    shardViaHashLevels : Map of container names to sharding config with:
  71          *                         'base'   : base of hash characters, 16 or 36
  72          *                         'levels' : the number of hash levels (and digits)
  73          *                         'repeat' : hash subdirectories are prefixed with all the
  74          *                                    parent hash directory names (e.g. "a/ab/abc")
  75          */
  76         public function __construct( array $config ) {
  77                 parent::__construct( $config );
  78                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  79                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  80                 }
  81                 // Required settings
  82                 $this->auth = new CF_Authentication(
  83                         $config['swiftUser'],
  84                         $config['swiftKey'],
  85                         null, // account; unused
  86                         $config['swiftAuthUrl']
  87                 );
  88                 // Optional settings
  89                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  90                         ? $config['swiftAuthTTL']
  91                         : 5 * 60; // some sane number
  92                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  93                         ? $config['swiftAnonUser']
  94                         : '';
  95                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  96                         ? $config['shardViaHashLevels']
  97                         : '';
  98                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
  99                         ? $config['swiftUseCDN']
 100                         : false;
 101                 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
 102                         ? $config['swiftCDNExpiry']
 103                         : 3600; // hour
 104                 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
 105                         ? $config['swiftCDNPurgable']
 106                         : true;
 107                 // Cache container info to mask latency
 108                 $this->memCache = wfGetMainCache();
 109         }
 110
 111         /**
 112          * @see FileBackendStore::resolveContainerPath()
 113          * @return null
 114          */
 115         protected function resolveContainerPath( $container, $relStoragePath ) {
 116                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 117                         return null; // too long for Swift
 118                 }
 119                 return $relStoragePath;
 120         }
 121
 122         /**
 123          * @see FileBackendStore::isPathUsableInternal()
 124          * @return bool
 125          */
 126         public function isPathUsableInternal( $storagePath ) {
 127                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 128                 if ( $rel === null ) {
 129                         return false; // invalid
 130                 }
 131
 132                 try {
 133                         $this->getContainer( $container );
 134                         return true; // container exists
 135                 } catch ( NoSuchContainerException $e ) {
 136                 } catch ( CloudFilesException $e ) { // some other exception?
 137                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 138                 }
 139
 140                 return false;
 141         }
 142
 143         /**
 144          * @see FileBackendStore::doCreateInternal()
 145          * @return Status
 146          */
 147         protected function doCreateInternal( array $params ) {
 148                 $status = Status::newGood();
 149
 150                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 151                 if ( $dstRel === null ) {
 152                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 153                         return $status;
 154                 }
 155
 156                 // (a) Check the destination container and object
 157                 try {
 158                         $dContObj = $this->getContainer( $dstCont );
 159                         if ( empty( $params['overwrite'] ) &&
 160                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 161                         {
 162                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 163                                 return $status;
 164                         }
 165                 } catch ( NoSuchContainerException $e ) {
 166                         $status->fatal( 'backend-fail-create', $params['dst'] );
 167                         return $status;
 168                 } catch ( CloudFilesException $e ) { // some other exception?
 169                         $this->handleException( $e, $status, __METHOD__, $params );
 170                         return $status;
 171                 }
 172
 173                 // (b) Get a SHA-1 hash of the object
 174                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 175
 176                 // (c) Actually create the object
 177                 try {
 178                         // Create a fresh CF_Object with no fields preloaded.
 179                         // We don't want to preserve headers, metadata, and such.
 180                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 181                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 182                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 183                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 184                         // The MD5 here will be checked within Swift against its own MD5.
 185                         $obj->set_etag( md5( $params['content'] ) );
 186                         // Use the same content type as StreamFile for security
 187                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 188                         if ( !empty( $params['async'] ) ) { // deferred
 189                                 $handle = $obj->write_async( $params['content'] );
 190                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle );
 191                                 $status->value->affectedObjects[] = $obj;
 192                         } else { // actually write the object in Swift
 193                                 $obj->write( $params['content'] );
 194                                 $this->purgeCDNCache( array( $obj ) );
 195                         }
 196                 } catch ( CDNNotEnabledException $e ) {
 197                         // CDN not enabled; nothing to see here
 198                 } catch ( BadContentTypeException $e ) {
 199                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 200                 } catch ( CloudFilesException $e ) { // some other exception?
 201                         $this->handleException( $e, $status, __METHOD__, $params );
 202                 }
 203
 204                 return $status;
 205         }
 206
 207         /**
 208          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 209          */
 210         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 211                 try {
 212                         $cfOp->getLastResponse();
 213                 } catch ( BadContentTypeException $e ) {
 214                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 215                 }
 216         }
 217
 218         /**
 219          * @see FileBackendStore::doStoreInternal()
 220          * @return Status
 221          */
 222         protected function doStoreInternal( array $params ) {
 223                 $status = Status::newGood();
 224
 225                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 226                 if ( $dstRel === null ) {
 227                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 228                         return $status;
 229                 }
 230
 231                 // (a) Check the destination container and object
 232                 try {
 233                         $dContObj = $this->getContainer( $dstCont );
 234                         if ( empty( $params['overwrite'] ) &&
 235                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 236                         {
 237                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 238                                 return $status;
 239                         }
 240                 } catch ( NoSuchContainerException $e ) {
 241                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 242                         return $status;
 243                 } catch ( CloudFilesException $e ) { // some other exception?
 244                         $this->handleException( $e, $status, __METHOD__, $params );
 245                         return $status;
 246                 }
 247
 248                 // (b) Get a SHA-1 hash of the object
 249                 $sha1Hash = sha1_file( $params['src'] );
 250                 if ( $sha1Hash === false ) { // source doesn't exist?
 251                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 252                         return $status;
 253                 }
 254                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 255
 256                 // (c) Actually store the object
 257                 try {
 258                         // Create a fresh CF_Object with no fields preloaded.
 259                         // We don't want to preserve headers, metadata, and such.
 260                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 261                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 262                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 263                         // The MD5 here will be checked within Swift against its own MD5.
 264                         $obj->set_etag( md5_file( $params['src'] ) );
 265                         // Use the same content type as StreamFile for security
 266                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 267                         if ( !empty( $params['async'] ) ) { // deferred
 268                                 wfSuppressWarnings();
 269                                 $fp = fopen( $params['src'], 'rb' );
 270                                 wfRestoreWarnings();
 271                                 if ( !$fp ) {
 272                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 273                                 } else {
 274                                         $handle = $obj->write_async( $fp, filesize( $params['src'] ), true );
 275                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle );
 276                                         $status->value->resourcesToClose[] = $fp;
 277                                         $status->value->affectedObjects[] = $obj;
 278                                 }
 279                         } else { // actually write the object in Swift
 280                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 281                                 $this->purgeCDNCache( array( $obj ) );
 282                         }
 283                 } catch ( CDNNotEnabledException $e ) {
 284                         // CDN not enabled; nothing to see here
 285                 } catch ( BadContentTypeException $e ) {
 286                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 287                 } catch ( IOException $e ) {
 288                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 289                 } catch ( CloudFilesException $e ) { // some other exception?
 290                         $this->handleException( $e, $status, __METHOD__, $params );
 291                 }
 292
 293                 return $status;
 294         }
 295
 296         /**
 297          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 298          */
 299         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 300                 try {
 301                         $cfOp->getLastResponse();
 302                 } catch ( BadContentTypeException $e ) {
 303                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 304                 } catch ( IOException $e ) {
 305                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 306                 }
 307         }
 308
 309         /**
 310          * @see FileBackendStore::doCopyInternal()
 311          * @return Status
 312          */
 313         protected function doCopyInternal( array $params ) {
 314                 $status = Status::newGood();
 315
 316                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 317                 if ( $srcRel === null ) {
 318                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 319                         return $status;
 320                 }
 321
 322                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 323                 if ( $dstRel === null ) {
 324                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 325                         return $status;
 326                 }
 327
 328                 // (a) Check the source/destination containers and destination object
 329                 try {
 330                         $sContObj = $this->getContainer( $srcCont );
 331                         $dContObj = $this->getContainer( $dstCont );
 332                         if ( empty( $params['overwrite'] ) &&
 333                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 334                         {
 335                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 336                                 return $status;
 337                         }
 338                 } catch ( NoSuchContainerException $e ) {
 339                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 340                         return $status;
 341                 } catch ( CloudFilesException $e ) { // some other exception?
 342                         $this->handleException( $e, $status, __METHOD__, $params );
 343                         return $status;
 344                 }
 345
 346                 // (b) Actually copy the file to the destination
 347                 try {
 348                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 349                         if ( !empty( $params['async'] ) ) { // deferred
 350                                 $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel );
 351                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle );
 352                                 $status->value->affectedObjects[] = $dstObj;
 353                         } else { // actually write the object in Swift
 354                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 355                                 $this->purgeCDNCache( array( $dstObj ) );
 356                         }
 357                 } catch ( CDNNotEnabledException $e ) {
 358                         // CDN not enabled; nothing to see here
 359                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 360                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 361                 } catch ( CloudFilesException $e ) { // some other exception?
 362                         $this->handleException( $e, $status, __METHOD__, $params );
 363                 }
 364
 365                 return $status;
 366         }
 367
 368         /**
 369          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 370          */
 371         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 372                 try {
 373                         $cfOp->getLastResponse();
 374                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 375                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 376                 }
 377         }
 378
 379         /**
 380          * @see FileBackendStore::doMoveInternal()
 381          * @return Status
 382          */
 383         protected function doMoveInternal( array $params ) {
 384                 $status = Status::newGood();
 385
 386                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 387                 if ( $srcRel === null ) {
 388                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 389                         return $status;
 390                 }
 391
 392                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 393                 if ( $dstRel === null ) {
 394                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 395                         return $status;
 396                 }
 397
 398                 // (a) Check the source/destination containers and destination object
 399                 try {
 400                         $sContObj = $this->getContainer( $srcCont );
 401                         $dContObj = $this->getContainer( $dstCont );
 402                         if ( empty( $params['overwrite'] ) &&
 403                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 404                         {
 405                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 406                                 return $status;
 407                         }
 408                 } catch ( NoSuchContainerException $e ) {
 409                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 410                         return $status;
 411                 } catch ( CloudFilesException $e ) { // some other exception?
 412                         $this->handleException( $e, $status, __METHOD__, $params );
 413                         return $status;
 414                 }
 415
 416                 // (b) Actually move the file to the destination
 417                 try {
 418                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 419                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 420                         if ( !empty( $params['async'] ) ) { // deferred
 421                                 $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel );
 422                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle );
 423                                 $status->value->affectedObjects[] = $srcObj;
 424                                 $status->value->affectedObjects[] = $dstObj;
 425                         } else { // actually write the object in Swift
 426                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel );
 427                                 $this->purgeCDNCache( array( $srcObj, $dstObj ) );
 428                         }
 429                 } catch ( CDNNotEnabledException $e ) {
 430                         // CDN not enabled; nothing to see here
 431                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 432                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 433                 } catch ( CloudFilesException $e ) { // some other exception?
 434                         $this->handleException( $e, $status, __METHOD__, $params );
 435                 }
 436
 437                 return $status;
 438         }
 439
 440         /**
 441          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 442          */
 443         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 444                 try {
 445                         $cfOp->getLastResponse();
 446                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 447                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 448                 }
 449         }
 450
 451         /**
 452          * @see FileBackendStore::doDeleteInternal()
 453          * @return Status
 454          */
 455         protected function doDeleteInternal( array $params ) {
 456                 $status = Status::newGood();
 457
 458                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 459                 if ( $srcRel === null ) {
 460                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 461                         return $status;
 462                 }
 463
 464                 try {
 465                         $sContObj = $this->getContainer( $srcCont );
 466                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 467                         if ( !empty( $params['async'] ) ) { // deferred
 468                                 $handle = $sContObj->delete_object_async( $srcRel );
 469                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle );
 470                                 $status->value->affectedObjects[] = $srcObj;
 471                         } else { // actually write the object in Swift
 472                                 $sContObj->delete_object( $srcRel );
 473                                 $this->purgeCDNCache( array( $srcObj ) );
 474                         }
 475                 } catch ( CDNNotEnabledException $e ) {
 476                         // CDN not enabled; nothing to see here
 477                 } catch ( NoSuchContainerException $e ) {
 478                         $status->fatal( 'backend-fail-delete', $params['src'] );
 479                 } catch ( NoSuchObjectException $e ) {
 480                         if ( empty( $params['ignoreMissingSource'] ) ) {
 481                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 482                         }
 483                 } catch ( CloudFilesException $e ) { // some other exception?
 484                         $this->handleException( $e, $status, __METHOD__, $params );
 485                 }
 486
 487                 return $status;
 488         }
 489
 490         /**
 491          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 492          */
 493         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 494                 try {
 495                         $cfOp->getLastResponse();
 496                 } catch ( NoSuchContainerException $e ) {
 497                         $status->fatal( 'backend-fail-delete', $params['src'] );
 498                 } catch ( NoSuchObjectException $e ) {
 499                         if ( empty( $params['ignoreMissingSource'] ) ) {
 500                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 501                         }
 502                 }
 503         }
 504
 505         /**
 506          * @see FileBackendStore::doPrepareInternal()
 507          * @return Status
 508          */
 509         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 510                 $status = Status::newGood();
 511
 512                 // (a) Check if container already exists
 513                 try {
 514                         $contObj = $this->getContainer( $fullCont );
 515                         // NoSuchContainerException not thrown: container must exist
 516                         return $status; // already exists
 517                 } catch ( NoSuchContainerException $e ) {
 518                         // NoSuchContainerException thrown: container does not exist
 519                 } catch ( CloudFilesException $e ) { // some other exception?
 520                         $this->handleException( $e, $status, __METHOD__, $params );
 521                         return $status;
 522                 }
 523
 524                 // (b) Create container as needed
 525                 try {
 526                         $contObj = $this->createContainer( $fullCont );
 527                         // Make container public to end-users...
 528                         if ( $this->swiftAnonUser != '' ) {
 529                                 $status->merge( $this->setContainerAccess(
 530                                         $contObj,
 531                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 532                                         array( $this->auth->username ) // write
 533                                 ) );
 534                         }
 535                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 536                                 $contObj->make_public( $this->swiftCDNExpiry );
 537                         }
 538                 } catch ( CDNNotEnabledException $e ) {
 539                         // CDN not enabled; nothing to see here
 540                 } catch ( CloudFilesException $e ) { // some other exception?
 541                         $this->handleException( $e, $status, __METHOD__, $params );
 542                         return $status;
 543                 }
 544
 545                 return $status;
 546         }
 547
 548         /**
 549          * @see FileBackendStore::doSecureInternal()
 550          * @return Status
 551          */
 552         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 553                 $status = Status::newGood();
 554
 555                 // Restrict container from end-users...
 556                 try {
 557                         // doPrepareInternal() should have been called,
 558                         // so the Swift container should already exist...
 559                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 560                         // NoSuchContainerException not thrown: container must exist
 561
 562                         // Make container private to end-users...
 563                         if ( $this->swiftAnonUser != '' && !isset( $contObj->mw_wasSecured ) ) {
 564                                 $status->merge( $this->setContainerAccess(
 565                                         $contObj,
 566                                         array( $this->auth->username ), // read
 567                                         array( $this->auth->username ) // write
 568                                 ) );
 569                                 // @TODO: when php-cloudfiles supports container
 570                                 // metadata, we can make use of that to avoid RTTs
 571                                 $contObj->mw_wasSecured = true; // avoid useless RTTs
 572                         }
 573                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 574                                 $contObj->make_private();
 575                         }
 576                 } catch ( CDNNotEnabledException $e ) {
 577                         // CDN not enabled; nothing to see here
 578                 } catch ( CloudFilesException $e ) { // some other exception?
 579                         $this->handleException( $e, $status, __METHOD__, $params );
 580                 }
 581
 582                 return $status;
 583         }
 584
 585         /**
 586          * @see FileBackendStore::doCleanInternal()
 587          * @return Status
 588          */
 589         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 590                 $status = Status::newGood();
 591
 592                 // Only containers themselves can be removed, all else is virtual
 593                 if ( $dir != '' ) {
 594                         return $status; // nothing to do
 595                 }
 596
 597                 // (a) Check the container
 598                 try {
 599                         $contObj = $this->getContainer( $fullCont, true );
 600                 } catch ( NoSuchContainerException $e ) {
 601                         return $status; // ok, nothing to do
 602                 } catch ( CloudFilesException $e ) { // some other exception?
 603                         $this->handleException( $e, $status, __METHOD__, $params );
 604                         return $status;
 605                 }
 606
 607                 // (b) Delete the container if empty
 608                 if ( $contObj->object_count == 0 ) {
 609                         try {
 610                                 $this->deleteContainer( $fullCont );
 611                         } catch ( NoSuchContainerException $e ) {
 612                                 return $status; // race?
 613                         } catch ( NonEmptyContainerException $e ) {
 614                                 return $status; // race? consistency delay?
 615                         } catch ( CloudFilesException $e ) { // some other exception?
 616                                 $this->handleException( $e, $status, __METHOD__, $params );
 617                                 return $status;
 618                         }
 619                 }
 620
 621                 return $status;
 622         }
 623
 624         /**
 625          * @see FileBackendStore::doFileExists()
 626          * @return array|bool|null
 627          */
 628         protected function doGetFileStat( array $params ) {
 629                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 630                 if ( $srcRel === null ) {
 631                         return false; // invalid storage path
 632                 }
 633
 634                 $stat = false;
 635                 try {
 636                         $contObj = $this->getContainer( $srcCont );
 637                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 638                         $this->addMissingMetadata( $srcObj, $params['src'] );
 639                         $stat = array(
 640                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 641                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 642                                 'size'  => (int)$srcObj->content_length,
 643                                 'sha1'  => $srcObj->metadata['Sha1base36']
 644                         );
 645                 } catch ( NoSuchContainerException $e ) {
 646                 } catch ( NoSuchObjectException $e ) {
 647                 } catch ( CloudFilesException $e ) { // some other exception?
 648                         $stat = null;
 649                         $this->handleException( $e, null, __METHOD__, $params );
 650                 }
 651
 652                 return $stat;
 653         }
 654
 655         /**
 656          * Fill in any missing object metadata and save it to Swift
 657          *
 658          * @param $obj CF_Object
 659          * @param $path string Storage path to object
 660          * @return bool Success
 661          * @throws Exception cloudfiles exceptions
 662          */
 663         protected function addMissingMetadata( CF_Object $obj, $path ) {
 664                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 665                         return true; // nothing to do
 666                 }
 667                 $status = Status::newGood();
 668                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 669                 if ( $status->isOK() ) {
 670                         # Do not stat the file in getLocalCopy() to avoid infinite loops
 671                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
 672                         if ( $tmpFile ) {
 673                                 $hash = $tmpFile->getSha1Base36();
 674                                 if ( $hash !== false ) {
 675                                         $obj->metadata['Sha1base36'] = $hash;
 676                                         $obj->sync_metadata(); // save to Swift
 677                                         return true; // success
 678                                 }
 679                         }
 680                 }
 681                 $obj->metadata['Sha1base36'] = false;
 682                 return false; // failed
 683         }
 684
 685         /**
 686          * @see FileBackend::getFileContents()
 687          * @return bool|null|string
 688          */
 689         public function getFileContents( array $params ) {
 690                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 691                 if ( $srcRel === null ) {
 692                         return false; // invalid storage path
 693                 }
 694
 695                 if ( !$this->fileExists( $params ) ) {
 696                         return null;
 697                 }
 698
 699                 $data = false;
 700                 try {
 701                         $sContObj = $this->getContainer( $srcCont );
 702                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 703                         $data = $obj->read( $this->headersFromParams( $params ) );
 704                 } catch ( NoSuchContainerException $e ) {
 705                 } catch ( CloudFilesException $e ) { // some other exception?
 706                         $this->handleException( $e, null, __METHOD__, $params );
 707                 }
 708
 709                 return $data;
 710         }
 711
 712         /**
 713          * @see FileBackendStore::doDirectoryExists()
 714          * @return bool|null
 715          */
 716         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 717                 try {
 718                         $container = $this->getContainer( $fullCont );
 719                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 720                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 721                 } catch ( NoSuchContainerException $e ) {
 722                         return false;
 723                 } catch ( CloudFilesException $e ) { // some other exception?
 724                         $this->handleException( $e, null, __METHOD__,
 725                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 726                 }
 727
 728                 return null; // error
 729         }
 730
 731         /**
 732          * @see FileBackendStore::getDirectoryListInternal()
 733          * @return SwiftFileBackendDirList
 734          */
 735         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 736                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 737         }
 738
 739         /**
 740          * @see FileBackendStore::getFileListInternal()
 741          * @return SwiftFileBackendFileList
 742          */
 743         public function getFileListInternal( $fullCont, $dir, array $params ) {
 744                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 745         }
 746
 747         /**
 748          * Do not call this function outside of SwiftFileBackendFileList
 749          *
 750          * @param $fullCont string Resolved container name
 751          * @param $dir string Resolved storage directory with no trailing slash
 752          * @param $after string|null Storage path of file to list items after
 753          * @param $limit integer Max number of items to list
 754          * @param $params Array Includes flag for 'topOnly'
 755          * @return Array List of relative paths of dirs directly under $dir
 756          */
 757         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 758                 $dirs = array();
 759                 if ( $after === INF ) {
 760                         return $dirs; // nothing more
 761                 }
 762                 wfProfileIn( __METHOD__ . '-' . $this->name );
 763
 764                 try {
 765                         $container = $this->getContainer( $fullCont );
 766                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 767                         // Non-recursive: only list dirs right under $dir
 768                         if ( !empty( $params['topOnly'] ) ) {
 769                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 770                                 foreach ( $objects as $object ) { // files and dirs
 771                                         if ( substr( $object, -1 ) === '/' ) {
 772                                                 $dirs[] = $object; // directories end in '/'
 773                                         }
 774                                 }
 775                         // Recursive: list all dirs under $dir and its subdirs
 776                         } else {
 777                                 // Get directory from last item of prior page
 778                                 $lastDir = $this->getParentDir( $after ); // must be first page
 779                                 $objects = $container->list_objects( $limit, $after, $prefix );
 780                                 foreach ( $objects as $object ) { // files
 781                                         $objectDir = $this->getParentDir( $object ); // directory of object
 782                                         if ( $objectDir !== false ) { // file has a parent dir
 783                                                 // Swift stores paths in UTF-8, using binary sorting.
 784                                                 // See function "create_container_table" in common/db.py.
 785                                                 // If a directory is not "greater" than the last one,
 786                                                 // then it was already listed by the calling iterator.
 787                                                 if ( $objectDir > $lastDir ) {
 788                                                         $pDir = $objectDir;
 789                                                         do { // add dir and all its parent dirs
 790                                                                 $dirs[] = "{$pDir}/";
 791                                                                 $pDir = $this->getParentDir( $pDir );
 792                                                         } while ( $pDir !== false // sanity
 793                                                                 && $pDir > $lastDir // not done already
 794                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 795                                                         );
 796                                                 }
 797                                                 $lastDir = $objectDir;
 798                                         }
 799                                 }
 800                         }
 801                         if ( count( $objects ) < $limit ) {
 802                                 $after = INF; // avoid a second RTT
 803                         } else {
 804                                 $after = end( $objects ); // update last item
 805                         }
 806                 } catch ( NoSuchContainerException $e ) {
 807                 } catch ( CloudFilesException $e ) { // some other exception?
 808                         $this->handleException( $e, null, __METHOD__,
 809                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 810                 }
 811
 812                 wfProfileOut( __METHOD__ . '-' . $this->name );
 813                 return $dirs;
 814         }
 815
 816         protected function getParentDir( $path ) {
 817                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 818         }
 819
 820         /**
 821          * Do not call this function outside of SwiftFileBackendFileList
 822          *
 823          * @param $fullCont string Resolved container name
 824          * @param $dir string Resolved storage directory with no trailing slash
 825          * @param $after string|null Storage path of file to list items after
 826          * @param $limit integer Max number of items to list
 827          * @param $params Array Includes flag for 'topOnly'
 828          * @return Array List of relative paths of files under $dir
 829          */
 830         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 831                 $files = array();
 832                 if ( $after === INF ) {
 833                         return $files; // nothing more
 834                 }
 835                 wfProfileIn( __METHOD__ . '-' . $this->name );
 836
 837                 try {
 838                         $container = $this->getContainer( $fullCont );
 839                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 840                         // Non-recursive: only list files right under $dir
 841                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 842                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 843                                 foreach ( $objects as $object ) {
 844                                         if ( substr( $object, -1 ) !== '/' ) {
 845                                                 $files[] = $object; // directories end in '/'
 846                                         }
 847                                 }
 848                         // Recursive: list all files under $dir and its subdirs
 849                         } else { // files
 850                                 $objects = $container->list_objects( $limit, $after, $prefix );
 851                                 $files = $objects;
 852                         }
 853                         if ( count( $objects ) < $limit ) {
 854                                 $after = INF; // avoid a second RTT
 855                         } else {
 856                                 $after = end( $objects ); // update last item
 857                         }
 858                 } catch ( NoSuchContainerException $e ) {
 859                 } catch ( CloudFilesException $e ) { // some other exception?
 860                         $this->handleException( $e, null, __METHOD__,
 861                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 862                 }
 863
 864                 wfProfileOut( __METHOD__ . '-' . $this->name );
 865                 return $files;
 866         }
 867
 868         /**
 869          * @see FileBackendStore::doGetFileSha1base36()
 870          * @return bool
 871          */
 872         protected function doGetFileSha1base36( array $params ) {
 873                 $stat = $this->getFileStat( $params );
 874                 if ( $stat ) {
 875                         return $stat['sha1'];
 876                 } else {
 877                         return false;
 878                 }
 879         }
 880
 881         /**
 882          * @see FileBackendStore::doStreamFile()
 883          * @return Status
 884          */
 885         protected function doStreamFile( array $params ) {
 886                 $status = Status::newGood();
 887
 888                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 889                 if ( $srcRel === null ) {
 890                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 891                 }
 892
 893                 try {
 894                         $cont = $this->getContainer( $srcCont );
 895                 } catch ( NoSuchContainerException $e ) {
 896                         $status->fatal( 'backend-fail-stream', $params['src'] );
 897                         return $status;
 898                 } catch ( CloudFilesException $e ) { // some other exception?
 899                         $this->handleException( $e, $status, __METHOD__, $params );
 900                         return $status;
 901                 }
 902
 903                 try {
 904                         $output = fopen( 'php://output', 'wb' );
 905                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
 906                         $obj->stream( $output, $this->headersFromParams( $params ) );
 907                 } catch ( CloudFilesException $e ) { // some other exception?
 908                         $this->handleException( $e, $status, __METHOD__, $params );
 909                 }
 910
 911                 return $status;
 912         }
 913
 914         /**
 915          * @see FileBackendStore::getLocalCopy()
 916          * @return null|TempFSFile
 917          */
 918         public function getLocalCopy( array $params ) {
 919                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 920                 if ( $srcRel === null ) {
 921                         return null;
 922                 }
 923
 924                 # Check the recursion guard to avoid loops when filling metadata
 925                 if ( empty( $params['nostat'] ) && !$this->fileExists( $params ) ) {
 926                         return null;
 927                 }
 928
 929                 $tmpFile = null;
 930                 try {
 931                         $sContObj = $this->getContainer( $srcCont );
 932                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 933                         // Get source file extension
 934                         $ext = FileBackend::extensionFromPath( $srcRel );
 935                         // Create a new temporary file...
 936                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 937                         if ( $tmpFile ) {
 938                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 939                                 if ( $handle ) {
 940                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 941                                         fclose( $handle );
 942                                 } else {
 943                                         $tmpFile = null; // couldn't open temp file
 944                                 }
 945                         }
 946                 } catch ( NoSuchContainerException $e ) {
 947                         $tmpFile = null;
 948                 } catch ( CloudFilesException $e ) { // some other exception?
 949                         $tmpFile = null;
 950                         $this->handleException( $e, null, __METHOD__, $params );
 951                 }
 952
 953                 return $tmpFile;
 954         }
 955
 956         /**
 957          * @see FileBackendStore::directoriesAreVirtual()
 958          * @return bool
 959          */
 960         protected function directoriesAreVirtual() {
 961                 return true;
 962         }
 963
 964         /**
 965          * Get headers to send to Swift when reading a file based
 966          * on a FileBackend params array, e.g. that of getLocalCopy().
 967          * $params is currently only checked for a 'latest' flag.
 968          *
 969          * @param $params Array
 970          * @return Array
 971          */
 972         protected function headersFromParams( array $params ) {
 973                 $hdrs = array();
 974                 if ( !empty( $params['latest'] ) ) {
 975                         $hdrs[] = 'X-Newest: true';
 976                 }
 977                 return $hdrs;
 978         }
 979
 980         /**
 981          * @see FileBackendStore::doExecuteOpHandlesInternal()
 982          * @return Array List of corresponding Status objects
 983          */
 984         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
 985                 $statuses = array();
 986
 987                 $cfOps = array(); // list of CF_Async_Op objects
 988                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
 989                         $cfOps[$index] = $fileOpHandle->cfOp;
 990                 }
 991                 $batch = new CF_Async_Op_Batch( $cfOps );
 992
 993                 $cfOps = $batch->execute();
 994                 foreach ( $cfOps as $index => $cfOp ) {
 995                         $status = Status::newGood();
 996                         try { // catch exceptions; update status
 997                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
 998                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
 999                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
1000                         } catch ( CloudFilesException $e ) { // some other exception?
1001                                 $this->handleException( $e, $status,
1002                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
1003                         }
1004                         $statuses[$index] = $status;
1005                 }
1006
1007                 return $statuses;
1008         }
1009
1010         /**
1011          * Set read/write permissions for a Swift container
1012          *
1013          * @param $contObj CF_Container Swift container
1014          * @param $readGrps Array Swift users who can read (account:user)
1015          * @param $writeGrps Array Swift users who can write (account:user)
1016          * @return Status
1017          */
1018         protected function setContainerAccess(
1019                 CF_Container $contObj, array $readGrps, array $writeGrps
1020         ) {
1021                 $creds = $contObj->cfs_auth->export_credentials();
1022
1023                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1024
1025                 // Note: 10 second timeout consistent with php-cloudfiles
1026                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1027                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1028                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1029                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1030
1031                 return $req->execute(); // should return 204
1032         }
1033
1034         /**
1035          * Purge the CDN cache of affected objects if CDN caching is enabled.
1036          * This is for Rackspace/Akamai CDNs.
1037          *
1038          * @param $objects Array List of CF_Object items
1039          * @return void
1040          */
1041         public function purgeCDNCache( array $objects ) {
1042                 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
1043                         foreach ( $objects as $object ) {
1044                                 try {
1045                                         $object->purge_from_cdn();
1046                                 } catch ( CDNNotEnabledException $e ) {
1047                                         // CDN not enabled; nothing to see here
1048                                 } catch ( CloudFilesException $e ) {
1049                                         $this->handleException( $e, null, __METHOD__,
1050                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1051                                 }
1052                         }
1053                 }
1054         }
1055
1056         /**
1057          * Get a connection to the Swift proxy
1058          *
1059          * @return CF_Connection|bool False on failure
1060          * @throws CloudFilesException
1061          */
1062         protected function getConnection() {
1063                 if ( $this->connException instanceof Exception ) {
1064                         throw $this->connException; // failed last attempt
1065                 }
1066                 // Session keys expire after a while, so we renew them periodically
1067                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
1068                         $this->conn->close(); // close active cURL connections
1069                         $this->conn = null;
1070                 }
1071                 // Authenticate with proxy and get a session key...
1072                 if ( !$this->conn ) {
1073                         $this->connStarted = 0;
1074                         $this->connContainers = array();
1075                         try {
1076                                 $this->auth->authenticate();
1077                                 $this->conn = new CF_Connection( $this->auth );
1078                                 $this->connStarted = time();
1079                         } catch ( CloudFilesException $e ) {
1080                                 $this->connException = $e; // don't keep re-trying
1081                                 throw $e; // throw it back
1082                         }
1083                 }
1084                 return $this->conn;
1085         }
1086
1087         /**
1088          * @see FileBackendStore::doClearCache()
1089          */
1090         protected function doClearCache( array $paths = null ) {
1091                 $this->connContainers = array(); // clear container object cache
1092         }
1093
1094         /**
1095          * Get a Swift container object, possibly from process cache.
1096          * Use $reCache if the file count or byte count is needed.
1097          *
1098          * @param $container string Container name
1099          * @param $bypassCache bool Bypass all caches and load from Swift
1100          * @return CF_Container
1101          * @throws CloudFilesException
1102          */
1103         protected function getContainer( $container, $bypassCache = false ) {
1104                 $conn = $this->getConnection(); // Swift proxy connection
1105                 if ( $bypassCache ) { // purge cache
1106                         unset( $this->connContainers[$container] );
1107                 } elseif ( !isset( $this->connContainers[$container] ) ) {
1108                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1109                 }
1110                 if ( !isset( $this->connContainers[$container] ) ) {
1111                         $contObj = $conn->get_container( $container );
1112                         // NoSuchContainerException not thrown: container must exist
1113                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
1114                                 reset( $this->connContainers );
1115                                 unset( $this->connContainers[key( $this->connContainers )] );
1116                         }
1117                         $this->connContainers[$container] = $contObj; // cache it
1118                         if ( !$bypassCache ) {
1119                                 $this->setContainerCache( $container, // update persistent cache
1120                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1121                                 );
1122                         }
1123                 }
1124                 return $this->connContainers[$container];
1125         }
1126
1127         /**
1128          * Create a Swift container
1129          *
1130          * @param $container string Container name
1131          * @return CF_Container
1132          * @throws CloudFilesException
1133          */
1134         protected function createContainer( $container ) {
1135                 $conn = $this->getConnection(); // Swift proxy connection
1136                 $contObj = $conn->create_container( $container );
1137                 $this->connContainers[$container] = $contObj; // cache it
1138                 return $contObj;
1139         }
1140
1141         /**
1142          * Delete a Swift container
1143          *
1144          * @param $container string Container name
1145          * @return void
1146          * @throws CloudFilesException
1147          */
1148         protected function deleteContainer( $container ) {
1149                 $conn = $this->getConnection(); // Swift proxy connection
1150                 unset( $this->connContainers[$container] ); // purge cache
1151                 $conn->delete_container( $container );
1152         }
1153
1154         /**
1155          * @see FileBackendStore::doPrimeContainerCache()
1156          * @return void
1157          */
1158         protected function doPrimeContainerCache( array $containerInfo ) {
1159                 try {
1160                         $conn = $this->getConnection(); // Swift proxy connection
1161                         foreach ( $containerInfo as $container => $info ) {
1162                                 $this->connContainers[$container] = new CF_Container(
1163                                         $conn->cfs_auth,
1164                                         $conn->cfs_http,
1165                                         $container,
1166                                         $info['count'],
1167                                         $info['bytes']
1168                                 );
1169                         }
1170                 } catch ( CloudFilesException $e ) { // some other exception?
1171                         $this->handleException( $e, null, __METHOD__, array() );
1172                 }
1173         }
1174
1175         /**
1176          * Log an unexpected exception for this backend.
1177          * This also sets the Status object to have a fatal error.
1178          *
1179          * @param $e Exception
1180          * @param $status Status|null
1181          * @param $func string
1182          * @param $params Array
1183          * @return void
1184          */
1185         protected function handleException( Exception $e, $status, $func, array $params ) {
1186                 if ( $status instanceof Status ) {
1187                         if ( $e instanceof AuthenticationException ) {
1188                                 $status->fatal( 'backend-fail-connect', $this->name );
1189                         } else {
1190                                 $status->fatal( 'backend-fail-internal', $this->name );
1191                         }
1192                 }
1193                 if ( $e->getMessage() ) {
1194                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1195                 }
1196                 wfDebugLog( 'SwiftBackend',
1197                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1198                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1199                 );
1200         }
1201 }
1202
1203 /**
1204  * @see FileBackendStoreOpHandle
1205  */
1206 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1207         /** @var CF_Async_Op */
1208         public $cfOp;
1209         /** @var Array */
1210         public $affectedObjects = array();
1211
1212         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1213                 $this->backend = $backend;
1214                 $this->params = $params;
1215                 $this->call = $call;
1216                 $this->cfOp = $cfOp;
1217         }
1218 }
1219
1220 /**
1221  * SwiftFileBackend helper class to page through listings.
1222  * Swift also has a listing limit of 10,000 objects for sanity.
1223  * Do not use this class from places outside SwiftFileBackend.
1224  *
1225  * @ingroup FileBackend
1226  */
1227 abstract class SwiftFileBackendList implements Iterator {
1228         /** @var Array */
1229         protected $bufferIter = array();
1230         protected $bufferAfter = null; // string; list items *after* this path
1231         protected $pos = 0; // integer
1232         /** @var Array */
1233         protected $params = array();
1234
1235         /** @var SwiftFileBackend */
1236         protected $backend;
1237         protected $container; // string; container name
1238         protected $dir; // string; storage directory
1239         protected $suffixStart; // integer
1240
1241         const PAGE_SIZE = 5000; // file listing buffer size
1242
1243         /**
1244          * @param $backend SwiftFileBackend
1245          * @param $fullCont string Resolved container name
1246          * @param $dir string Resolved directory relative to container
1247          * @param $params Array
1248          */
1249         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1250                 $this->backend = $backend;
1251                 $this->container = $fullCont;
1252                 $this->dir = $dir;
1253                 if ( substr( $this->dir, -1 ) === '/' ) {
1254                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1255                 }
1256                 if ( $this->dir == '' ) { // whole container
1257                         $this->suffixStart = 0;
1258                 } else { // dir within container
1259                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1260                 }
1261                 $this->params = $params;
1262         }
1263
1264         /**
1265          * @see Iterator::key()
1266          * @return integer
1267          */
1268         public function key() {
1269                 return $this->pos;
1270         }
1271
1272         /**
1273          * @see Iterator::next()
1274          * @return void
1275          */
1276         public function next() {
1277                 // Advance to the next file in the page
1278                 next( $this->bufferIter );
1279                 ++$this->pos;
1280                 // Check if there are no files left in this page and
1281                 // advance to the next page if this page was not empty.
1282                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1283                         $this->bufferIter = $this->pageFromList(
1284                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1285                         ); // updates $this->bufferAfter
1286                 }
1287         }
1288
1289         /**
1290          * @see Iterator::rewind()
1291          * @return void
1292          */
1293         public function rewind() {
1294                 $this->pos = 0;
1295                 $this->bufferAfter = null;
1296                 $this->bufferIter = $this->pageFromList(
1297                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1298                 ); // updates $this->bufferAfter
1299         }
1300
1301         /**
1302          * @see Iterator::valid()
1303          * @return bool
1304          */
1305         public function valid() {
1306                 if ( $this->bufferIter === null ) {
1307                         return false; // some failure?
1308                 } else {
1309                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1310                 }
1311         }
1312
1313         /**
1314          * Get the given list portion (page)
1315          *
1316          * @param $container string Resolved container name
1317          * @param $dir string Resolved path relative to container
1318          * @param $after string|null
1319          * @param $limit integer
1320          * @param $params Array
1321          * @return Traversable|Array|null Returns null on failure
1322          */
1323         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1324 }
1325
1326 /**
1327  * Iterator for listing directories
1328  */
1329 class SwiftFileBackendDirList extends SwiftFileBackendList {
1330         /**
1331          * @see Iterator::current()
1332          * @return string|bool String (relative path) or false
1333          */
1334         public function current() {
1335                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1336         }
1337
1338         /**
1339          * @see SwiftFileBackendList::pageFromList()
1340          * @return Array|null
1341          */
1342         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1343                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1344         }
1345 }
1346
1347 /**
1348  * Iterator for listing regular files
1349  */
1350 class SwiftFileBackendFileList extends SwiftFileBackendList {
1351         /**
1352          * @see Iterator::current()
1353          * @return string|bool String (relative path) or false
1354          */
1355         public function current() {
1356                 return substr( current( $this->bufferIter ), $this->suffixStart );
1357         }
1358
1359         /**
1360          * @see SwiftFileBackendList::pageFromList()
1361          * @return Array|null
1362          */
1363         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1364                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1365         }
1366 }