includes/filebackend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  45         protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
  46         protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
  47
  48         /** @var CF_Connection */
  49         protected $conn; // Swift connection handle
  50         protected $connStarted = 0; // integer UNIX timestamp
  51         protected $connException; // CloudFiles exception
  52
  53         /** @var ProcessCacheLRU */
  54         protected $connContainerCache; // container object cache
  55
  56         /**
  57          * @see FileBackendStore::__construct()
  58          * Additional $config params include:
  59          *   - swiftAuthUrl       : Swift authentication server URL
  60          *   - swiftUser          : Swift user used by MediaWiki (account:username)
  61          *   - swiftKey           : Swift authentication key for the above user
  62          *   - swiftAuthTTL       : Swift authentication TTL (seconds)
  63          *   - swiftAnonUser      : Swift user used for end-user requests (account:username).
  64          *                          If set, then views of public containers are assumed to go
  65          *                          through this user. If not set, then public containers are
  66          *                          accessible to unauthenticated requests via ".r:*" in the ACL.
  67          *   - swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  68          *   - swiftCDNExpiry     : How long (in seconds) to store content in the CDN.
  69          *                          If files may likely change, this should probably not exceed
  70          *                          a few days. For example, deletions may take this long to apply.
  71          *                          If object purging is enabled, however, this is not an issue.
  72          *   - swiftCDNPurgable   : Whether object purge requests are allowed by the CDN.
  73          *   - shardViaHashLevels : Map of container names to sharding config with:
  74          *                             - base   : base of hash characters, 16 or 36
  75          *                             - levels : the number of hash levels (and digits)
  76          *                             - repeat : hash subdirectories are prefixed with all the
  77          *                                        parent hash directory names (e.g. "a/ab/abc")
  78          */
  79         public function __construct( array $config ) {
  80                 parent::__construct( $config );
  81                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  82                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  83                 }
  84                 // Required settings
  85                 $this->auth = new CF_Authentication(
  86                         $config['swiftUser'],
  87                         $config['swiftKey'],
  88                         null, // account; unused
  89                         $config['swiftAuthUrl']
  90                 );
  91                 // Optional settings
  92                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  93                         ? $config['swiftAuthTTL']
  94                         : 5 * 60; // some sane number
  95                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  96                         ? $config['swiftAnonUser']
  97                         : '';
  98                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  99                         ? $config['shardViaHashLevels']
 100                         : '';
 101                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
 102                         ? $config['swiftUseCDN']
 103                         : false;
 104                 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
 105                         ? $config['swiftCDNExpiry']
 106                         : 3600; // hour
 107                 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
 108                         ? $config['swiftCDNPurgable']
 109                         : true;
 110                 // Cache container info to mask latency
 111                 $this->memCache = wfGetMainCache();
 112                 // Process cache for container info
 113                 $this->connContainerCache = new ProcessCacheLRU( 300 );
 114         }
 115
 116         /**
 117          * @see FileBackendStore::resolveContainerPath()
 118          * @return null
 119          */
 120         protected function resolveContainerPath( $container, $relStoragePath ) {
 121                 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
 122                         return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
 123                 } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 124                         return null; // too long for Swift
 125                 }
 126                 return $relStoragePath;
 127         }
 128
 129         /**
 130          * @see FileBackendStore::isPathUsableInternal()
 131          * @return bool
 132          */
 133         public function isPathUsableInternal( $storagePath ) {
 134                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 135                 if ( $rel === null ) {
 136                         return false; // invalid
 137                 }
 138
 139                 try {
 140                         $this->getContainer( $container );
 141                         return true; // container exists
 142                 } catch ( NoSuchContainerException $e ) {
 143                 } catch ( CloudFilesException $e ) { // some other exception?
 144                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 145                 }
 146
 147                 return false;
 148         }
 149
 150         /**
 151          * @see FileBackendStore::doCreateInternal()
 152          * @return Status
 153          */
 154         protected function doCreateInternal( array $params ) {
 155                 $status = Status::newGood();
 156
 157                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 158                 if ( $dstRel === null ) {
 159                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 160                         return $status;
 161                 }
 162
 163                 // (a) Check the destination container and object
 164                 try {
 165                         $dContObj = $this->getContainer( $dstCont );
 166                         if ( empty( $params['overwrite'] ) &&
 167                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 168                         {
 169                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 170                                 return $status;
 171                         }
 172                 } catch ( NoSuchContainerException $e ) {
 173                         $status->fatal( 'backend-fail-create', $params['dst'] );
 174                         return $status;
 175                 } catch ( CloudFilesException $e ) { // some other exception?
 176                         $this->handleException( $e, $status, __METHOD__, $params );
 177                         return $status;
 178                 }
 179
 180                 // (b) Get a SHA-1 hash of the object
 181                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 182
 183                 // (c) Actually create the object
 184                 try {
 185                         // Create a fresh CF_Object with no fields preloaded.
 186                         // We don't want to preserve headers, metadata, and such.
 187                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 188                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 189                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 190                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 191                         // The MD5 here will be checked within Swift against its own MD5.
 192                         $obj->set_etag( md5( $params['content'] ) );
 193                         // Use the same content type as StreamFile for security
 194                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 195                         if ( !strlen( $obj->content_type ) ) { // special case
 196                                 $obj->content_type = 'unknown/unknown';
 197                         }
 198                         if ( !empty( $params['async'] ) ) { // deferred
 199                                 $handle = $obj->write_async( $params['content'] );
 200                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle );
 201                                 $status->value->affectedObjects[] = $obj;
 202                         } else { // actually write the object in Swift
 203                                 $obj->write( $params['content'] );
 204                                 $this->purgeCDNCache( array( $obj ) );
 205                         }
 206                 } catch ( CDNNotEnabledException $e ) {
 207                         // CDN not enabled; nothing to see here
 208                 } catch ( BadContentTypeException $e ) {
 209                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 210                 } catch ( CloudFilesException $e ) { // some other exception?
 211                         $this->handleException( $e, $status, __METHOD__, $params );
 212                 }
 213
 214                 return $status;
 215         }
 216
 217         /**
 218          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 219          */
 220         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 221                 try {
 222                         $cfOp->getLastResponse();
 223                 } catch ( BadContentTypeException $e ) {
 224                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 225                 }
 226         }
 227
 228         /**
 229          * @see FileBackendStore::doStoreInternal()
 230          * @return Status
 231          */
 232         protected function doStoreInternal( array $params ) {
 233                 $status = Status::newGood();
 234
 235                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 236                 if ( $dstRel === null ) {
 237                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 238                         return $status;
 239                 }
 240
 241                 // (a) Check the destination container and object
 242                 try {
 243                         $dContObj = $this->getContainer( $dstCont );
 244                         if ( empty( $params['overwrite'] ) &&
 245                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 246                         {
 247                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 248                                 return $status;
 249                         }
 250                 } catch ( NoSuchContainerException $e ) {
 251                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 252                         return $status;
 253                 } catch ( CloudFilesException $e ) { // some other exception?
 254                         $this->handleException( $e, $status, __METHOD__, $params );
 255                         return $status;
 256                 }
 257
 258                 // (b) Get a SHA-1 hash of the object
 259                 $sha1Hash = sha1_file( $params['src'] );
 260                 if ( $sha1Hash === false ) { // source doesn't exist?
 261                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 262                         return $status;
 263                 }
 264                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 265
 266                 // (c) Actually store the object
 267                 try {
 268                         // Create a fresh CF_Object with no fields preloaded.
 269                         // We don't want to preserve headers, metadata, and such.
 270                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 271                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 272                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 273                         // The MD5 here will be checked within Swift against its own MD5.
 274                         $obj->set_etag( md5_file( $params['src'] ) );
 275                         // Use the same content type as StreamFile for security
 276                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 277                         if ( !strlen( $obj->content_type ) ) { // special case
 278                                 $obj->content_type = 'unknown/unknown';
 279                         }
 280                         if ( !empty( $params['async'] ) ) { // deferred
 281                                 wfSuppressWarnings();
 282                                 $fp = fopen( $params['src'], 'rb' );
 283                                 wfRestoreWarnings();
 284                                 if ( !$fp ) {
 285                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 286                                 } else {
 287                                         $handle = $obj->write_async( $fp, filesize( $params['src'] ), true );
 288                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle );
 289                                         $status->value->resourcesToClose[] = $fp;
 290                                         $status->value->affectedObjects[] = $obj;
 291                                 }
 292                         } else { // actually write the object in Swift
 293                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 294                                 $this->purgeCDNCache( array( $obj ) );
 295                         }
 296                 } catch ( CDNNotEnabledException $e ) {
 297                         // CDN not enabled; nothing to see here
 298                 } catch ( BadContentTypeException $e ) {
 299                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 300                 } catch ( IOException $e ) {
 301                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 302                 } catch ( CloudFilesException $e ) { // some other exception?
 303                         $this->handleException( $e, $status, __METHOD__, $params );
 304                 }
 305
 306                 return $status;
 307         }
 308
 309         /**
 310          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 311          */
 312         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 313                 try {
 314                         $cfOp->getLastResponse();
 315                 } catch ( BadContentTypeException $e ) {
 316                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 317                 } catch ( IOException $e ) {
 318                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 319                 }
 320         }
 321
 322         /**
 323          * @see FileBackendStore::doCopyInternal()
 324          * @return Status
 325          */
 326         protected function doCopyInternal( array $params ) {
 327                 $status = Status::newGood();
 328
 329                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 330                 if ( $srcRel === null ) {
 331                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 332                         return $status;
 333                 }
 334
 335                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 336                 if ( $dstRel === null ) {
 337                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 338                         return $status;
 339                 }
 340
 341                 // (a) Check the source/destination containers and destination object
 342                 try {
 343                         $sContObj = $this->getContainer( $srcCont );
 344                         $dContObj = $this->getContainer( $dstCont );
 345                         if ( empty( $params['overwrite'] ) &&
 346                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 347                         {
 348                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 349                                 return $status;
 350                         }
 351                 } catch ( NoSuchContainerException $e ) {
 352                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 353                         return $status;
 354                 } catch ( CloudFilesException $e ) { // some other exception?
 355                         $this->handleException( $e, $status, __METHOD__, $params );
 356                         return $status;
 357                 }
 358
 359                 // (b) Actually copy the file to the destination
 360                 try {
 361                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 362                         if ( !empty( $params['async'] ) ) { // deferred
 363                                 $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel );
 364                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle );
 365                                 $status->value->affectedObjects[] = $dstObj;
 366                         } else { // actually write the object in Swift
 367                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 368                                 $this->purgeCDNCache( array( $dstObj ) );
 369                         }
 370                 } catch ( CDNNotEnabledException $e ) {
 371                         // CDN not enabled; nothing to see here
 372                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 373                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 374                 } catch ( CloudFilesException $e ) { // some other exception?
 375                         $this->handleException( $e, $status, __METHOD__, $params );
 376                 }
 377
 378                 return $status;
 379         }
 380
 381         /**
 382          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 383          */
 384         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 385                 try {
 386                         $cfOp->getLastResponse();
 387                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 388                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 389                 }
 390         }
 391
 392         /**
 393          * @see FileBackendStore::doMoveInternal()
 394          * @return Status
 395          */
 396         protected function doMoveInternal( array $params ) {
 397                 $status = Status::newGood();
 398
 399                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 400                 if ( $srcRel === null ) {
 401                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 402                         return $status;
 403                 }
 404
 405                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 406                 if ( $dstRel === null ) {
 407                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 408                         return $status;
 409                 }
 410
 411                 // (a) Check the source/destination containers and destination object
 412                 try {
 413                         $sContObj = $this->getContainer( $srcCont );
 414                         $dContObj = $this->getContainer( $dstCont );
 415                         if ( empty( $params['overwrite'] ) &&
 416                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 417                         {
 418                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 419                                 return $status;
 420                         }
 421                 } catch ( NoSuchContainerException $e ) {
 422                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 423                         return $status;
 424                 } catch ( CloudFilesException $e ) { // some other exception?
 425                         $this->handleException( $e, $status, __METHOD__, $params );
 426                         return $status;
 427                 }
 428
 429                 // (b) Actually move the file to the destination
 430                 try {
 431                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 432                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 433                         if ( !empty( $params['async'] ) ) { // deferred
 434                                 $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel );
 435                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle );
 436                                 $status->value->affectedObjects[] = $srcObj;
 437                                 $status->value->affectedObjects[] = $dstObj;
 438                         } else { // actually write the object in Swift
 439                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel );
 440                                 $this->purgeCDNCache( array( $srcObj, $dstObj ) );
 441                         }
 442                 } catch ( CDNNotEnabledException $e ) {
 443                         // CDN not enabled; nothing to see here
 444                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 445                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 446                 } catch ( CloudFilesException $e ) { // some other exception?
 447                         $this->handleException( $e, $status, __METHOD__, $params );
 448                 }
 449
 450                 return $status;
 451         }
 452
 453         /**
 454          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 455          */
 456         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 457                 try {
 458                         $cfOp->getLastResponse();
 459                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 460                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 461                 }
 462         }
 463
 464         /**
 465          * @see FileBackendStore::doDeleteInternal()
 466          * @return Status
 467          */
 468         protected function doDeleteInternal( array $params ) {
 469                 $status = Status::newGood();
 470
 471                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 472                 if ( $srcRel === null ) {
 473                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 474                         return $status;
 475                 }
 476
 477                 try {
 478                         $sContObj = $this->getContainer( $srcCont );
 479                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 480                         if ( !empty( $params['async'] ) ) { // deferred
 481                                 $handle = $sContObj->delete_object_async( $srcRel );
 482                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle );
 483                                 $status->value->affectedObjects[] = $srcObj;
 484                         } else { // actually write the object in Swift
 485                                 $sContObj->delete_object( $srcRel );
 486                                 $this->purgeCDNCache( array( $srcObj ) );
 487                         }
 488                 } catch ( CDNNotEnabledException $e ) {
 489                         // CDN not enabled; nothing to see here
 490                 } catch ( NoSuchContainerException $e ) {
 491                         $status->fatal( 'backend-fail-delete', $params['src'] );
 492                 } catch ( NoSuchObjectException $e ) {
 493                         if ( empty( $params['ignoreMissingSource'] ) ) {
 494                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 495                         }
 496                 } catch ( CloudFilesException $e ) { // some other exception?
 497                         $this->handleException( $e, $status, __METHOD__, $params );
 498                 }
 499
 500                 return $status;
 501         }
 502
 503         /**
 504          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 505          */
 506         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 507                 try {
 508                         $cfOp->getLastResponse();
 509                 } catch ( NoSuchContainerException $e ) {
 510                         $status->fatal( 'backend-fail-delete', $params['src'] );
 511                 } catch ( NoSuchObjectException $e ) {
 512                         if ( empty( $params['ignoreMissingSource'] ) ) {
 513                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 514                         }
 515                 }
 516         }
 517
 518         /**
 519          * @see FileBackendStore::doPrepareInternal()
 520          * @return Status
 521          */
 522         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 523                 $status = Status::newGood();
 524
 525                 // (a) Check if container already exists
 526                 try {
 527                         $contObj = $this->getContainer( $fullCont );
 528                         // NoSuchContainerException not thrown: container must exist
 529                         return $status; // already exists
 530                 } catch ( NoSuchContainerException $e ) {
 531                         // NoSuchContainerException thrown: container does not exist
 532                 } catch ( CloudFilesException $e ) { // some other exception?
 533                         $this->handleException( $e, $status, __METHOD__, $params );
 534                         return $status;
 535                 }
 536
 537                 // (b) Create container as needed
 538                 try {
 539                         $contObj = $this->createContainer( $fullCont );
 540                         if ( !empty( $params['noAccess'] ) ) {
 541                                 // Make container private to end-users...
 542                                 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 543                         } else {
 544                                 // Make container public to end-users...
 545                                 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 546                         }
 547                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 548                                 $contObj->make_public( $this->swiftCDNExpiry );
 549                         }
 550                 } catch ( CDNNotEnabledException $e ) {
 551                         // CDN not enabled; nothing to see here
 552                 } catch ( CloudFilesException $e ) { // some other exception?
 553                         $this->handleException( $e, $status, __METHOD__, $params );
 554                         return $status;
 555                 }
 556
 557                 return $status;
 558         }
 559
 560         /**
 561          * @see FileBackendStore::doSecureInternal()
 562          * @return Status
 563          */
 564         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 565                 $status = Status::newGood();
 566                 if ( empty( $params['noAccess'] ) ) {
 567                         return $status; // nothing to do
 568                 }
 569
 570                 // Restrict container from end-users...
 571                 try {
 572                         // doPrepareInternal() should have been called,
 573                         // so the Swift container should already exist...
 574                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 575                         // NoSuchContainerException not thrown: container must exist
 576
 577                         // Make container private to end-users...
 578                         $status->merge( $this->setContainerAccess(
 579                                 $contObj,
 580                                 array( $this->auth->username ), // read
 581                                 array( $this->auth->username ) // write
 582                         ) );
 583                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 584                                 $contObj->make_private();
 585                         }
 586                 } catch ( CDNNotEnabledException $e ) {
 587                         // CDN not enabled; nothing to see here
 588                 } catch ( CloudFilesException $e ) { // some other exception?
 589                         $this->handleException( $e, $status, __METHOD__, $params );
 590                 }
 591
 592                 return $status;
 593         }
 594
 595         /**
 596          * @see FileBackendStore::doPublishInternal()
 597          * @return Status
 598          */
 599         protected function doPublishInternal( $fullCont, $dir, array $params ) {
 600                 $status = Status::newGood();
 601
 602                 // Unrestrict container from end-users...
 603                 try {
 604                         // doPrepareInternal() should have been called,
 605                         // so the Swift container should already exist...
 606                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 607                         // NoSuchContainerException not thrown: container must exist
 608
 609                         // Make container public to end-users...
 610                         if ( $this->swiftAnonUser != '' ) {
 611                                 $status->merge( $this->setContainerAccess(
 612                                         $contObj,
 613                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 614                                         array( $this->auth->username, $this->swiftAnonUser ) // write
 615                                 ) );
 616                         } else {
 617                                 $status->merge( $this->setContainerAccess(
 618                                         $contObj,
 619                                         array( $this->auth->username, '.r:*' ), // read
 620                                         array( $this->auth->username ) // write
 621                                 ) );
 622                         }
 623                         if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
 624                                 $contObj->make_public();
 625                         }
 626                 } catch ( CDNNotEnabledException $e ) {
 627                         // CDN not enabled; nothing to see here
 628                 } catch ( CloudFilesException $e ) { // some other exception?
 629                         $this->handleException( $e, $status, __METHOD__, $params );
 630                 }
 631
 632                 return $status;
 633         }
 634
 635         /**
 636          * @see FileBackendStore::doCleanInternal()
 637          * @return Status
 638          */
 639         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 640                 $status = Status::newGood();
 641
 642                 // Only containers themselves can be removed, all else is virtual
 643                 if ( $dir != '' ) {
 644                         return $status; // nothing to do
 645                 }
 646
 647                 // (a) Check the container
 648                 try {
 649                         $contObj = $this->getContainer( $fullCont, true );
 650                 } catch ( NoSuchContainerException $e ) {
 651                         return $status; // ok, nothing to do
 652                 } catch ( CloudFilesException $e ) { // some other exception?
 653                         $this->handleException( $e, $status, __METHOD__, $params );
 654                         return $status;
 655                 }
 656
 657                 // (b) Delete the container if empty
 658                 if ( $contObj->object_count == 0 ) {
 659                         try {
 660                                 $this->deleteContainer( $fullCont );
 661                         } catch ( NoSuchContainerException $e ) {
 662                                 return $status; // race?
 663                         } catch ( NonEmptyContainerException $e ) {
 664                                 return $status; // race? consistency delay?
 665                         } catch ( CloudFilesException $e ) { // some other exception?
 666                                 $this->handleException( $e, $status, __METHOD__, $params );
 667                                 return $status;
 668                         }
 669                 }
 670
 671                 return $status;
 672         }
 673
 674         /**
 675          * @see FileBackendStore::doFileExists()
 676          * @return array|bool|null
 677          */
 678         protected function doGetFileStat( array $params ) {
 679                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 680                 if ( $srcRel === null ) {
 681                         return false; // invalid storage path
 682                 }
 683
 684                 $stat = false;
 685                 try {
 686                         $contObj = $this->getContainer( $srcCont );
 687                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 688                         $this->addMissingMetadata( $srcObj, $params['src'] );
 689                         $stat = array(
 690                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 691                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 692                                 'size'  => (int)$srcObj->content_length,
 693                                 'sha1'  => $srcObj->metadata['Sha1base36']
 694                         );
 695                 } catch ( NoSuchContainerException $e ) {
 696                 } catch ( NoSuchObjectException $e ) {
 697                 } catch ( CloudFilesException $e ) { // some other exception?
 698                         $stat = null;
 699                         $this->handleException( $e, null, __METHOD__, $params );
 700                 }
 701
 702                 return $stat;
 703         }
 704
 705         /**
 706          * Fill in any missing object metadata and save it to Swift
 707          *
 708          * @param $obj CF_Object
 709          * @param $path string Storage path to object
 710          * @return bool Success
 711          * @throws Exception cloudfiles exceptions
 712          */
 713         protected function addMissingMetadata( CF_Object $obj, $path ) {
 714                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 715                         return true; // nothing to do
 716                 }
 717                 $status = Status::newGood();
 718                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 719                 if ( $status->isOK() ) {
 720                         # Do not stat the file in getLocalCopy() to avoid infinite loops
 721                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
 722                         if ( $tmpFile ) {
 723                                 $hash = $tmpFile->getSha1Base36();
 724                                 if ( $hash !== false ) {
 725                                         $obj->metadata['Sha1base36'] = $hash;
 726                                         $obj->sync_metadata(); // save to Swift
 727                                         return true; // success
 728                                 }
 729                         }
 730                 }
 731                 $obj->metadata['Sha1base36'] = false;
 732                 return false; // failed
 733         }
 734
 735         /**
 736          * @see FileBackend::getFileContents()
 737          * @return bool|null|string
 738          */
 739         public function getFileContents( array $params ) {
 740                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 741                 if ( $srcRel === null ) {
 742                         return false; // invalid storage path
 743                 }
 744
 745                 if ( !$this->fileExists( $params ) ) {
 746                         return null;
 747                 }
 748
 749                 $data = false;
 750                 try {
 751                         $sContObj = $this->getContainer( $srcCont );
 752                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 753                         $data = $obj->read( $this->headersFromParams( $params ) );
 754                 } catch ( NoSuchContainerException $e ) {
 755                 } catch ( CloudFilesException $e ) { // some other exception?
 756                         $this->handleException( $e, null, __METHOD__, $params );
 757                 }
 758
 759                 return $data;
 760         }
 761
 762         /**
 763          * @see FileBackendStore::doDirectoryExists()
 764          * @return bool|null
 765          */
 766         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 767                 try {
 768                         $container = $this->getContainer( $fullCont );
 769                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 770                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 771                 } catch ( NoSuchContainerException $e ) {
 772                         return false;
 773                 } catch ( CloudFilesException $e ) { // some other exception?
 774                         $this->handleException( $e, null, __METHOD__,
 775                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 776                 }
 777
 778                 return null; // error
 779         }
 780
 781         /**
 782          * @see FileBackendStore::getDirectoryListInternal()
 783          * @return SwiftFileBackendDirList
 784          */
 785         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 786                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 787         }
 788
 789         /**
 790          * @see FileBackendStore::getFileListInternal()
 791          * @return SwiftFileBackendFileList
 792          */
 793         public function getFileListInternal( $fullCont, $dir, array $params ) {
 794                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 795         }
 796
 797         /**
 798          * Do not call this function outside of SwiftFileBackendFileList
 799          *
 800          * @param $fullCont string Resolved container name
 801          * @param $dir string Resolved storage directory with no trailing slash
 802          * @param $after string|null Storage path of file to list items after
 803          * @param $limit integer Max number of items to list
 804          * @param $params Array Includes flag for 'topOnly'
 805          * @return Array List of relative paths of dirs directly under $dir
 806          */
 807         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 808                 $dirs = array();
 809                 if ( $after === INF ) {
 810                         return $dirs; // nothing more
 811                 }
 812                 wfProfileIn( __METHOD__ . '-' . $this->name );
 813
 814                 try {
 815                         $container = $this->getContainer( $fullCont );
 816                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 817                         // Non-recursive: only list dirs right under $dir
 818                         if ( !empty( $params['topOnly'] ) ) {
 819                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 820                                 foreach ( $objects as $object ) { // files and dirs
 821                                         if ( substr( $object, -1 ) === '/' ) {
 822                                                 $dirs[] = $object; // directories end in '/'
 823                                         }
 824                                 }
 825                         // Recursive: list all dirs under $dir and its subdirs
 826                         } else {
 827                                 // Get directory from last item of prior page
 828                                 $lastDir = $this->getParentDir( $after ); // must be first page
 829                                 $objects = $container->list_objects( $limit, $after, $prefix );
 830                                 foreach ( $objects as $object ) { // files
 831                                         $objectDir = $this->getParentDir( $object ); // directory of object
 832                                         if ( $objectDir !== false ) { // file has a parent dir
 833                                                 // Swift stores paths in UTF-8, using binary sorting.
 834                                                 // See function "create_container_table" in common/db.py.
 835                                                 // If a directory is not "greater" than the last one,
 836                                                 // then it was already listed by the calling iterator.
 837                                                 if ( $objectDir > $lastDir ) {
 838                                                         $pDir = $objectDir;
 839                                                         do { // add dir and all its parent dirs
 840                                                                 $dirs[] = "{$pDir}/";
 841                                                                 $pDir = $this->getParentDir( $pDir );
 842                                                         } while ( $pDir !== false // sanity
 843                                                                 && $pDir > $lastDir // not done already
 844                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 845                                                         );
 846                                                 }
 847                                                 $lastDir = $objectDir;
 848                                         }
 849                                 }
 850                         }
 851                         if ( count( $objects ) < $limit ) {
 852                                 $after = INF; // avoid a second RTT
 853                         } else {
 854                                 $after = end( $objects ); // update last item
 855                         }
 856                 } catch ( NoSuchContainerException $e ) {
 857                 } catch ( CloudFilesException $e ) { // some other exception?
 858                         $this->handleException( $e, null, __METHOD__,
 859                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 860                 }
 861
 862                 wfProfileOut( __METHOD__ . '-' . $this->name );
 863                 return $dirs;
 864         }
 865
 866         protected function getParentDir( $path ) {
 867                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 868         }
 869
 870         /**
 871          * Do not call this function outside of SwiftFileBackendFileList
 872          *
 873          * @param $fullCont string Resolved container name
 874          * @param $dir string Resolved storage directory with no trailing slash
 875          * @param $after string|null Storage path of file to list items after
 876          * @param $limit integer Max number of items to list
 877          * @param $params Array Includes flag for 'topOnly'
 878          * @return Array List of relative paths of files under $dir
 879          */
 880         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 881                 $files = array();
 882                 if ( $after === INF ) {
 883                         return $files; // nothing more
 884                 }
 885                 wfProfileIn( __METHOD__ . '-' . $this->name );
 886
 887                 try {
 888                         $container = $this->getContainer( $fullCont );
 889                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 890                         // Non-recursive: only list files right under $dir
 891                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 892                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 893                                 foreach ( $objects as $object ) {
 894                                         if ( substr( $object, -1 ) !== '/' ) {
 895                                                 $files[] = $object; // directories end in '/'
 896                                         }
 897                                 }
 898                         // Recursive: list all files under $dir and its subdirs
 899                         } else { // files
 900                                 $objects = $container->list_objects( $limit, $after, $prefix );
 901                                 $files = $objects;
 902                         }
 903                         if ( count( $objects ) < $limit ) {
 904                                 $after = INF; // avoid a second RTT
 905                         } else {
 906                                 $after = end( $objects ); // update last item
 907                         }
 908                 } catch ( NoSuchContainerException $e ) {
 909                 } catch ( CloudFilesException $e ) { // some other exception?
 910                         $this->handleException( $e, null, __METHOD__,
 911                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 912                 }
 913
 914                 wfProfileOut( __METHOD__ . '-' . $this->name );
 915                 return $files;
 916         }
 917
 918         /**
 919          * @see FileBackendStore::doGetFileSha1base36()
 920          * @return bool
 921          */
 922         protected function doGetFileSha1base36( array $params ) {
 923                 $stat = $this->getFileStat( $params );
 924                 if ( $stat ) {
 925                         return $stat['sha1'];
 926                 } else {
 927                         return false;
 928                 }
 929         }
 930
 931         /**
 932          * @see FileBackendStore::doStreamFile()
 933          * @return Status
 934          */
 935         protected function doStreamFile( array $params ) {
 936                 $status = Status::newGood();
 937
 938                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 939                 if ( $srcRel === null ) {
 940                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 941                 }
 942
 943                 try {
 944                         $cont = $this->getContainer( $srcCont );
 945                 } catch ( NoSuchContainerException $e ) {
 946                         $status->fatal( 'backend-fail-stream', $params['src'] );
 947                         return $status;
 948                 } catch ( CloudFilesException $e ) { // some other exception?
 949                         $this->handleException( $e, $status, __METHOD__, $params );
 950                         return $status;
 951                 }
 952
 953                 try {
 954                         $output = fopen( 'php://output', 'wb' );
 955                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
 956                         $obj->stream( $output, $this->headersFromParams( $params ) );
 957                 } catch ( CloudFilesException $e ) { // some other exception?
 958                         $this->handleException( $e, $status, __METHOD__, $params );
 959                 }
 960
 961                 return $status;
 962         }
 963
 964         /**
 965          * @see FileBackendStore::getLocalCopy()
 966          * @return null|TempFSFile
 967          */
 968         public function getLocalCopy( array $params ) {
 969                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 970                 if ( $srcRel === null ) {
 971                         return null;
 972                 }
 973
 974                 # Check the recursion guard to avoid loops when filling metadata
 975                 if ( empty( $params['nostat'] ) && !$this->fileExists( $params ) ) {
 976                         return null;
 977                 }
 978
 979                 $tmpFile = null;
 980                 try {
 981                         $sContObj = $this->getContainer( $srcCont );
 982                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 983                         // Get source file extension
 984                         $ext = FileBackend::extensionFromPath( $srcRel );
 985                         // Create a new temporary file...
 986                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 987                         if ( $tmpFile ) {
 988                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 989                                 if ( $handle ) {
 990                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 991                                         fclose( $handle );
 992                                 } else {
 993                                         $tmpFile = null; // couldn't open temp file
 994                                 }
 995                         }
 996                 } catch ( NoSuchContainerException $e ) {
 997                         $tmpFile = null;
 998                 } catch ( CloudFilesException $e ) { // some other exception?
 999                         $tmpFile = null;
1000                         $this->handleException( $e, null, __METHOD__, $params );
1001                 }
1002
1003                 return $tmpFile;
1004         }
1005
1006         /**
1007          * @see FileBackendStore::directoriesAreVirtual()
1008          * @return bool
1009          */
1010         protected function directoriesAreVirtual() {
1011                 return true;
1012         }
1013
1014         /**
1015          * Get headers to send to Swift when reading a file based
1016          * on a FileBackend params array, e.g. that of getLocalCopy().
1017          * $params is currently only checked for a 'latest' flag.
1018          *
1019          * @param $params Array
1020          * @return Array
1021          */
1022         protected function headersFromParams( array $params ) {
1023                 $hdrs = array();
1024                 if ( !empty( $params['latest'] ) ) {
1025                         $hdrs[] = 'X-Newest: true';
1026                 }
1027                 return $hdrs;
1028         }
1029
1030         /**
1031          * @see FileBackendStore::doExecuteOpHandlesInternal()
1032          * @return Array List of corresponding Status objects
1033          */
1034         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1035                 $statuses = array();
1036
1037                 $cfOps = array(); // list of CF_Async_Op objects
1038                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1039                         $cfOps[$index] = $fileOpHandle->cfOp;
1040                 }
1041                 $batch = new CF_Async_Op_Batch( $cfOps );
1042
1043                 $cfOps = $batch->execute();
1044                 foreach ( $cfOps as $index => $cfOp ) {
1045                         $status = Status::newGood();
1046                         try { // catch exceptions; update status
1047                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
1048                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
1049                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
1050                         } catch ( CloudFilesException $e ) { // some other exception?
1051                                 $this->handleException( $e, $status,
1052                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
1053                         }
1054                         $statuses[$index] = $status;
1055                 }
1056
1057                 return $statuses;
1058         }
1059
1060         /**
1061          * Set read/write permissions for a Swift container.
1062          *
1063          * $readGrps is a list of the possible criteria for a request to have
1064          * access to read a container. Each item is one of the following formats:
1065          *   - account:user       : Grants access if the request is by the given user
1066          *   - .r:<regex>         : Grants access if the request is from a referrer host that
1067          *                          matches the expression and the request is not for a listing.
1068          *                          Setting this to '*' effectively makes a container public.
1069          *   - .rlistings:<regex> : Grants access if the request is from a referrer host that
1070          *                          matches the expression and the request for a listing.
1071          *
1072          * $writeGrps is a list of the possible criteria for a request to have
1073          * access to write to a container. Each item is of the following format:
1074          *   - account:user       : Grants access if the request is by the given user
1075          *
1076          * @see http://swift.openstack.org/misc.html#acls
1077          *
1078          * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
1079          * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
1080          *
1081          * @param $contObj CF_Container Swift container
1082          * @param $readGrps Array List of read access routes
1083          * @param $writeGrps Array List of write access routes
1084          * @return Status
1085          */
1086         protected function setContainerAccess(
1087                 CF_Container $contObj, array $readGrps, array $writeGrps
1088         ) {
1089                 $creds = $contObj->cfs_auth->export_credentials();
1090
1091                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1092
1093                 // Note: 10 second timeout consistent with php-cloudfiles
1094                 $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1095                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1096                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1097                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1098
1099                 return $req->execute(); // should return 204
1100         }
1101
1102         /**
1103          * Purge the CDN cache of affected objects if CDN caching is enabled.
1104          * This is for Rackspace/Akamai CDNs.
1105          *
1106          * @param $objects Array List of CF_Object items
1107          * @return void
1108          */
1109         public function purgeCDNCache( array $objects ) {
1110                 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
1111                         foreach ( $objects as $object ) {
1112                                 try {
1113                                         $object->purge_from_cdn();
1114                                 } catch ( CDNNotEnabledException $e ) {
1115                                         // CDN not enabled; nothing to see here
1116                                 } catch ( CloudFilesException $e ) {
1117                                         $this->handleException( $e, null, __METHOD__,
1118                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1119                                 }
1120                         }
1121                 }
1122         }
1123
1124         /**
1125          * Get a connection to the Swift proxy
1126          *
1127          * @return CF_Connection|bool False on failure
1128          * @throws CloudFilesException
1129          */
1130         protected function getConnection() {
1131                 if ( $this->connException instanceof Exception ) {
1132                         throw $this->connException; // failed last attempt
1133                 }
1134                 // Session keys expire after a while, so we renew them periodically
1135                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
1136                         $this->conn->close(); // close active cURL connections
1137                         $this->conn = null;
1138                 }
1139                 // Authenticate with proxy and get a session key...
1140                 if ( !$this->conn ) {
1141                         $this->connStarted = 0;
1142                         $this->connContainerCache->clear();
1143                         try {
1144                                 $this->auth->authenticate();
1145                                 $this->conn = new CF_Connection( $this->auth );
1146                                 $this->connStarted = time();
1147                         } catch ( CloudFilesException $e ) {
1148                                 $this->connException = $e; // don't keep re-trying
1149                                 throw $e; // throw it back
1150                         }
1151                 }
1152                 return $this->conn;
1153         }
1154
1155         /**
1156          * @see FileBackendStore::doClearCache()
1157          */
1158         protected function doClearCache( array $paths = null ) {
1159                 $this->connContainerCache->clear(); // clear container object cache
1160         }
1161
1162         /**
1163          * Get a Swift container object, possibly from process cache.
1164          * Use $reCache if the file count or byte count is needed.
1165          *
1166          * @param $container string Container name
1167          * @param $bypassCache bool Bypass all caches and load from Swift
1168          * @return CF_Container
1169          * @throws CloudFilesException
1170          */
1171         protected function getContainer( $container, $bypassCache = false ) {
1172                 $conn = $this->getConnection(); // Swift proxy connection
1173                 if ( $bypassCache ) { // purge cache
1174                         $this->connContainerCache->clear( $container );
1175                 } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1176                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1177                 }
1178                 if ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1179                         $contObj = $conn->get_container( $container );
1180                         // NoSuchContainerException not thrown: container must exist
1181                         $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it
1182                         if ( !$bypassCache ) {
1183                                 $this->setContainerCache( $container, // update persistent cache
1184                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1185                                 );
1186                         }
1187                 }
1188                 return $this->connContainerCache->get( $container, 'obj' );
1189         }
1190
1191         /**
1192          * Create a Swift container
1193          *
1194          * @param $container string Container name
1195          * @return CF_Container
1196          * @throws CloudFilesException
1197          */
1198         protected function createContainer( $container ) {
1199                 $conn = $this->getConnection(); // Swift proxy connection
1200                 $contObj = $conn->create_container( $container );
1201                 $this->connContainerCache->set( $container, 'obj', $contObj ); // cache
1202                 return $contObj;
1203         }
1204
1205         /**
1206          * Delete a Swift container
1207          *
1208          * @param $container string Container name
1209          * @return void
1210          * @throws CloudFilesException
1211          */
1212         protected function deleteContainer( $container ) {
1213                 $conn = $this->getConnection(); // Swift proxy connection
1214                 $this->connContainerCache->clear( $container ); // purge
1215                 $conn->delete_container( $container );
1216         }
1217
1218         /**
1219          * @see FileBackendStore::doPrimeContainerCache()
1220          * @return void
1221          */
1222         protected function doPrimeContainerCache( array $containerInfo ) {
1223                 try {
1224                         $conn = $this->getConnection(); // Swift proxy connection
1225                         foreach ( $containerInfo as $container => $info ) {
1226                                 $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http,
1227                                         $container, $info['count'], $info['bytes'] );
1228                                 $this->connContainerCache->set( $container, 'obj', $contObj );
1229                         }
1230                 } catch ( CloudFilesException $e ) { // some other exception?
1231                         $this->handleException( $e, null, __METHOD__, array() );
1232                 }
1233         }
1234
1235         /**
1236          * Log an unexpected exception for this backend.
1237          * This also sets the Status object to have a fatal error.
1238          *
1239          * @param $e Exception
1240          * @param $status Status|null
1241          * @param $func string
1242          * @param $params Array
1243          * @return void
1244          */
1245         protected function handleException( Exception $e, $status, $func, array $params ) {
1246                 if ( $status instanceof Status ) {
1247                         if ( $e instanceof AuthenticationException ) {
1248                                 $status->fatal( 'backend-fail-connect', $this->name );
1249                         } else {
1250                                 $status->fatal( 'backend-fail-internal', $this->name );
1251                         }
1252                 }
1253                 if ( $e->getMessage() ) {
1254                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1255                 }
1256                 wfDebugLog( 'SwiftBackend',
1257                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1258                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1259                 );
1260         }
1261 }
1262
1263 /**
1264  * @see FileBackendStoreOpHandle
1265  */
1266 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1267         /** @var CF_Async_Op */
1268         public $cfOp;
1269         /** @var Array */
1270         public $affectedObjects = array();
1271
1272         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1273                 $this->backend = $backend;
1274                 $this->params = $params;
1275                 $this->call = $call;
1276                 $this->cfOp = $cfOp;
1277         }
1278 }
1279
1280 /**
1281  * SwiftFileBackend helper class to page through listings.
1282  * Swift also has a listing limit of 10,000 objects for sanity.
1283  * Do not use this class from places outside SwiftFileBackend.
1284  *
1285  * @ingroup FileBackend
1286  */
1287 abstract class SwiftFileBackendList implements Iterator {
1288         /** @var Array */
1289         protected $bufferIter = array();
1290         protected $bufferAfter = null; // string; list items *after* this path
1291         protected $pos = 0; // integer
1292         /** @var Array */
1293         protected $params = array();
1294
1295         /** @var SwiftFileBackend */
1296         protected $backend;
1297         protected $container; // string; container name
1298         protected $dir; // string; storage directory
1299         protected $suffixStart; // integer
1300
1301         const PAGE_SIZE = 9000; // file listing buffer size
1302
1303         /**
1304          * @param $backend SwiftFileBackend
1305          * @param $fullCont string Resolved container name
1306          * @param $dir string Resolved directory relative to container
1307          * @param $params Array
1308          */
1309         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1310                 $this->backend = $backend;
1311                 $this->container = $fullCont;
1312                 $this->dir = $dir;
1313                 if ( substr( $this->dir, -1 ) === '/' ) {
1314                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1315                 }
1316                 if ( $this->dir == '' ) { // whole container
1317                         $this->suffixStart = 0;
1318                 } else { // dir within container
1319                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1320                 }
1321                 $this->params = $params;
1322         }
1323
1324         /**
1325          * @see Iterator::key()
1326          * @return integer
1327          */
1328         public function key() {
1329                 return $this->pos;
1330         }
1331
1332         /**
1333          * @see Iterator::next()
1334          * @return void
1335          */
1336         public function next() {
1337                 // Advance to the next file in the page
1338                 next( $this->bufferIter );
1339                 ++$this->pos;
1340                 // Check if there are no files left in this page and
1341                 // advance to the next page if this page was not empty.
1342                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1343                         $this->bufferIter = $this->pageFromList(
1344                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1345                         ); // updates $this->bufferAfter
1346                 }
1347         }
1348
1349         /**
1350          * @see Iterator::rewind()
1351          * @return void
1352          */
1353         public function rewind() {
1354                 $this->pos = 0;
1355                 $this->bufferAfter = null;
1356                 $this->bufferIter = $this->pageFromList(
1357                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1358                 ); // updates $this->bufferAfter
1359         }
1360
1361         /**
1362          * @see Iterator::valid()
1363          * @return bool
1364          */
1365         public function valid() {
1366                 if ( $this->bufferIter === null ) {
1367                         return false; // some failure?
1368                 } else {
1369                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1370                 }
1371         }
1372
1373         /**
1374          * Get the given list portion (page)
1375          *
1376          * @param $container string Resolved container name
1377          * @param $dir string Resolved path relative to container
1378          * @param $after string|null
1379          * @param $limit integer
1380          * @param $params Array
1381          * @return Traversable|Array|null Returns null on failure
1382          */
1383         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1384 }
1385
1386 /**
1387  * Iterator for listing directories
1388  */
1389 class SwiftFileBackendDirList extends SwiftFileBackendList {
1390         /**
1391          * @see Iterator::current()
1392          * @return string|bool String (relative path) or false
1393          */
1394         public function current() {
1395                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1396         }
1397
1398         /**
1399          * @see SwiftFileBackendList::pageFromList()
1400          * @return Array|null
1401          */
1402         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1403                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1404         }
1405 }
1406
1407 /**
1408  * Iterator for listing regular files
1409  */
1410 class SwiftFileBackendFileList extends SwiftFileBackendList {
1411         /**
1412          * @see Iterator::current()
1413          * @return string|bool String (relative path) or false
1414          */
1415         public function current() {
1416                 return substr( current( $this->bufferIter ), $this->suffixStart );
1417         }
1418
1419         /**
1420          * @see SwiftFileBackendList::pageFromList()
1421          * @return Array|null
1422          */
1423         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1424                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1425         }
1426 }