includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  45         protected $maxContCacheSize = 300; // integer; max containers with entries
  46
  47         /** @var CF_Connection */
  48         protected $conn; // Swift connection handle
  49         protected $connStarted = 0; // integer UNIX timestamp
  50         protected $connContainers = array(); // container object cache
  51         protected $connException; // CloudFiles exception
  52
  53         /**
  54          * @see FileBackendStore::__construct()
  55          * Additional $config params include:
  56          *    swiftAuthUrl       : Swift authentication server URL
  57          *    swiftUser          : Swift user used by MediaWiki (account:username)
  58          *    swiftKey           : Swift authentication key for the above user
  59          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  60          *    swiftAnonUser      : Swift user used for end-user requests (account:username)
  61          *    swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  62          *    shardViaHashLevels : Map of container names to sharding config with:
  63          *                         'base'   : base of hash characters, 16 or 36
  64          *                         'levels' : the number of hash levels (and digits)
  65          *                         'repeat' : hash subdirectories are prefixed with all the
  66          *                                    parent hash directory names (e.g. "a/ab/abc")
  67          */
  68         public function __construct( array $config ) {
  69                 parent::__construct( $config );
  70                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  71                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  72                 }
  73                 // Required settings
  74                 $this->auth = new CF_Authentication(
  75                         $config['swiftUser'],
  76                         $config['swiftKey'],
  77                         null, // account; unused
  78                         $config['swiftAuthUrl']
  79                 );
  80                 // Optional settings
  81                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  82                         ? $config['swiftAuthTTL']
  83                         : 5 * 60; // some sane number
  84                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  85                         ? $config['swiftAnonUser']
  86                         : '';
  87                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  88                         ? $config['shardViaHashLevels']
  89                         : '';
  90                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
  91                         ? $config['swiftUseCDN']
  92                         : false;
  93                 // Cache container info to mask latency
  94                 $this->memCache = wfGetMainCache();
  95         }
  96
  97         /**
  98          * @see FileBackendStore::resolveContainerPath()
  99          * @return null
 100          */
 101         protected function resolveContainerPath( $container, $relStoragePath ) {
 102                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 103                         return null; // too long for Swift
 104                 }
 105                 return $relStoragePath;
 106         }
 107
 108         /**
 109          * @see FileBackendStore::isPathUsableInternal()
 110          * @return bool
 111          */
 112         public function isPathUsableInternal( $storagePath ) {
 113                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 114                 if ( $rel === null ) {
 115                         return false; // invalid
 116                 }
 117
 118                 try {
 119                         $this->getContainer( $container );
 120                         return true; // container exists
 121                 } catch ( NoSuchContainerException $e ) {
 122                 } catch ( CloudFilesException $e ) { // some other exception?
 123                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 124                 }
 125
 126                 return false;
 127         }
 128
 129         /**
 130          * @see FileBackendStore::doCreateInternal()
 131          * @return Status
 132          */
 133         protected function doCreateInternal( array $params ) {
 134                 $status = Status::newGood();
 135
 136                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 137                 if ( $dstRel === null ) {
 138                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 139                         return $status;
 140                 }
 141
 142                 // (a) Check the destination container and object
 143                 try {
 144                         $dContObj = $this->getContainer( $dstCont );
 145                         if ( empty( $params['overwrite'] ) &&
 146                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 147                         {
 148                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 149                                 return $status;
 150                         }
 151                 } catch ( NoSuchContainerException $e ) {
 152                         $status->fatal( 'backend-fail-create', $params['dst'] );
 153                         return $status;
 154                 } catch ( CloudFilesException $e ) { // some other exception?
 155                         $this->handleException( $e, $status, __METHOD__, $params );
 156                         return $status;
 157                 }
 158
 159                 // (b) Get a SHA-1 hash of the object
 160                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 161
 162                 // (c) Actually create the object
 163                 try {
 164                         // Create a fresh CF_Object with no fields preloaded.
 165                         // We don't want to preserve headers, metadata, and such.
 166                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 167                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 168                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 169                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 170                         // The MD5 here will be checked within Swift against its own MD5.
 171                         $obj->set_etag( md5( $params['content'] ) );
 172                         // Use the same content type as StreamFile for security
 173                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 174                         if ( !empty( $params['async'] ) ) { // deferred
 175                                 $handle = $obj->write_async( $params['content'] );
 176                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle );
 177                                 $status->value->affectedObjects[] = $obj;
 178                         } else { // actually write the object in Swift
 179                                 $obj->write( $params['content'] );
 180                                 $this->purgeCDNCache( array( $obj ) );
 181                         }
 182                 } catch ( CDNNotEnabledException $e ) {
 183                         // CDN not enabled; nothing to see here
 184                 } catch ( BadContentTypeException $e ) {
 185                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 186                 } catch ( CloudFilesException $e ) { // some other exception?
 187                         $this->handleException( $e, $status, __METHOD__, $params );
 188                 }
 189
 190                 return $status;
 191         }
 192
 193         /**
 194          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 195          */
 196         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 197                 try {
 198                         $cfOp->getLastResponse();
 199                 } catch ( BadContentTypeException $e ) {
 200                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 201                 }
 202         }
 203
 204         /**
 205          * @see FileBackendStore::doStoreInternal()
 206          * @return Status
 207          */
 208         protected function doStoreInternal( array $params ) {
 209                 $status = Status::newGood();
 210
 211                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 212                 if ( $dstRel === null ) {
 213                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 214                         return $status;
 215                 }
 216
 217                 // (a) Check the destination container and object
 218                 try {
 219                         $dContObj = $this->getContainer( $dstCont );
 220                         if ( empty( $params['overwrite'] ) &&
 221                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 222                         {
 223                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 224                                 return $status;
 225                         }
 226                 } catch ( NoSuchContainerException $e ) {
 227                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 228                         return $status;
 229                 } catch ( CloudFilesException $e ) { // some other exception?
 230                         $this->handleException( $e, $status, __METHOD__, $params );
 231                         return $status;
 232                 }
 233
 234                 // (b) Get a SHA-1 hash of the object
 235                 $sha1Hash = sha1_file( $params['src'] );
 236                 if ( $sha1Hash === false ) { // source doesn't exist?
 237                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 238                         return $status;
 239                 }
 240                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 241
 242                 // (c) Actually store the object
 243                 try {
 244                         // Create a fresh CF_Object with no fields preloaded.
 245                         // We don't want to preserve headers, metadata, and such.
 246                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 247                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 248                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 249                         // The MD5 here will be checked within Swift against its own MD5.
 250                         $obj->set_etag( md5_file( $params['src'] ) );
 251                         // Use the same content type as StreamFile for security
 252                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 253                         if ( !empty( $params['async'] ) ) { // deferred
 254                                 wfSuppressWarnings();
 255                                 $fp = fopen( $params['src'], 'rb' );
 256                                 wfRestoreWarnings();
 257                                 if ( !$fp ) {
 258                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 259                                 } else {
 260                                         $handle = $obj->write_async( $fp, filesize( $params['src'] ), true );
 261                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle );
 262                                         $status->value->resourcesToClose[] = $fp;
 263                                         $status->value->affectedObjects[] = $obj;
 264                                 }
 265                         } else { // actually write the object in Swift
 266                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 267                                 $this->purgeCDNCache( array( $obj ) );
 268                         }
 269                 } catch ( CDNNotEnabledException $e ) {
 270                         // CDN not enabled; nothing to see here
 271                 } catch ( BadContentTypeException $e ) {
 272                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 273                 } catch ( IOException $e ) {
 274                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 275                 } catch ( CloudFilesException $e ) { // some other exception?
 276                         $this->handleException( $e, $status, __METHOD__, $params );
 277                 }
 278
 279                 return $status;
 280         }
 281
 282         /**
 283          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 284          */
 285         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 286                 try {
 287                         $cfOp->getLastResponse();
 288                 } catch ( BadContentTypeException $e ) {
 289                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 290                 } catch ( IOException $e ) {
 291                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 292                 }
 293         }
 294
 295         /**
 296          * @see FileBackendStore::doCopyInternal()
 297          * @return Status
 298          */
 299         protected function doCopyInternal( array $params ) {
 300                 $status = Status::newGood();
 301
 302                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 303                 if ( $srcRel === null ) {
 304                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 305                         return $status;
 306                 }
 307
 308                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 309                 if ( $dstRel === null ) {
 310                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 311                         return $status;
 312                 }
 313
 314                 // (a) Check the source/destination containers and destination object
 315                 try {
 316                         $sContObj = $this->getContainer( $srcCont );
 317                         $dContObj = $this->getContainer( $dstCont );
 318                         if ( empty( $params['overwrite'] ) &&
 319                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 320                         {
 321                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 322                                 return $status;
 323                         }
 324                 } catch ( NoSuchContainerException $e ) {
 325                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 326                         return $status;
 327                 } catch ( CloudFilesException $e ) { // some other exception?
 328                         $this->handleException( $e, $status, __METHOD__, $params );
 329                         return $status;
 330                 }
 331
 332                 // (b) Actually copy the file to the destination
 333                 try {
 334                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 335                         if ( !empty( $params['async'] ) ) { // deferred
 336                                 $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel );
 337                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle );
 338                                 $status->value->affectedObjects[] = $dstObj;
 339                         } else { // actually write the object in Swift
 340                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 341                                 $this->purgeCDNCache( array( $dstObj ) );
 342                         }
 343                 } catch ( CDNNotEnabledException $e ) {
 344                         // CDN not enabled; nothing to see here
 345                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 346                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 347                 } catch ( CloudFilesException $e ) { // some other exception?
 348                         $this->handleException( $e, $status, __METHOD__, $params );
 349                 }
 350
 351                 return $status;
 352         }
 353
 354         /**
 355          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 356          */
 357         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 358                 try {
 359                         $cfOp->getLastResponse();
 360                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 361                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 362                 }
 363         }
 364
 365         /**
 366          * @see FileBackendStore::doMoveInternal()
 367          * @return Status
 368          */
 369         protected function doMoveInternal( array $params ) {
 370                 $status = Status::newGood();
 371
 372                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 373                 if ( $srcRel === null ) {
 374                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 375                         return $status;
 376                 }
 377
 378                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 379                 if ( $dstRel === null ) {
 380                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 381                         return $status;
 382                 }
 383
 384                 // (a) Check the source/destination containers and destination object
 385                 try {
 386                         $sContObj = $this->getContainer( $srcCont );
 387                         $dContObj = $this->getContainer( $dstCont );
 388                         if ( empty( $params['overwrite'] ) &&
 389                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 390                         {
 391                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 392                                 return $status;
 393                         }
 394                 } catch ( NoSuchContainerException $e ) {
 395                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 396                         return $status;
 397                 } catch ( CloudFilesException $e ) { // some other exception?
 398                         $this->handleException( $e, $status, __METHOD__, $params );
 399                         return $status;
 400                 }
 401
 402                 // (b) Actually move the file to the destination
 403                 try {
 404                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 405                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 406                         if ( !empty( $params['async'] ) ) { // deferred
 407                                 $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel );
 408                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle );
 409                                 $status->value->affectedObjects[] = $srcObj;
 410                                 $status->value->affectedObjects[] = $dstObj;
 411                         } else { // actually write the object in Swift
 412                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel );
 413                                 $this->purgeCDNCache( array( $srcObj, $dstObj ) );
 414                         }
 415                 } catch ( CDNNotEnabledException $e ) {
 416                         // CDN not enabled; nothing to see here
 417                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 418                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 419                 } catch ( CloudFilesException $e ) { // some other exception?
 420                         $this->handleException( $e, $status, __METHOD__, $params );
 421                 }
 422
 423                 return $status;
 424         }
 425
 426         /**
 427          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 428          */
 429         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 430                 try {
 431                         $cfOp->getLastResponse();
 432                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 433                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 434                 }
 435         }
 436
 437         /**
 438          * @see FileBackendStore::doDeleteInternal()
 439          * @return Status
 440          */
 441         protected function doDeleteInternal( array $params ) {
 442                 $status = Status::newGood();
 443
 444                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 445                 if ( $srcRel === null ) {
 446                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 447                         return $status;
 448                 }
 449
 450                 try {
 451                         $sContObj = $this->getContainer( $srcCont );
 452                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 453                         if ( !empty( $params['async'] ) ) { // deferred
 454                                 $handle = $sContObj->delete_object_async( $srcRel );
 455                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle );
 456                                 $status->value->affectedObjects[] = $srcObj;
 457                         } else { // actually write the object in Swift
 458                                 $sContObj->delete_object( $srcRel );
 459                                 $this->purgeCDNCache( array( $srcObj ) );
 460                         }
 461                 } catch ( CDNNotEnabledException $e ) {
 462                         // CDN not enabled; nothing to see here
 463                 } catch ( NoSuchContainerException $e ) {
 464                         $status->fatal( 'backend-fail-delete', $params['src'] );
 465                 } catch ( NoSuchObjectException $e ) {
 466                         if ( empty( $params['ignoreMissingSource'] ) ) {
 467                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 468                         }
 469                 } catch ( CloudFilesException $e ) { // some other exception?
 470                         $this->handleException( $e, $status, __METHOD__, $params );
 471                 }
 472
 473                 return $status;
 474         }
 475
 476         /**
 477          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 478          */
 479         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 480                 try {
 481                         $cfOp->getLastResponse();
 482                 } catch ( NoSuchContainerException $e ) {
 483                         $status->fatal( 'backend-fail-delete', $params['src'] );
 484                 } catch ( NoSuchObjectException $e ) {
 485                         if ( empty( $params['ignoreMissingSource'] ) ) {
 486                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 487                         }
 488                 }
 489         }
 490
 491         /**
 492          * @see FileBackendStore::doPrepareInternal()
 493          * @return Status
 494          */
 495         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 496                 $status = Status::newGood();
 497
 498                 // (a) Check if container already exists
 499                 try {
 500                         $contObj = $this->getContainer( $fullCont );
 501                         // NoSuchContainerException not thrown: container must exist
 502                         return $status; // already exists
 503                 } catch ( NoSuchContainerException $e ) {
 504                         // NoSuchContainerException thrown: container does not exist
 505                 } catch ( CloudFilesException $e ) { // some other exception?
 506                         $this->handleException( $e, $status, __METHOD__, $params );
 507                         return $status;
 508                 }
 509
 510                 // (b) Create container as needed
 511                 try {
 512                         $contObj = $this->createContainer( $fullCont );
 513                         // Make container public to end-users...
 514                         if ( $this->swiftAnonUser != '' ) {
 515                                 $status->merge( $this->setContainerAccess(
 516                                         $contObj,
 517                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 518                                         array( $this->auth->username ) // write
 519                                 ) );
 520                         }
 521                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 522                                 $contObj->make_public();
 523                         }
 524                 } catch ( CDNNotEnabledException $e ) {
 525                         // CDN not enabled; nothing to see here
 526                 } catch ( CloudFilesException $e ) { // some other exception?
 527                         $this->handleException( $e, $status, __METHOD__, $params );
 528                         return $status;
 529                 }
 530
 531                 return $status;
 532         }
 533
 534         /**
 535          * @see FileBackendStore::doSecureInternal()
 536          * @return Status
 537          */
 538         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 539                 $status = Status::newGood();
 540
 541                 // Restrict container from end-users...
 542                 try {
 543                         // doPrepareInternal() should have been called,
 544                         // so the Swift container should already exist...
 545                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 546                         // NoSuchContainerException not thrown: container must exist
 547
 548                         // Make container private to end-users...
 549                         if ( $this->swiftAnonUser != '' && !isset( $contObj->mw_wasSecured ) ) {
 550                                 $status->merge( $this->setContainerAccess(
 551                                         $contObj,
 552                                         array( $this->auth->username ), // read
 553                                         array( $this->auth->username ) // write
 554                                 ) );
 555                                 // @TODO: when php-cloudfiles supports container
 556                                 // metadata, we can make use of that to avoid RTTs
 557                                 $contObj->mw_wasSecured = true; // avoid useless RTTs
 558                         }
 559                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 560                                 $contObj->make_private();
 561                         }
 562                 } catch ( CDNNotEnabledException $e ) {
 563                         // CDN not enabled; nothing to see here
 564                 } catch ( CloudFilesException $e ) { // some other exception?
 565                         $this->handleException( $e, $status, __METHOD__, $params );
 566                 }
 567
 568                 return $status;
 569         }
 570
 571         /**
 572          * @see FileBackendStore::doCleanInternal()
 573          * @return Status
 574          */
 575         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 576                 $status = Status::newGood();
 577
 578                 // Only containers themselves can be removed, all else is virtual
 579                 if ( $dir != '' ) {
 580                         return $status; // nothing to do
 581                 }
 582
 583                 // (a) Check the container
 584                 try {
 585                         $contObj = $this->getContainer( $fullCont, true );
 586                 } catch ( NoSuchContainerException $e ) {
 587                         return $status; // ok, nothing to do
 588                 } catch ( CloudFilesException $e ) { // some other exception?
 589                         $this->handleException( $e, $status, __METHOD__, $params );
 590                         return $status;
 591                 }
 592
 593                 // (b) Delete the container if empty
 594                 if ( $contObj->object_count == 0 ) {
 595                         try {
 596                                 $this->deleteContainer( $fullCont );
 597                         } catch ( NoSuchContainerException $e ) {
 598                                 return $status; // race?
 599                         } catch ( NonEmptyContainerException $e ) {
 600                                 return $status; // race? consistency delay?
 601                         } catch ( CloudFilesException $e ) { // some other exception?
 602                                 $this->handleException( $e, $status, __METHOD__, $params );
 603                                 return $status;
 604                         }
 605                 }
 606
 607                 return $status;
 608         }
 609
 610         /**
 611          * @see FileBackendStore::doFileExists()
 612          * @return array|bool|null
 613          */
 614         protected function doGetFileStat( array $params ) {
 615                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 616                 if ( $srcRel === null ) {
 617                         return false; // invalid storage path
 618                 }
 619
 620                 $stat = false;
 621                 try {
 622                         $contObj = $this->getContainer( $srcCont );
 623                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 624                         $this->addMissingMetadata( $srcObj, $params['src'] );
 625                         $stat = array(
 626                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 627                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 628                                 'size'  => $srcObj->content_length,
 629                                 'sha1'  => $srcObj->metadata['Sha1base36']
 630                         );
 631                 } catch ( NoSuchContainerException $e ) {
 632                 } catch ( NoSuchObjectException $e ) {
 633                 } catch ( CloudFilesException $e ) { // some other exception?
 634                         $stat = null;
 635                         $this->handleException( $e, null, __METHOD__, $params );
 636                 }
 637
 638                 return $stat;
 639         }
 640
 641         /**
 642          * Fill in any missing object metadata and save it to Swift
 643          *
 644          * @param $obj CF_Object
 645          * @param $path string Storage path to object
 646          * @return bool Success
 647          * @throws Exception cloudfiles exceptions
 648          */
 649         protected function addMissingMetadata( CF_Object $obj, $path ) {
 650                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 651                         return true; // nothing to do
 652                 }
 653                 $status = Status::newGood();
 654                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 655                 if ( $status->isOK() ) {
 656                         # Do not stat the file in getLocalCopy() to avoid infinite loops
 657                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
 658                         if ( $tmpFile ) {
 659                                 $hash = $tmpFile->getSha1Base36();
 660                                 if ( $hash !== false ) {
 661                                         $obj->metadata['Sha1base36'] = $hash;
 662                                         $obj->sync_metadata(); // save to Swift
 663                                         return true; // success
 664                                 }
 665                         }
 666                 }
 667                 $obj->metadata['Sha1base36'] = false;
 668                 return false; // failed
 669         }
 670
 671         /**
 672          * @see FileBackend::getFileContents()
 673          * @return bool|null|string
 674          */
 675         public function getFileContents( array $params ) {
 676                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 677                 if ( $srcRel === null ) {
 678                         return false; // invalid storage path
 679                 }
 680
 681                 if ( !$this->fileExists( $params ) ) {
 682                         return null;
 683                 }
 684
 685                 $data = false;
 686                 try {
 687                         $sContObj = $this->getContainer( $srcCont );
 688                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 689                         $data = $obj->read( $this->headersFromParams( $params ) );
 690                 } catch ( NoSuchContainerException $e ) {
 691                 } catch ( CloudFilesException $e ) { // some other exception?
 692                         $this->handleException( $e, null, __METHOD__, $params );
 693                 }
 694
 695                 return $data;
 696         }
 697
 698         /**
 699          * @see FileBackendStore::doDirectoryExists()
 700          * @return bool|null
 701          */
 702         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 703                 try {
 704                         $container = $this->getContainer( $fullCont );
 705                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 706                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 707                 } catch ( NoSuchContainerException $e ) {
 708                         return false;
 709                 } catch ( CloudFilesException $e ) { // some other exception?
 710                         $this->handleException( $e, null, __METHOD__,
 711                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 712                 }
 713
 714                 return null; // error
 715         }
 716
 717         /**
 718          * @see FileBackendStore::getDirectoryListInternal()
 719          * @return SwiftFileBackendDirList
 720          */
 721         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 722                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 723         }
 724
 725         /**
 726          * @see FileBackendStore::getFileListInternal()
 727          * @return SwiftFileBackendFileList
 728          */
 729         public function getFileListInternal( $fullCont, $dir, array $params ) {
 730                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 731         }
 732
 733         /**
 734          * Do not call this function outside of SwiftFileBackendFileList
 735          *
 736          * @param $fullCont string Resolved container name
 737          * @param $dir string Resolved storage directory with no trailing slash
 738          * @param $after string|null Storage path of file to list items after
 739          * @param $limit integer Max number of items to list
 740          * @param $params Array Includes flag for 'topOnly'
 741          * @return Array List of relative paths of dirs directly under $dir
 742          */
 743         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 744                 $dirs = array();
 745
 746                 try {
 747                         $container = $this->getContainer( $fullCont );
 748                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 749                         // Non-recursive: only list dirs right under $dir
 750                         if ( !empty( $params['topOnly'] ) ) {
 751                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 752                                 foreach ( $objects as $object ) { // files and dirs
 753                                         if ( substr( $object, -1 ) === '/' ) {
 754                                                 $dirs[] = $object; // directories end in '/'
 755                                         }
 756                                         $after = $object; // update last item
 757                                 }
 758                         // Recursive: list all dirs under $dir and its subdirs
 759                         } else {
 760                                 // Get directory from last item of prior page
 761                                 $lastDir = $this->getParentDir( $after ); // must be first page
 762                                 $objects = $container->list_objects( $limit, $after, $prefix );
 763                                 foreach ( $objects as $object ) { // files
 764                                         $objectDir = $this->getParentDir( $object ); // directory of object
 765                                         if ( $objectDir !== false ) { // file has a parent dir
 766                                                 // Swift stores paths in UTF-8, using binary sorting.
 767                                                 // See function "create_container_table" in common/db.py.
 768                                                 // If a directory is not "greater" than the last one,
 769                                                 // then it was already listed by the calling iterator.
 770                                                 if ( $objectDir > $lastDir ) {
 771                                                         $pDir = $objectDir;
 772                                                         do { // add dir and all its parent dirs
 773                                                                 $dirs[] = "{$pDir}/";
 774                                                                 $pDir = $this->getParentDir( $pDir );
 775                                                         } while ( $pDir !== false // sanity
 776                                                                 && $pDir > $lastDir // not done already
 777                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 778                                                         );
 779                                                 }
 780                                                 $lastDir = $objectDir;
 781                                         }
 782                                         $after = $object; // update last item
 783                                 }
 784                         }
 785                 } catch ( NoSuchContainerException $e ) {
 786                 } catch ( CloudFilesException $e ) { // some other exception?
 787                         $this->handleException( $e, null, __METHOD__,
 788                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 789                 }
 790
 791                 return $dirs;
 792         }
 793
 794         protected function getParentDir( $path ) {
 795                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 796         }
 797
 798         /**
 799          * Do not call this function outside of SwiftFileBackendFileList
 800          *
 801          * @param $fullCont string Resolved container name
 802          * @param $dir string Resolved storage directory with no trailing slash
 803          * @param $after string|null Storage path of file to list items after
 804          * @param $limit integer Max number of items to list
 805          * @param $params Array Includes flag for 'topOnly'
 806          * @return Array List of relative paths of files under $dir
 807          */
 808         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 809                 $files = array();
 810
 811                 try {
 812                         $container = $this->getContainer( $fullCont );
 813                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 814                         // Non-recursive: only list files right under $dir
 815                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 816                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 817                                 foreach ( $objects as $object ) {
 818                                         if ( substr( $object, -1 ) !== '/' ) {
 819                                                 $files[] = $object; // directories end in '/'
 820                                         }
 821                                 }
 822                         // Recursive: list all files under $dir and its subdirs
 823                         } else { // files
 824                                 $files = $container->list_objects( $limit, $after, $prefix );
 825                         }
 826                         $after = end( $files ); // update last item
 827                         reset( $files ); // reset pointer
 828                 } catch ( NoSuchContainerException $e ) {
 829                 } catch ( CloudFilesException $e ) { // some other exception?
 830                         $this->handleException( $e, null, __METHOD__,
 831                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 832                 }
 833
 834                 return $files;
 835         }
 836
 837         /**
 838          * @see FileBackendStore::doGetFileSha1base36()
 839          * @return bool
 840          */
 841         protected function doGetFileSha1base36( array $params ) {
 842                 $stat = $this->getFileStat( $params );
 843                 if ( $stat ) {
 844                         return $stat['sha1'];
 845                 } else {
 846                         return false;
 847                 }
 848         }
 849
 850         /**
 851          * @see FileBackendStore::doStreamFile()
 852          * @return Status
 853          */
 854         protected function doStreamFile( array $params ) {
 855                 $status = Status::newGood();
 856
 857                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 858                 if ( $srcRel === null ) {
 859                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 860                 }
 861
 862                 try {
 863                         $cont = $this->getContainer( $srcCont );
 864                 } catch ( NoSuchContainerException $e ) {
 865                         $status->fatal( 'backend-fail-stream', $params['src'] );
 866                         return $status;
 867                 } catch ( CloudFilesException $e ) { // some other exception?
 868                         $this->handleException( $e, $status, __METHOD__, $params );
 869                         return $status;
 870                 }
 871
 872                 try {
 873                         $output = fopen( 'php://output', 'wb' );
 874                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
 875                         $obj->stream( $output, $this->headersFromParams( $params ) );
 876                 } catch ( CloudFilesException $e ) { // some other exception?
 877                         $this->handleException( $e, $status, __METHOD__, $params );
 878                 }
 879
 880                 return $status;
 881         }
 882
 883         /**
 884          * @see FileBackendStore::getLocalCopy()
 885          * @return null|TempFSFile
 886          */
 887         public function getLocalCopy( array $params ) {
 888                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 889                 if ( $srcRel === null ) {
 890                         return null;
 891                 }
 892
 893                 # Check the recursion guard to avoid loops when filling metadata
 894                 if ( empty( $params['nostat'] ) && !$this->fileExists( $params ) ) {
 895                         return null;
 896                 }
 897
 898                 $tmpFile = null;
 899                 try {
 900                         $sContObj = $this->getContainer( $srcCont );
 901                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 902                         // Get source file extension
 903                         $ext = FileBackend::extensionFromPath( $srcRel );
 904                         // Create a new temporary file...
 905                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 906                         if ( $tmpFile ) {
 907                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 908                                 if ( $handle ) {
 909                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 910                                         fclose( $handle );
 911                                 } else {
 912                                         $tmpFile = null; // couldn't open temp file
 913                                 }
 914                         }
 915                 } catch ( NoSuchContainerException $e ) {
 916                         $tmpFile = null;
 917                 } catch ( CloudFilesException $e ) { // some other exception?
 918                         $tmpFile = null;
 919                         $this->handleException( $e, null, __METHOD__, $params );
 920                 }
 921
 922                 return $tmpFile;
 923         }
 924
 925         /**
 926          * @see FileBackendStore::directoriesAreVirtual()
 927          * @return bool
 928          */
 929         protected function directoriesAreVirtual() {
 930                 return true;
 931         }
 932
 933         /**
 934          * Get headers to send to Swift when reading a file based
 935          * on a FileBackend params array, e.g. that of getLocalCopy().
 936          * $params is currently only checked for a 'latest' flag.
 937          *
 938          * @param $params Array
 939          * @return Array
 940          */
 941         protected function headersFromParams( array $params ) {
 942                 $hdrs = array();
 943                 if ( !empty( $params['latest'] ) ) {
 944                         $hdrs[] = 'X-Newest: true';
 945                 }
 946                 return $hdrs;
 947         }
 948
 949         /**
 950          * @see FileBackendStore::doExecuteOpHandlesInternal()
 951          * @return Array List of corresponding Status objects
 952          */
 953         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
 954                 $statuses = array();
 955
 956                 $cfOps = array(); // list of CF_Async_Op objects
 957                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
 958                         $cfOps[$index] = $fileOpHandle->cfOp;
 959                 }
 960                 $batch = new CF_Async_Op_Batch( $cfOps );
 961
 962                 $cfOps = $batch->execute();
 963                 foreach ( $cfOps as $index => $cfOp ) {
 964                         $status = Status::newGood();
 965                         try { // catch exceptions; update status
 966                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
 967                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
 968                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
 969                         } catch ( CloudFilesException $e ) { // some other exception?
 970                                 $this->handleException( $e, $status,
 971                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
 972                         }
 973                         $statuses[$index] = $status;
 974                 }
 975
 976                 return $statuses;
 977         }
 978
 979         /**
 980          * Set read/write permissions for a Swift container
 981          *
 982          * @param $contObj CF_Container Swift container
 983          * @param $readGrps Array Swift users who can read (account:user)
 984          * @param $writeGrps Array Swift users who can write (account:user)
 985          * @return Status
 986          */
 987         protected function setContainerAccess(
 988                 CF_Container $contObj, array $readGrps, array $writeGrps
 989         ) {
 990                 $creds = $contObj->cfs_auth->export_credentials();
 991
 992                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
 993
 994                 // Note: 10 second timeout consistent with php-cloudfiles
 995                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
 996                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
 997                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
 998                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
 999
1000                 return $req->execute(); // should return 204
1001         }
1002
1003         /**
1004          * Purge the CDN cache of affected objects if CDN caching is enabled
1005          *
1006          * @param $objects Array List of CF_Object items
1007          * @return void
1008          */
1009         public function purgeCDNCache( array $objects ) {
1010                 if ( $this->swiftUseCDN ) { // Rackspace style CDN
1011                         foreach ( $objects as $object ) {
1012                                 try {
1013                                         $object->purge_from_cdn();
1014                                 } catch ( CDNNotEnabledException $e ) {
1015                                         // CDN not enabled; nothing to see here
1016                                 } catch ( CloudFilesException $e ) {
1017                                         $this->handleException( $e, null, __METHOD__,
1018                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1019                                 }
1020                         }
1021                 }
1022         }
1023
1024         /**
1025          * Get a connection to the Swift proxy
1026          *
1027          * @return CF_Connection|bool False on failure
1028          * @throws CloudFilesException
1029          */
1030         protected function getConnection() {
1031                 if ( $this->connException instanceof Exception ) {
1032                         throw $this->connException; // failed last attempt
1033                 }
1034                 // Session keys expire after a while, so we renew them periodically
1035                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
1036                         $this->conn->close(); // close active cURL connections
1037                         $this->conn = null;
1038                 }
1039                 // Authenticate with proxy and get a session key...
1040                 if ( !$this->conn ) {
1041                         $this->connStarted = 0;
1042                         $this->connContainers = array();
1043                         try {
1044                                 $this->auth->authenticate();
1045                                 $this->conn = new CF_Connection( $this->auth );
1046                                 $this->connStarted = time();
1047                         } catch ( CloudFilesException $e ) {
1048                                 $this->connException = $e; // don't keep re-trying
1049                                 throw $e; // throw it back
1050                         }
1051                 }
1052                 return $this->conn;
1053         }
1054
1055         /**
1056          * @see FileBackendStore::doClearCache()
1057          */
1058         protected function doClearCache( array $paths = null ) {
1059                 $this->connContainers = array(); // clear container object cache
1060         }
1061
1062         /**
1063          * Get a Swift container object, possibly from process cache.
1064          * Use $reCache if the file count or byte count is needed.
1065          *
1066          * @param $container string Container name
1067          * @param $bypassCache bool Bypass all caches and load from Swift
1068          * @return CF_Container
1069          * @throws CloudFilesException
1070          */
1071         protected function getContainer( $container, $bypassCache = false ) {
1072                 $conn = $this->getConnection(); // Swift proxy connection
1073                 if ( $bypassCache ) { // purge cache
1074                         unset( $this->connContainers[$container] );
1075                 } elseif ( !isset( $this->connContainers[$container] ) ) {
1076                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1077                 }
1078                 if ( !isset( $this->connContainers[$container] ) ) {
1079                         $contObj = $conn->get_container( $container );
1080                         // NoSuchContainerException not thrown: container must exist
1081                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
1082                                 reset( $this->connContainers );
1083                                 unset( $this->connContainers[key( $this->connContainers )] );
1084                         }
1085                         $this->connContainers[$container] = $contObj; // cache it
1086                         if ( !$bypassCache ) {
1087                                 $this->setContainerCache( $container, // update persistent cache
1088                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1089                                 );
1090                         }
1091                 }
1092                 return $this->connContainers[$container];
1093         }
1094
1095         /**
1096          * Create a Swift container
1097          *
1098          * @param $container string Container name
1099          * @return CF_Container
1100          * @throws InvalidResponseException
1101          */
1102         protected function createContainer( $container ) {
1103                 $conn = $this->getConnection(); // Swift proxy connection
1104                 $contObj = $conn->create_container( $container );
1105                 $this->connContainers[$container] = $contObj; // cache it
1106                 return $contObj;
1107         }
1108
1109         /**
1110          * Delete a Swift container
1111          *
1112          * @param $container string Container name
1113          * @return void
1114          * @throws InvalidResponseException
1115          */
1116         protected function deleteContainer( $container ) {
1117                 $conn = $this->getConnection(); // Swift proxy connection
1118                 $conn->delete_container( $container );
1119                 unset( $this->connContainers[$container] ); // purge cache
1120         }
1121
1122         /**
1123          * @see FileBackendStore::doPrimeContainerCache()
1124          * @return void
1125          */
1126         protected function doPrimeContainerCache( array $containerInfo ) {
1127                 try {
1128                         $conn = $this->getConnection(); // Swift proxy connection
1129                         foreach ( $containerInfo as $container => $info ) {
1130                                 $this->connContainers[$container] = new CF_Container(
1131                                         $conn->cfs_auth,
1132                                         $conn->cfs_http,
1133                                         $container,
1134                                         $info['count'],
1135                                         $info['bytes']
1136                                 );
1137                         }
1138                 } catch ( CloudFilesException $e ) { // some other exception?
1139                         $this->handleException( $e, null, __METHOD__, array() );
1140                 }
1141         }
1142
1143         /**
1144          * Log an unexpected exception for this backend.
1145          * This also sets the Status object to have a fatal error.
1146          *
1147          * @param $e Exception
1148          * @param $status Status|null
1149          * @param $func string
1150          * @param $params Array
1151          * @return void
1152          */
1153         protected function handleException( Exception $e, $status, $func, array $params ) {
1154                 if ( $status instanceof Status ) {
1155                         if ( $e instanceof AuthenticationException ) {
1156                                 $status->fatal( 'backend-fail-connect', $this->name );
1157                         } else {
1158                                 $status->fatal( 'backend-fail-internal', $this->name );
1159                         }
1160                 }
1161                 if ( $e->getMessage() ) {
1162                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1163                 }
1164                 wfDebugLog( 'SwiftBackend',
1165                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1166                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1167                 );
1168         }
1169 }
1170
1171 /**
1172  * @see FileBackendStoreOpHandle
1173  */
1174 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1175         /** @var CF_Async_Op */
1176         public $cfOp;
1177         /** @var Array */
1178         public $affectedObjects = array();
1179
1180         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1181                 $this->backend = $backend;
1182                 $this->params = $params;
1183                 $this->call = $call;
1184                 $this->cfOp = $cfOp;
1185         }
1186 }
1187
1188 /**
1189  * SwiftFileBackend helper class to page through listings.
1190  * Swift also has a listing limit of 10,000 objects for sanity.
1191  * Do not use this class from places outside SwiftFileBackend.
1192  *
1193  * @ingroup FileBackend
1194  */
1195 abstract class SwiftFileBackendList implements Iterator {
1196         /** @var Array */
1197         protected $bufferIter = array();
1198         protected $bufferAfter = null; // string; list items *after* this path
1199         protected $pos = 0; // integer
1200         /** @var Array */
1201         protected $params = array();
1202
1203         /** @var SwiftFileBackend */
1204         protected $backend;
1205         protected $container; // string; container name
1206         protected $dir; // string; storage directory
1207         protected $suffixStart; // integer
1208
1209         const PAGE_SIZE = 5000; // file listing buffer size
1210
1211         /**
1212          * @param $backend SwiftFileBackend
1213          * @param $fullCont string Resolved container name
1214          * @param $dir string Resolved directory relative to container
1215          * @param $params Array
1216          */
1217         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1218                 $this->backend = $backend;
1219                 $this->container = $fullCont;
1220                 $this->dir = $dir;
1221                 if ( substr( $this->dir, -1 ) === '/' ) {
1222                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1223                 }
1224                 if ( $this->dir == '' ) { // whole container
1225                         $this->suffixStart = 0;
1226                 } else { // dir within container
1227                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1228                 }
1229                 $this->params = $params;
1230         }
1231
1232         /**
1233          * @see Iterator::key()
1234          * @return integer
1235          */
1236         public function key() {
1237                 return $this->pos;
1238         }
1239
1240         /**
1241          * @see Iterator::next()
1242          * @return void
1243          */
1244         public function next() {
1245                 // Advance to the next file in the page
1246                 next( $this->bufferIter );
1247                 ++$this->pos;
1248                 // Check if there are no files left in this page and
1249                 // advance to the next page if this page was not empty.
1250                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1251                         $this->bufferIter = $this->pageFromList(
1252                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1253                         ); // updates $this->bufferAfter
1254                 }
1255         }
1256
1257         /**
1258          * @see Iterator::rewind()
1259          * @return void
1260          */
1261         public function rewind() {
1262                 $this->pos = 0;
1263                 $this->bufferAfter = null;
1264                 $this->bufferIter = $this->pageFromList(
1265                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1266                 ); // updates $this->bufferAfter
1267         }
1268
1269         /**
1270          * @see Iterator::valid()
1271          * @return bool
1272          */
1273         public function valid() {
1274                 if ( $this->bufferIter === null ) {
1275                         return false; // some failure?
1276                 } else {
1277                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1278                 }
1279         }
1280
1281         /**
1282          * Get the given list portion (page)
1283          *
1284          * @param $container string Resolved container name
1285          * @param $dir string Resolved path relative to container
1286          * @param $after string|null
1287          * @param $limit integer
1288          * @param $params Array
1289          * @return Traversable|Array|null Returns null on failure
1290          */
1291         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1292 }
1293
1294 /**
1295  * Iterator for listing directories
1296  */
1297 class SwiftFileBackendDirList extends SwiftFileBackendList {
1298         /**
1299          * @see Iterator::current()
1300          * @return string|bool String (relative path) or false
1301          */
1302         public function current() {
1303                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1304         }
1305
1306         /**
1307          * @see SwiftFileBackendList::pageFromList()
1308          * @return Array|null
1309          */
1310         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1311                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1312         }
1313 }
1314
1315 /**
1316  * Iterator for listing regular files
1317  */
1318 class SwiftFileBackendFileList extends SwiftFileBackendList {
1319         /**
1320          * @see Iterator::current()
1321          * @return string|bool String (relative path) or false
1322          */
1323         public function current() {
1324                 return substr( current( $this->bufferIter ), $this->suffixStart );
1325         }
1326
1327         /**
1328          * @see SwiftFileBackendList::pageFromList()
1329          * @return Array|null
1330          */
1331         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1332                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1333         }
1334 }