includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $maxContCacheSize = 300; // integer; max containers with entries
  45
  46         /** @var CF_Connection */
  47         protected $conn; // Swift connection handle
  48         protected $connStarted = 0; // integer UNIX timestamp
  49         protected $connContainers = array(); // container object cache
  50
  51         /**
  52          * @see FileBackendStore::__construct()
  53          * Additional $config params include:
  54          *    swiftAuthUrl       : Swift authentication server URL
  55          *    swiftUser          : Swift user used by MediaWiki (account:username)
  56          *    swiftKey           : Swift authentication key for the above user
  57          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  58          *    swiftAnonUser      : Swift user used for end-user requests (account:username)
  59          *    shardViaHashLevels : Map of container names to sharding config with:
  60          *                         'base'   : base of hash characters, 16 or 36
  61          *                         'levels' : the number of hash levels (and digits)
  62          *                         'repeat' : hash subdirectories are prefixed with all the
  63          *                                    parent hash directory names (e.g. "a/ab/abc")
  64          */
  65         public function __construct( array $config ) {
  66                 parent::__construct( $config );
  67                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  68                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  69                 }
  70                 // Required settings
  71                 $this->auth = new CF_Authentication(
  72                         $config['swiftUser'],
  73                         $config['swiftKey'],
  74                         null, // account; unused
  75                         $config['swiftAuthUrl']
  76                 );
  77                 // Optional settings
  78                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  79                         ? $config['swiftAuthTTL']
  80                         : 5 * 60; // some sane number
  81                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  82                         ? $config['swiftAnonUser']
  83                         : '';
  84                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  85                         ? $config['shardViaHashLevels']
  86                         : '';
  87                 // Cache container info to mask latency
  88                 $this->memCache = wfGetMainCache();
  89         }
  90
  91         /**
  92          * @see FileBackendStore::resolveContainerPath()
  93          * @return null
  94          */
  95         protected function resolveContainerPath( $container, $relStoragePath ) {
  96                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
  97                         return null; // too long for Swift
  98                 }
  99                 return $relStoragePath;
 100         }
 101
 102         /**
 103          * @see FileBackendStore::isPathUsableInternal()
 104          * @return bool
 105          */
 106         public function isPathUsableInternal( $storagePath ) {
 107                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 108                 if ( $rel === null ) {
 109                         return false; // invalid
 110                 }
 111
 112                 try {
 113                         $this->getContainer( $container );
 114                         return true; // container exists
 115                 } catch ( NoSuchContainerException $e ) {
 116                 } catch ( CloudFilesException $e ) { // some other exception?
 117                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 118                 }
 119
 120                 return false;
 121         }
 122
 123         /**
 124          * @see FileBackendStore::doCreateInternal()
 125          * @return Status
 126          */
 127         protected function doCreateInternal( array $params ) {
 128                 $status = Status::newGood();
 129
 130                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 131                 if ( $dstRel === null ) {
 132                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 133                         return $status;
 134                 }
 135
 136                 // (a) Check the destination container and object
 137                 try {
 138                         $dContObj = $this->getContainer( $dstCont );
 139                         if ( empty( $params['overwrite'] ) &&
 140                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 141                         {
 142                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 143                                 return $status;
 144                         }
 145                 } catch ( NoSuchContainerException $e ) {
 146                         $status->fatal( 'backend-fail-create', $params['dst'] );
 147                         return $status;
 148                 } catch ( CloudFilesException $e ) { // some other exception?
 149                         $this->handleException( $e, $status, __METHOD__, $params );
 150                         return $status;
 151                 }
 152
 153                 // (b) Get a SHA-1 hash of the object
 154                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 155
 156                 // (c) Actually create the object
 157                 try {
 158                         // Create a fresh CF_Object with no fields preloaded.
 159                         // We don't want to preserve headers, metadata, and such.
 160                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 161                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 162                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 163                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 164                         // The MD5 here will be checked within Swift against its own MD5.
 165                         $obj->set_etag( md5( $params['content'] ) );
 166                         // Use the same content type as StreamFile for security
 167                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 168                         if ( !empty( $params['async'] ) ) { // deferred
 169                                 $handle = $obj->write_async( $params['content'] );
 170                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle );
 171                         } else { // actually write the object in Swift
 172                                 $obj->write( $params['content'] );
 173                         }
 174                 } catch ( BadContentTypeException $e ) {
 175                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 176                 } catch ( CloudFilesException $e ) { // some other exception?
 177                         $this->handleException( $e, $status, __METHOD__, $params );
 178                 }
 179
 180                 return $status;
 181         }
 182
 183         /**
 184          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 185          */
 186         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 187                 try {
 188                         $cfOp->getLastResponse();
 189                 } catch ( BadContentTypeException $e ) {
 190                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 191                 }
 192         }
 193
 194         /**
 195          * @see FileBackendStore::doStoreInternal()
 196          * @return Status
 197          */
 198         protected function doStoreInternal( array $params ) {
 199                 $status = Status::newGood();
 200
 201                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 202                 if ( $dstRel === null ) {
 203                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 204                         return $status;
 205                 }
 206
 207                 // (a) Check the destination container and object
 208                 try {
 209                         $dContObj = $this->getContainer( $dstCont );
 210                         if ( empty( $params['overwrite'] ) &&
 211                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 212                         {
 213                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 214                                 return $status;
 215                         }
 216                 } catch ( NoSuchContainerException $e ) {
 217                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 218                         return $status;
 219                 } catch ( CloudFilesException $e ) { // some other exception?
 220                         $this->handleException( $e, $status, __METHOD__, $params );
 221                         return $status;
 222                 }
 223
 224                 // (b) Get a SHA-1 hash of the object
 225                 $sha1Hash = sha1_file( $params['src'] );
 226                 if ( $sha1Hash === false ) { // source doesn't exist?
 227                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 228                         return $status;
 229                 }
 230                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 231
 232                 // (c) Actually store the object
 233                 try {
 234                         // Create a fresh CF_Object with no fields preloaded.
 235                         // We don't want to preserve headers, metadata, and such.
 236                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 237                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 238                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 239                         // The MD5 here will be checked within Swift against its own MD5.
 240                         $obj->set_etag( md5_file( $params['src'] ) );
 241                         // Use the same content type as StreamFile for security
 242                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 243                         if ( !empty( $params['async'] ) ) { // deferred
 244                                 wfSuppressWarnings();
 245                                 $fp = fopen( $params['src'], 'rb' );
 246                                 wfRestoreWarnings();
 247                                 if ( !$fp ) {
 248                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 249                                 } else {
 250                                         $handle = $obj->write_async( $fp, filesize( $params['src'] ), true );
 251                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle );
 252                                         $status->value->resourcesToClose[] = $fp;
 253                                 }
 254                         } else { // actually write the object in Swift
 255                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 256                         }
 257                 } catch ( BadContentTypeException $e ) {
 258                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 259                 } catch ( IOException $e ) {
 260                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 261                 } catch ( CloudFilesException $e ) { // some other exception?
 262                         $this->handleException( $e, $status, __METHOD__, $params );
 263                 }
 264
 265                 return $status;
 266         }
 267
 268         /**
 269          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 270          */
 271         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 272                 try {
 273                         $cfOp->getLastResponse();
 274                 } catch ( BadContentTypeException $e ) {
 275                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 276                 } catch ( IOException $e ) {
 277                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 278                 }
 279         }
 280
 281         /**
 282          * @see FileBackendStore::doCopyInternal()
 283          * @return Status
 284          */
 285         protected function doCopyInternal( array $params ) {
 286                 $status = Status::newGood();
 287
 288                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 289                 if ( $srcRel === null ) {
 290                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 291                         return $status;
 292                 }
 293
 294                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 295                 if ( $dstRel === null ) {
 296                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 297                         return $status;
 298                 }
 299
 300                 // (a) Check the source/destination containers and destination object
 301                 try {
 302                         $sContObj = $this->getContainer( $srcCont );
 303                         $dContObj = $this->getContainer( $dstCont );
 304                         if ( empty( $params['overwrite'] ) &&
 305                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 306                         {
 307                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 308                                 return $status;
 309                         }
 310                 } catch ( NoSuchContainerException $e ) {
 311                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 312                         return $status;
 313                 } catch ( CloudFilesException $e ) { // some other exception?
 314                         $this->handleException( $e, $status, __METHOD__, $params );
 315                         return $status;
 316                 }
 317
 318                 // (b) Actually copy the file to the destination
 319                 try {
 320                         if ( !empty( $params['async'] ) ) { // deferred
 321                                 $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel );
 322                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle );
 323                         } else { // actually write the object in Swift
 324                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 325                         }
 326                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 327                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 328                 } catch ( CloudFilesException $e ) { // some other exception?
 329                         $this->handleException( $e, $status, __METHOD__, $params );
 330                 }
 331
 332                 return $status;
 333         }
 334
 335         /**
 336          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 337          */
 338         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 339                 try {
 340                         $cfOp->getLastResponse();
 341                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 342                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 343                 }
 344         }
 345
 346         /**
 347          * @see FileBackendStore::doMoveInternal()
 348          * @return Status
 349          */
 350         protected function doMoveInternal( array $params ) {
 351                 $status = Status::newGood();
 352
 353                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 354                 if ( $srcRel === null ) {
 355                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 356                         return $status;
 357                 }
 358
 359                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 360                 if ( $dstRel === null ) {
 361                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 362                         return $status;
 363                 }
 364
 365                 // (a) Check the source/destination containers and destination object
 366                 try {
 367                         $sContObj = $this->getContainer( $srcCont );
 368                         $dContObj = $this->getContainer( $dstCont );
 369                         if ( empty( $params['overwrite'] ) &&
 370                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 371                         {
 372                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 373                                 return $status;
 374                         }
 375                 } catch ( NoSuchContainerException $e ) {
 376                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 377                         return $status;
 378                 } catch ( CloudFilesException $e ) { // some other exception?
 379                         $this->handleException( $e, $status, __METHOD__, $params );
 380                         return $status;
 381                 }
 382
 383                 // (b) Actually move the file to the destination
 384                 try {
 385                         if ( !empty( $params['async'] ) ) { // deferred
 386                                 $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel );
 387                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle );
 388                         } else { // actually write the object in Swift
 389                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel );
 390                         }
 391                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 392                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 393                 } catch ( CloudFilesException $e ) { // some other exception?
 394                         $this->handleException( $e, $status, __METHOD__, $params );
 395                 }
 396
 397                 return $status;
 398         }
 399
 400         /**
 401          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 402          */
 403         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 404                 try {
 405                         $cfOp->getLastResponse();
 406                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 407                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 408                 }
 409         }
 410
 411         /**
 412          * @see FileBackendStore::doDeleteInternal()
 413          * @return Status
 414          */
 415         protected function doDeleteInternal( array $params ) {
 416                 $status = Status::newGood();
 417
 418                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 419                 if ( $srcRel === null ) {
 420                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 421                         return $status;
 422                 }
 423
 424                 try {
 425                         $sContObj = $this->getContainer( $srcCont );
 426                         if ( !empty( $params['async'] ) ) { // deferred
 427                                 $handle = $sContObj->delete_object_async( $srcRel );
 428                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle );
 429                         } else { // actually write the object in Swift
 430                                 $sContObj->delete_object( $srcRel );
 431                         }
 432                 } catch ( NoSuchContainerException $e ) {
 433                         $status->fatal( 'backend-fail-delete', $params['src'] );
 434                 } catch ( NoSuchObjectException $e ) {
 435                         if ( empty( $params['ignoreMissingSource'] ) ) {
 436                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 437                         }
 438                 } catch ( CloudFilesException $e ) { // some other exception?
 439                         $this->handleException( $e, $status, __METHOD__, $params );
 440                 }
 441
 442                 return $status;
 443         }
 444
 445         /**
 446          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 447          */
 448         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 449                 try {
 450                         $cfOp->getLastResponse();
 451                 } catch ( NoSuchContainerException $e ) {
 452                         $status->fatal( 'backend-fail-delete', $params['src'] );
 453                 } catch ( NoSuchObjectException $e ) {
 454                         if ( empty( $params['ignoreMissingSource'] ) ) {
 455                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 456                         }
 457                 }
 458         }
 459
 460         /**
 461          * @see FileBackendStore::doPrepareInternal()
 462          * @return Status
 463          */
 464         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 465                 $status = Status::newGood();
 466
 467                 // (a) Check if container already exists
 468                 try {
 469                         $contObj = $this->getContainer( $fullCont );
 470                         // NoSuchContainerException not thrown: container must exist
 471                         return $status; // already exists
 472                 } catch ( NoSuchContainerException $e ) {
 473                         // NoSuchContainerException thrown: container does not exist
 474                 } catch ( CloudFilesException $e ) { // some other exception?
 475                         $this->handleException( $e, $status, __METHOD__, $params );
 476                         return $status;
 477                 }
 478
 479                 // (b) Create container as needed
 480                 try {
 481                         $contObj = $this->createContainer( $fullCont );
 482                         if ( $this->swiftAnonUser != '' ) {
 483                                 // Make container public to end-users...
 484                                 $status->merge( $this->setContainerAccess(
 485                                         $contObj,
 486                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 487                                         array( $this->auth->username ) // write
 488                                 ) );
 489                         }
 490                 } catch ( CloudFilesException $e ) { // some other exception?
 491                         $this->handleException( $e, $status, __METHOD__, $params );
 492                         return $status;
 493                 }
 494
 495                 return $status;
 496         }
 497
 498         /**
 499          * @see FileBackendStore::doSecureInternal()
 500          * @return Status
 501          */
 502         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 503                 $status = Status::newGood();
 504
 505                 if ( $this->swiftAnonUser != '' ) {
 506                         // Restrict container from end-users...
 507                         try {
 508                                 // doPrepareInternal() should have been called,
 509                                 // so the Swift container should already exist...
 510                                 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 511                                 // NoSuchContainerException not thrown: container must exist
 512                                 if ( !isset( $contObj->mw_wasSecured ) ) {
 513                                         $status->merge( $this->setContainerAccess(
 514                                                 $contObj,
 515                                                 array( $this->auth->username ), // read
 516                                                 array( $this->auth->username ) // write
 517                                         ) );
 518                                         // @TODO: when php-cloudfiles supports container
 519                                         // metadata, we can make use of that to avoid RTTs
 520                                         $contObj->mw_wasSecured = true; // avoid useless RTTs
 521                                 }
 522                         } catch ( CloudFilesException $e ) { // some other exception?
 523                                 $this->handleException( $e, $status, __METHOD__, $params );
 524                         }
 525                 }
 526
 527                 return $status;
 528         }
 529
 530         /**
 531          * @see FileBackendStore::doCleanInternal()
 532          * @return Status
 533          */
 534         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 535                 $status = Status::newGood();
 536
 537                 // Only containers themselves can be removed, all else is virtual
 538                 if ( $dir != '' ) {
 539                         return $status; // nothing to do
 540                 }
 541
 542                 // (a) Check the container
 543                 try {
 544                         $contObj = $this->getContainer( $fullCont, true );
 545                 } catch ( NoSuchContainerException $e ) {
 546                         return $status; // ok, nothing to do
 547                 } catch ( CloudFilesException $e ) { // some other exception?
 548                         $this->handleException( $e, $status, __METHOD__, $params );
 549                         return $status;
 550                 }
 551
 552                 // (b) Delete the container if empty
 553                 if ( $contObj->object_count == 0 ) {
 554                         try {
 555                                 $this->deleteContainer( $fullCont );
 556                         } catch ( NoSuchContainerException $e ) {
 557                                 return $status; // race?
 558                         } catch ( NonEmptyContainerException $e ) {
 559                                 return $status; // race? consistency delay?
 560                         } catch ( CloudFilesException $e ) { // some other exception?
 561                                 $this->handleException( $e, $status, __METHOD__, $params );
 562                                 return $status;
 563                         }
 564                 }
 565
 566                 return $status;
 567         }
 568
 569         /**
 570          * @see FileBackendStore::doFileExists()
 571          * @return array|bool|null
 572          */
 573         protected function doGetFileStat( array $params ) {
 574                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 575                 if ( $srcRel === null ) {
 576                         return false; // invalid storage path
 577                 }
 578
 579                 $stat = false;
 580                 try {
 581                         $contObj = $this->getContainer( $srcCont );
 582                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 583                         $this->addMissingMetadata( $srcObj, $params['src'] );
 584                         $stat = array(
 585                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 586                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 587                                 'size'  => $srcObj->content_length,
 588                                 'sha1'  => $srcObj->metadata['Sha1base36']
 589                         );
 590                 } catch ( NoSuchContainerException $e ) {
 591                 } catch ( NoSuchObjectException $e ) {
 592                 } catch ( CloudFilesException $e ) { // some other exception?
 593                         $stat = null;
 594                         $this->handleException( $e, null, __METHOD__, $params );
 595                 }
 596
 597                 return $stat;
 598         }
 599
 600         /**
 601          * Fill in any missing object metadata and save it to Swift
 602          *
 603          * @param $obj CF_Object
 604          * @param $path string Storage path to object
 605          * @return bool Success
 606          * @throws Exception cloudfiles exceptions
 607          */
 608         protected function addMissingMetadata( CF_Object $obj, $path ) {
 609                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 610                         return true; // nothing to do
 611                 }
 612                 $status = Status::newGood();
 613                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 614                 if ( $status->isOK() ) {
 615                         # Do not stat the file in getLocalCopy() to avoid infinite loops
 616                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
 617                         if ( $tmpFile ) {
 618                                 $hash = $tmpFile->getSha1Base36();
 619                                 if ( $hash !== false ) {
 620                                         $obj->metadata['Sha1base36'] = $hash;
 621                                         $obj->sync_metadata(); // save to Swift
 622                                         return true; // success
 623                                 }
 624                         }
 625                 }
 626                 $obj->metadata['Sha1base36'] = false;
 627                 return false; // failed
 628         }
 629
 630         /**
 631          * @see FileBackend::getFileContents()
 632          * @return bool|null|string
 633          */
 634         public function getFileContents( array $params ) {
 635                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 636                 if ( $srcRel === null ) {
 637                         return false; // invalid storage path
 638                 }
 639
 640                 if ( !$this->fileExists( $params ) ) {
 641                         return null;
 642                 }
 643
 644                 $data = false;
 645                 try {
 646                         $sContObj = $this->getContainer( $srcCont );
 647                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request
 648                         $data = $obj->read( $this->headersFromParams( $params ) );
 649                 } catch ( NoSuchContainerException $e ) {
 650                 } catch ( CloudFilesException $e ) { // some other exception?
 651                         $this->handleException( $e, null, __METHOD__, $params );
 652                 }
 653
 654                 return $data;
 655         }
 656
 657         /**
 658          * @see FileBackendStore::doDirectoryExists()
 659          * @return bool|null
 660          */
 661         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 662                 try {
 663                         $container = $this->getContainer( $fullCont );
 664                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 665                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 666                 } catch ( NoSuchContainerException $e ) {
 667                         return false;
 668                 } catch ( CloudFilesException $e ) { // some other exception?
 669                         $this->handleException( $e, null, __METHOD__,
 670                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 671                 }
 672
 673                 return null; // error
 674         }
 675
 676         /**
 677          * @see FileBackendStore::getDirectoryListInternal()
 678          * @return SwiftFileBackendDirList
 679          */
 680         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 681                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 682         }
 683
 684         /**
 685          * @see FileBackendStore::getFileListInternal()
 686          * @return SwiftFileBackendFileList
 687          */
 688         public function getFileListInternal( $fullCont, $dir, array $params ) {
 689                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 690         }
 691
 692         /**
 693          * Do not call this function outside of SwiftFileBackendFileList
 694          *
 695          * @param $fullCont string Resolved container name
 696          * @param $dir string Resolved storage directory with no trailing slash
 697          * @param $after string|null Storage path of file to list items after
 698          * @param $limit integer Max number of items to list
 699          * @param $params Array Includes flag for 'topOnly'
 700          * @return Array List of relative paths of dirs directly under $dir
 701          */
 702         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 703                 $dirs = array();
 704
 705                 try {
 706                         $container = $this->getContainer( $fullCont );
 707                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 708                         // Non-recursive: only list dirs right under $dir
 709                         if ( !empty( $params['topOnly'] ) ) {
 710                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 711                                 foreach ( $objects as $object ) { // files and dirs
 712                                         if ( substr( $object, -1 ) === '/' ) {
 713                                                 $dirs[] = $object; // directories end in '/'
 714                                         }
 715                                         $after = $object; // update last item
 716                                 }
 717                         // Recursive: list all dirs under $dir and its subdirs
 718                         } else {
 719                                 // Get directory from last item of prior page
 720                                 $lastDir = $this->getParentDir( $after ); // must be first page
 721                                 $objects = $container->list_objects( $limit, $after, $prefix );
 722                                 foreach ( $objects as $object ) { // files
 723                                         $objectDir = $this->getParentDir( $object ); // directory of object
 724                                         if ( $objectDir !== false ) { // file has a parent dir
 725                                                 // Swift stores paths in UTF-8, using binary sorting.
 726                                                 // See function "create_container_table" in common/db.py.
 727                                                 // If a directory is not "greater" than the last one,
 728                                                 // then it was already listed by the calling iterator.
 729                                                 if ( $objectDir > $lastDir ) {
 730                                                         $pDir = $objectDir;
 731                                                         do { // add dir and all its parent dirs
 732                                                                 $dirs[] = "{$pDir}/";
 733                                                                 $pDir = $this->getParentDir( $pDir );
 734                                                         } while ( $pDir !== false // sanity
 735                                                                 && $pDir > $lastDir // not done already
 736                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 737                                                         );
 738                                                 }
 739                                                 $lastDir = $objectDir;
 740                                         }
 741                                         $after = $object; // update last item
 742                                 }
 743                         }
 744                 } catch ( NoSuchContainerException $e ) {
 745                 } catch ( CloudFilesException $e ) { // some other exception?
 746                         $this->handleException( $e, null, __METHOD__,
 747                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 748                 }
 749
 750                 return $dirs;
 751         }
 752
 753         protected function getParentDir( $path ) {
 754                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 755         }
 756
 757         /**
 758          * Do not call this function outside of SwiftFileBackendFileList
 759          *
 760          * @param $fullCont string Resolved container name
 761          * @param $dir string Resolved storage directory with no trailing slash
 762          * @param $after string|null Storage path of file to list items after
 763          * @param $limit integer Max number of items to list
 764          * @param $params Array Includes flag for 'topOnly'
 765          * @return Array List of relative paths of files under $dir
 766          */
 767         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 768                 $files = array();
 769
 770                 try {
 771                         $container = $this->getContainer( $fullCont );
 772                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 773                         // Non-recursive: only list files right under $dir
 774                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 775                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 776                                 foreach ( $objects as $object ) {
 777                                         if ( substr( $object, -1 ) !== '/' ) {
 778                                                 $files[] = $object; // directories end in '/'
 779                                         }
 780                                 }
 781                         // Recursive: list all files under $dir and its subdirs
 782                         } else { // files
 783                                 $files = $container->list_objects( $limit, $after, $prefix );
 784                         }
 785                         $after = end( $files ); // update last item
 786                         reset( $files ); // reset pointer
 787                 } catch ( NoSuchContainerException $e ) {
 788                 } catch ( CloudFilesException $e ) { // some other exception?
 789                         $this->handleException( $e, null, __METHOD__,
 790                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 791                 }
 792
 793                 return $files;
 794         }
 795
 796         /**
 797          * @see FileBackendStore::doGetFileSha1base36()
 798          * @return bool
 799          */
 800         protected function doGetFileSha1base36( array $params ) {
 801                 $stat = $this->getFileStat( $params );
 802                 if ( $stat ) {
 803                         return $stat['sha1'];
 804                 } else {
 805                         return false;
 806                 }
 807         }
 808
 809         /**
 810          * @see FileBackendStore::doStreamFile()
 811          * @return Status
 812          */
 813         protected function doStreamFile( array $params ) {
 814                 $status = Status::newGood();
 815
 816                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 817                 if ( $srcRel === null ) {
 818                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 819                 }
 820
 821                 try {
 822                         $cont = $this->getContainer( $srcCont );
 823                 } catch ( NoSuchContainerException $e ) {
 824                         $status->fatal( 'backend-fail-stream', $params['src'] );
 825                         return $status;
 826                 } catch ( CloudFilesException $e ) { // some other exception?
 827                         $this->handleException( $e, $status, __METHOD__, $params );
 828                         return $status;
 829                 }
 830
 831                 try {
 832                         $output = fopen( 'php://output', 'wb' );
 833                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
 834                         $obj->stream( $output, $this->headersFromParams( $params ) );
 835                 } catch ( CloudFilesException $e ) { // some other exception?
 836                         $this->handleException( $e, $status, __METHOD__, $params );
 837                 }
 838
 839                 return $status;
 840         }
 841
 842         /**
 843          * @see FileBackendStore::getLocalCopy()
 844          * @return null|TempFSFile
 845          */
 846         public function getLocalCopy( array $params ) {
 847                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 848                 if ( $srcRel === null ) {
 849                         return null;
 850                 }
 851
 852                 # Check the recursion guard to avoid loops when filling metadata
 853                 if ( empty( $params['nostat'] ) && !$this->fileExists( $params ) ) {
 854                         return null;
 855                 }
 856
 857                 $tmpFile = null;
 858                 try {
 859                         $sContObj = $this->getContainer( $srcCont );
 860                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 861                         // Get source file extension
 862                         $ext = FileBackend::extensionFromPath( $srcRel );
 863                         // Create a new temporary file...
 864                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 865                         if ( $tmpFile ) {
 866                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 867                                 if ( $handle ) {
 868                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 869                                         fclose( $handle );
 870                                 } else {
 871                                         $tmpFile = null; // couldn't open temp file
 872                                 }
 873                         }
 874                 } catch ( NoSuchContainerException $e ) {
 875                         $tmpFile = null;
 876                 } catch ( CloudFilesException $e ) { // some other exception?
 877                         $tmpFile = null;
 878                         $this->handleException( $e, null, __METHOD__, $params );
 879                 }
 880
 881                 return $tmpFile;
 882         }
 883
 884         /**
 885          * @see FileBackendStore::directoriesAreVirtual()
 886          * @return bool
 887          */
 888         protected function directoriesAreVirtual() {
 889                 return true;
 890         }
 891
 892         /**
 893          * Get headers to send to Swift when reading a file based
 894          * on a FileBackend params array, e.g. that of getLocalCopy().
 895          * $params is currently only checked for a 'latest' flag.
 896          *
 897          * @param $params Array
 898          * @return Array
 899          */
 900         protected function headersFromParams( array $params ) {
 901                 $hdrs = array();
 902                 if ( !empty( $params['latest'] ) ) {
 903                         $hdrs[] = 'X-Newest: true';
 904                 }
 905                 return $hdrs;
 906         }
 907
 908         /**
 909          * @see FileBackendStore::doExecuteOpHandlesInternal()
 910          * @return Array List of corresponding Status objects
 911          */
 912         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
 913                 $statuses = array();
 914
 915                 $cfOps = array(); // list of CF_Async_Op objects
 916                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
 917                         $cfOps[$index] = $fileOpHandle->cfOp;
 918                 }
 919                 $batch = new CF_Async_Op_Batch( $cfOps );
 920
 921                 $cfOps = $batch->execute();
 922                 foreach ( $cfOps as $index => $cfOp ) {
 923                         $status = Status::newGood();
 924                         try { // catch exceptions; update status
 925                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
 926                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
 927                         } catch ( CloudFilesException $e ) { // some other exception?
 928                                 $this->handleException( $e, $status,
 929                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
 930                         }
 931                         $statuses[$index] = $status;
 932                 }
 933
 934                 foreach ( $fileOpHandles as $fileOpHandle ) {
 935                         $fileOpHandle->closeResources();
 936                 }
 937
 938                 return $statuses;
 939         }
 940
 941         /**
 942          * Set read/write permissions for a Swift container
 943          *
 944          * @param $contObj CF_Container Swift container
 945          * @param $readGrps Array Swift users who can read (account:user)
 946          * @param $writeGrps Array Swift users who can write (account:user)
 947          * @return Status
 948          */
 949         protected function setContainerAccess(
 950                 CF_Container $contObj, array $readGrps, array $writeGrps
 951         ) {
 952                 $creds = $contObj->cfs_auth->export_credentials();
 953
 954                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
 955
 956                 // Note: 10 second timeout consistent with php-cloudfiles
 957                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
 958                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
 959                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
 960                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
 961
 962                 return $req->execute(); // should return 204
 963         }
 964
 965         /**
 966          * Get a connection to the Swift proxy
 967          *
 968          * @return CF_Connection|bool False on failure
 969          * @throws InvalidResponseException
 970          */
 971         protected function getConnection() {
 972                 if ( $this->conn === false ) {
 973                         throw new InvalidResponseException; // failed last attempt
 974                 }
 975                 // Session keys expire after a while, so we renew them periodically
 976                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
 977                         $this->conn->close(); // close active cURL connections
 978                         $this->conn = null;
 979                 }
 980                 // Authenticate with proxy and get a session key...
 981                 if ( $this->conn === null ) {
 982                         $this->connContainers = array();
 983                         try {
 984                                 $this->auth->authenticate();
 985                                 $this->conn = new CF_Connection( $this->auth );
 986                                 $this->connStarted = time();
 987                         } catch ( AuthenticationException $e ) {
 988                                 $this->conn = false; // don't keep re-trying
 989                         } catch ( InvalidResponseException $e ) {
 990                                 $this->conn = false; // don't keep re-trying
 991                         }
 992                 }
 993                 if ( !$this->conn ) {
 994                         throw new InvalidResponseException; // auth/connection problem
 995                 }
 996                 return $this->conn;
 997         }
 998
 999         /**
1000          * @see FileBackendStore::doClearCache()
1001          */
1002         protected function doClearCache( array $paths = null ) {
1003                 $this->connContainers = array(); // clear container object cache
1004         }
1005
1006         /**
1007          * Get a Swift container object, possibly from process cache.
1008          * Use $reCache if the file count or byte count is needed.
1009          *
1010          * @param $container string Container name
1011          * @param $bypassCache bool Bypass all caches and load from Swift
1012          * @return CF_Container
1013          * @throws NoSuchContainerException
1014          * @throws InvalidResponseException
1015          */
1016         protected function getContainer( $container, $bypassCache = false ) {
1017                 $conn = $this->getConnection(); // Swift proxy connection
1018                 if ( $bypassCache ) { // purge cache
1019                         unset( $this->connContainers[$container] );
1020                 } elseif ( !isset( $this->connContainers[$container] ) ) {
1021                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1022                 }
1023                 if ( !isset( $this->connContainers[$container] ) ) {
1024                         $contObj = $conn->get_container( $container );
1025                         // NoSuchContainerException not thrown: container must exist
1026                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
1027                                 reset( $this->connContainers );
1028                                 unset( $this->connContainers[key( $this->connContainers )] );
1029                         }
1030                         $this->connContainers[$container] = $contObj; // cache it
1031                         if ( !$bypassCache ) {
1032                                 $this->setContainerCache( $container, // update persistent cache
1033                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1034                                 );
1035                         }
1036                 }
1037                 return $this->connContainers[$container];
1038         }
1039
1040         /**
1041          * Create a Swift container
1042          *
1043          * @param $container string Container name
1044          * @return CF_Container
1045          * @throws InvalidResponseException
1046          */
1047         protected function createContainer( $container ) {
1048                 $conn = $this->getConnection(); // Swift proxy connection
1049                 $contObj = $conn->create_container( $container );
1050                 $this->connContainers[$container] = $contObj; // cache it
1051                 return $contObj;
1052         }
1053
1054         /**
1055          * Delete a Swift container
1056          *
1057          * @param $container string Container name
1058          * @return void
1059          * @throws InvalidResponseException
1060          */
1061         protected function deleteContainer( $container ) {
1062                 $conn = $this->getConnection(); // Swift proxy connection
1063                 $conn->delete_container( $container );
1064                 unset( $this->connContainers[$container] ); // purge cache
1065         }
1066
1067         /**
1068          * @see FileBackendStore::doPrimeContainerCache()
1069          * @return void
1070          */
1071         protected function doPrimeContainerCache( array $containerInfo ) {
1072                 try {
1073                         $conn = $this->getConnection(); // Swift proxy connection
1074                         foreach ( $containerInfo as $container => $info ) {
1075                                 $this->connContainers[$container] = new CF_Container(
1076                                         $conn->cfs_auth,
1077                                         $conn->cfs_http,
1078                                         $container,
1079                                         $info['count'],
1080                                         $info['bytes']
1081                                 );
1082                         }
1083                 } catch ( CloudFilesException $e ) { // some other exception?
1084                         $this->handleException( $e, null, __METHOD__, array() );
1085                 }
1086         }
1087
1088         /**
1089          * Log an unexpected exception for this backend.
1090          * This also sets the Status object to have a fatal error.
1091          *
1092          * @param $e Exception
1093          * @param $status Status|null
1094          * @param $func string
1095          * @param $params Array
1096          * @return void
1097          */
1098         protected function handleException( Exception $e, $status, $func, array $params ) {
1099                 if ( $status instanceof Status ) {
1100                         if ( $e instanceof AuthenticationException ) {
1101                                 $status->fatal( 'backend-fail-connect', $this->name );
1102                         } else {
1103                                 $status->fatal( 'backend-fail-internal', $this->name );
1104                         }
1105                 }
1106                 if ( $e->getMessage() ) {
1107                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1108                 }
1109                 wfDebugLog( 'SwiftBackend',
1110                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1111                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1112                 );
1113         }
1114 }
1115
1116 /**
1117  * @see FileBackendStoreOpHandle
1118  */
1119 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1120         /** @var CF_Async_Op */
1121         public $cfOp;
1122
1123         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1124                 $this->backend = $backend;
1125                 $this->params = $params;
1126                 $this->call = $call;
1127                 $this->cfOp = $cfOp;
1128         }
1129 }
1130
1131 /**
1132  * SwiftFileBackend helper class to page through listings.
1133  * Swift also has a listing limit of 10,000 objects for sanity.
1134  * Do not use this class from places outside SwiftFileBackend.
1135  *
1136  * @ingroup FileBackend
1137  */
1138 abstract class SwiftFileBackendList implements Iterator {
1139         /** @var Array */
1140         protected $bufferIter = array();
1141         protected $bufferAfter = null; // string; list items *after* this path
1142         protected $pos = 0; // integer
1143         /** @var Array */
1144         protected $params = array();
1145
1146         /** @var SwiftFileBackend */
1147         protected $backend;
1148         protected $container; // string; container name
1149         protected $dir; // string; storage directory
1150         protected $suffixStart; // integer
1151
1152         const PAGE_SIZE = 5000; // file listing buffer size
1153
1154         /**
1155          * @param $backend SwiftFileBackend
1156          * @param $fullCont string Resolved container name
1157          * @param $dir string Resolved directory relative to container
1158          * @param $params Array
1159          */
1160         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1161                 $this->backend = $backend;
1162                 $this->container = $fullCont;
1163                 $this->dir = $dir;
1164                 if ( substr( $this->dir, -1 ) === '/' ) {
1165                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1166                 }
1167                 if ( $this->dir == '' ) { // whole container
1168                         $this->suffixStart = 0;
1169                 } else { // dir within container
1170                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1171                 }
1172                 $this->params = $params;
1173         }
1174
1175         /**
1176          * @see Iterator::key()
1177          * @return integer
1178          */
1179         public function key() {
1180                 return $this->pos;
1181         }
1182
1183         /**
1184          * @see Iterator::next()
1185          * @return void
1186          */
1187         public function next() {
1188                 // Advance to the next file in the page
1189                 next( $this->bufferIter );
1190                 ++$this->pos;
1191                 // Check if there are no files left in this page and
1192                 // advance to the next page if this page was not empty.
1193                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1194                         $this->bufferIter = $this->pageFromList(
1195                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1196                         ); // updates $this->bufferAfter
1197                 }
1198         }
1199
1200         /**
1201          * @see Iterator::rewind()
1202          * @return void
1203          */
1204         public function rewind() {
1205                 $this->pos = 0;
1206                 $this->bufferAfter = null;
1207                 $this->bufferIter = $this->pageFromList(
1208                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1209                 ); // updates $this->bufferAfter
1210         }
1211
1212         /**
1213          * @see Iterator::valid()
1214          * @return bool
1215          */
1216         public function valid() {
1217                 if ( $this->bufferIter === null ) {
1218                         return false; // some failure?
1219                 } else {
1220                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1221                 }
1222         }
1223
1224         /**
1225          * Get the given list portion (page)
1226          *
1227          * @param $container string Resolved container name
1228          * @param $dir string Resolved path relative to container
1229          * @param $after string|null
1230          * @param $limit integer
1231          * @param $params Array
1232          * @return Traversable|Array|null Returns null on failure
1233          */
1234         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1235 }
1236
1237 /**
1238  * Iterator for listing directories
1239  */
1240 class SwiftFileBackendDirList extends SwiftFileBackendList {
1241         /**
1242          * @see Iterator::current()
1243          * @return string|bool String (relative path) or false
1244          */
1245         public function current() {
1246                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1247         }
1248
1249         /**
1250          * @see SwiftFileBackendList::pageFromList()
1251          * @return Array|null
1252          */
1253         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1254                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1255         }
1256 }
1257
1258 /**
1259  * Iterator for listing regular files
1260  */
1261 class SwiftFileBackendFileList extends SwiftFileBackendList {
1262         /**
1263          * @see Iterator::current()
1264          * @return string|bool String (relative path) or false
1265          */
1266         public function current() {
1267                 return substr( current( $this->bufferIter ), $this->suffixStart );
1268         }
1269
1270         /**
1271          * @see SwiftFileBackendList::pageFromList()
1272          * @return Array|null
1273          */
1274         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1275                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1276         }
1277 }