includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $maxContCacheSize = 300; // integer; max containers with entries
  45
  46         /** @var CF_Connection */
  47         protected $conn; // Swift connection handle
  48         protected $connStarted = 0; // integer UNIX timestamp
  49         protected $connContainers = array(); // container object cache
  50
  51         /**
  52          * @see FileBackendStore::__construct()
  53          * Additional $config params include:
  54          *    swiftAuthUrl       : Swift authentication server URL
  55          *    swiftUser          : Swift user used by MediaWiki (account:username)
  56          *    swiftKey           : Swift authentication key for the above user
  57          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  58          *    swiftAnonUser      : Swift user used for end-user requests (account:username)
  59          *    shardViaHashLevels : Map of container names to sharding config with:
  60          *                         'base'   : base of hash characters, 16 or 36
  61          *                         'levels' : the number of hash levels (and digits)
  62          *                         'repeat' : hash subdirectories are prefixed with all the
  63          *                                    parent hash directory names (e.g. "a/ab/abc")
  64          */
  65         public function __construct( array $config ) {
  66                 parent::__construct( $config );
  67                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  68                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  69                 }
  70                 // Required settings
  71                 $this->auth = new CF_Authentication(
  72                         $config['swiftUser'],
  73                         $config['swiftKey'],
  74                         null, // account; unused
  75                         $config['swiftAuthUrl']
  76                 );
  77                 // Optional settings
  78                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  79                         ? $config['swiftAuthTTL']
  80                         : 5 * 60; // some sane number
  81                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  82                         ? $config['swiftAnonUser']
  83                         : '';
  84                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  85                         ? $config['shardViaHashLevels']
  86                         : '';
  87                 // Cache container info to mask latency
  88                 $this->memCache = wfGetMainCache();
  89         }
  90
  91         /**
  92          * @see FileBackendStore::resolveContainerPath()
  93          * @return null
  94          */
  95         protected function resolveContainerPath( $container, $relStoragePath ) {
  96                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
  97                         return null; // too long for Swift
  98                 }
  99                 return $relStoragePath;
 100         }
 101
 102         /**
 103          * @see FileBackendStore::isPathUsableInternal()
 104          * @return bool
 105          */
 106         public function isPathUsableInternal( $storagePath ) {
 107                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 108                 if ( $rel === null ) {
 109                         return false; // invalid
 110                 }
 111
 112                 try {
 113                         $this->getContainer( $container );
 114                         return true; // container exists
 115                 } catch ( NoSuchContainerException $e ) {
 116                 } catch ( CloudFilesException $e ) { // some other exception?
 117                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 118                 }
 119
 120                 return false;
 121         }
 122
 123         /**
 124          * @see FileBackendStore::doCreateInternal()
 125          * @return Status
 126          */
 127         protected function doCreateInternal( array $params ) {
 128                 $status = Status::newGood();
 129
 130                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 131                 if ( $dstRel === null ) {
 132                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 133                         return $status;
 134                 }
 135
 136                 // (a) Check the destination container and object
 137                 try {
 138                         $dContObj = $this->getContainer( $dstCont );
 139                         if ( empty( $params['overwrite'] ) &&
 140                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 141                         {
 142                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 143                                 return $status;
 144                         }
 145                 } catch ( NoSuchContainerException $e ) {
 146                         $status->fatal( 'backend-fail-create', $params['dst'] );
 147                         return $status;
 148                 } catch ( CloudFilesException $e ) { // some other exception?
 149                         $this->handleException( $e, $status, __METHOD__, $params );
 150                         return $status;
 151                 }
 152
 153                 // (b) Get a SHA-1 hash of the object
 154                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 155
 156                 // (c) Actually create the object
 157                 try {
 158                         // Create a fresh CF_Object with no fields preloaded.
 159                         // We don't want to preserve headers, metadata, and such.
 160                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 161                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 162                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 163                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 164                         // The MD5 here will be checked within Swift against its own MD5.
 165                         $obj->set_etag( md5( $params['content'] ) );
 166                         // Use the same content type as StreamFile for security
 167                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 168                         if ( !empty( $params['async'] ) ) { // deferred
 169                                 $handle = $obj->write_async( $params['content'] );
 170                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle );
 171                         } else { // actually write the object in Swift
 172                                 $obj->write( $params['content'] );
 173                         }
 174                 } catch ( BadContentTypeException $e ) {
 175                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 176                 } catch ( CloudFilesException $e ) { // some other exception?
 177                         $this->handleException( $e, $status, __METHOD__, $params );
 178                 }
 179
 180                 return $status;
 181         }
 182
 183         /**
 184          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 185          */
 186         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 187                 try {
 188                         $cfOp->getLastResponse();
 189                 } catch ( BadContentTypeException $e ) {
 190                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 191                 }
 192         }
 193
 194         /**
 195          * @see FileBackendStore::doStoreInternal()
 196          * @return Status
 197          */
 198         protected function doStoreInternal( array $params ) {
 199                 $status = Status::newGood();
 200
 201                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 202                 if ( $dstRel === null ) {
 203                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 204                         return $status;
 205                 }
 206
 207                 // (a) Check the destination container and object
 208                 try {
 209                         $dContObj = $this->getContainer( $dstCont );
 210                         if ( empty( $params['overwrite'] ) &&
 211                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 212                         {
 213                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 214                                 return $status;
 215                         }
 216                 } catch ( NoSuchContainerException $e ) {
 217                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 218                         return $status;
 219                 } catch ( CloudFilesException $e ) { // some other exception?
 220                         $this->handleException( $e, $status, __METHOD__, $params );
 221                         return $status;
 222                 }
 223
 224                 // (b) Get a SHA-1 hash of the object
 225                 $sha1Hash = sha1_file( $params['src'] );
 226                 if ( $sha1Hash === false ) { // source doesn't exist?
 227                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 228                         return $status;
 229                 }
 230                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 231
 232                 // (c) Actually store the object
 233                 try {
 234                         // Create a fresh CF_Object with no fields preloaded.
 235                         // We don't want to preserve headers, metadata, and such.
 236                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 237                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 238                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 239                         // The MD5 here will be checked within Swift against its own MD5.
 240                         $obj->set_etag( md5_file( $params['src'] ) );
 241                         // Use the same content type as StreamFile for security
 242                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 243                         if ( !empty( $params['async'] ) ) { // deferred
 244                                 wfSuppressWarnings();
 245                                 $fp = fopen( $params['src'], 'rb' );
 246                                 wfRestoreWarnings();
 247                                 if ( !$fp ) {
 248                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 249                                 } else {
 250                                         $handle = $obj->write_async( $fp, filesize( $params['src'] ), true );
 251                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle );
 252                                         $status->value->resourcesToClose[] = $fp;
 253                                 }
 254                         } else { // actually write the object in Swift
 255                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 256                         }
 257                 } catch ( BadContentTypeException $e ) {
 258                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 259                 } catch ( IOException $e ) {
 260                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 261                 } catch ( CloudFilesException $e ) { // some other exception?
 262                         $this->handleException( $e, $status, __METHOD__, $params );
 263                 }
 264
 265                 return $status;
 266         }
 267
 268         /**
 269          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 270          */
 271         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 272                 try {
 273                         $cfOp->getLastResponse();
 274                 } catch ( BadContentTypeException $e ) {
 275                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 276                 } catch ( IOException $e ) {
 277                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 278                 }
 279         }
 280
 281         /**
 282          * @see FileBackendStore::doCopyInternal()
 283          * @return Status
 284          */
 285         protected function doCopyInternal( array $params ) {
 286                 $status = Status::newGood();
 287
 288                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 289                 if ( $srcRel === null ) {
 290                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 291                         return $status;
 292                 }
 293
 294                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 295                 if ( $dstRel === null ) {
 296                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 297                         return $status;
 298                 }
 299
 300                 // (a) Check the source/destination containers and destination object
 301                 try {
 302                         $sContObj = $this->getContainer( $srcCont );
 303                         $dContObj = $this->getContainer( $dstCont );
 304                         if ( empty( $params['overwrite'] ) &&
 305                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 306                         {
 307                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 308                                 return $status;
 309                         }
 310                 } catch ( NoSuchContainerException $e ) {
 311                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 312                         return $status;
 313                 } catch ( CloudFilesException $e ) { // some other exception?
 314                         $this->handleException( $e, $status, __METHOD__, $params );
 315                         return $status;
 316                 }
 317
 318                 // (b) Actually copy the file to the destination
 319                 try {
 320                         if ( !empty( $params['async'] ) ) { // deferred
 321                                 $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel );
 322                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle );
 323                         } else { // actually write the object in Swift
 324                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 325                         }
 326                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 327                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 328                 } catch ( CloudFilesException $e ) { // some other exception?
 329                         $this->handleException( $e, $status, __METHOD__, $params );
 330                 }
 331
 332                 return $status;
 333         }
 334
 335         /**
 336          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 337          */
 338         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 339                 try {
 340                         $cfOp->getLastResponse();
 341                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 342                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 343                 }
 344         }
 345
 346         /**
 347          * @see FileBackendStore::doMoveInternal()
 348          * @return Status
 349          */
 350         protected function doMoveInternal( array $params ) {
 351                 $status = Status::newGood();
 352
 353                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 354                 if ( $srcRel === null ) {
 355                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 356                         return $status;
 357                 }
 358
 359                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 360                 if ( $dstRel === null ) {
 361                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 362                         return $status;
 363                 }
 364
 365                 // (a) Check the source/destination containers and destination object
 366                 try {
 367                         $sContObj = $this->getContainer( $srcCont );
 368                         $dContObj = $this->getContainer( $dstCont );
 369                         if ( empty( $params['overwrite'] ) &&
 370                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 371                         {
 372                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 373                                 return $status;
 374                         }
 375                 } catch ( NoSuchContainerException $e ) {
 376                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 377                         return $status;
 378                 } catch ( CloudFilesException $e ) { // some other exception?
 379                         $this->handleException( $e, $status, __METHOD__, $params );
 380                         return $status;
 381                 }
 382
 383                 // (b) Actually move the file to the destination
 384                 try {
 385                         if ( !empty( $params['async'] ) ) { // deferred
 386                                 $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel );
 387                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle );
 388                         } else { // actually write the object in Swift
 389                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel );
 390                         }
 391                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 392                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 393                 } catch ( CloudFilesException $e ) { // some other exception?
 394                         $this->handleException( $e, $status, __METHOD__, $params );
 395                 }
 396
 397                 return $status;
 398         }
 399
 400         /**
 401          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 402          */
 403         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 404                 try {
 405                         $cfOp->getLastResponse();
 406                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 407                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 408                 }
 409         }
 410
 411         /**
 412          * @see FileBackendStore::doDeleteInternal()
 413          * @return Status
 414          */
 415         protected function doDeleteInternal( array $params ) {
 416                 $status = Status::newGood();
 417
 418                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 419                 if ( $srcRel === null ) {
 420                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 421                         return $status;
 422                 }
 423
 424                 try {
 425                         $sContObj = $this->getContainer( $srcCont );
 426                         if ( !empty( $params['async'] ) ) { // deferred
 427                                 $handle = $sContObj->delete_object_async( $srcRel );
 428                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle );
 429                         } else { // actually write the object in Swift
 430                                 $sContObj->delete_object( $srcRel );
 431                         }
 432                 } catch ( NoSuchContainerException $e ) {
 433                         $status->fatal( 'backend-fail-delete', $params['src'] );
 434                 } catch ( NoSuchObjectException $e ) {
 435                         if ( empty( $params['ignoreMissingSource'] ) ) {
 436                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 437                         }
 438                 } catch ( CloudFilesException $e ) { // some other exception?
 439                         $this->handleException( $e, $status, __METHOD__, $params );
 440                 }
 441
 442                 return $status;
 443         }
 444
 445         /**
 446          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 447          */
 448         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 449                 try {
 450                         $cfOp->getLastResponse();
 451                 } catch ( NoSuchContainerException $e ) {
 452                         $status->fatal( 'backend-fail-delete', $params['src'] );
 453                 } catch ( NoSuchObjectException $e ) {
 454                         if ( empty( $params['ignoreMissingSource'] ) ) {
 455                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 456                         }
 457                 }
 458         }
 459
 460         /**
 461          * @see FileBackendStore::doPrepareInternal()
 462          * @return Status
 463          */
 464         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 465                 $status = Status::newGood();
 466
 467                 // (a) Check if container already exists
 468                 try {
 469                         $contObj = $this->getContainer( $fullCont );
 470                         // NoSuchContainerException not thrown: container must exist
 471                         return $status; // already exists
 472                 } catch ( NoSuchContainerException $e ) {
 473                         // NoSuchContainerException thrown: container does not exist
 474                 } catch ( CloudFilesException $e ) { // some other exception?
 475                         $this->handleException( $e, $status, __METHOD__, $params );
 476                         return $status;
 477                 }
 478
 479                 // (b) Create container as needed
 480                 try {
 481                         $contObj = $this->createContainer( $fullCont );
 482                         if ( $this->swiftAnonUser != '' ) {
 483                                 // Make container public to end-users...
 484                                 $status->merge( $this->setContainerAccess(
 485                                         $contObj,
 486                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 487                                         array( $this->auth->username ) // write
 488                                 ) );
 489                         }
 490                 } catch ( CloudFilesException $e ) { // some other exception?
 491                         $this->handleException( $e, $status, __METHOD__, $params );
 492                         return $status;
 493                 }
 494
 495                 return $status;
 496         }
 497
 498         /**
 499          * @see FileBackendStore::doSecureInternal()
 500          * @return Status
 501          */
 502         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 503                 $status = Status::newGood();
 504
 505                 if ( $this->swiftAnonUser != '' ) {
 506                         // Restrict container from end-users...
 507                         try {
 508                                 // doPrepareInternal() should have been called,
 509                                 // so the Swift container should already exist...
 510                                 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 511                                 // NoSuchContainerException not thrown: container must exist
 512                                 if ( !isset( $contObj->mw_wasSecured ) ) {
 513                                         $status->merge( $this->setContainerAccess(
 514                                                 $contObj,
 515                                                 array( $this->auth->username ), // read
 516                                                 array( $this->auth->username ) // write
 517                                         ) );
 518                                         // @TODO: when php-cloudfiles supports container
 519                                         // metadata, we can make use of that to avoid RTTs
 520                                         $contObj->mw_wasSecured = true; // avoid useless RTTs
 521                                 }
 522                         } catch ( CloudFilesException $e ) { // some other exception?
 523                                 $this->handleException( $e, $status, __METHOD__, $params );
 524                         }
 525                 }
 526
 527                 return $status;
 528         }
 529
 530         /**
 531          * @see FileBackendStore::doCleanInternal()
 532          * @return Status
 533          */
 534         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 535                 $status = Status::newGood();
 536
 537                 // Only containers themselves can be removed, all else is virtual
 538                 if ( $dir != '' ) {
 539                         return $status; // nothing to do
 540                 }
 541
 542                 // (a) Check the container
 543                 try {
 544                         $contObj = $this->getContainer( $fullCont, true );
 545                 } catch ( NoSuchContainerException $e ) {
 546                         return $status; // ok, nothing to do
 547                 } catch ( CloudFilesException $e ) { // some other exception?
 548                         $this->handleException( $e, $status, __METHOD__, $params );
 549                         return $status;
 550                 }
 551
 552                 // (b) Delete the container if empty
 553                 if ( $contObj->object_count == 0 ) {
 554                         try {
 555                                 $this->deleteContainer( $fullCont );
 556                         } catch ( NoSuchContainerException $e ) {
 557                                 return $status; // race?
 558                         } catch ( NonEmptyContainerException $e ) {
 559                                 return $status; // race? consistency delay?
 560                         } catch ( CloudFilesException $e ) { // some other exception?
 561                                 $this->handleException( $e, $status, __METHOD__, $params );
 562                                 return $status;
 563                         }
 564                 }
 565
 566                 return $status;
 567         }
 568
 569         /**
 570          * @see FileBackendStore::doFileExists()
 571          * @return array|bool|null
 572          */
 573         protected function doGetFileStat( array $params ) {
 574                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 575                 if ( $srcRel === null ) {
 576                         return false; // invalid storage path
 577                 }
 578
 579                 $stat = false;
 580                 try {
 581                         $contObj = $this->getContainer( $srcCont );
 582                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 583                         $this->addMissingMetadata( $srcObj, $params['src'] );
 584                         $stat = array(
 585                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 586                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 587                                 'size'  => $srcObj->content_length,
 588                                 'sha1'  => $srcObj->metadata['Sha1base36']
 589                         );
 590                 } catch ( NoSuchContainerException $e ) {
 591                 } catch ( NoSuchObjectException $e ) {
 592                 } catch ( CloudFilesException $e ) { // some other exception?
 593                         $stat = null;
 594                         $this->handleException( $e, null, __METHOD__, $params );
 595                 }
 596
 597                 return $stat;
 598         }
 599
 600         /**
 601          * Fill in any missing object metadata and save it to Swift
 602          *
 603          * @param $obj CF_Object
 604          * @param $path string Storage path to object
 605          * @return bool Success
 606          * @throws Exception cloudfiles exceptions
 607          */
 608         protected function addMissingMetadata( CF_Object $obj, $path ) {
 609                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 610                         return true; // nothing to do
 611                 }
 612                 $status = Status::newGood();
 613                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 614                 if ( $status->isOK() ) {
 615                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
 616                         if ( $tmpFile ) {
 617                                 $hash = $tmpFile->getSha1Base36();
 618                                 if ( $hash !== false ) {
 619                                         $obj->metadata['Sha1base36'] = $hash;
 620                                         $obj->sync_metadata(); // save to Swift
 621                                         return true; // success
 622                                 }
 623                         }
 624                 }
 625                 $obj->metadata['Sha1base36'] = false;
 626                 return false; // failed
 627         }
 628
 629         /**
 630          * @see FileBackend::getFileContents()
 631          * @return bool|null|string
 632          */
 633         public function getFileContents( array $params ) {
 634                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 635                 if ( $srcRel === null ) {
 636                         return false; // invalid storage path
 637                 }
 638
 639                 if ( !$this->fileExists( $params ) ) {
 640                         return null;
 641                 }
 642
 643                 $data = false;
 644                 try {
 645                         $sContObj = $this->getContainer( $srcCont );
 646                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request
 647                         $data = $obj->read( $this->headersFromParams( $params ) );
 648                 } catch ( NoSuchContainerException $e ) {
 649                 } catch ( CloudFilesException $e ) { // some other exception?
 650                         $this->handleException( $e, null, __METHOD__, $params );
 651                 }
 652
 653                 return $data;
 654         }
 655
 656         /**
 657          * @see FileBackendStore::doDirectoryExists()
 658          * @return bool|null
 659          */
 660         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 661                 try {
 662                         $container = $this->getContainer( $fullCont );
 663                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 664                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 665                 } catch ( NoSuchContainerException $e ) {
 666                         return false;
 667                 } catch ( CloudFilesException $e ) { // some other exception?
 668                         $this->handleException( $e, null, __METHOD__,
 669                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 670                 }
 671
 672                 return null; // error
 673         }
 674
 675         /**
 676          * @see FileBackendStore::getDirectoryListInternal()
 677          * @return SwiftFileBackendDirList
 678          */
 679         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 680                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 681         }
 682
 683         /**
 684          * @see FileBackendStore::getFileListInternal()
 685          * @return SwiftFileBackendFileList
 686          */
 687         public function getFileListInternal( $fullCont, $dir, array $params ) {
 688                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 689         }
 690
 691         /**
 692          * Do not call this function outside of SwiftFileBackendFileList
 693          *
 694          * @param $fullCont string Resolved container name
 695          * @param $dir string Resolved storage directory with no trailing slash
 696          * @param $after string|null Storage path of file to list items after
 697          * @param $limit integer Max number of items to list
 698          * @param $params Array Includes flag for 'topOnly'
 699          * @return Array List of relative paths of dirs directly under $dir
 700          */
 701         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 702                 $dirs = array();
 703
 704                 try {
 705                         $container = $this->getContainer( $fullCont );
 706                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 707                         // Non-recursive: only list dirs right under $dir
 708                         if ( !empty( $params['topOnly'] ) ) {
 709                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 710                                 foreach ( $objects as $object ) { // files and dirs
 711                                         if ( substr( $object, -1 ) === '/' ) {
 712                                                 $dirs[] = $object; // directories end in '/'
 713                                         }
 714                                         $after = $object; // update last item
 715                                 }
 716                         // Recursive: list all dirs under $dir and its subdirs
 717                         } else {
 718                                 // Get directory from last item of prior page
 719                                 $lastDir = $this->getParentDir( $after ); // must be first page
 720                                 $objects = $container->list_objects( $limit, $after, $prefix );
 721                                 foreach ( $objects as $object ) { // files
 722                                         $objectDir = $this->getParentDir( $object ); // directory of object
 723                                         if ( $objectDir !== false ) { // file has a parent dir
 724                                                 // Swift stores paths in UTF-8, using binary sorting.
 725                                                 // See function "create_container_table" in common/db.py.
 726                                                 // If a directory is not "greater" than the last one,
 727                                                 // then it was already listed by the calling iterator.
 728                                                 if ( $objectDir > $lastDir ) {
 729                                                         $pDir = $objectDir;
 730                                                         do { // add dir and all its parent dirs
 731                                                                 $dirs[] = "{$pDir}/";
 732                                                                 $pDir = $this->getParentDir( $pDir );
 733                                                         } while ( $pDir !== false // sanity
 734                                                                 && $pDir > $lastDir // not done already
 735                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 736                                                         );
 737                                                 }
 738                                                 $lastDir = $objectDir;
 739                                         }
 740                                         $after = $object; // update last item
 741                                 }
 742                         }
 743                 } catch ( NoSuchContainerException $e ) {
 744                 } catch ( CloudFilesException $e ) { // some other exception?
 745                         $this->handleException( $e, null, __METHOD__,
 746                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 747                 }
 748
 749                 return $dirs;
 750         }
 751
 752         protected function getParentDir( $path ) {
 753                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 754         }
 755
 756         /**
 757          * Do not call this function outside of SwiftFileBackendFileList
 758          *
 759          * @param $fullCont string Resolved container name
 760          * @param $dir string Resolved storage directory with no trailing slash
 761          * @param $after string|null Storage path of file to list items after
 762          * @param $limit integer Max number of items to list
 763          * @param $params Array Includes flag for 'topOnly'
 764          * @return Array List of relative paths of files under $dir
 765          */
 766         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 767                 $files = array();
 768
 769                 try {
 770                         $container = $this->getContainer( $fullCont );
 771                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 772                         // Non-recursive: only list files right under $dir
 773                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 774                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 775                                 foreach ( $objects as $object ) {
 776                                         if ( substr( $object, -1 ) !== '/' ) {
 777                                                 $files[] = $object; // directories end in '/'
 778                                         }
 779                                 }
 780                         // Recursive: list all files under $dir and its subdirs
 781                         } else { // files
 782                                 $files = $container->list_objects( $limit, $after, $prefix );
 783                         }
 784                         $after = end( $files ); // update last item
 785                         reset( $files ); // reset pointer
 786                 } catch ( NoSuchContainerException $e ) {
 787                 } catch ( CloudFilesException $e ) { // some other exception?
 788                         $this->handleException( $e, null, __METHOD__,
 789                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 790                 }
 791
 792                 return $files;
 793         }
 794
 795         /**
 796          * @see FileBackendStore::doGetFileSha1base36()
 797          * @return bool
 798          */
 799         protected function doGetFileSha1base36( array $params ) {
 800                 $stat = $this->getFileStat( $params );
 801                 if ( $stat ) {
 802                         return $stat['sha1'];
 803                 } else {
 804                         return false;
 805                 }
 806         }
 807
 808         /**
 809          * @see FileBackendStore::doStreamFile()
 810          * @return Status
 811          */
 812         protected function doStreamFile( array $params ) {
 813                 $status = Status::newGood();
 814
 815                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 816                 if ( $srcRel === null ) {
 817                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 818                 }
 819
 820                 try {
 821                         $cont = $this->getContainer( $srcCont );
 822                 } catch ( NoSuchContainerException $e ) {
 823                         $status->fatal( 'backend-fail-stream', $params['src'] );
 824                         return $status;
 825                 } catch ( CloudFilesException $e ) { // some other exception?
 826                         $this->handleException( $e, $status, __METHOD__, $params );
 827                         return $status;
 828                 }
 829
 830                 try {
 831                         $output = fopen( 'php://output', 'wb' );
 832                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
 833                         $obj->stream( $output, $this->headersFromParams( $params ) );
 834                 } catch ( CloudFilesException $e ) { // some other exception?
 835                         $this->handleException( $e, $status, __METHOD__, $params );
 836                 }
 837
 838                 return $status;
 839         }
 840
 841         /**
 842          * @see FileBackendStore::getLocalCopy()
 843          * @return null|TempFSFile
 844          */
 845         public function getLocalCopy( array $params ) {
 846                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 847                 if ( $srcRel === null ) {
 848                         return null;
 849                 }
 850
 851                 if ( !$this->fileExists( $params ) ) {
 852                         return null;
 853                 }
 854
 855                 $tmpFile = null;
 856                 try {
 857                         $sContObj = $this->getContainer( $srcCont );
 858                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 859                         // Get source file extension
 860                         $ext = FileBackend::extensionFromPath( $srcRel );
 861                         // Create a new temporary file...
 862                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 863                         if ( $tmpFile ) {
 864                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 865                                 if ( $handle ) {
 866                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 867                                         fclose( $handle );
 868                                 } else {
 869                                         $tmpFile = null; // couldn't open temp file
 870                                 }
 871                         }
 872                 } catch ( NoSuchContainerException $e ) {
 873                         $tmpFile = null;
 874                 } catch ( CloudFilesException $e ) { // some other exception?
 875                         $tmpFile = null;
 876                         $this->handleException( $e, null, __METHOD__, $params );
 877                 }
 878
 879                 return $tmpFile;
 880         }
 881
 882         /**
 883          * @see FileBackendStore::directoriesAreVirtual()
 884          * @return bool
 885          */
 886         protected function directoriesAreVirtual() {
 887                 return true;
 888         }
 889
 890         /**
 891          * Get headers to send to Swift when reading a file based
 892          * on a FileBackend params array, e.g. that of getLocalCopy().
 893          * $params is currently only checked for a 'latest' flag.
 894          *
 895          * @param $params Array
 896          * @return Array
 897          */
 898         protected function headersFromParams( array $params ) {
 899                 $hdrs = array();
 900                 if ( !empty( $params['latest'] ) ) {
 901                         $hdrs[] = 'X-Newest: true';
 902                 }
 903                 return $hdrs;
 904         }
 905
 906         /**
 907          * @see FileBackendStore::doExecuteOpHandlesInternal()
 908          * @return Array List of corresponding Status objects
 909          */
 910         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
 911                 $statuses = array();
 912
 913                 $cfOps = array(); // list of CF_Async_Op objects
 914                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
 915                         $cfOps[$index] = $fileOpHandle->cfOp;
 916                 }
 917                 $batch = new CF_Async_Op_Batch( $cfOps );
 918
 919                 $cfOps = $batch->execute();
 920                 foreach ( $cfOps as $index => $cfOp ) {
 921                         $status = Status::newGood();
 922                         try { // catch exceptions; update status
 923                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
 924                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
 925                         } catch ( CloudFilesException $e ) { // some other exception?
 926                                 $this->handleException( $e, $status,
 927                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
 928                         }
 929                         $statuses[$index] = $status;
 930                 }
 931
 932                 foreach ( $fileOpHandles as $fileOpHandle ) {
 933                         $fileOpHandle->closeResources();
 934                 }
 935
 936                 return $statuses;
 937         }
 938
 939         /**
 940          * Set read/write permissions for a Swift container
 941          *
 942          * @param $contObj CF_Container Swift container
 943          * @param $readGrps Array Swift users who can read (account:user)
 944          * @param $writeGrps Array Swift users who can write (account:user)
 945          * @return Status
 946          */
 947         protected function setContainerAccess(
 948                 CF_Container $contObj, array $readGrps, array $writeGrps
 949         ) {
 950                 $creds = $contObj->cfs_auth->export_credentials();
 951
 952                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
 953
 954                 // Note: 10 second timeout consistent with php-cloudfiles
 955                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
 956                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
 957                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
 958                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
 959
 960                 return $req->execute(); // should return 204
 961         }
 962
 963         /**
 964          * Get a connection to the Swift proxy
 965          *
 966          * @return CF_Connection|bool False on failure
 967          * @throws InvalidResponseException
 968          */
 969         protected function getConnection() {
 970                 if ( $this->conn === false ) {
 971                         throw new InvalidResponseException; // failed last attempt
 972                 }
 973                 // Session keys expire after a while, so we renew them periodically
 974                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
 975                         $this->conn->close(); // close active cURL connections
 976                         $this->conn = null;
 977                 }
 978                 // Authenticate with proxy and get a session key...
 979                 if ( $this->conn === null ) {
 980                         $this->connContainers = array();
 981                         try {
 982                                 $this->auth->authenticate();
 983                                 $this->conn = new CF_Connection( $this->auth );
 984                                 $this->connStarted = time();
 985                         } catch ( AuthenticationException $e ) {
 986                                 $this->conn = false; // don't keep re-trying
 987                         } catch ( InvalidResponseException $e ) {
 988                                 $this->conn = false; // don't keep re-trying
 989                         }
 990                 }
 991                 if ( !$this->conn ) {
 992                         throw new InvalidResponseException; // auth/connection problem
 993                 }
 994                 return $this->conn;
 995         }
 996
 997         /**
 998          * @see FileBackendStore::doClearCache()
 999          */
1000         protected function doClearCache( array $paths = null ) {
1001                 $this->connContainers = array(); // clear container object cache
1002         }
1003
1004         /**
1005          * Get a Swift container object, possibly from process cache.
1006          * Use $reCache if the file count or byte count is needed.
1007          *
1008          * @param $container string Container name
1009          * @param $bypassCache bool Bypass all caches and load from Swift
1010          * @return CF_Container
1011          * @throws NoSuchContainerException
1012          * @throws InvalidResponseException
1013          */
1014         protected function getContainer( $container, $bypassCache = false ) {
1015                 $conn = $this->getConnection(); // Swift proxy connection
1016                 if ( $bypassCache ) { // purge cache
1017                         unset( $this->connContainers[$container] );
1018                 } elseif ( !isset( $this->connContainers[$container] ) ) {
1019                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1020                 }
1021                 if ( !isset( $this->connContainers[$container] ) ) {
1022                         $contObj = $conn->get_container( $container );
1023                         // NoSuchContainerException not thrown: container must exist
1024                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
1025                                 reset( $this->connContainers );
1026                                 unset( $this->connContainers[key( $this->connContainers )] );
1027                         }
1028                         $this->connContainers[$container] = $contObj; // cache it
1029                         if ( !$bypassCache ) {
1030                                 $this->setContainerCache( $container, // update persistent cache
1031                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1032                                 );
1033                         }
1034                 }
1035                 return $this->connContainers[$container];
1036         }
1037
1038         /**
1039          * Create a Swift container
1040          *
1041          * @param $container string Container name
1042          * @return CF_Container
1043          * @throws InvalidResponseException
1044          */
1045         protected function createContainer( $container ) {
1046                 $conn = $this->getConnection(); // Swift proxy connection
1047                 $contObj = $conn->create_container( $container );
1048                 $this->connContainers[$container] = $contObj; // cache it
1049                 return $contObj;
1050         }
1051
1052         /**
1053          * Delete a Swift container
1054          *
1055          * @param $container string Container name
1056          * @return void
1057          * @throws InvalidResponseException
1058          */
1059         protected function deleteContainer( $container ) {
1060                 $conn = $this->getConnection(); // Swift proxy connection
1061                 $conn->delete_container( $container );
1062                 unset( $this->connContainers[$container] ); // purge cache
1063         }
1064
1065         /**
1066          * @see FileBackendStore::doPrimeContainerCache()
1067          * @return void
1068          */
1069         protected function doPrimeContainerCache( array $containerInfo ) {
1070                 try {
1071                         $conn = $this->getConnection(); // Swift proxy connection
1072                         foreach ( $containerInfo as $container => $info ) {
1073                                 $this->connContainers[$container] = new CF_Container(
1074                                         $conn->cfs_auth,
1075                                         $conn->cfs_http,
1076                                         $container,
1077                                         $info['count'],
1078                                         $info['bytes']
1079                                 );
1080                         }
1081                 } catch ( CloudFilesException $e ) { // some other exception?
1082                         $this->handleException( $e, null, __METHOD__, array() );
1083                 }
1084         }
1085
1086         /**
1087          * Log an unexpected exception for this backend.
1088          * This also sets the Status object to have a fatal error.
1089          *
1090          * @param $e Exception
1091          * @param $status Status|null
1092          * @param $func string
1093          * @param $params Array
1094          * @return void
1095          */
1096         protected function handleException( Exception $e, $status, $func, array $params ) {
1097                 if ( $status instanceof Status ) {
1098                         if ( $e instanceof AuthenticationException ) {
1099                                 $status->fatal( 'backend-fail-connect', $this->name );
1100                         } else {
1101                                 $status->fatal( 'backend-fail-internal', $this->name );
1102                         }
1103                 }
1104                 if ( $e->getMessage() ) {
1105                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1106                 }
1107                 wfDebugLog( 'SwiftBackend',
1108                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1109                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1110                 );
1111         }
1112 }
1113
1114 /**
1115  * @see FileBackendStoreOpHandle
1116  */
1117 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1118         /** @var CF_Async_Op */
1119         public $cfOp;
1120
1121         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1122                 $this->backend = $backend;
1123                 $this->params = $params;
1124                 $this->call = $call;
1125                 $this->cfOp = $cfOp;
1126         }
1127 }
1128
1129 /**
1130  * SwiftFileBackend helper class to page through listings.
1131  * Swift also has a listing limit of 10,000 objects for sanity.
1132  * Do not use this class from places outside SwiftFileBackend.
1133  *
1134  * @ingroup FileBackend
1135  */
1136 abstract class SwiftFileBackendList implements Iterator {
1137         /** @var Array */
1138         protected $bufferIter = array();
1139         protected $bufferAfter = null; // string; list items *after* this path
1140         protected $pos = 0; // integer
1141         /** @var Array */
1142         protected $params = array();
1143
1144         /** @var SwiftFileBackend */
1145         protected $backend;
1146         protected $container; // string; container name
1147         protected $dir; // string; storage directory
1148         protected $suffixStart; // integer
1149
1150         const PAGE_SIZE = 5000; // file listing buffer size
1151
1152         /**
1153          * @param $backend SwiftFileBackend
1154          * @param $fullCont string Resolved container name
1155          * @param $dir string Resolved directory relative to container
1156          * @param $params Array
1157          */
1158         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1159                 $this->backend = $backend;
1160                 $this->container = $fullCont;
1161                 $this->dir = $dir;
1162                 if ( substr( $this->dir, -1 ) === '/' ) {
1163                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1164                 }
1165                 if ( $this->dir == '' ) { // whole container
1166                         $this->suffixStart = 0;
1167                 } else { // dir within container
1168                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1169                 }
1170                 $this->params = $params;
1171         }
1172
1173         /**
1174          * @see Iterator::key()
1175          * @return integer
1176          */
1177         public function key() {
1178                 return $this->pos;
1179         }
1180
1181         /**
1182          * @see Iterator::next()
1183          * @return void
1184          */
1185         public function next() {
1186                 // Advance to the next file in the page
1187                 next( $this->bufferIter );
1188                 ++$this->pos;
1189                 // Check if there are no files left in this page and
1190                 // advance to the next page if this page was not empty.
1191                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1192                         $this->bufferIter = $this->pageFromList(
1193                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1194                         ); // updates $this->bufferAfter
1195                 }
1196         }
1197
1198         /**
1199          * @see Iterator::rewind()
1200          * @return void
1201          */
1202         public function rewind() {
1203                 $this->pos = 0;
1204                 $this->bufferAfter = null;
1205                 $this->bufferIter = $this->pageFromList(
1206                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1207                 ); // updates $this->bufferAfter
1208         }
1209
1210         /**
1211          * @see Iterator::valid()
1212          * @return bool
1213          */
1214         public function valid() {
1215                 if ( $this->bufferIter === null ) {
1216                         return false; // some failure?
1217                 } else {
1218                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1219                 }
1220         }
1221
1222         /**
1223          * Get the given list portion (page)
1224          *
1225          * @param $container string Resolved container name
1226          * @param $dir string Resolved path relative to container
1227          * @param $after string|null
1228          * @param $limit integer
1229          * @param $params Array
1230          * @return Traversable|Array|null Returns null on failure
1231          */
1232         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1233 }
1234
1235 /**
1236  * Iterator for listing directories
1237  */
1238 class SwiftFileBackendDirList extends SwiftFileBackendList {
1239         /**
1240          * @see Iterator::current()
1241          * @return string|bool String (relative path) or false
1242          */
1243         public function current() {
1244                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1245         }
1246
1247         /**
1248          * @see SwiftFileBackendList::pageFromList()
1249          * @return Array|null
1250          */
1251         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1252                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1253         }
1254 }
1255
1256 /**
1257  * Iterator for listing regular files
1258  */
1259 class SwiftFileBackendFileList extends SwiftFileBackendList {
1260         /**
1261          * @see Iterator::current()
1262          * @return string|bool String (relative path) or false
1263          */
1264         public function current() {
1265                 return substr( current( $this->bufferIter ), $this->suffixStart );
1266         }
1267
1268         /**
1269          * @see SwiftFileBackendList::pageFromList()
1270          * @return Array|null
1271          */
1272         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1273                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1274         }
1275 }