includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $maxContCacheSize = 300; // integer; max containers with entries
  45
  46         /** @var CF_Connection */
  47         protected $conn; // Swift connection handle
  48         protected $connStarted = 0; // integer UNIX timestamp
  49         protected $connContainers = array(); // container object cache
  50         protected $connException; // CloudFiles exception
  51
  52         /**
  53          * @see FileBackendStore::__construct()
  54          * Additional $config params include:
  55          *    swiftAuthUrl       : Swift authentication server URL
  56          *    swiftUser          : Swift user used by MediaWiki (account:username)
  57          *    swiftKey           : Swift authentication key for the above user
  58          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  59          *    swiftAnonUser      : Swift user used for end-user requests (account:username)
  60          *    shardViaHashLevels : Map of container names to sharding config with:
  61          *                         'base'   : base of hash characters, 16 or 36
  62          *                         'levels' : the number of hash levels (and digits)
  63          *                         'repeat' : hash subdirectories are prefixed with all the
  64          *                                    parent hash directory names (e.g. "a/ab/abc")
  65          */
  66         public function __construct( array $config ) {
  67                 parent::__construct( $config );
  68                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  69                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  70                 }
  71                 // Required settings
  72                 $this->auth = new CF_Authentication(
  73                         $config['swiftUser'],
  74                         $config['swiftKey'],
  75                         null, // account; unused
  76                         $config['swiftAuthUrl']
  77                 );
  78                 // Optional settings
  79                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  80                         ? $config['swiftAuthTTL']
  81                         : 5 * 60; // some sane number
  82                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  83                         ? $config['swiftAnonUser']
  84                         : '';
  85                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  86                         ? $config['shardViaHashLevels']
  87                         : '';
  88                 // Cache container info to mask latency
  89                 $this->memCache = wfGetMainCache();
  90         }
  91
  92         /**
  93          * @see FileBackendStore::resolveContainerPath()
  94          * @return null
  95          */
  96         protected function resolveContainerPath( $container, $relStoragePath ) {
  97                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
  98                         return null; // too long for Swift
  99                 }
 100                 return $relStoragePath;
 101         }
 102
 103         /**
 104          * @see FileBackendStore::isPathUsableInternal()
 105          * @return bool
 106          */
 107         public function isPathUsableInternal( $storagePath ) {
 108                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 109                 if ( $rel === null ) {
 110                         return false; // invalid
 111                 }
 112
 113                 try {
 114                         $this->getContainer( $container );
 115                         return true; // container exists
 116                 } catch ( NoSuchContainerException $e ) {
 117                 } catch ( CloudFilesException $e ) { // some other exception?
 118                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 119                 }
 120
 121                 return false;
 122         }
 123
 124         /**
 125          * @see FileBackendStore::doCreateInternal()
 126          * @return Status
 127          */
 128         protected function doCreateInternal( array $params ) {
 129                 $status = Status::newGood();
 130
 131                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 132                 if ( $dstRel === null ) {
 133                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 134                         return $status;
 135                 }
 136
 137                 // (a) Check the destination container and object
 138                 try {
 139                         $dContObj = $this->getContainer( $dstCont );
 140                         if ( empty( $params['overwrite'] ) &&
 141                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 142                         {
 143                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 144                                 return $status;
 145                         }
 146                 } catch ( NoSuchContainerException $e ) {
 147                         $status->fatal( 'backend-fail-create', $params['dst'] );
 148                         return $status;
 149                 } catch ( CloudFilesException $e ) { // some other exception?
 150                         $this->handleException( $e, $status, __METHOD__, $params );
 151                         return $status;
 152                 }
 153
 154                 // (b) Get a SHA-1 hash of the object
 155                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 156
 157                 // (c) Actually create the object
 158                 try {
 159                         // Create a fresh CF_Object with no fields preloaded.
 160                         // We don't want to preserve headers, metadata, and such.
 161                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 162                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 163                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 164                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 165                         // The MD5 here will be checked within Swift against its own MD5.
 166                         $obj->set_etag( md5( $params['content'] ) );
 167                         // Use the same content type as StreamFile for security
 168                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 169                         if ( !empty( $params['async'] ) ) { // deferred
 170                                 $handle = $obj->write_async( $params['content'] );
 171                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle );
 172                         } else { // actually write the object in Swift
 173                                 $obj->write( $params['content'] );
 174                         }
 175                 } catch ( BadContentTypeException $e ) {
 176                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 177                 } catch ( CloudFilesException $e ) { // some other exception?
 178                         $this->handleException( $e, $status, __METHOD__, $params );
 179                 }
 180
 181                 return $status;
 182         }
 183
 184         /**
 185          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 186          */
 187         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 188                 try {
 189                         $cfOp->getLastResponse();
 190                 } catch ( BadContentTypeException $e ) {
 191                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 192                 }
 193         }
 194
 195         /**
 196          * @see FileBackendStore::doStoreInternal()
 197          * @return Status
 198          */
 199         protected function doStoreInternal( array $params ) {
 200                 $status = Status::newGood();
 201
 202                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 203                 if ( $dstRel === null ) {
 204                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 205                         return $status;
 206                 }
 207
 208                 // (a) Check the destination container and object
 209                 try {
 210                         $dContObj = $this->getContainer( $dstCont );
 211                         if ( empty( $params['overwrite'] ) &&
 212                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 213                         {
 214                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 215                                 return $status;
 216                         }
 217                 } catch ( NoSuchContainerException $e ) {
 218                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 219                         return $status;
 220                 } catch ( CloudFilesException $e ) { // some other exception?
 221                         $this->handleException( $e, $status, __METHOD__, $params );
 222                         return $status;
 223                 }
 224
 225                 // (b) Get a SHA-1 hash of the object
 226                 $sha1Hash = sha1_file( $params['src'] );
 227                 if ( $sha1Hash === false ) { // source doesn't exist?
 228                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 229                         return $status;
 230                 }
 231                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 232
 233                 // (c) Actually store the object
 234                 try {
 235                         // Create a fresh CF_Object with no fields preloaded.
 236                         // We don't want to preserve headers, metadata, and such.
 237                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 238                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 239                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 240                         // The MD5 here will be checked within Swift against its own MD5.
 241                         $obj->set_etag( md5_file( $params['src'] ) );
 242                         // Use the same content type as StreamFile for security
 243                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 244                         if ( !empty( $params['async'] ) ) { // deferred
 245                                 wfSuppressWarnings();
 246                                 $fp = fopen( $params['src'], 'rb' );
 247                                 wfRestoreWarnings();
 248                                 if ( !$fp ) {
 249                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 250                                 } else {
 251                                         $handle = $obj->write_async( $fp, filesize( $params['src'] ), true );
 252                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle );
 253                                         $status->value->resourcesToClose[] = $fp;
 254                                 }
 255                         } else { // actually write the object in Swift
 256                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 257                         }
 258                 } catch ( BadContentTypeException $e ) {
 259                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 260                 } catch ( IOException $e ) {
 261                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 262                 } catch ( CloudFilesException $e ) { // some other exception?
 263                         $this->handleException( $e, $status, __METHOD__, $params );
 264                 }
 265
 266                 return $status;
 267         }
 268
 269         /**
 270          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 271          */
 272         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 273                 try {
 274                         $cfOp->getLastResponse();
 275                 } catch ( BadContentTypeException $e ) {
 276                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 277                 } catch ( IOException $e ) {
 278                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 279                 }
 280         }
 281
 282         /**
 283          * @see FileBackendStore::doCopyInternal()
 284          * @return Status
 285          */
 286         protected function doCopyInternal( array $params ) {
 287                 $status = Status::newGood();
 288
 289                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 290                 if ( $srcRel === null ) {
 291                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 292                         return $status;
 293                 }
 294
 295                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 296                 if ( $dstRel === null ) {
 297                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 298                         return $status;
 299                 }
 300
 301                 // (a) Check the source/destination containers and destination object
 302                 try {
 303                         $sContObj = $this->getContainer( $srcCont );
 304                         $dContObj = $this->getContainer( $dstCont );
 305                         if ( empty( $params['overwrite'] ) &&
 306                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 307                         {
 308                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 309                                 return $status;
 310                         }
 311                 } catch ( NoSuchContainerException $e ) {
 312                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 313                         return $status;
 314                 } catch ( CloudFilesException $e ) { // some other exception?
 315                         $this->handleException( $e, $status, __METHOD__, $params );
 316                         return $status;
 317                 }
 318
 319                 // (b) Actually copy the file to the destination
 320                 try {
 321                         if ( !empty( $params['async'] ) ) { // deferred
 322                                 $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel );
 323                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle );
 324                         } else { // actually write the object in Swift
 325                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 326                         }
 327                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 328                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 329                 } catch ( CloudFilesException $e ) { // some other exception?
 330                         $this->handleException( $e, $status, __METHOD__, $params );
 331                 }
 332
 333                 return $status;
 334         }
 335
 336         /**
 337          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 338          */
 339         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 340                 try {
 341                         $cfOp->getLastResponse();
 342                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 343                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 344                 }
 345         }
 346
 347         /**
 348          * @see FileBackendStore::doMoveInternal()
 349          * @return Status
 350          */
 351         protected function doMoveInternal( array $params ) {
 352                 $status = Status::newGood();
 353
 354                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 355                 if ( $srcRel === null ) {
 356                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 357                         return $status;
 358                 }
 359
 360                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 361                 if ( $dstRel === null ) {
 362                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 363                         return $status;
 364                 }
 365
 366                 // (a) Check the source/destination containers and destination object
 367                 try {
 368                         $sContObj = $this->getContainer( $srcCont );
 369                         $dContObj = $this->getContainer( $dstCont );
 370                         if ( empty( $params['overwrite'] ) &&
 371                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 372                         {
 373                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 374                                 return $status;
 375                         }
 376                 } catch ( NoSuchContainerException $e ) {
 377                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 378                         return $status;
 379                 } catch ( CloudFilesException $e ) { // some other exception?
 380                         $this->handleException( $e, $status, __METHOD__, $params );
 381                         return $status;
 382                 }
 383
 384                 // (b) Actually move the file to the destination
 385                 try {
 386                         if ( !empty( $params['async'] ) ) { // deferred
 387                                 $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel );
 388                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle );
 389                         } else { // actually write the object in Swift
 390                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel );
 391                         }
 392                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 393                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 394                 } catch ( CloudFilesException $e ) { // some other exception?
 395                         $this->handleException( $e, $status, __METHOD__, $params );
 396                 }
 397
 398                 return $status;
 399         }
 400
 401         /**
 402          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 403          */
 404         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 405                 try {
 406                         $cfOp->getLastResponse();
 407                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 408                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 409                 }
 410         }
 411
 412         /**
 413          * @see FileBackendStore::doDeleteInternal()
 414          * @return Status
 415          */
 416         protected function doDeleteInternal( array $params ) {
 417                 $status = Status::newGood();
 418
 419                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 420                 if ( $srcRel === null ) {
 421                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 422                         return $status;
 423                 }
 424
 425                 try {
 426                         $sContObj = $this->getContainer( $srcCont );
 427                         if ( !empty( $params['async'] ) ) { // deferred
 428                                 $handle = $sContObj->delete_object_async( $srcRel );
 429                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle );
 430                         } else { // actually write the object in Swift
 431                                 $sContObj->delete_object( $srcRel );
 432                         }
 433                 } catch ( NoSuchContainerException $e ) {
 434                         $status->fatal( 'backend-fail-delete', $params['src'] );
 435                 } catch ( NoSuchObjectException $e ) {
 436                         if ( empty( $params['ignoreMissingSource'] ) ) {
 437                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 438                         }
 439                 } catch ( CloudFilesException $e ) { // some other exception?
 440                         $this->handleException( $e, $status, __METHOD__, $params );
 441                 }
 442
 443                 return $status;
 444         }
 445
 446         /**
 447          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 448          */
 449         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 450                 try {
 451                         $cfOp->getLastResponse();
 452                 } catch ( NoSuchContainerException $e ) {
 453                         $status->fatal( 'backend-fail-delete', $params['src'] );
 454                 } catch ( NoSuchObjectException $e ) {
 455                         if ( empty( $params['ignoreMissingSource'] ) ) {
 456                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 457                         }
 458                 }
 459         }
 460
 461         /**
 462          * @see FileBackendStore::doPrepareInternal()
 463          * @return Status
 464          */
 465         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 466                 $status = Status::newGood();
 467
 468                 // (a) Check if container already exists
 469                 try {
 470                         $contObj = $this->getContainer( $fullCont );
 471                         // NoSuchContainerException not thrown: container must exist
 472                         return $status; // already exists
 473                 } catch ( NoSuchContainerException $e ) {
 474                         // NoSuchContainerException thrown: container does not exist
 475                 } catch ( CloudFilesException $e ) { // some other exception?
 476                         $this->handleException( $e, $status, __METHOD__, $params );
 477                         return $status;
 478                 }
 479
 480                 // (b) Create container as needed
 481                 try {
 482                         $contObj = $this->createContainer( $fullCont );
 483                         if ( $this->swiftAnonUser != '' ) {
 484                                 // Make container public to end-users...
 485                                 $status->merge( $this->setContainerAccess(
 486                                         $contObj,
 487                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 488                                         array( $this->auth->username ) // write
 489                                 ) );
 490                         }
 491                 } catch ( CloudFilesException $e ) { // some other exception?
 492                         $this->handleException( $e, $status, __METHOD__, $params );
 493                         return $status;
 494                 }
 495
 496                 return $status;
 497         }
 498
 499         /**
 500          * @see FileBackendStore::doSecureInternal()
 501          * @return Status
 502          */
 503         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 504                 $status = Status::newGood();
 505
 506                 if ( $this->swiftAnonUser != '' ) {
 507                         // Restrict container from end-users...
 508                         try {
 509                                 // doPrepareInternal() should have been called,
 510                                 // so the Swift container should already exist...
 511                                 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 512                                 // NoSuchContainerException not thrown: container must exist
 513                                 if ( !isset( $contObj->mw_wasSecured ) ) {
 514                                         $status->merge( $this->setContainerAccess(
 515                                                 $contObj,
 516                                                 array( $this->auth->username ), // read
 517                                                 array( $this->auth->username ) // write
 518                                         ) );
 519                                         // @TODO: when php-cloudfiles supports container
 520                                         // metadata, we can make use of that to avoid RTTs
 521                                         $contObj->mw_wasSecured = true; // avoid useless RTTs
 522                                 }
 523                         } catch ( CloudFilesException $e ) { // some other exception?
 524                                 $this->handleException( $e, $status, __METHOD__, $params );
 525                         }
 526                 }
 527
 528                 return $status;
 529         }
 530
 531         /**
 532          * @see FileBackendStore::doCleanInternal()
 533          * @return Status
 534          */
 535         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 536                 $status = Status::newGood();
 537
 538                 // Only containers themselves can be removed, all else is virtual
 539                 if ( $dir != '' ) {
 540                         return $status; // nothing to do
 541                 }
 542
 543                 // (a) Check the container
 544                 try {
 545                         $contObj = $this->getContainer( $fullCont, true );
 546                 } catch ( NoSuchContainerException $e ) {
 547                         return $status; // ok, nothing to do
 548                 } catch ( CloudFilesException $e ) { // some other exception?
 549                         $this->handleException( $e, $status, __METHOD__, $params );
 550                         return $status;
 551                 }
 552
 553                 // (b) Delete the container if empty
 554                 if ( $contObj->object_count == 0 ) {
 555                         try {
 556                                 $this->deleteContainer( $fullCont );
 557                         } catch ( NoSuchContainerException $e ) {
 558                                 return $status; // race?
 559                         } catch ( NonEmptyContainerException $e ) {
 560                                 return $status; // race? consistency delay?
 561                         } catch ( CloudFilesException $e ) { // some other exception?
 562                                 $this->handleException( $e, $status, __METHOD__, $params );
 563                                 return $status;
 564                         }
 565                 }
 566
 567                 return $status;
 568         }
 569
 570         /**
 571          * @see FileBackendStore::doFileExists()
 572          * @return array|bool|null
 573          */
 574         protected function doGetFileStat( array $params ) {
 575                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 576                 if ( $srcRel === null ) {
 577                         return false; // invalid storage path
 578                 }
 579
 580                 $stat = false;
 581                 try {
 582                         $contObj = $this->getContainer( $srcCont );
 583                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 584                         $this->addMissingMetadata( $srcObj, $params['src'] );
 585                         $stat = array(
 586                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 587                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 588                                 'size'  => $srcObj->content_length,
 589                                 'sha1'  => $srcObj->metadata['Sha1base36']
 590                         );
 591                 } catch ( NoSuchContainerException $e ) {
 592                 } catch ( NoSuchObjectException $e ) {
 593                 } catch ( CloudFilesException $e ) { // some other exception?
 594                         $stat = null;
 595                         $this->handleException( $e, null, __METHOD__, $params );
 596                 }
 597
 598                 return $stat;
 599         }
 600
 601         /**
 602          * Fill in any missing object metadata and save it to Swift
 603          *
 604          * @param $obj CF_Object
 605          * @param $path string Storage path to object
 606          * @return bool Success
 607          * @throws Exception cloudfiles exceptions
 608          */
 609         protected function addMissingMetadata( CF_Object $obj, $path ) {
 610                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 611                         return true; // nothing to do
 612                 }
 613                 $status = Status::newGood();
 614                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 615                 if ( $status->isOK() ) {
 616                         # Do not stat the file in getLocalCopy() to avoid infinite loops
 617                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
 618                         if ( $tmpFile ) {
 619                                 $hash = $tmpFile->getSha1Base36();
 620                                 if ( $hash !== false ) {
 621                                         $obj->metadata['Sha1base36'] = $hash;
 622                                         $obj->sync_metadata(); // save to Swift
 623                                         return true; // success
 624                                 }
 625                         }
 626                 }
 627                 $obj->metadata['Sha1base36'] = false;
 628                 return false; // failed
 629         }
 630
 631         /**
 632          * @see FileBackend::getFileContents()
 633          * @return bool|null|string
 634          */
 635         public function getFileContents( array $params ) {
 636                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 637                 if ( $srcRel === null ) {
 638                         return false; // invalid storage path
 639                 }
 640
 641                 if ( !$this->fileExists( $params ) ) {
 642                         return null;
 643                 }
 644
 645                 $data = false;
 646                 try {
 647                         $sContObj = $this->getContainer( $srcCont );
 648                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request
 649                         $data = $obj->read( $this->headersFromParams( $params ) );
 650                 } catch ( NoSuchContainerException $e ) {
 651                 } catch ( CloudFilesException $e ) { // some other exception?
 652                         $this->handleException( $e, null, __METHOD__, $params );
 653                 }
 654
 655                 return $data;
 656         }
 657
 658         /**
 659          * @see FileBackendStore::doDirectoryExists()
 660          * @return bool|null
 661          */
 662         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 663                 try {
 664                         $container = $this->getContainer( $fullCont );
 665                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 666                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 667                 } catch ( NoSuchContainerException $e ) {
 668                         return false;
 669                 } catch ( CloudFilesException $e ) { // some other exception?
 670                         $this->handleException( $e, null, __METHOD__,
 671                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 672                 }
 673
 674                 return null; // error
 675         }
 676
 677         /**
 678          * @see FileBackendStore::getDirectoryListInternal()
 679          * @return SwiftFileBackendDirList
 680          */
 681         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 682                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 683         }
 684
 685         /**
 686          * @see FileBackendStore::getFileListInternal()
 687          * @return SwiftFileBackendFileList
 688          */
 689         public function getFileListInternal( $fullCont, $dir, array $params ) {
 690                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 691         }
 692
 693         /**
 694          * Do not call this function outside of SwiftFileBackendFileList
 695          *
 696          * @param $fullCont string Resolved container name
 697          * @param $dir string Resolved storage directory with no trailing slash
 698          * @param $after string|null Storage path of file to list items after
 699          * @param $limit integer Max number of items to list
 700          * @param $params Array Includes flag for 'topOnly'
 701          * @return Array List of relative paths of dirs directly under $dir
 702          */
 703         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 704                 $dirs = array();
 705
 706                 try {
 707                         $container = $this->getContainer( $fullCont );
 708                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 709                         // Non-recursive: only list dirs right under $dir
 710                         if ( !empty( $params['topOnly'] ) ) {
 711                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 712                                 foreach ( $objects as $object ) { // files and dirs
 713                                         if ( substr( $object, -1 ) === '/' ) {
 714                                                 $dirs[] = $object; // directories end in '/'
 715                                         }
 716                                         $after = $object; // update last item
 717                                 }
 718                         // Recursive: list all dirs under $dir and its subdirs
 719                         } else {
 720                                 // Get directory from last item of prior page
 721                                 $lastDir = $this->getParentDir( $after ); // must be first page
 722                                 $objects = $container->list_objects( $limit, $after, $prefix );
 723                                 foreach ( $objects as $object ) { // files
 724                                         $objectDir = $this->getParentDir( $object ); // directory of object
 725                                         if ( $objectDir !== false ) { // file has a parent dir
 726                                                 // Swift stores paths in UTF-8, using binary sorting.
 727                                                 // See function "create_container_table" in common/db.py.
 728                                                 // If a directory is not "greater" than the last one,
 729                                                 // then it was already listed by the calling iterator.
 730                                                 if ( $objectDir > $lastDir ) {
 731                                                         $pDir = $objectDir;
 732                                                         do { // add dir and all its parent dirs
 733                                                                 $dirs[] = "{$pDir}/";
 734                                                                 $pDir = $this->getParentDir( $pDir );
 735                                                         } while ( $pDir !== false // sanity
 736                                                                 && $pDir > $lastDir // not done already
 737                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 738                                                         );
 739                                                 }
 740                                                 $lastDir = $objectDir;
 741                                         }
 742                                         $after = $object; // update last item
 743                                 }
 744                         }
 745                 } catch ( NoSuchContainerException $e ) {
 746                 } catch ( CloudFilesException $e ) { // some other exception?
 747                         $this->handleException( $e, null, __METHOD__,
 748                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 749                 }
 750
 751                 return $dirs;
 752         }
 753
 754         protected function getParentDir( $path ) {
 755                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 756         }
 757
 758         /**
 759          * Do not call this function outside of SwiftFileBackendFileList
 760          *
 761          * @param $fullCont string Resolved container name
 762          * @param $dir string Resolved storage directory with no trailing slash
 763          * @param $after string|null Storage path of file to list items after
 764          * @param $limit integer Max number of items to list
 765          * @param $params Array Includes flag for 'topOnly'
 766          * @return Array List of relative paths of files under $dir
 767          */
 768         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 769                 $files = array();
 770
 771                 try {
 772                         $container = $this->getContainer( $fullCont );
 773                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 774                         // Non-recursive: only list files right under $dir
 775                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 776                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 777                                 foreach ( $objects as $object ) {
 778                                         if ( substr( $object, -1 ) !== '/' ) {
 779                                                 $files[] = $object; // directories end in '/'
 780                                         }
 781                                 }
 782                         // Recursive: list all files under $dir and its subdirs
 783                         } else { // files
 784                                 $files = $container->list_objects( $limit, $after, $prefix );
 785                         }
 786                         $after = end( $files ); // update last item
 787                         reset( $files ); // reset pointer
 788                 } catch ( NoSuchContainerException $e ) {
 789                 } catch ( CloudFilesException $e ) { // some other exception?
 790                         $this->handleException( $e, null, __METHOD__,
 791                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 792                 }
 793
 794                 return $files;
 795         }
 796
 797         /**
 798          * @see FileBackendStore::doGetFileSha1base36()
 799          * @return bool
 800          */
 801         protected function doGetFileSha1base36( array $params ) {
 802                 $stat = $this->getFileStat( $params );
 803                 if ( $stat ) {
 804                         return $stat['sha1'];
 805                 } else {
 806                         return false;
 807                 }
 808         }
 809
 810         /**
 811          * @see FileBackendStore::doStreamFile()
 812          * @return Status
 813          */
 814         protected function doStreamFile( array $params ) {
 815                 $status = Status::newGood();
 816
 817                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 818                 if ( $srcRel === null ) {
 819                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 820                 }
 821
 822                 try {
 823                         $cont = $this->getContainer( $srcCont );
 824                 } catch ( NoSuchContainerException $e ) {
 825                         $status->fatal( 'backend-fail-stream', $params['src'] );
 826                         return $status;
 827                 } catch ( CloudFilesException $e ) { // some other exception?
 828                         $this->handleException( $e, $status, __METHOD__, $params );
 829                         return $status;
 830                 }
 831
 832                 try {
 833                         $output = fopen( 'php://output', 'wb' );
 834                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
 835                         $obj->stream( $output, $this->headersFromParams( $params ) );
 836                 } catch ( CloudFilesException $e ) { // some other exception?
 837                         $this->handleException( $e, $status, __METHOD__, $params );
 838                 }
 839
 840                 return $status;
 841         }
 842
 843         /**
 844          * @see FileBackendStore::getLocalCopy()
 845          * @return null|TempFSFile
 846          */
 847         public function getLocalCopy( array $params ) {
 848                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 849                 if ( $srcRel === null ) {
 850                         return null;
 851                 }
 852
 853                 # Check the recursion guard to avoid loops when filling metadata
 854                 if ( empty( $params['nostat'] ) && !$this->fileExists( $params ) ) {
 855                         return null;
 856                 }
 857
 858                 $tmpFile = null;
 859                 try {
 860                         $sContObj = $this->getContainer( $srcCont );
 861                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 862                         // Get source file extension
 863                         $ext = FileBackend::extensionFromPath( $srcRel );
 864                         // Create a new temporary file...
 865                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 866                         if ( $tmpFile ) {
 867                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 868                                 if ( $handle ) {
 869                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 870                                         fclose( $handle );
 871                                 } else {
 872                                         $tmpFile = null; // couldn't open temp file
 873                                 }
 874                         }
 875                 } catch ( NoSuchContainerException $e ) {
 876                         $tmpFile = null;
 877                 } catch ( CloudFilesException $e ) { // some other exception?
 878                         $tmpFile = null;
 879                         $this->handleException( $e, null, __METHOD__, $params );
 880                 }
 881
 882                 return $tmpFile;
 883         }
 884
 885         /**
 886          * @see FileBackendStore::directoriesAreVirtual()
 887          * @return bool
 888          */
 889         protected function directoriesAreVirtual() {
 890                 return true;
 891         }
 892
 893         /**
 894          * Get headers to send to Swift when reading a file based
 895          * on a FileBackend params array, e.g. that of getLocalCopy().
 896          * $params is currently only checked for a 'latest' flag.
 897          *
 898          * @param $params Array
 899          * @return Array
 900          */
 901         protected function headersFromParams( array $params ) {
 902                 $hdrs = array();
 903                 if ( !empty( $params['latest'] ) ) {
 904                         $hdrs[] = 'X-Newest: true';
 905                 }
 906                 return $hdrs;
 907         }
 908
 909         /**
 910          * @see FileBackendStore::doExecuteOpHandlesInternal()
 911          * @return Array List of corresponding Status objects
 912          */
 913         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
 914                 $statuses = array();
 915
 916                 $cfOps = array(); // list of CF_Async_Op objects
 917                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
 918                         $cfOps[$index] = $fileOpHandle->cfOp;
 919                 }
 920                 $batch = new CF_Async_Op_Batch( $cfOps );
 921
 922                 $cfOps = $batch->execute();
 923                 foreach ( $cfOps as $index => $cfOp ) {
 924                         $status = Status::newGood();
 925                         try { // catch exceptions; update status
 926                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
 927                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
 928                         } catch ( CloudFilesException $e ) { // some other exception?
 929                                 $this->handleException( $e, $status,
 930                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
 931                         }
 932                         $statuses[$index] = $status;
 933                 }
 934
 935                 foreach ( $fileOpHandles as $fileOpHandle ) {
 936                         $fileOpHandle->closeResources();
 937                 }
 938
 939                 return $statuses;
 940         }
 941
 942         /**
 943          * Set read/write permissions for a Swift container
 944          *
 945          * @param $contObj CF_Container Swift container
 946          * @param $readGrps Array Swift users who can read (account:user)
 947          * @param $writeGrps Array Swift users who can write (account:user)
 948          * @return Status
 949          */
 950         protected function setContainerAccess(
 951                 CF_Container $contObj, array $readGrps, array $writeGrps
 952         ) {
 953                 $creds = $contObj->cfs_auth->export_credentials();
 954
 955                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
 956
 957                 // Note: 10 second timeout consistent with php-cloudfiles
 958                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
 959                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
 960                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
 961                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
 962
 963                 return $req->execute(); // should return 204
 964         }
 965
 966         /**
 967          * Get a connection to the Swift proxy
 968          *
 969          * @return CF_Connection|bool False on failure
 970          * @throws CloudFilesException
 971          */
 972         protected function getConnection() {
 973                 if ( $this->connException instanceof Exception ) {
 974                         throw $this->connException; // failed last attempt
 975                 }
 976                 // Session keys expire after a while, so we renew them periodically
 977                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
 978                         $this->conn->close(); // close active cURL connections
 979                         $this->conn = null;
 980                 }
 981                 // Authenticate with proxy and get a session key...
 982                 if ( !$this->conn ) {
 983                         $this->connStarted = 0;
 984                         $this->connContainers = array();
 985                         try {
 986                                 $this->auth->authenticate();
 987                                 $this->conn = new CF_Connection( $this->auth );
 988                                 $this->connStarted = time();
 989                         } catch ( CloudFilesException $e ) {
 990                                 $this->connException = $e; // don't keep re-trying
 991                                 throw $e; // throw it back
 992                         }
 993                 }
 994                 return $this->conn;
 995         }
 996
 997         /**
 998          * @see FileBackendStore::doClearCache()
 999          */
1000         protected function doClearCache( array $paths = null ) {
1001                 $this->connContainers = array(); // clear container object cache
1002         }
1003
1004         /**
1005          * Get a Swift container object, possibly from process cache.
1006          * Use $reCache if the file count or byte count is needed.
1007          *
1008          * @param $container string Container name
1009          * @param $bypassCache bool Bypass all caches and load from Swift
1010          * @return CF_Container
1011          * @throws CloudFilesException
1012          */
1013         protected function getContainer( $container, $bypassCache = false ) {
1014                 $conn = $this->getConnection(); // Swift proxy connection
1015                 if ( $bypassCache ) { // purge cache
1016                         unset( $this->connContainers[$container] );
1017                 } elseif ( !isset( $this->connContainers[$container] ) ) {
1018                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1019                 }
1020                 if ( !isset( $this->connContainers[$container] ) ) {
1021                         $contObj = $conn->get_container( $container );
1022                         // NoSuchContainerException not thrown: container must exist
1023                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
1024                                 reset( $this->connContainers );
1025                                 unset( $this->connContainers[key( $this->connContainers )] );
1026                         }
1027                         $this->connContainers[$container] = $contObj; // cache it
1028                         if ( !$bypassCache ) {
1029                                 $this->setContainerCache( $container, // update persistent cache
1030                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1031                                 );
1032                         }
1033                 }
1034                 return $this->connContainers[$container];
1035         }
1036
1037         /**
1038          * Create a Swift container
1039          *
1040          * @param $container string Container name
1041          * @return CF_Container
1042          * @throws InvalidResponseException
1043          */
1044         protected function createContainer( $container ) {
1045                 $conn = $this->getConnection(); // Swift proxy connection
1046                 $contObj = $conn->create_container( $container );
1047                 $this->connContainers[$container] = $contObj; // cache it
1048                 return $contObj;
1049         }
1050
1051         /**
1052          * Delete a Swift container
1053          *
1054          * @param $container string Container name
1055          * @return void
1056          * @throws InvalidResponseException
1057          */
1058         protected function deleteContainer( $container ) {
1059                 $conn = $this->getConnection(); // Swift proxy connection
1060                 $conn->delete_container( $container );
1061                 unset( $this->connContainers[$container] ); // purge cache
1062         }
1063
1064         /**
1065          * @see FileBackendStore::doPrimeContainerCache()
1066          * @return void
1067          */
1068         protected function doPrimeContainerCache( array $containerInfo ) {
1069                 try {
1070                         $conn = $this->getConnection(); // Swift proxy connection
1071                         foreach ( $containerInfo as $container => $info ) {
1072                                 $this->connContainers[$container] = new CF_Container(
1073                                         $conn->cfs_auth,
1074                                         $conn->cfs_http,
1075                                         $container,
1076                                         $info['count'],
1077                                         $info['bytes']
1078                                 );
1079                         }
1080                 } catch ( CloudFilesException $e ) { // some other exception?
1081                         $this->handleException( $e, null, __METHOD__, array() );
1082                 }
1083         }
1084
1085         /**
1086          * Log an unexpected exception for this backend.
1087          * This also sets the Status object to have a fatal error.
1088          *
1089          * @param $e Exception
1090          * @param $status Status|null
1091          * @param $func string
1092          * @param $params Array
1093          * @return void
1094          */
1095         protected function handleException( Exception $e, $status, $func, array $params ) {
1096                 if ( $status instanceof Status ) {
1097                         if ( $e instanceof AuthenticationException ) {
1098                                 $status->fatal( 'backend-fail-connect', $this->name );
1099                         } else {
1100                                 $status->fatal( 'backend-fail-internal', $this->name );
1101                         }
1102                 }
1103                 if ( $e->getMessage() ) {
1104                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1105                 }
1106                 wfDebugLog( 'SwiftBackend',
1107                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1108                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1109                 );
1110         }
1111 }
1112
1113 /**
1114  * @see FileBackendStoreOpHandle
1115  */
1116 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1117         /** @var CF_Async_Op */
1118         public $cfOp;
1119
1120         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1121                 $this->backend = $backend;
1122                 $this->params = $params;
1123                 $this->call = $call;
1124                 $this->cfOp = $cfOp;
1125         }
1126 }
1127
1128 /**
1129  * SwiftFileBackend helper class to page through listings.
1130  * Swift also has a listing limit of 10,000 objects for sanity.
1131  * Do not use this class from places outside SwiftFileBackend.
1132  *
1133  * @ingroup FileBackend
1134  */
1135 abstract class SwiftFileBackendList implements Iterator {
1136         /** @var Array */
1137         protected $bufferIter = array();
1138         protected $bufferAfter = null; // string; list items *after* this path
1139         protected $pos = 0; // integer
1140         /** @var Array */
1141         protected $params = array();
1142
1143         /** @var SwiftFileBackend */
1144         protected $backend;
1145         protected $container; // string; container name
1146         protected $dir; // string; storage directory
1147         protected $suffixStart; // integer
1148
1149         const PAGE_SIZE = 5000; // file listing buffer size
1150
1151         /**
1152          * @param $backend SwiftFileBackend
1153          * @param $fullCont string Resolved container name
1154          * @param $dir string Resolved directory relative to container
1155          * @param $params Array
1156          */
1157         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1158                 $this->backend = $backend;
1159                 $this->container = $fullCont;
1160                 $this->dir = $dir;
1161                 if ( substr( $this->dir, -1 ) === '/' ) {
1162                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1163                 }
1164                 if ( $this->dir == '' ) { // whole container
1165                         $this->suffixStart = 0;
1166                 } else { // dir within container
1167                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1168                 }
1169                 $this->params = $params;
1170         }
1171
1172         /**
1173          * @see Iterator::key()
1174          * @return integer
1175          */
1176         public function key() {
1177                 return $this->pos;
1178         }
1179
1180         /**
1181          * @see Iterator::next()
1182          * @return void
1183          */
1184         public function next() {
1185                 // Advance to the next file in the page
1186                 next( $this->bufferIter );
1187                 ++$this->pos;
1188                 // Check if there are no files left in this page and
1189                 // advance to the next page if this page was not empty.
1190                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1191                         $this->bufferIter = $this->pageFromList(
1192                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1193                         ); // updates $this->bufferAfter
1194                 }
1195         }
1196
1197         /**
1198          * @see Iterator::rewind()
1199          * @return void
1200          */
1201         public function rewind() {
1202                 $this->pos = 0;
1203                 $this->bufferAfter = null;
1204                 $this->bufferIter = $this->pageFromList(
1205                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1206                 ); // updates $this->bufferAfter
1207         }
1208
1209         /**
1210          * @see Iterator::valid()
1211          * @return bool
1212          */
1213         public function valid() {
1214                 if ( $this->bufferIter === null ) {
1215                         return false; // some failure?
1216                 } else {
1217                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1218                 }
1219         }
1220
1221         /**
1222          * Get the given list portion (page)
1223          *
1224          * @param $container string Resolved container name
1225          * @param $dir string Resolved path relative to container
1226          * @param $after string|null
1227          * @param $limit integer
1228          * @param $params Array
1229          * @return Traversable|Array|null Returns null on failure
1230          */
1231         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1232 }
1233
1234 /**
1235  * Iterator for listing directories
1236  */
1237 class SwiftFileBackendDirList extends SwiftFileBackendList {
1238         /**
1239          * @see Iterator::current()
1240          * @return string|bool String (relative path) or false
1241          */
1242         public function current() {
1243                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1244         }
1245
1246         /**
1247          * @see SwiftFileBackendList::pageFromList()
1248          * @return Array|null
1249          */
1250         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1251                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1252         }
1253 }
1254
1255 /**
1256  * Iterator for listing regular files
1257  */
1258 class SwiftFileBackendFileList extends SwiftFileBackendList {
1259         /**
1260          * @see Iterator::current()
1261          * @return string|bool String (relative path) or false
1262          */
1263         public function current() {
1264                 return substr( current( $this->bufferIter ), $this->suffixStart );
1265         }
1266
1267         /**
1268          * @see SwiftFileBackendList::pageFromList()
1269          * @return Array|null
1270          */
1271         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1272                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1273         }
1274 }