includes/filerepo/backend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  44         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  45         protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
  46         protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
  47
  48         protected $maxContCacheSize = 300; // integer; max containers with entries
  49
  50         /** @var CF_Connection */
  51         protected $conn; // Swift connection handle
  52         protected $connStarted = 0; // integer UNIX timestamp
  53         protected $connContainers = array(); // container object cache
  54         protected $connException; // CloudFiles exception
  55
  56         /**
  57          * @see FileBackendStore::__construct()
  58          * Additional $config params include:
  59          *    swiftAuthUrl       : Swift authentication server URL
  60          *    swiftUser          : Swift user used by MediaWiki (account:username)
  61          *    swiftKey           : Swift authentication key for the above user
  62          *    swiftAuthTTL       : Swift authentication TTL (seconds)
  63          *    swiftAnonUser      : Swift user used for end-user requests (account:username)
  64          *    swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  65          *    swiftCDNExpiry     : How long (in seconds) to store content in the CDN.
  66          *                         If files may likely change, this should probably not exceed
  67          *                         a few days. For example, deletions may take this long to apply.
  68          *                         If object purging is enabled, however, this is not an issue.
  69          *    swiftCDNPurgable   : Whether object purge requests are allowed by the CDN.
  70          *    shardViaHashLevels : Map of container names to sharding config with:
  71          *                         'base'   : base of hash characters, 16 or 36
  72          *                         'levels' : the number of hash levels (and digits)
  73          *                         'repeat' : hash subdirectories are prefixed with all the
  74          *                                    parent hash directory names (e.g. "a/ab/abc")
  75          */
  76         public function __construct( array $config ) {
  77                 parent::__construct( $config );
  78                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
  79                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
  80                 }
  81                 // Required settings
  82                 $this->auth = new CF_Authentication(
  83                         $config['swiftUser'],
  84                         $config['swiftKey'],
  85                         null, // account; unused
  86                         $config['swiftAuthUrl']
  87                 );
  88                 // Optional settings
  89                 $this->authTTL = isset( $config['swiftAuthTTL'] )
  90                         ? $config['swiftAuthTTL']
  91                         : 5 * 60; // some sane number
  92                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
  93                         ? $config['swiftAnonUser']
  94                         : '';
  95                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
  96                         ? $config['shardViaHashLevels']
  97                         : '';
  98                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
  99                         ? $config['swiftUseCDN']
 100                         : false;
 101                 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
 102                         ? $config['swiftCDNExpiry']
 103                         : 3600; // hour
 104                 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
 105                         ? $config['swiftCDNPurgable']
 106                         : true;
 107                 // Cache container info to mask latency
 108                 $this->memCache = wfGetMainCache();
 109         }
 110
 111         /**
 112          * @see FileBackendStore::resolveContainerPath()
 113          * @return null
 114          */
 115         protected function resolveContainerPath( $container, $relStoragePath ) {
 116                 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 117                         return null; // too long for Swift
 118                 }
 119                 return $relStoragePath;
 120         }
 121
 122         /**
 123          * @see FileBackendStore::isPathUsableInternal()
 124          * @return bool
 125          */
 126         public function isPathUsableInternal( $storagePath ) {
 127                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 128                 if ( $rel === null ) {
 129                         return false; // invalid
 130                 }
 131
 132                 try {
 133                         $this->getContainer( $container );
 134                         return true; // container exists
 135                 } catch ( NoSuchContainerException $e ) {
 136                 } catch ( CloudFilesException $e ) { // some other exception?
 137                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 138                 }
 139
 140                 return false;
 141         }
 142
 143         /**
 144          * @see FileBackendStore::doCreateInternal()
 145          * @return Status
 146          */
 147         protected function doCreateInternal( array $params ) {
 148                 $status = Status::newGood();
 149
 150                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 151                 if ( $dstRel === null ) {
 152                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 153                         return $status;
 154                 }
 155
 156                 // (a) Check the destination container and object
 157                 try {
 158                         $dContObj = $this->getContainer( $dstCont );
 159                         if ( empty( $params['overwrite'] ) &&
 160                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 161                         {
 162                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 163                                 return $status;
 164                         }
 165                 } catch ( NoSuchContainerException $e ) {
 166                         $status->fatal( 'backend-fail-create', $params['dst'] );
 167                         return $status;
 168                 } catch ( CloudFilesException $e ) { // some other exception?
 169                         $this->handleException( $e, $status, __METHOD__, $params );
 170                         return $status;
 171                 }
 172
 173                 // (b) Get a SHA-1 hash of the object
 174                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 175
 176                 // (c) Actually create the object
 177                 try {
 178                         // Create a fresh CF_Object with no fields preloaded.
 179                         // We don't want to preserve headers, metadata, and such.
 180                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 181                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 182                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 183                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 184                         // The MD5 here will be checked within Swift against its own MD5.
 185                         $obj->set_etag( md5( $params['content'] ) );
 186                         // Use the same content type as StreamFile for security
 187                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 188                         if ( !empty( $params['async'] ) ) { // deferred
 189                                 $handle = $obj->write_async( $params['content'] );
 190                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle );
 191                                 $status->value->affectedObjects[] = $obj;
 192                         } else { // actually write the object in Swift
 193                                 $obj->write( $params['content'] );
 194                                 $this->purgeCDNCache( array( $obj ) );
 195                         }
 196                 } catch ( CDNNotEnabledException $e ) {
 197                         // CDN not enabled; nothing to see here
 198                 } catch ( BadContentTypeException $e ) {
 199                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 200                 } catch ( CloudFilesException $e ) { // some other exception?
 201                         $this->handleException( $e, $status, __METHOD__, $params );
 202                 }
 203
 204                 return $status;
 205         }
 206
 207         /**
 208          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 209          */
 210         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 211                 try {
 212                         $cfOp->getLastResponse();
 213                 } catch ( BadContentTypeException $e ) {
 214                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 215                 }
 216         }
 217
 218         /**
 219          * @see FileBackendStore::doStoreInternal()
 220          * @return Status
 221          */
 222         protected function doStoreInternal( array $params ) {
 223                 $status = Status::newGood();
 224
 225                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 226                 if ( $dstRel === null ) {
 227                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 228                         return $status;
 229                 }
 230
 231                 // (a) Check the destination container and object
 232                 try {
 233                         $dContObj = $this->getContainer( $dstCont );
 234                         if ( empty( $params['overwrite'] ) &&
 235                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 236                         {
 237                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 238                                 return $status;
 239                         }
 240                 } catch ( NoSuchContainerException $e ) {
 241                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 242                         return $status;
 243                 } catch ( CloudFilesException $e ) { // some other exception?
 244                         $this->handleException( $e, $status, __METHOD__, $params );
 245                         return $status;
 246                 }
 247
 248                 // (b) Get a SHA-1 hash of the object
 249                 $sha1Hash = sha1_file( $params['src'] );
 250                 if ( $sha1Hash === false ) { // source doesn't exist?
 251                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 252                         return $status;
 253                 }
 254                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 255
 256                 // (c) Actually store the object
 257                 try {
 258                         // Create a fresh CF_Object with no fields preloaded.
 259                         // We don't want to preserve headers, metadata, and such.
 260                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 261                         // Note: metadata keys stored as [Upper case char][[Lower case char]...]
 262                         $obj->metadata = array( 'Sha1base36' => $sha1Hash );
 263                         // The MD5 here will be checked within Swift against its own MD5.
 264                         $obj->set_etag( md5_file( $params['src'] ) );
 265                         // Use the same content type as StreamFile for security
 266                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 267                         if ( !empty( $params['async'] ) ) { // deferred
 268                                 wfSuppressWarnings();
 269                                 $fp = fopen( $params['src'], 'rb' );
 270                                 wfRestoreWarnings();
 271                                 if ( !$fp ) {
 272                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 273                                 } else {
 274                                         $handle = $obj->write_async( $fp, filesize( $params['src'] ), true );
 275                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle );
 276                                         $status->value->resourcesToClose[] = $fp;
 277                                         $status->value->affectedObjects[] = $obj;
 278                                 }
 279                         } else { // actually write the object in Swift
 280                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 281                                 $this->purgeCDNCache( array( $obj ) );
 282                         }
 283                 } catch ( CDNNotEnabledException $e ) {
 284                         // CDN not enabled; nothing to see here
 285                 } catch ( BadContentTypeException $e ) {
 286                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 287                 } catch ( IOException $e ) {
 288                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 289                 } catch ( CloudFilesException $e ) { // some other exception?
 290                         $this->handleException( $e, $status, __METHOD__, $params );
 291                 }
 292
 293                 return $status;
 294         }
 295
 296         /**
 297          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 298          */
 299         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 300                 try {
 301                         $cfOp->getLastResponse();
 302                 } catch ( BadContentTypeException $e ) {
 303                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 304                 } catch ( IOException $e ) {
 305                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 306                 }
 307         }
 308
 309         /**
 310          * @see FileBackendStore::doCopyInternal()
 311          * @return Status
 312          */
 313         protected function doCopyInternal( array $params ) {
 314                 $status = Status::newGood();
 315
 316                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 317                 if ( $srcRel === null ) {
 318                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 319                         return $status;
 320                 }
 321
 322                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 323                 if ( $dstRel === null ) {
 324                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 325                         return $status;
 326                 }
 327
 328                 // (a) Check the source/destination containers and destination object
 329                 try {
 330                         $sContObj = $this->getContainer( $srcCont );
 331                         $dContObj = $this->getContainer( $dstCont );
 332                         if ( empty( $params['overwrite'] ) &&
 333                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 334                         {
 335                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 336                                 return $status;
 337                         }
 338                 } catch ( NoSuchContainerException $e ) {
 339                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 340                         return $status;
 341                 } catch ( CloudFilesException $e ) { // some other exception?
 342                         $this->handleException( $e, $status, __METHOD__, $params );
 343                         return $status;
 344                 }
 345
 346                 // (b) Actually copy the file to the destination
 347                 try {
 348                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 349                         if ( !empty( $params['async'] ) ) { // deferred
 350                                 $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel );
 351                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle );
 352                                 $status->value->affectedObjects[] = $dstObj;
 353                         } else { // actually write the object in Swift
 354                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
 355                                 $this->purgeCDNCache( array( $dstObj ) );
 356                         }
 357                 } catch ( CDNNotEnabledException $e ) {
 358                         // CDN not enabled; nothing to see here
 359                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 360                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 361                 } catch ( CloudFilesException $e ) { // some other exception?
 362                         $this->handleException( $e, $status, __METHOD__, $params );
 363                 }
 364
 365                 return $status;
 366         }
 367
 368         /**
 369          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 370          */
 371         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 372                 try {
 373                         $cfOp->getLastResponse();
 374                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 375                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 376                 }
 377         }
 378
 379         /**
 380          * @see FileBackendStore::doMoveInternal()
 381          * @return Status
 382          */
 383         protected function doMoveInternal( array $params ) {
 384                 $status = Status::newGood();
 385
 386                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 387                 if ( $srcRel === null ) {
 388                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 389                         return $status;
 390                 }
 391
 392                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 393                 if ( $dstRel === null ) {
 394                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 395                         return $status;
 396                 }
 397
 398                 // (a) Check the source/destination containers and destination object
 399                 try {
 400                         $sContObj = $this->getContainer( $srcCont );
 401                         $dContObj = $this->getContainer( $dstCont );
 402                         if ( empty( $params['overwrite'] ) &&
 403                                 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
 404                         {
 405                                 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
 406                                 return $status;
 407                         }
 408                 } catch ( NoSuchContainerException $e ) {
 409                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 410                         return $status;
 411                 } catch ( CloudFilesException $e ) { // some other exception?
 412                         $this->handleException( $e, $status, __METHOD__, $params );
 413                         return $status;
 414                 }
 415
 416                 // (b) Actually move the file to the destination
 417                 try {
 418                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 419                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 420                         if ( !empty( $params['async'] ) ) { // deferred
 421                                 $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel );
 422                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle );
 423                                 $status->value->affectedObjects[] = $srcObj;
 424                                 $status->value->affectedObjects[] = $dstObj;
 425                         } else { // actually write the object in Swift
 426                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel );
 427                                 $this->purgeCDNCache( array( $srcObj, $dstObj ) );
 428                         }
 429                 } catch ( CDNNotEnabledException $e ) {
 430                         // CDN not enabled; nothing to see here
 431                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 432                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 433                 } catch ( CloudFilesException $e ) { // some other exception?
 434                         $this->handleException( $e, $status, __METHOD__, $params );
 435                 }
 436
 437                 return $status;
 438         }
 439
 440         /**
 441          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 442          */
 443         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 444                 try {
 445                         $cfOp->getLastResponse();
 446                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 447                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 448                 }
 449         }
 450
 451         /**
 452          * @see FileBackendStore::doDeleteInternal()
 453          * @return Status
 454          */
 455         protected function doDeleteInternal( array $params ) {
 456                 $status = Status::newGood();
 457
 458                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 459                 if ( $srcRel === null ) {
 460                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 461                         return $status;
 462                 }
 463
 464                 try {
 465                         $sContObj = $this->getContainer( $srcCont );
 466                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 467                         if ( !empty( $params['async'] ) ) { // deferred
 468                                 $handle = $sContObj->delete_object_async( $srcRel );
 469                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle );
 470                                 $status->value->affectedObjects[] = $srcObj;
 471                         } else { // actually write the object in Swift
 472                                 $sContObj->delete_object( $srcRel );
 473                                 $this->purgeCDNCache( array( $srcObj ) );
 474                         }
 475                 } catch ( CDNNotEnabledException $e ) {
 476                         // CDN not enabled; nothing to see here
 477                 } catch ( NoSuchContainerException $e ) {
 478                         $status->fatal( 'backend-fail-delete', $params['src'] );
 479                 } catch ( NoSuchObjectException $e ) {
 480                         if ( empty( $params['ignoreMissingSource'] ) ) {
 481                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 482                         }
 483                 } catch ( CloudFilesException $e ) { // some other exception?
 484                         $this->handleException( $e, $status, __METHOD__, $params );
 485                 }
 486
 487                 return $status;
 488         }
 489
 490         /**
 491          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 492          */
 493         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 494                 try {
 495                         $cfOp->getLastResponse();
 496                 } catch ( NoSuchContainerException $e ) {
 497                         $status->fatal( 'backend-fail-delete', $params['src'] );
 498                 } catch ( NoSuchObjectException $e ) {
 499                         if ( empty( $params['ignoreMissingSource'] ) ) {
 500                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 501                         }
 502                 }
 503         }
 504
 505         /**
 506          * @see FileBackendStore::doPrepareInternal()
 507          * @return Status
 508          */
 509         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 510                 $status = Status::newGood();
 511
 512                 // (a) Check if container already exists
 513                 try {
 514                         $contObj = $this->getContainer( $fullCont );
 515                         // NoSuchContainerException not thrown: container must exist
 516                         return $status; // already exists
 517                 } catch ( NoSuchContainerException $e ) {
 518                         // NoSuchContainerException thrown: container does not exist
 519                 } catch ( CloudFilesException $e ) { // some other exception?
 520                         $this->handleException( $e, $status, __METHOD__, $params );
 521                         return $status;
 522                 }
 523
 524                 // (b) Create container as needed
 525                 try {
 526                         $contObj = $this->createContainer( $fullCont );
 527                         if ( !empty( $params['noAccess'] ) ) {
 528                                 // Make container private to end-users...
 529                                 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 530                         } else {
 531                                 // Make container public to end-users...
 532                                 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 533                         }
 534                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 535                                 $contObj->make_public( $this->swiftCDNExpiry );
 536                         }
 537                 } catch ( CDNNotEnabledException $e ) {
 538                         // CDN not enabled; nothing to see here
 539                 } catch ( CloudFilesException $e ) { // some other exception?
 540                         $this->handleException( $e, $status, __METHOD__, $params );
 541                         return $status;
 542                 }
 543
 544                 return $status;
 545         }
 546
 547         /**
 548          * @see FileBackendStore::doSecureInternal()
 549          * @return Status
 550          */
 551         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 552                 $status = Status::newGood();
 553                 if ( empty( $params['noAccess'] ) ) {
 554                         return $status; // nothing to do
 555                 }
 556
 557                 // Restrict container from end-users...
 558                 try {
 559                         // doPrepareInternal() should have been called,
 560                         // so the Swift container should already exist...
 561                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 562                         // NoSuchContainerException not thrown: container must exist
 563
 564                         // Make container private to end-users...
 565                         $status->merge( $this->setContainerAccess(
 566                                 $contObj,
 567                                 array( $this->auth->username ), // read
 568                                 array( $this->auth->username ) // write
 569                         ) );
 570                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 571                                 $contObj->make_private();
 572                         }
 573                 } catch ( CDNNotEnabledException $e ) {
 574                         // CDN not enabled; nothing to see here
 575                 } catch ( CloudFilesException $e ) { // some other exception?
 576                         $this->handleException( $e, $status, __METHOD__, $params );
 577                 }
 578
 579                 return $status;
 580         }
 581
 582         /**
 583          * @see FileBackendStore::doPublishInternal()
 584          * @return Status
 585          */
 586         protected function doPublishInternal( $fullCont, $dir, array $params ) {
 587                 $status = Status::newGood();
 588
 589                 // Unrestrict container from end-users...
 590                 try {
 591                         // doPrepareInternal() should have been called,
 592                         // so the Swift container should already exist...
 593                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 594                         // NoSuchContainerException not thrown: container must exist
 595
 596                         // Make container public to end-users...
 597                         if ( $this->swiftAnonUser != '' ) {
 598                                 $status->merge( $this->setContainerAccess(
 599                                         $contObj,
 600                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 601                                         array( $this->auth->username, $this->swiftAnonUser ) // write
 602                                 ) );
 603                         } else {
 604                                 $status->merge( $this->setContainerAccess(
 605                                         $contObj,
 606                                         array( $this->auth->username, '.r:*' ), // read
 607                                         array( $this->auth->username ) // write
 608                                 ) );
 609                         }
 610                         if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
 611                                 $contObj->make_public();
 612                         }
 613                 } catch ( CDNNotEnabledException $e ) {
 614                         // CDN not enabled; nothing to see here
 615                 } catch ( CloudFilesException $e ) { // some other exception?
 616                         $this->handleException( $e, $status, __METHOD__, $params );
 617                 }
 618
 619                 return $status;
 620         }
 621
 622         /**
 623          * @see FileBackendStore::doCleanInternal()
 624          * @return Status
 625          */
 626         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 627                 $status = Status::newGood();
 628
 629                 // Only containers themselves can be removed, all else is virtual
 630                 if ( $dir != '' ) {
 631                         return $status; // nothing to do
 632                 }
 633
 634                 // (a) Check the container
 635                 try {
 636                         $contObj = $this->getContainer( $fullCont, true );
 637                 } catch ( NoSuchContainerException $e ) {
 638                         return $status; // ok, nothing to do
 639                 } catch ( CloudFilesException $e ) { // some other exception?
 640                         $this->handleException( $e, $status, __METHOD__, $params );
 641                         return $status;
 642                 }
 643
 644                 // (b) Delete the container if empty
 645                 if ( $contObj->object_count == 0 ) {
 646                         try {
 647                                 $this->deleteContainer( $fullCont );
 648                         } catch ( NoSuchContainerException $e ) {
 649                                 return $status; // race?
 650                         } catch ( NonEmptyContainerException $e ) {
 651                                 return $status; // race? consistency delay?
 652                         } catch ( CloudFilesException $e ) { // some other exception?
 653                                 $this->handleException( $e, $status, __METHOD__, $params );
 654                                 return $status;
 655                         }
 656                 }
 657
 658                 return $status;
 659         }
 660
 661         /**
 662          * @see FileBackendStore::doFileExists()
 663          * @return array|bool|null
 664          */
 665         protected function doGetFileStat( array $params ) {
 666                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 667                 if ( $srcRel === null ) {
 668                         return false; // invalid storage path
 669                 }
 670
 671                 $stat = false;
 672                 try {
 673                         $contObj = $this->getContainer( $srcCont );
 674                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 675                         $this->addMissingMetadata( $srcObj, $params['src'] );
 676                         $stat = array(
 677                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 678                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 679                                 'size'  => (int)$srcObj->content_length,
 680                                 'sha1'  => $srcObj->metadata['Sha1base36']
 681                         );
 682                 } catch ( NoSuchContainerException $e ) {
 683                 } catch ( NoSuchObjectException $e ) {
 684                 } catch ( CloudFilesException $e ) { // some other exception?
 685                         $stat = null;
 686                         $this->handleException( $e, null, __METHOD__, $params );
 687                 }
 688
 689                 return $stat;
 690         }
 691
 692         /**
 693          * Fill in any missing object metadata and save it to Swift
 694          *
 695          * @param $obj CF_Object
 696          * @param $path string Storage path to object
 697          * @return bool Success
 698          * @throws Exception cloudfiles exceptions
 699          */
 700         protected function addMissingMetadata( CF_Object $obj, $path ) {
 701                 if ( isset( $obj->metadata['Sha1base36'] ) ) {
 702                         return true; // nothing to do
 703                 }
 704                 $status = Status::newGood();
 705                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 706                 if ( $status->isOK() ) {
 707                         # Do not stat the file in getLocalCopy() to avoid infinite loops
 708                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
 709                         if ( $tmpFile ) {
 710                                 $hash = $tmpFile->getSha1Base36();
 711                                 if ( $hash !== false ) {
 712                                         $obj->metadata['Sha1base36'] = $hash;
 713                                         $obj->sync_metadata(); // save to Swift
 714                                         return true; // success
 715                                 }
 716                         }
 717                 }
 718                 $obj->metadata['Sha1base36'] = false;
 719                 return false; // failed
 720         }
 721
 722         /**
 723          * @see FileBackend::getFileContents()
 724          * @return bool|null|string
 725          */
 726         public function getFileContents( array $params ) {
 727                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 728                 if ( $srcRel === null ) {
 729                         return false; // invalid storage path
 730                 }
 731
 732                 if ( !$this->fileExists( $params ) ) {
 733                         return null;
 734                 }
 735
 736                 $data = false;
 737                 try {
 738                         $sContObj = $this->getContainer( $srcCont );
 739                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 740                         $data = $obj->read( $this->headersFromParams( $params ) );
 741                 } catch ( NoSuchContainerException $e ) {
 742                 } catch ( CloudFilesException $e ) { // some other exception?
 743                         $this->handleException( $e, null, __METHOD__, $params );
 744                 }
 745
 746                 return $data;
 747         }
 748
 749         /**
 750          * @see FileBackendStore::doDirectoryExists()
 751          * @return bool|null
 752          */
 753         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 754                 try {
 755                         $container = $this->getContainer( $fullCont );
 756                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 757                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 758                 } catch ( NoSuchContainerException $e ) {
 759                         return false;
 760                 } catch ( CloudFilesException $e ) { // some other exception?
 761                         $this->handleException( $e, null, __METHOD__,
 762                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 763                 }
 764
 765                 return null; // error
 766         }
 767
 768         /**
 769          * @see FileBackendStore::getDirectoryListInternal()
 770          * @return SwiftFileBackendDirList
 771          */
 772         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 773                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 774         }
 775
 776         /**
 777          * @see FileBackendStore::getFileListInternal()
 778          * @return SwiftFileBackendFileList
 779          */
 780         public function getFileListInternal( $fullCont, $dir, array $params ) {
 781                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 782         }
 783
 784         /**
 785          * Do not call this function outside of SwiftFileBackendFileList
 786          *
 787          * @param $fullCont string Resolved container name
 788          * @param $dir string Resolved storage directory with no trailing slash
 789          * @param $after string|null Storage path of file to list items after
 790          * @param $limit integer Max number of items to list
 791          * @param $params Array Includes flag for 'topOnly'
 792          * @return Array List of relative paths of dirs directly under $dir
 793          */
 794         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 795                 $dirs = array();
 796                 if ( $after === INF ) {
 797                         return $dirs; // nothing more
 798                 }
 799                 wfProfileIn( __METHOD__ . '-' . $this->name );
 800
 801                 try {
 802                         $container = $this->getContainer( $fullCont );
 803                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 804                         // Non-recursive: only list dirs right under $dir
 805                         if ( !empty( $params['topOnly'] ) ) {
 806                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 807                                 foreach ( $objects as $object ) { // files and dirs
 808                                         if ( substr( $object, -1 ) === '/' ) {
 809                                                 $dirs[] = $object; // directories end in '/'
 810                                         }
 811                                 }
 812                         // Recursive: list all dirs under $dir and its subdirs
 813                         } else {
 814                                 // Get directory from last item of prior page
 815                                 $lastDir = $this->getParentDir( $after ); // must be first page
 816                                 $objects = $container->list_objects( $limit, $after, $prefix );
 817                                 foreach ( $objects as $object ) { // files
 818                                         $objectDir = $this->getParentDir( $object ); // directory of object
 819                                         if ( $objectDir !== false ) { // file has a parent dir
 820                                                 // Swift stores paths in UTF-8, using binary sorting.
 821                                                 // See function "create_container_table" in common/db.py.
 822                                                 // If a directory is not "greater" than the last one,
 823                                                 // then it was already listed by the calling iterator.
 824                                                 if ( $objectDir > $lastDir ) {
 825                                                         $pDir = $objectDir;
 826                                                         do { // add dir and all its parent dirs
 827                                                                 $dirs[] = "{$pDir}/";
 828                                                                 $pDir = $this->getParentDir( $pDir );
 829                                                         } while ( $pDir !== false // sanity
 830                                                                 && $pDir > $lastDir // not done already
 831                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
 832                                                         );
 833                                                 }
 834                                                 $lastDir = $objectDir;
 835                                         }
 836                                 }
 837                         }
 838                         if ( count( $objects ) < $limit ) {
 839                                 $after = INF; // avoid a second RTT
 840                         } else {
 841                                 $after = end( $objects ); // update last item
 842                         }
 843                 } catch ( NoSuchContainerException $e ) {
 844                 } catch ( CloudFilesException $e ) { // some other exception?
 845                         $this->handleException( $e, null, __METHOD__,
 846                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 847                 }
 848
 849                 wfProfileOut( __METHOD__ . '-' . $this->name );
 850                 return $dirs;
 851         }
 852
 853         protected function getParentDir( $path ) {
 854                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
 855         }
 856
 857         /**
 858          * Do not call this function outside of SwiftFileBackendFileList
 859          *
 860          * @param $fullCont string Resolved container name
 861          * @param $dir string Resolved storage directory with no trailing slash
 862          * @param $after string|null Storage path of file to list items after
 863          * @param $limit integer Max number of items to list
 864          * @param $params Array Includes flag for 'topOnly'
 865          * @return Array List of relative paths of files under $dir
 866          */
 867         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 868                 $files = array();
 869                 if ( $after === INF ) {
 870                         return $files; // nothing more
 871                 }
 872                 wfProfileIn( __METHOD__ . '-' . $this->name );
 873
 874                 try {
 875                         $container = $this->getContainer( $fullCont );
 876                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 877                         // Non-recursive: only list files right under $dir
 878                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
 879                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 880                                 foreach ( $objects as $object ) {
 881                                         if ( substr( $object, -1 ) !== '/' ) {
 882                                                 $files[] = $object; // directories end in '/'
 883                                         }
 884                                 }
 885                         // Recursive: list all files under $dir and its subdirs
 886                         } else { // files
 887                                 $objects = $container->list_objects( $limit, $after, $prefix );
 888                                 $files = $objects;
 889                         }
 890                         if ( count( $objects ) < $limit ) {
 891                                 $after = INF; // avoid a second RTT
 892                         } else {
 893                                 $after = end( $objects ); // update last item
 894                         }
 895                 } catch ( NoSuchContainerException $e ) {
 896                 } catch ( CloudFilesException $e ) { // some other exception?
 897                         $this->handleException( $e, null, __METHOD__,
 898                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 899                 }
 900
 901                 wfProfileOut( __METHOD__ . '-' . $this->name );
 902                 return $files;
 903         }
 904
 905         /**
 906          * @see FileBackendStore::doGetFileSha1base36()
 907          * @return bool
 908          */
 909         protected function doGetFileSha1base36( array $params ) {
 910                 $stat = $this->getFileStat( $params );
 911                 if ( $stat ) {
 912                         return $stat['sha1'];
 913                 } else {
 914                         return false;
 915                 }
 916         }
 917
 918         /**
 919          * @see FileBackendStore::doStreamFile()
 920          * @return Status
 921          */
 922         protected function doStreamFile( array $params ) {
 923                 $status = Status::newGood();
 924
 925                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 926                 if ( $srcRel === null ) {
 927                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 928                 }
 929
 930                 try {
 931                         $cont = $this->getContainer( $srcCont );
 932                 } catch ( NoSuchContainerException $e ) {
 933                         $status->fatal( 'backend-fail-stream', $params['src'] );
 934                         return $status;
 935                 } catch ( CloudFilesException $e ) { // some other exception?
 936                         $this->handleException( $e, $status, __METHOD__, $params );
 937                         return $status;
 938                 }
 939
 940                 try {
 941                         $output = fopen( 'php://output', 'wb' );
 942                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
 943                         $obj->stream( $output, $this->headersFromParams( $params ) );
 944                 } catch ( CloudFilesException $e ) { // some other exception?
 945                         $this->handleException( $e, $status, __METHOD__, $params );
 946                 }
 947
 948                 return $status;
 949         }
 950
 951         /**
 952          * @see FileBackendStore::getLocalCopy()
 953          * @return null|TempFSFile
 954          */
 955         public function getLocalCopy( array $params ) {
 956                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 957                 if ( $srcRel === null ) {
 958                         return null;
 959                 }
 960
 961                 # Check the recursion guard to avoid loops when filling metadata
 962                 if ( empty( $params['nostat'] ) && !$this->fileExists( $params ) ) {
 963                         return null;
 964                 }
 965
 966                 $tmpFile = null;
 967                 try {
 968                         $sContObj = $this->getContainer( $srcCont );
 969                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 970                         // Get source file extension
 971                         $ext = FileBackend::extensionFromPath( $srcRel );
 972                         // Create a new temporary file...
 973                         $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
 974                         if ( $tmpFile ) {
 975                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
 976                                 if ( $handle ) {
 977                                         $obj->stream( $handle, $this->headersFromParams( $params ) );
 978                                         fclose( $handle );
 979                                 } else {
 980                                         $tmpFile = null; // couldn't open temp file
 981                                 }
 982                         }
 983                 } catch ( NoSuchContainerException $e ) {
 984                         $tmpFile = null;
 985                 } catch ( CloudFilesException $e ) { // some other exception?
 986                         $tmpFile = null;
 987                         $this->handleException( $e, null, __METHOD__, $params );
 988                 }
 989
 990                 return $tmpFile;
 991         }
 992
 993         /**
 994          * @see FileBackendStore::directoriesAreVirtual()
 995          * @return bool
 996          */
 997         protected function directoriesAreVirtual() {
 998                 return true;
 999         }
1000
1001         /**
1002          * Get headers to send to Swift when reading a file based
1003          * on a FileBackend params array, e.g. that of getLocalCopy().
1004          * $params is currently only checked for a 'latest' flag.
1005          *
1006          * @param $params Array
1007          * @return Array
1008          */
1009         protected function headersFromParams( array $params ) {
1010                 $hdrs = array();
1011                 if ( !empty( $params['latest'] ) ) {
1012                         $hdrs[] = 'X-Newest: true';
1013                 }
1014                 return $hdrs;
1015         }
1016
1017         /**
1018          * @see FileBackendStore::doExecuteOpHandlesInternal()
1019          * @return Array List of corresponding Status objects
1020          */
1021         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1022                 $statuses = array();
1023
1024                 $cfOps = array(); // list of CF_Async_Op objects
1025                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1026                         $cfOps[$index] = $fileOpHandle->cfOp;
1027                 }
1028                 $batch = new CF_Async_Op_Batch( $cfOps );
1029
1030                 $cfOps = $batch->execute();
1031                 foreach ( $cfOps as $index => $cfOp ) {
1032                         $status = Status::newGood();
1033                         try { // catch exceptions; update status
1034                                 $function = '_getResponse' . $fileOpHandles[$index]->call;
1035                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
1036                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
1037                         } catch ( CloudFilesException $e ) { // some other exception?
1038                                 $this->handleException( $e, $status,
1039                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
1040                         }
1041                         $statuses[$index] = $status;
1042                 }
1043
1044                 return $statuses;
1045         }
1046
1047         /**
1048          * Set read/write permissions for a Swift container.
1049          *
1050          * $readGrps is a list of the possible criteria for a request to have
1051          * access to write to a container. Each item is of the following format:
1052          *   account:user       - Grants access if the request is by this user
1053          *
1054          * $readGrps is a list of the possible criteria for a request to have
1055          * access to read a container. Each item is one of the following formats:
1056          *   .r:<regex>         - Grants access if the request is from a referrer host that
1057          *                        matches the expression and the request is not for a listing.
1058          *                        Setting this to '*' effectively makes a container public.
1059          *   .rlistings:<regex> - Grants access if the request is from a referrer host that
1060          *                        matches the expression and the request for a listing.
1061          *
1062          * @see http://swift.openstack.org/misc.html#acls
1063          *
1064          * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
1065          * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
1066          *
1067          * @param $contObj CF_Container Swift container
1068          * @param $readGrps Array List of read access routes
1069          * @param $writeGrps Array List of write access routes
1070          * @return Status
1071          */
1072         protected function setContainerAccess(
1073                 CF_Container $contObj, array $readGrps, array $writeGrps
1074         ) {
1075                 $creds = $contObj->cfs_auth->export_credentials();
1076
1077                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1078
1079                 // Note: 10 second timeout consistent with php-cloudfiles
1080                 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1081                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1082                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1083                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1084
1085                 return $req->execute(); // should return 204
1086         }
1087
1088         /**
1089          * Purge the CDN cache of affected objects if CDN caching is enabled.
1090          * This is for Rackspace/Akamai CDNs.
1091          *
1092          * @param $objects Array List of CF_Object items
1093          * @return void
1094          */
1095         public function purgeCDNCache( array $objects ) {
1096                 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
1097                         foreach ( $objects as $object ) {
1098                                 try {
1099                                         $object->purge_from_cdn();
1100                                 } catch ( CDNNotEnabledException $e ) {
1101                                         // CDN not enabled; nothing to see here
1102                                 } catch ( CloudFilesException $e ) {
1103                                         $this->handleException( $e, null, __METHOD__,
1104                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1105                                 }
1106                         }
1107                 }
1108         }
1109
1110         /**
1111          * Get a connection to the Swift proxy
1112          *
1113          * @return CF_Connection|bool False on failure
1114          * @throws CloudFilesException
1115          */
1116         protected function getConnection() {
1117                 if ( $this->connException instanceof Exception ) {
1118                         throw $this->connException; // failed last attempt
1119                 }
1120                 // Session keys expire after a while, so we renew them periodically
1121                 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
1122                         $this->conn->close(); // close active cURL connections
1123                         $this->conn = null;
1124                 }
1125                 // Authenticate with proxy and get a session key...
1126                 if ( !$this->conn ) {
1127                         $this->connStarted = 0;
1128                         $this->connContainers = array();
1129                         try {
1130                                 $this->auth->authenticate();
1131                                 $this->conn = new CF_Connection( $this->auth );
1132                                 $this->connStarted = time();
1133                         } catch ( CloudFilesException $e ) {
1134                                 $this->connException = $e; // don't keep re-trying
1135                                 throw $e; // throw it back
1136                         }
1137                 }
1138                 return $this->conn;
1139         }
1140
1141         /**
1142          * @see FileBackendStore::doClearCache()
1143          */
1144         protected function doClearCache( array $paths = null ) {
1145                 $this->connContainers = array(); // clear container object cache
1146         }
1147
1148         /**
1149          * Get a Swift container object, possibly from process cache.
1150          * Use $reCache if the file count or byte count is needed.
1151          *
1152          * @param $container string Container name
1153          * @param $bypassCache bool Bypass all caches and load from Swift
1154          * @return CF_Container
1155          * @throws CloudFilesException
1156          */
1157         protected function getContainer( $container, $bypassCache = false ) {
1158                 $conn = $this->getConnection(); // Swift proxy connection
1159                 if ( $bypassCache ) { // purge cache
1160                         unset( $this->connContainers[$container] );
1161                 } elseif ( !isset( $this->connContainers[$container] ) ) {
1162                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1163                 }
1164                 if ( !isset( $this->connContainers[$container] ) ) {
1165                         $contObj = $conn->get_container( $container );
1166                         // NoSuchContainerException not thrown: container must exist
1167                         if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache?
1168                                 reset( $this->connContainers );
1169                                 unset( $this->connContainers[key( $this->connContainers )] );
1170                         }
1171                         $this->connContainers[$container] = $contObj; // cache it
1172                         if ( !$bypassCache ) {
1173                                 $this->setContainerCache( $container, // update persistent cache
1174                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1175                                 );
1176                         }
1177                 }
1178                 return $this->connContainers[$container];
1179         }
1180
1181         /**
1182          * Create a Swift container
1183          *
1184          * @param $container string Container name
1185          * @return CF_Container
1186          * @throws CloudFilesException
1187          */
1188         protected function createContainer( $container ) {
1189                 $conn = $this->getConnection(); // Swift proxy connection
1190                 $contObj = $conn->create_container( $container );
1191                 $this->connContainers[$container] = $contObj; // cache it
1192                 return $contObj;
1193         }
1194
1195         /**
1196          * Delete a Swift container
1197          *
1198          * @param $container string Container name
1199          * @return void
1200          * @throws CloudFilesException
1201          */
1202         protected function deleteContainer( $container ) {
1203                 $conn = $this->getConnection(); // Swift proxy connection
1204                 unset( $this->connContainers[$container] ); // purge cache
1205                 $conn->delete_container( $container );
1206         }
1207
1208         /**
1209          * @see FileBackendStore::doPrimeContainerCache()
1210          * @return void
1211          */
1212         protected function doPrimeContainerCache( array $containerInfo ) {
1213                 try {
1214                         $conn = $this->getConnection(); // Swift proxy connection
1215                         foreach ( $containerInfo as $container => $info ) {
1216                                 $this->connContainers[$container] = new CF_Container(
1217                                         $conn->cfs_auth,
1218                                         $conn->cfs_http,
1219                                         $container,
1220                                         $info['count'],
1221                                         $info['bytes']
1222                                 );
1223                         }
1224                 } catch ( CloudFilesException $e ) { // some other exception?
1225                         $this->handleException( $e, null, __METHOD__, array() );
1226                 }
1227         }
1228
1229         /**
1230          * Log an unexpected exception for this backend.
1231          * This also sets the Status object to have a fatal error.
1232          *
1233          * @param $e Exception
1234          * @param $status Status|null
1235          * @param $func string
1236          * @param $params Array
1237          * @return void
1238          */
1239         protected function handleException( Exception $e, $status, $func, array $params ) {
1240                 if ( $status instanceof Status ) {
1241                         if ( $e instanceof AuthenticationException ) {
1242                                 $status->fatal( 'backend-fail-connect', $this->name );
1243                         } else {
1244                                 $status->fatal( 'backend-fail-internal', $this->name );
1245                         }
1246                 }
1247                 if ( $e->getMessage() ) {
1248                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1249                 }
1250                 wfDebugLog( 'SwiftBackend',
1251                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1252                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1253                 );
1254         }
1255 }
1256
1257 /**
1258  * @see FileBackendStoreOpHandle
1259  */
1260 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1261         /** @var CF_Async_Op */
1262         public $cfOp;
1263         /** @var Array */
1264         public $affectedObjects = array();
1265
1266         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1267                 $this->backend = $backend;
1268                 $this->params = $params;
1269                 $this->call = $call;
1270                 $this->cfOp = $cfOp;
1271         }
1272 }
1273
1274 /**
1275  * SwiftFileBackend helper class to page through listings.
1276  * Swift also has a listing limit of 10,000 objects for sanity.
1277  * Do not use this class from places outside SwiftFileBackend.
1278  *
1279  * @ingroup FileBackend
1280  */
1281 abstract class SwiftFileBackendList implements Iterator {
1282         /** @var Array */
1283         protected $bufferIter = array();
1284         protected $bufferAfter = null; // string; list items *after* this path
1285         protected $pos = 0; // integer
1286         /** @var Array */
1287         protected $params = array();
1288
1289         /** @var SwiftFileBackend */
1290         protected $backend;
1291         protected $container; // string; container name
1292         protected $dir; // string; storage directory
1293         protected $suffixStart; // integer
1294
1295         const PAGE_SIZE = 5000; // file listing buffer size
1296
1297         /**
1298          * @param $backend SwiftFileBackend
1299          * @param $fullCont string Resolved container name
1300          * @param $dir string Resolved directory relative to container
1301          * @param $params Array
1302          */
1303         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1304                 $this->backend = $backend;
1305                 $this->container = $fullCont;
1306                 $this->dir = $dir;
1307                 if ( substr( $this->dir, -1 ) === '/' ) {
1308                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1309                 }
1310                 if ( $this->dir == '' ) { // whole container
1311                         $this->suffixStart = 0;
1312                 } else { // dir within container
1313                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1314                 }
1315                 $this->params = $params;
1316         }
1317
1318         /**
1319          * @see Iterator::key()
1320          * @return integer
1321          */
1322         public function key() {
1323                 return $this->pos;
1324         }
1325
1326         /**
1327          * @see Iterator::next()
1328          * @return void
1329          */
1330         public function next() {
1331                 // Advance to the next file in the page
1332                 next( $this->bufferIter );
1333                 ++$this->pos;
1334                 // Check if there are no files left in this page and
1335                 // advance to the next page if this page was not empty.
1336                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1337                         $this->bufferIter = $this->pageFromList(
1338                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1339                         ); // updates $this->bufferAfter
1340                 }
1341         }
1342
1343         /**
1344          * @see Iterator::rewind()
1345          * @return void
1346          */
1347         public function rewind() {
1348                 $this->pos = 0;
1349                 $this->bufferAfter = null;
1350                 $this->bufferIter = $this->pageFromList(
1351                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1352                 ); // updates $this->bufferAfter
1353         }
1354
1355         /**
1356          * @see Iterator::valid()
1357          * @return bool
1358          */
1359         public function valid() {
1360                 if ( $this->bufferIter === null ) {
1361                         return false; // some failure?
1362                 } else {
1363                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1364                 }
1365         }
1366
1367         /**
1368          * Get the given list portion (page)
1369          *
1370          * @param $container string Resolved container name
1371          * @param $dir string Resolved path relative to container
1372          * @param $after string|null
1373          * @param $limit integer
1374          * @param $params Array
1375          * @return Traversable|Array|null Returns null on failure
1376          */
1377         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1378 }
1379
1380 /**
1381  * Iterator for listing directories
1382  */
1383 class SwiftFileBackendDirList extends SwiftFileBackendList {
1384         /**
1385          * @see Iterator::current()
1386          * @return string|bool String (relative path) or false
1387          */
1388         public function current() {
1389                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1390         }
1391
1392         /**
1393          * @see SwiftFileBackendList::pageFromList()
1394          * @return Array|null
1395          */
1396         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1397                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1398         }
1399 }
1400
1401 /**
1402  * Iterator for listing regular files
1403  */
1404 class SwiftFileBackendFileList extends SwiftFileBackendList {
1405         /**
1406          * @see Iterator::current()
1407          * @return string|bool String (relative path) or false
1408          */
1409         public function current() {
1410                 return substr( current( $this->bufferIter ), $this->suffixStart );
1411         }
1412
1413         /**
1414          * @see SwiftFileBackendList::pageFromList()
1415          * @return Array|null
1416          */
1417         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1418                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1419         }
1420 }