includes/filebackend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftTempUrlKey; // string; shared secret value for making temp urls
  44         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  45         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  46         protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
  47         protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
  48
  49         // Rados Gateway specific options
  50         protected $rgwS3AccessKey; // string; S3 access key
  51         protected $rgwS3SecretKey; // string; S3 authentication key
  52
  53         /** @var CF_Connection */
  54         protected $conn; // Swift connection handle
  55         protected $sessionStarted = 0; // integer UNIX timestamp
  56
  57         /** @var CloudFilesException */
  58         protected $connException;
  59         protected $connErrorTime = 0; // UNIX timestamp
  60
  61         /** @var BagOStuff */
  62         protected $srvCache;
  63
  64         /** @var ProcessCacheLRU */
  65         protected $connContainerCache; // container object cache
  66
  67         /**
  68          * @see FileBackendStore::__construct()
  69          * Additional $config params include:
  70          *   - swiftAuthUrl       : Swift authentication server URL
  71          *   - swiftUser          : Swift user used by MediaWiki (account:username)
  72          *   - swiftKey           : Swift authentication key for the above user
  73          *   - swiftAuthTTL       : Swift authentication TTL (seconds)
  74          *   - swiftTempUrlKey    : Swift "X-Account-Meta-Temp-URL-Key" value on the account.
  75          *                          Do not set this until it has been set in the backend.
  76          *   - swiftAnonUser      : Swift user used for end-user requests (account:username).
  77          *                          If set, then views of public containers are assumed to go
  78          *                          through this user. If not set, then public containers are
  79          *                          accessible to unauthenticated requests via ".r:*" in the ACL.
  80          *   - swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  81          *   - swiftCDNExpiry     : How long (in seconds) to store content in the CDN.
  82          *                          If files may likely change, this should probably not exceed
  83          *                          a few days. For example, deletions may take this long to apply.
  84          *                          If object purging is enabled, however, this is not an issue.
  85          *   - swiftCDNPurgable   : Whether object purge requests are allowed by the CDN.
  86          *   - shardViaHashLevels : Map of container names to sharding config with:
  87          *                             - base   : base of hash characters, 16 or 36
  88          *                             - levels : the number of hash levels (and digits)
  89          *                             - repeat : hash subdirectories are prefixed with all the
  90          *                                        parent hash directory names (e.g. "a/ab/abc")
  91          *   - cacheAuthInfo      : Whether to cache authentication tokens in APC, XCache, ect.
  92          *                          If those are not available, then the main cache will be used.
  93          *                          This is probably insecure in shared hosting environments.
  94          *   - rgwS3AccessKey     : Ragos Gateway S3 "access key" value on the account.
  95          *                          Do not set this until it has been set in the backend.
  96          *                          This is used for generating expiring pre-authenticated URLs.
  97          *                          Only use this when using rgw and to work around
  98          *                          http://tracker.newdream.net/issues/3454.
  99          *   - rgwS3SecretKey     : Ragos Gateway S3 "secret key" value on the account.
 100          *                          Do not set this until it has been set in the backend.
 101          *                          This is used for generating expiring pre-authenticated URLs.
 102          *                          Only use this when using rgw and to work around
 103          *                          http://tracker.newdream.net/issues/3454.
 104          */
 105         public function __construct( array $config ) {
 106                 parent::__construct( $config );
 107                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
 108                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
 109                 }
 110                 // Required settings
 111                 $this->auth = new CF_Authentication(
 112                         $config['swiftUser'],
 113                         $config['swiftKey'],
 114                         null, // account; unused
 115                         $config['swiftAuthUrl']
 116                 );
 117                 // Optional settings
 118                 $this->authTTL = isset( $config['swiftAuthTTL'] )
 119                         ? $config['swiftAuthTTL']
 120                         : 5 * 60; // some sane number
 121                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
 122                         ? $config['swiftAnonUser']
 123                         : '';
 124                 $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] )
 125                         ? $config['swiftTempUrlKey']
 126                         : '';
 127                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
 128                         ? $config['shardViaHashLevels']
 129                         : '';
 130                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
 131                         ? $config['swiftUseCDN']
 132                         : false;
 133                 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
 134                         ? $config['swiftCDNExpiry']
 135                         : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
 136                 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
 137                         ? $config['swiftCDNPurgable']
 138                         : true;
 139                 $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] )
 140                         ? $config['rgwS3AccessKey']
 141                         : '';
 142                 $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] )
 143                         ? $config['rgwS3SecretKey']
 144                         : '';
 145                 // Cache container information to mask latency
 146                 $this->memCache = wfGetMainCache();
 147                 // Process cache for container info
 148                 $this->connContainerCache = new ProcessCacheLRU( 300 );
 149                 // Cache auth token information to avoid RTTs
 150                 if ( !empty( $config['cacheAuthInfo'] ) ) {
 151                         if ( PHP_SAPI === 'cli' ) {
 152                                 $this->srvCache = wfGetMainCache(); // preferrably memcached
 153                         } else {
 154                                 try { // look for APC, XCache, WinCache, ect...
 155                                         $this->srvCache = ObjectCache::newAccelerator( array() );
 156                                 } catch ( Exception $e ) {}
 157                         }
 158                 }
 159                 $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
 160         }
 161
 162         /**
 163          * @see FileBackendStore::resolveContainerPath()
 164          * @return null
 165          */
 166         protected function resolveContainerPath( $container, $relStoragePath ) {
 167                 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
 168                         return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
 169                 } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 170                         return null; // too long for Swift
 171                 }
 172                 return $relStoragePath;
 173         }
 174
 175         /**
 176          * @see FileBackendStore::isPathUsableInternal()
 177          * @return bool
 178          */
 179         public function isPathUsableInternal( $storagePath ) {
 180                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 181                 if ( $rel === null ) {
 182                         return false; // invalid
 183                 }
 184
 185                 try {
 186                         $this->getContainer( $container );
 187                         return true; // container exists
 188                 } catch ( NoSuchContainerException $e ) {
 189                 } catch ( CloudFilesException $e ) { // some other exception?
 190                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 191                 }
 192
 193                 return false;
 194         }
 195
 196         /**
 197          * @param string $disposition Content-Disposition header value
 198          * @return string Truncated Content-Disposition header value to meet Swift limits
 199          */
 200         protected function truncDisp( $disposition ) {
 201                 $res = '';
 202                 foreach ( explode( ';', $disposition ) as $part ) {
 203                         $part = trim( $part );
 204                         $new = ( $res === '' ) ? $part : "{$res};{$part}";
 205                         if ( strlen( $new ) <= 255 ) {
 206                                 $res = $new;
 207                         } else {
 208                                 break; // too long; sigh
 209                         }
 210                 }
 211                 return $res;
 212         }
 213
 214         /**
 215          * @see FileBackendStore::doCreateInternal()
 216          * @return Status
 217          */
 218         protected function doCreateInternal( array $params ) {
 219                 $status = Status::newGood();
 220
 221                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 222                 if ( $dstRel === null ) {
 223                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 224                         return $status;
 225                 }
 226
 227                 // (a) Check the destination container and object
 228                 try {
 229                         $dContObj = $this->getContainer( $dstCont );
 230                 } catch ( NoSuchContainerException $e ) {
 231                         $status->fatal( 'backend-fail-create', $params['dst'] );
 232                         return $status;
 233                 } catch ( CloudFilesException $e ) { // some other exception?
 234                         $this->handleException( $e, $status, __METHOD__, $params );
 235                         return $status;
 236                 }
 237
 238                 // (b) Get a SHA-1 hash of the object
 239                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 240
 241                 // (c) Actually create the object
 242                 try {
 243                         // Create a fresh CF_Object with no fields preloaded.
 244                         // We don't want to preserve headers, metadata, and such.
 245                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 246                         $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
 247                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 248                         // The MD5 here will be checked within Swift against its own MD5.
 249                         $obj->set_etag( md5( $params['content'] ) );
 250                         // Use the same content type as StreamFile for security
 251                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 252                         if ( !strlen( $obj->content_type ) ) { // special case
 253                                 $obj->content_type = 'unknown/unknown';
 254                         }
 255                         // Set the Content-Disposition header if requested
 256                         if ( isset( $params['disposition'] ) ) {
 257                                 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 258                         }
 259                         // Set any other custom headers if requested
 260                         if ( isset( $params['headers'] ) ) {
 261                                 $obj->headers += $params['headers'];
 262                         }
 263                         if ( !empty( $params['async'] ) ) { // deferred
 264                                 $op = $obj->write_async( $params['content'] );
 265                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op );
 266                                 $status->value->affectedObjects[] = $obj;
 267                         } else { // actually write the object in Swift
 268                                 $obj->write( $params['content'] );
 269                                 $this->purgeCDNCache( array( $obj ) );
 270                         }
 271                 } catch ( CDNNotEnabledException $e ) {
 272                         // CDN not enabled; nothing to see here
 273                 } catch ( BadContentTypeException $e ) {
 274                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 275                 } catch ( CloudFilesException $e ) { // some other exception?
 276                         $this->handleException( $e, $status, __METHOD__, $params );
 277                 }
 278
 279                 return $status;
 280         }
 281
 282         /**
 283          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 284          */
 285         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 286                 try {
 287                         $cfOp->getLastResponse();
 288                 } catch ( BadContentTypeException $e ) {
 289                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 290                 }
 291         }
 292
 293         /**
 294          * @see FileBackendStore::doStoreInternal()
 295          * @return Status
 296          */
 297         protected function doStoreInternal( array $params ) {
 298                 $status = Status::newGood();
 299
 300                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 301                 if ( $dstRel === null ) {
 302                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 303                         return $status;
 304                 }
 305
 306                 // (a) Check the destination container and object
 307                 try {
 308                         $dContObj = $this->getContainer( $dstCont );
 309                 } catch ( NoSuchContainerException $e ) {
 310                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 311                         return $status;
 312                 } catch ( CloudFilesException $e ) { // some other exception?
 313                         $this->handleException( $e, $status, __METHOD__, $params );
 314                         return $status;
 315                 }
 316
 317                 // (b) Get a SHA-1 hash of the object
 318                 wfSuppressWarnings();
 319                 $sha1Hash = sha1_file( $params['src'] );
 320                 wfRestoreWarnings();
 321                 if ( $sha1Hash === false ) { // source doesn't exist?
 322                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 323                         return $status;
 324                 }
 325                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 326
 327                 // (c) Actually store the object
 328                 try {
 329                         // Create a fresh CF_Object with no fields preloaded.
 330                         // We don't want to preserve headers, metadata, and such.
 331                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 332                         $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
 333                         // The MD5 here will be checked within Swift against its own MD5.
 334                         $obj->set_etag( md5_file( $params['src'] ) );
 335                         // Use the same content type as StreamFile for security
 336                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 337                         if ( !strlen( $obj->content_type ) ) { // special case
 338                                 $obj->content_type = 'unknown/unknown';
 339                         }
 340                         // Set the Content-Disposition header if requested
 341                         if ( isset( $params['disposition'] ) ) {
 342                                 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 343                         }
 344                         // Set any other custom headers if requested
 345                         if ( isset( $params['headers'] ) ) {
 346                                 $obj->headers += $params['headers'];
 347                         }
 348                         if ( !empty( $params['async'] ) ) { // deferred
 349                                 wfSuppressWarnings();
 350                                 $fp = fopen( $params['src'], 'rb' );
 351                                 wfRestoreWarnings();
 352                                 if ( !$fp ) {
 353                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 354                                 } else {
 355                                         $op = $obj->write_async( $fp, filesize( $params['src'] ), true );
 356                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op );
 357                                         $status->value->resourcesToClose[] = $fp;
 358                                         $status->value->affectedObjects[] = $obj;
 359                                 }
 360                         } else { // actually write the object in Swift
 361                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 362                                 $this->purgeCDNCache( array( $obj ) );
 363                         }
 364                 } catch ( CDNNotEnabledException $e ) {
 365                         // CDN not enabled; nothing to see here
 366                 } catch ( BadContentTypeException $e ) {
 367                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 368                 } catch ( IOException $e ) {
 369                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 370                 } catch ( CloudFilesException $e ) { // some other exception?
 371                         $this->handleException( $e, $status, __METHOD__, $params );
 372                 }
 373
 374                 return $status;
 375         }
 376
 377         /**
 378          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 379          */
 380         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 381                 try {
 382                         $cfOp->getLastResponse();
 383                 } catch ( BadContentTypeException $e ) {
 384                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 385                 } catch ( IOException $e ) {
 386                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 387                 }
 388         }
 389
 390         /**
 391          * @see FileBackendStore::doCopyInternal()
 392          * @return Status
 393          */
 394         protected function doCopyInternal( array $params ) {
 395                 $status = Status::newGood();
 396
 397                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 398                 if ( $srcRel === null ) {
 399                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 400                         return $status;
 401                 }
 402
 403                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 404                 if ( $dstRel === null ) {
 405                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 406                         return $status;
 407                 }
 408
 409                 // (a) Check the source/destination containers and destination object
 410                 try {
 411                         $sContObj = $this->getContainer( $srcCont );
 412                         $dContObj = $this->getContainer( $dstCont );
 413                 } catch ( NoSuchContainerException $e ) {
 414                         if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) {
 415                                 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 416                         }
 417                         return $status;
 418                 } catch ( CloudFilesException $e ) { // some other exception?
 419                         $this->handleException( $e, $status, __METHOD__, $params );
 420                         return $status;
 421                 }
 422
 423                 // (b) Actually copy the file to the destination
 424                 try {
 425                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 426                         $hdrs = array(); // source file headers to override with new values
 427                         if ( isset( $params['disposition'] ) ) {
 428                                 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 429                         }
 430                         if ( !empty( $params['async'] ) ) { // deferred
 431                                 $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
 432                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op );
 433                                 $status->value->affectedObjects[] = $dstObj;
 434                         } else { // actually write the object in Swift
 435                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
 436                                 $this->purgeCDNCache( array( $dstObj ) );
 437                         }
 438                 } catch ( CDNNotEnabledException $e ) {
 439                         // CDN not enabled; nothing to see here
 440                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 441                         if ( empty( $params['ignoreMissingSource'] ) ) {
 442                                 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 443                         }
 444                 } catch ( CloudFilesException $e ) { // some other exception?
 445                         $this->handleException( $e, $status, __METHOD__, $params );
 446                 }
 447
 448                 return $status;
 449         }
 450
 451         /**
 452          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 453          */
 454         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 455                 try {
 456                         $cfOp->getLastResponse();
 457                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 458                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 459                 }
 460         }
 461
 462         /**
 463          * @see FileBackendStore::doMoveInternal()
 464          * @return Status
 465          */
 466         protected function doMoveInternal( array $params ) {
 467                 $status = Status::newGood();
 468
 469                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 470                 if ( $srcRel === null ) {
 471                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 472                         return $status;
 473                 }
 474
 475                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 476                 if ( $dstRel === null ) {
 477                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 478                         return $status;
 479                 }
 480
 481                 // (a) Check the source/destination containers and destination object
 482                 try {
 483                         $sContObj = $this->getContainer( $srcCont );
 484                         $dContObj = $this->getContainer( $dstCont );
 485                 } catch ( NoSuchContainerException $e ) {
 486                         if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) {
 487                                 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 488                         }
 489                         return $status;
 490                 } catch ( CloudFilesException $e ) { // some other exception?
 491                         $this->handleException( $e, $status, __METHOD__, $params );
 492                         return $status;
 493                 }
 494
 495                 // (b) Actually move the file to the destination
 496                 try {
 497                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 498                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 499                         $hdrs = array(); // source file headers to override with new values
 500                         if ( isset( $params['disposition'] ) ) {
 501                                 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 502                         }
 503                         if ( !empty( $params['async'] ) ) { // deferred
 504                                 $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
 505                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op );
 506                                 $status->value->affectedObjects[] = $srcObj;
 507                                 $status->value->affectedObjects[] = $dstObj;
 508                         } else { // actually write the object in Swift
 509                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
 510                                 $this->purgeCDNCache( array( $srcObj ) );
 511                                 $this->purgeCDNCache( array( $dstObj ) );
 512                         }
 513                 } catch ( CDNNotEnabledException $e ) {
 514                         // CDN not enabled; nothing to see here
 515                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 516                         if ( empty( $params['ignoreMissingSource'] ) ) {
 517                                 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 518                         }
 519                 } catch ( CloudFilesException $e ) { // some other exception?
 520                         $this->handleException( $e, $status, __METHOD__, $params );
 521                 }
 522
 523                 return $status;
 524         }
 525
 526         /**
 527          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 528          */
 529         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 530                 try {
 531                         $cfOp->getLastResponse();
 532                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 533                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 534                 }
 535         }
 536
 537         /**
 538          * @see FileBackendStore::doDeleteInternal()
 539          * @return Status
 540          */
 541         protected function doDeleteInternal( array $params ) {
 542                 $status = Status::newGood();
 543
 544                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 545                 if ( $srcRel === null ) {
 546                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 547                         return $status;
 548                 }
 549
 550                 try {
 551                         $sContObj = $this->getContainer( $srcCont );
 552                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 553                         if ( !empty( $params['async'] ) ) { // deferred
 554                                 $op = $sContObj->delete_object_async( $srcRel );
 555                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $op );
 556                                 $status->value->affectedObjects[] = $srcObj;
 557                         } else { // actually write the object in Swift
 558                                 $sContObj->delete_object( $srcRel );
 559                                 $this->purgeCDNCache( array( $srcObj ) );
 560                         }
 561                 } catch ( CDNNotEnabledException $e ) {
 562                         // CDN not enabled; nothing to see here
 563                 } catch ( NoSuchContainerException $e ) {
 564                         if ( empty( $params['ignoreMissingSource'] ) ) {
 565                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 566                         }
 567                 } catch ( NoSuchObjectException $e ) {
 568                         if ( empty( $params['ignoreMissingSource'] ) ) {
 569                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 570                         }
 571                 } catch ( CloudFilesException $e ) { // some other exception?
 572                         $this->handleException( $e, $status, __METHOD__, $params );
 573                 }
 574
 575                 return $status;
 576         }
 577
 578         /**
 579          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 580          */
 581         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 582                 try {
 583                         $cfOp->getLastResponse();
 584                 } catch ( NoSuchContainerException $e ) {
 585                         $status->fatal( 'backend-fail-delete', $params['src'] );
 586                 } catch ( NoSuchObjectException $e ) {
 587                         if ( empty( $params['ignoreMissingSource'] ) ) {
 588                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 589                         }
 590                 }
 591         }
 592
 593         /**
 594          * @see FileBackendStore::doDescribeInternal()
 595          * @return Status
 596          */
 597         protected function doDescribeInternal( array $params ) {
 598                 $status = Status::newGood();
 599
 600                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 601                 if ( $srcRel === null ) {
 602                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 603                         return $status;
 604                 }
 605
 606                 $hdrs = isset( $params['headers'] ) ? $params['headers'] : array();
 607                 // Set the Content-Disposition header if requested
 608                 if ( isset( $params['disposition'] ) ) {
 609                         $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 610                 }
 611
 612                 try {
 613                         $sContObj = $this->getContainer( $srcCont );
 614                         // Get the latest version of the current metadata
 615                         $srcObj = $sContObj->get_object( $srcRel,
 616                                 $this->headersFromParams( array( 'latest' => true ) ) );
 617                         // Merge in the metadata updates...
 618                         $srcObj->headers = $hdrs + $srcObj->headers;
 619                         $srcObj->sync_metadata(); // save to Swift
 620                         $this->purgeCDNCache( array( $srcObj ) );
 621                 } catch ( CDNNotEnabledException $e ) {
 622                         // CDN not enabled; nothing to see here
 623                 } catch ( NoSuchContainerException $e ) {
 624                         $status->fatal( 'backend-fail-describe', $params['src'] );
 625                 } catch ( NoSuchObjectException $e ) {
 626                         $status->fatal( 'backend-fail-describe', $params['src'] );
 627                 } catch ( CloudFilesException $e ) { // some other exception?
 628                         $this->handleException( $e, $status, __METHOD__, $params );
 629                 }
 630
 631                 return $status;
 632         }
 633
 634         /**
 635          * @see FileBackendStore::doPrepareInternal()
 636          * @return Status
 637          */
 638         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 639                 $status = Status::newGood();
 640
 641                 // (a) Check if container already exists
 642                 try {
 643                         $this->getContainer( $fullCont );
 644                         // NoSuchContainerException not thrown: container must exist
 645                         return $status; // already exists
 646                 } catch ( NoSuchContainerException $e ) {
 647                         // NoSuchContainerException thrown: container does not exist
 648                 } catch ( CloudFilesException $e ) { // some other exception?
 649                         $this->handleException( $e, $status, __METHOD__, $params );
 650                         return $status;
 651                 }
 652
 653                 // (b) Create container as needed
 654                 try {
 655                         $contObj = $this->createContainer( $fullCont );
 656                         if ( !empty( $params['noAccess'] ) ) {
 657                                 // Make container private to end-users...
 658                                 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 659                         } else {
 660                                 // Make container public to end-users...
 661                                 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 662                         }
 663                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 664                                 $contObj->make_public( $this->swiftCDNExpiry );
 665                         }
 666                 } catch ( CDNNotEnabledException $e ) {
 667                         // CDN not enabled; nothing to see here
 668                 } catch ( CloudFilesException $e ) { // some other exception?
 669                         $this->handleException( $e, $status, __METHOD__, $params );
 670                         return $status;
 671                 }
 672
 673                 return $status;
 674         }
 675
 676         /**
 677          * @see FileBackendStore::doSecureInternal()
 678          * @return Status
 679          */
 680         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 681                 $status = Status::newGood();
 682                 if ( empty( $params['noAccess'] ) ) {
 683                         return $status; // nothing to do
 684                 }
 685
 686                 // Restrict container from end-users...
 687                 try {
 688                         // doPrepareInternal() should have been called,
 689                         // so the Swift container should already exist...
 690                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 691                         // NoSuchContainerException not thrown: container must exist
 692
 693                         // Make container private to end-users...
 694                         $status->merge( $this->setContainerAccess(
 695                                 $contObj,
 696                                 array( $this->auth->username ), // read
 697                                 array( $this->auth->username ) // write
 698                         ) );
 699                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 700                                 $contObj->make_private();
 701                         }
 702                 } catch ( CDNNotEnabledException $e ) {
 703                         // CDN not enabled; nothing to see here
 704                 } catch ( CloudFilesException $e ) { // some other exception?
 705                         $this->handleException( $e, $status, __METHOD__, $params );
 706                 }
 707
 708                 return $status;
 709         }
 710
 711         /**
 712          * @see FileBackendStore::doPublishInternal()
 713          * @return Status
 714          */
 715         protected function doPublishInternal( $fullCont, $dir, array $params ) {
 716                 $status = Status::newGood();
 717
 718                 // Unrestrict container from end-users...
 719                 try {
 720                         // doPrepareInternal() should have been called,
 721                         // so the Swift container should already exist...
 722                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 723                         // NoSuchContainerException not thrown: container must exist
 724
 725                         // Make container public to end-users...
 726                         if ( $this->swiftAnonUser != '' ) {
 727                                 $status->merge( $this->setContainerAccess(
 728                                         $contObj,
 729                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 730                                         array( $this->auth->username, $this->swiftAnonUser ) // write
 731                                 ) );
 732                         } else {
 733                                 $status->merge( $this->setContainerAccess(
 734                                         $contObj,
 735                                         array( $this->auth->username, '.r:*' ), // read
 736                                         array( $this->auth->username ) // write
 737                                 ) );
 738                         }
 739                         if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
 740                                 $contObj->make_public();
 741                         }
 742                 } catch ( CDNNotEnabledException $e ) {
 743                         // CDN not enabled; nothing to see here
 744                 } catch ( CloudFilesException $e ) { // some other exception?
 745                         $this->handleException( $e, $status, __METHOD__, $params );
 746                 }
 747
 748                 return $status;
 749         }
 750
 751         /**
 752          * @see FileBackendStore::doCleanInternal()
 753          * @return Status
 754          */
 755         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 756                 $status = Status::newGood();
 757
 758                 // Only containers themselves can be removed, all else is virtual
 759                 if ( $dir != '' ) {
 760                         return $status; // nothing to do
 761                 }
 762
 763                 // (a) Check the container
 764                 try {
 765                         $contObj = $this->getContainer( $fullCont, true );
 766                 } catch ( NoSuchContainerException $e ) {
 767                         return $status; // ok, nothing to do
 768                 } catch ( CloudFilesException $e ) { // some other exception?
 769                         $this->handleException( $e, $status, __METHOD__, $params );
 770                         return $status;
 771                 }
 772
 773                 // (b) Delete the container if empty
 774                 if ( $contObj->object_count == 0 ) {
 775                         try {
 776                                 $this->deleteContainer( $fullCont );
 777                         } catch ( NoSuchContainerException $e ) {
 778                                 return $status; // race?
 779                         } catch ( NonEmptyContainerException $e ) {
 780                                 return $status; // race? consistency delay?
 781                         } catch ( CloudFilesException $e ) { // some other exception?
 782                                 $this->handleException( $e, $status, __METHOD__, $params );
 783                                 return $status;
 784                         }
 785                 }
 786
 787                 return $status;
 788         }
 789
 790         /**
 791          * @see FileBackendStore::doFileExists()
 792          * @return array|bool|null
 793          */
 794         protected function doGetFileStat( array $params ) {
 795                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 796                 if ( $srcRel === null ) {
 797                         return false; // invalid storage path
 798                 }
 799
 800                 $stat = false;
 801                 try {
 802                         $contObj = $this->getContainer( $srcCont );
 803                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 804                         $this->addMissingMetadata( $srcObj, $params['src'] );
 805                         $stat = array(
 806                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 807                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 808                                 'size'  => (int)$srcObj->content_length,
 809                                 'sha1'  => $srcObj->getMetadataValue( 'Sha1base36' )
 810                         );
 811                 } catch ( NoSuchContainerException $e ) {
 812                 } catch ( NoSuchObjectException $e ) {
 813                 } catch ( CloudFilesException $e ) { // some other exception?
 814                         $stat = null;
 815                         $this->handleException( $e, null, __METHOD__, $params );
 816                 }
 817
 818                 return $stat;
 819         }
 820
 821         /**
 822          * Fill in any missing object metadata and save it to Swift
 823          *
 824          * @param $obj CF_Object
 825          * @param string $path Storage path to object
 826          * @return bool Success
 827          * @throws Exception cloudfiles exceptions
 828          */
 829         protected function addMissingMetadata( CF_Object $obj, $path ) {
 830                 if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) {
 831                         return true; // nothing to do
 832                 }
 833                 wfProfileIn( __METHOD__ );
 834                 trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING );
 835                 $status = Status::newGood();
 836                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 837                 if ( $status->isOK() ) {
 838                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
 839                         if ( $tmpFile ) {
 840                                 $hash = $tmpFile->getSha1Base36();
 841                                 if ( $hash !== false ) {
 842                                         $obj->setMetadataValues( array( 'Sha1base36' => $hash ) );
 843                                         $obj->sync_metadata(); // save to Swift
 844                                         wfProfileOut( __METHOD__ );
 845                                         return true; // success
 846                                 }
 847                         }
 848                 }
 849                 trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING );
 850                 $obj->setMetadataValues( array( 'Sha1base36' => false ) );
 851                 wfProfileOut( __METHOD__ );
 852                 return false; // failed
 853         }
 854
 855         /**
 856          * @see FileBackendStore::doGetFileContentsMulti()
 857          * @return Array
 858          */
 859         protected function doGetFileContentsMulti( array $params ) {
 860                 $contents = array();
 861
 862                 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
 863                 // Blindly create tmp files and stream to them, catching any exception if the file does
 864                 // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
 865                 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
 866                         $cfOps = array(); // (path => CF_Async_Op)
 867
 868                         foreach ( $pathBatch as $path ) { // each path in this concurrent batch
 869                                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
 870                                 if ( $srcRel === null ) {
 871                                         $contents[$path] = false;
 872                                         continue;
 873                                 }
 874                                 $data = false;
 875                                 try {
 876                                         $sContObj = $this->getContainer( $srcCont );
 877                                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 878                                         // Create a new temporary memory file...
 879                                         $handle = fopen( 'php://temp', 'wb' );
 880                                         if ( $handle ) {
 881                                                 $headers = $this->headersFromParams( $params );
 882                                                 if ( count( $pathBatch ) > 1 ) {
 883                                                         $cfOps[$path] = $obj->stream_async( $handle, $headers );
 884                                                         $cfOps[$path]->_file_handle = $handle; // close this later
 885                                                 } else {
 886                                                         $obj->stream( $handle, $headers );
 887                                                         rewind( $handle ); // start from the beginning
 888                                                         $data = stream_get_contents( $handle );
 889                                                         fclose( $handle );
 890                                                 }
 891                                         } else {
 892                                                 $data = false;
 893                                         }
 894                                 } catch ( NoSuchContainerException $e ) {
 895                                         $data = false;
 896                                 } catch ( NoSuchObjectException $e ) {
 897                                         $data = false;
 898                                 } catch ( CloudFilesException $e ) { // some other exception?
 899                                         $data = false;
 900                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
 901                                 }
 902                                 $contents[$path] = $data;
 903                         }
 904
 905                         $batch = new CF_Async_Op_Batch( $cfOps );
 906                         $cfOps = $batch->execute();
 907                         foreach ( $cfOps as $path => $cfOp ) {
 908                                 try {
 909                                         $cfOp->getLastResponse();
 910                                         rewind( $cfOp->_file_handle ); // start from the beginning
 911                                         $contents[$path] = stream_get_contents( $cfOp->_file_handle );
 912                                 } catch ( NoSuchContainerException $e ) {
 913                                         $contents[$path] = false;
 914                                 } catch ( NoSuchObjectException $e ) {
 915                                         $contents[$path] = false;
 916                                 } catch ( CloudFilesException $e ) { // some other exception?
 917                                         $contents[$path] = false;
 918                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
 919                                 }
 920                                 fclose( $cfOp->_file_handle ); // close open handle
 921                         }
 922                 }
 923
 924                 return $contents;
 925         }
 926
 927         /**
 928          * @see FileBackendStore::doDirectoryExists()
 929          * @return bool|null
 930          */
 931         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 932                 try {
 933                         $container = $this->getContainer( $fullCont );
 934                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 935                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 936                 } catch ( NoSuchContainerException $e ) {
 937                         return false;
 938                 } catch ( CloudFilesException $e ) { // some other exception?
 939                         $this->handleException( $e, null, __METHOD__,
 940                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 941                 }
 942
 943                 return null; // error
 944         }
 945
 946         /**
 947          * @see FileBackendStore::getDirectoryListInternal()
 948          * @return SwiftFileBackendDirList
 949          */
 950         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 951                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 952         }
 953
 954         /**
 955          * @see FileBackendStore::getFileListInternal()
 956          * @return SwiftFileBackendFileList
 957          */
 958         public function getFileListInternal( $fullCont, $dir, array $params ) {
 959                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 960         }
 961
 962         /**
 963          * Do not call this function outside of SwiftFileBackendFileList
 964          *
 965          * @param string $fullCont Resolved container name
 966          * @param string $dir Resolved storage directory with no trailing slash
 967          * @param string|null $after Storage path of file to list items after
 968          * @param $limit integer Max number of items to list
 969          * @param array $params Includes flag for 'topOnly'
 970          * @return Array List of relative paths of dirs directly under $dir
 971          */
 972         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 973                 $dirs = array();
 974                 if ( $after === INF ) {
 975                         return $dirs; // nothing more
 976                 }
 977                 wfProfileIn( __METHOD__ . '-' . $this->name );
 978
 979                 try {
 980                         $container = $this->getContainer( $fullCont );
 981                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 982                         // Non-recursive: only list dirs right under $dir
 983                         if ( !empty( $params['topOnly'] ) ) {
 984                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 985                                 foreach ( $objects as $object ) { // files and dirs
 986                                         if ( substr( $object, -1 ) === '/' ) {
 987                                                 $dirs[] = $object; // directories end in '/'
 988                                         }
 989                                 }
 990                         // Recursive: list all dirs under $dir and its subdirs
 991                         } else {
 992                                 // Get directory from last item of prior page
 993                                 $lastDir = $this->getParentDir( $after ); // must be first page
 994                                 $objects = $container->list_objects( $limit, $after, $prefix );
 995                                 foreach ( $objects as $object ) { // files
 996                                         $objectDir = $this->getParentDir( $object ); // directory of object
 997                                         if ( $objectDir !== false && $objectDir !== $dir ) {
 998                                                 // Swift stores paths in UTF-8, using binary sorting.
 999                                                 // See function "create_container_table" in common/db.py.
1000                                                 // If a directory is not "greater" than the last one,
1001                                                 // then it was already listed by the calling iterator.
1002                                                 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
1003                                                         $pDir = $objectDir;
1004                                                         do { // add dir and all its parent dirs
1005                                                                 $dirs[] = "{$pDir}/";
1006                                                                 $pDir = $this->getParentDir( $pDir );
1007                                                         } while ( $pDir !== false // sanity
1008                                                                 && strcmp( $pDir, $lastDir ) > 0 // not done already
1009                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
1010                                                         );
1011                                                 }
1012                                                 $lastDir = $objectDir;
1013                                         }
1014                                 }
1015                         }
1016                         if ( count( $objects ) < $limit ) {
1017                                 $after = INF; // avoid a second RTT
1018                         } else {
1019                                 $after = end( $objects ); // update last item
1020                         }
1021                 } catch ( NoSuchContainerException $e ) {
1022                 } catch ( CloudFilesException $e ) { // some other exception?
1023                         $this->handleException( $e, null, __METHOD__,
1024                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
1025                 }
1026
1027                 wfProfileOut( __METHOD__ . '-' . $this->name );
1028                 return $dirs;
1029         }
1030
1031         protected function getParentDir( $path ) {
1032                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
1033         }
1034
1035         /**
1036          * Do not call this function outside of SwiftFileBackendFileList
1037          *
1038          * @param string $fullCont Resolved container name
1039          * @param string $dir Resolved storage directory with no trailing slash
1040          * @param string|null $after Storage path of file to list items after
1041          * @param $limit integer Max number of items to list
1042          * @param array $params Includes flag for 'topOnly'
1043          * @return Array List of relative paths of files under $dir
1044          */
1045         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
1046                 $files = array();
1047                 if ( $after === INF ) {
1048                         return $files; // nothing more
1049                 }
1050                 wfProfileIn( __METHOD__ . '-' . $this->name );
1051
1052                 try {
1053                         $container = $this->getContainer( $fullCont );
1054                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
1055                         // Non-recursive: only list files right under $dir
1056                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
1057                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
1058                                 foreach ( $objects as $object ) {
1059                                         if ( substr( $object, -1 ) !== '/' ) {
1060                                                 $files[] = $object; // directories end in '/'
1061                                         }
1062                                 }
1063                         // Recursive: list all files under $dir and its subdirs
1064                         } else { // files
1065                                 $objects = $container->list_objects( $limit, $after, $prefix );
1066                                 $files = $objects;
1067                         }
1068                         if ( count( $objects ) < $limit ) {
1069                                 $after = INF; // avoid a second RTT
1070                         } else {
1071                                 $after = end( $objects ); // update last item
1072                         }
1073                 } catch ( NoSuchContainerException $e ) {
1074                 } catch ( CloudFilesException $e ) { // some other exception?
1075                         $this->handleException( $e, null, __METHOD__,
1076                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
1077                 }
1078
1079                 wfProfileOut( __METHOD__ . '-' . $this->name );
1080                 return $files;
1081         }
1082
1083         /**
1084          * @see FileBackendStore::doGetFileSha1base36()
1085          * @return bool
1086          */
1087         protected function doGetFileSha1base36( array $params ) {
1088                 $stat = $this->getFileStat( $params );
1089                 if ( $stat ) {
1090                         return $stat['sha1'];
1091                 } else {
1092                         return false;
1093                 }
1094         }
1095
1096         /**
1097          * @see FileBackendStore::doStreamFile()
1098          * @return Status
1099          */
1100         protected function doStreamFile( array $params ) {
1101                 $status = Status::newGood();
1102
1103                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1104                 if ( $srcRel === null ) {
1105                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1106                 }
1107
1108                 try {
1109                         $cont = $this->getContainer( $srcCont );
1110                 } catch ( NoSuchContainerException $e ) {
1111                         $status->fatal( 'backend-fail-stream', $params['src'] );
1112                         return $status;
1113                 } catch ( CloudFilesException $e ) { // some other exception?
1114                         $this->handleException( $e, $status, __METHOD__, $params );
1115                         return $status;
1116                 }
1117
1118                 try {
1119                         $output = fopen( 'php://output', 'wb' );
1120                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
1121                         $obj->stream( $output, $this->headersFromParams( $params ) );
1122                 } catch ( NoSuchObjectException $e ) {
1123                         $status->fatal( 'backend-fail-stream', $params['src'] );
1124                 } catch ( CloudFilesException $e ) { // some other exception?
1125                         $this->handleException( $e, $status, __METHOD__, $params );
1126                 }
1127
1128                 return $status;
1129         }
1130
1131         /**
1132          * @see FileBackendStore::doGetLocalCopyMulti()
1133          * @return null|TempFSFile
1134          */
1135         protected function doGetLocalCopyMulti( array $params ) {
1136                 $tmpFiles = array();
1137
1138                 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
1139                 // Blindly create tmp files and stream to them, catching any exception if the file does
1140                 // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
1141                 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
1142                         $cfOps = array(); // (path => CF_Async_Op)
1143
1144                         foreach ( $pathBatch as $path ) { // each path in this concurrent batch
1145                                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1146                                 if ( $srcRel === null ) {
1147                                         $tmpFiles[$path] = null;
1148                                         continue;
1149                                 }
1150                                 $tmpFile = null;
1151                                 try {
1152                                         $sContObj = $this->getContainer( $srcCont );
1153                                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
1154                                         // Get source file extension
1155                                         $ext = FileBackend::extensionFromPath( $path );
1156                                         // Create a new temporary file...
1157                                         $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
1158                                         if ( $tmpFile ) {
1159                                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
1160                                                 if ( $handle ) {
1161                                                         $headers = $this->headersFromParams( $params );
1162                                                         if ( count( $pathBatch ) > 1 ) {
1163                                                                 $cfOps[$path] = $obj->stream_async( $handle, $headers );
1164                                                                 $cfOps[$path]->_file_handle = $handle; // close this later
1165                                                         } else {
1166                                                                 $obj->stream( $handle, $headers );
1167                                                                 fclose( $handle );
1168                                                         }
1169                                                 } else {
1170                                                         $tmpFile = null;
1171                                                 }
1172                                         }
1173                                 } catch ( NoSuchContainerException $e ) {
1174                                         $tmpFile = null;
1175                                 } catch ( NoSuchObjectException $e ) {
1176                                         $tmpFile = null;
1177                                 } catch ( CloudFilesException $e ) { // some other exception?
1178                                         $tmpFile = null;
1179                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
1180                                 }
1181                                 $tmpFiles[$path] = $tmpFile;
1182                         }
1183
1184                         $batch = new CF_Async_Op_Batch( $cfOps );
1185                         $cfOps = $batch->execute();
1186                         foreach ( $cfOps as $path => $cfOp ) {
1187                                 try {
1188                                         $cfOp->getLastResponse();
1189                                 } catch ( NoSuchContainerException $e ) {
1190                                         $tmpFiles[$path] = null;
1191                                 } catch ( NoSuchObjectException $e ) {
1192                                         $tmpFiles[$path] = null;
1193                                 } catch ( CloudFilesException $e ) { // some other exception?
1194                                         $tmpFiles[$path] = null;
1195                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
1196                                 }
1197                                 fclose( $cfOp->_file_handle ); // close open handle
1198                         }
1199                 }
1200
1201                 return $tmpFiles;
1202         }
1203
1204         /**
1205          * @see FileBackendStore::getFileHttpUrl()
1206          * @return string|null
1207          */
1208         public function getFileHttpUrl( array $params ) {
1209                 if ( $this->swiftTempUrlKey != '' ||
1210                         ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) )
1211                 {
1212                         list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1213                         if ( $srcRel === null ) {
1214                                 return null; // invalid path
1215                         }
1216                         try {
1217                                 $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400;
1218                                 $sContObj = $this->getContainer( $srcCont );
1219                                 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
1220                                 if ( $this->swiftTempUrlKey != '' ) {
1221                                         return $obj->get_temp_url( $this->swiftTempUrlKey, $ttl, "GET" );
1222                                 } else { // give S3 API URL for rgw
1223                                         $expires = time() + $ttl;
1224                                         // Path for signature starts with the bucket
1225                                         $spath = '/' . rawurlencode( $srcCont ) . '/' .
1226                                                 str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1227                                         // Calculate the hash
1228                                         $signature = base64_encode( hash_hmac(
1229                                                 'sha1',
1230                                                 "GET\n\n\n{$expires}\n{$spath}",
1231                                                 $this->rgwS3SecretKey,
1232                                                 true // raw
1233                                         ) );
1234                                         // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1235                                         // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1236                                         return wfAppendQuery(
1237                                                 str_replace( '/swift/v1', '', // S3 API is the rgw default
1238                                                         $sContObj->cfs_http->getStorageUrl() . $spath ),
1239                                                 array(
1240                                                         'Signature'      => $signature,
1241                                                         'Expires'        => $expires,
1242                                                         'AWSAccessKeyId' => $this->rgwS3AccessKey )
1243                                         );
1244                                 }
1245                         } catch ( NoSuchContainerException $e ) {
1246                         } catch ( CloudFilesException $e ) { // some other exception?
1247                                 $this->handleException( $e, null, __METHOD__, $params );
1248                         }
1249                 }
1250                 return null;
1251         }
1252
1253         /**
1254          * @see FileBackendStore::directoriesAreVirtual()
1255          * @return bool
1256          */
1257         protected function directoriesAreVirtual() {
1258                 return true;
1259         }
1260
1261         /**
1262          * Get headers to send to Swift when reading a file based
1263          * on a FileBackend params array, e.g. that of getLocalCopy().
1264          * $params is currently only checked for a 'latest' flag.
1265          *
1266          * @param array $params
1267          * @return Array
1268          */
1269         protected function headersFromParams( array $params ) {
1270                 $hdrs = array();
1271                 if ( !empty( $params['latest'] ) ) {
1272                         $hdrs[] = 'X-Newest: true';
1273                 }
1274                 return $hdrs;
1275         }
1276
1277         /**
1278          * @see FileBackendStore::doExecuteOpHandlesInternal()
1279          * @return Array List of corresponding Status objects
1280          */
1281         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1282                 $statuses = array();
1283
1284                 $cfOps = array(); // list of CF_Async_Op objects
1285                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1286                         $cfOps[$index] = $fileOpHandle->cfOp;
1287                 }
1288                 $batch = new CF_Async_Op_Batch( $cfOps );
1289
1290                 $cfOps = $batch->execute();
1291                 foreach ( $cfOps as $index => $cfOp ) {
1292                         $status = Status::newGood();
1293                         $function = '_getResponse' . $fileOpHandles[$index]->call;
1294                         try { // catch exceptions; update status
1295                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
1296                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
1297                         } catch ( CloudFilesException $e ) { // some other exception?
1298                                 $this->handleException( $e, $status,
1299                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
1300                         }
1301                         $statuses[$index] = $status;
1302                 }
1303
1304                 return $statuses;
1305         }
1306
1307         /**
1308          * Set read/write permissions for a Swift container.
1309          *
1310          * $readGrps is a list of the possible criteria for a request to have
1311          * access to read a container. Each item is one of the following formats:
1312          *   - account:user        : Grants access if the request is by the given user
1313          *   - ".r:<regex>"        : Grants access if the request is from a referrer host that
1314          *                           matches the expression and the request is not for a listing.
1315          *                           Setting this to '*' effectively makes a container public.
1316          *   -".rlistings:<regex>" : Grants access if the request is from a referrer host that
1317          *                           matches the expression and the request for a listing.
1318          *
1319          * $writeGrps is a list of the possible criteria for a request to have
1320          * access to write to a container. Each item is of the following format:
1321          *   - account:user       : Grants access if the request is by the given user
1322          *
1323          * @see http://swift.openstack.org/misc.html#acls
1324          *
1325          * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
1326          * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
1327          *
1328          * @param $contObj CF_Container Swift container
1329          * @param array $readGrps List of read access routes
1330          * @param array $writeGrps List of write access routes
1331          * @return Status
1332          */
1333         protected function setContainerAccess(
1334                 CF_Container $contObj, array $readGrps, array $writeGrps
1335         ) {
1336                 $creds = $contObj->cfs_auth->export_credentials();
1337
1338                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1339
1340                 // Note: 10 second timeout consistent with php-cloudfiles
1341                 $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1342                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1343                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1344                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1345
1346                 return $req->execute(); // should return 204
1347         }
1348
1349         /**
1350          * Purge the CDN cache of affected objects if CDN caching is enabled.
1351          * This is for Rackspace/Akamai CDNs.
1352          *
1353          * @param array $objects List of CF_Object items
1354          * @return void
1355          */
1356         public function purgeCDNCache( array $objects ) {
1357                 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
1358                         foreach ( $objects as $object ) {
1359                                 try {
1360                                         $object->purge_from_cdn();
1361                                 } catch ( CDNNotEnabledException $e ) {
1362                                         // CDN not enabled; nothing to see here
1363                                 } catch ( CloudFilesException $e ) {
1364                                         $this->handleException( $e, null, __METHOD__,
1365                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1366                                 }
1367                         }
1368                 }
1369         }
1370
1371         /**
1372          * Get an authenticated connection handle to the Swift proxy
1373          *
1374          * @throws CloudFilesException
1375          * @throws CloudFilesException|Exception
1376          * @return CF_Connection|bool False on failure
1377          */
1378         protected function getConnection() {
1379                 if ( $this->connException instanceof CloudFilesException ) {
1380                         if ( ( time() - $this->connErrorTime ) < 60 ) {
1381                                 throw $this->connException; // failed last attempt; don't bother
1382                         } else { // actually retry this time
1383                                 $this->connException = null;
1384                                 $this->connErrorTime = 0;
1385                         }
1386                 }
1387                 // Session keys expire after a while, so we renew them periodically
1388                 $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL );
1389                 // Authenticate with proxy and get a session key...
1390                 if ( !$this->conn || $reAuth ) {
1391                         $this->sessionStarted = 0;
1392                         $this->connContainerCache->clear();
1393                         $cacheKey = $this->getCredsCacheKey( $this->auth->username );
1394                         $creds = $this->srvCache->get( $cacheKey ); // credentials
1395                         if ( is_array( $creds ) ) { // cache hit
1396                                 $this->auth->load_cached_credentials(
1397                                         $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] );
1398                                 $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case
1399                         } else { // cache miss
1400                                 try {
1401                                         $this->auth->authenticate();
1402                                         $creds = $this->auth->export_credentials();
1403                                         $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache
1404                                         $this->sessionStarted = time();
1405                                 } catch ( CloudFilesException $e ) {
1406                                         $this->connException = $e; // don't keep re-trying
1407                                         $this->connErrorTime = time();
1408                                         throw $e; // throw it back
1409                                 }
1410                         }
1411                         if ( $this->conn ) { // re-authorizing?
1412                                 $this->conn->close(); // close active cURL handles in CF_Http object
1413                         }
1414                         $this->conn = new CF_Connection( $this->auth );
1415                 }
1416                 return $this->conn;
1417         }
1418
1419         /**
1420          * Close the connection to the Swift proxy
1421          *
1422          * @return void
1423          */
1424         protected function closeConnection() {
1425                 if ( $this->conn ) {
1426                         $this->conn->close(); // close active cURL handles in CF_Http object
1427                         $this->conn = null;
1428                         $this->sessionStarted = 0;
1429                         $this->connContainerCache->clear();
1430                 }
1431         }
1432
1433         /**
1434          * Get the cache key for a container
1435          *
1436          * @param $username string
1437          * @return string
1438          */
1439         private function getCredsCacheKey( $username ) {
1440                 return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username );
1441         }
1442
1443         /**
1444          * Get a Swift container object, possibly from process cache.
1445          * Use $reCache if the file count or byte count is needed.
1446          *
1447          * @param string $container Container name
1448          * @param bool $bypassCache Bypass all caches and load from Swift
1449          * @return CF_Container
1450          * @throws CloudFilesException
1451          */
1452         protected function getContainer( $container, $bypassCache = false ) {
1453                 $conn = $this->getConnection(); // Swift proxy connection
1454                 if ( $bypassCache ) { // purge cache
1455                         $this->connContainerCache->clear( $container );
1456                 } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1457                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1458                 }
1459                 if ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1460                         $contObj = $conn->get_container( $container );
1461                         // NoSuchContainerException not thrown: container must exist
1462                         $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it
1463                         if ( !$bypassCache ) {
1464                                 $this->setContainerCache( $container, // update persistent cache
1465                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1466                                 );
1467                         }
1468                 }
1469                 return $this->connContainerCache->get( $container, 'obj' );
1470         }
1471
1472         /**
1473          * Create a Swift container
1474          *
1475          * @param string $container Container name
1476          * @return CF_Container
1477          * @throws CloudFilesException
1478          */
1479         protected function createContainer( $container ) {
1480                 $conn = $this->getConnection(); // Swift proxy connection
1481                 $contObj = $conn->create_container( $container );
1482                 $this->connContainerCache->set( $container, 'obj', $contObj ); // cache
1483                 return $contObj;
1484         }
1485
1486         /**
1487          * Delete a Swift container
1488          *
1489          * @param string $container Container name
1490          * @return void
1491          * @throws CloudFilesException
1492          */
1493         protected function deleteContainer( $container ) {
1494                 $conn = $this->getConnection(); // Swift proxy connection
1495                 $this->connContainerCache->clear( $container ); // purge
1496                 $conn->delete_container( $container );
1497         }
1498
1499         /**
1500          * @see FileBackendStore::doPrimeContainerCache()
1501          * @return void
1502          */
1503         protected function doPrimeContainerCache( array $containerInfo ) {
1504                 try {
1505                         $conn = $this->getConnection(); // Swift proxy connection
1506                         foreach ( $containerInfo as $container => $info ) {
1507                                 $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http,
1508                                         $container, $info['count'], $info['bytes'] );
1509                                 $this->connContainerCache->set( $container, 'obj', $contObj );
1510                         }
1511                 } catch ( CloudFilesException $e ) { // some other exception?
1512                         $this->handleException( $e, null, __METHOD__, array() );
1513                 }
1514         }
1515
1516         /**
1517          * Log an unexpected exception for this backend.
1518          * This also sets the Status object to have a fatal error.
1519          *
1520          * @param $e Exception
1521          * @param $status Status|null
1522          * @param $func string
1523          * @param array $params
1524          * @return void
1525          */
1526         protected function handleException( Exception $e, $status, $func, array $params ) {
1527                 if ( $status instanceof Status ) {
1528                         if ( $e instanceof AuthenticationException ) {
1529                                 $status->fatal( 'backend-fail-connect', $this->name );
1530                         } else {
1531                                 $status->fatal( 'backend-fail-internal', $this->name );
1532                         }
1533                 }
1534                 if ( $e->getMessage() ) {
1535                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1536                 }
1537                 if ( $e instanceof InvalidResponseException ) { // possibly a stale token
1538                         $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) );
1539                         $this->closeConnection(); // force a re-connect and re-auth next time
1540                 }
1541                 wfDebugLog( 'SwiftBackend',
1542                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1543                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1544                 );
1545         }
1546 }
1547
1548 /**
1549  * @see FileBackendStoreOpHandle
1550  */
1551 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1552         /** @var CF_Async_Op */
1553         public $cfOp;
1554         /** @var Array */
1555         public $affectedObjects = array();
1556
1557         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1558                 $this->backend = $backend;
1559                 $this->params = $params;
1560                 $this->call = $call;
1561                 $this->cfOp = $cfOp;
1562         }
1563 }
1564
1565 /**
1566  * SwiftFileBackend helper class to page through listings.
1567  * Swift also has a listing limit of 10,000 objects for sanity.
1568  * Do not use this class from places outside SwiftFileBackend.
1569  *
1570  * @ingroup FileBackend
1571  */
1572 abstract class SwiftFileBackendList implements Iterator {
1573         /** @var Array */
1574         protected $bufferIter = array();
1575         protected $bufferAfter = null; // string; list items *after* this path
1576         protected $pos = 0; // integer
1577         /** @var Array */
1578         protected $params = array();
1579
1580         /** @var SwiftFileBackend */
1581         protected $backend;
1582         protected $container; // string; container name
1583         protected $dir; // string; storage directory
1584         protected $suffixStart; // integer
1585
1586         const PAGE_SIZE = 9000; // file listing buffer size
1587
1588         /**
1589          * @param $backend SwiftFileBackend
1590          * @param string $fullCont Resolved container name
1591          * @param string $dir Resolved directory relative to container
1592          * @param array $params
1593          */
1594         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1595                 $this->backend = $backend;
1596                 $this->container = $fullCont;
1597                 $this->dir = $dir;
1598                 if ( substr( $this->dir, -1 ) === '/' ) {
1599                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1600                 }
1601                 if ( $this->dir == '' ) { // whole container
1602                         $this->suffixStart = 0;
1603                 } else { // dir within container
1604                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1605                 }
1606                 $this->params = $params;
1607         }
1608
1609         /**
1610          * @see Iterator::key()
1611          * @return integer
1612          */
1613         public function key() {
1614                 return $this->pos;
1615         }
1616
1617         /**
1618          * @see Iterator::next()
1619          * @return void
1620          */
1621         public function next() {
1622                 // Advance to the next file in the page
1623                 next( $this->bufferIter );
1624                 ++$this->pos;
1625                 // Check if there are no files left in this page and
1626                 // advance to the next page if this page was not empty.
1627                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1628                         $this->bufferIter = $this->pageFromList(
1629                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1630                         ); // updates $this->bufferAfter
1631                 }
1632         }
1633
1634         /**
1635          * @see Iterator::rewind()
1636          * @return void
1637          */
1638         public function rewind() {
1639                 $this->pos = 0;
1640                 $this->bufferAfter = null;
1641                 $this->bufferIter = $this->pageFromList(
1642                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1643                 ); // updates $this->bufferAfter
1644         }
1645
1646         /**
1647          * @see Iterator::valid()
1648          * @return bool
1649          */
1650         public function valid() {
1651                 if ( $this->bufferIter === null ) {
1652                         return false; // some failure?
1653                 } else {
1654                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1655                 }
1656         }
1657
1658         /**
1659          * Get the given list portion (page)
1660          *
1661          * @param string $container Resolved container name
1662          * @param string $dir Resolved path relative to container
1663          * @param $after string|null
1664          * @param $limit integer
1665          * @param array $params
1666          * @return Traversable|Array|null Returns null on failure
1667          */
1668         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1669 }
1670
1671 /**
1672  * Iterator for listing directories
1673  */
1674 class SwiftFileBackendDirList extends SwiftFileBackendList {
1675         /**
1676          * @see Iterator::current()
1677          * @return string|bool String (relative path) or false
1678          */
1679         public function current() {
1680                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1681         }
1682
1683         /**
1684          * @see SwiftFileBackendList::pageFromList()
1685          * @return Array|null
1686          */
1687         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1688                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1689         }
1690 }
1691
1692 /**
1693  * Iterator for listing regular files
1694  */
1695 class SwiftFileBackendFileList extends SwiftFileBackendList {
1696         /**
1697          * @see Iterator::current()
1698          * @return string|bool String (relative path) or false
1699          */
1700         public function current() {
1701                 return substr( current( $this->bufferIter ), $this->suffixStart );
1702         }
1703
1704         /**
1705          * @see SwiftFileBackendList::pageFromList()
1706          * @return Array|null
1707          */
1708         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1709                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1710         }
1711 }