includes/filebackend/SwiftFileBackend.php

   1 <?php
   2 /**
   3  * OpenStack Swift based file backend.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Russ Nelson
  23  * @author Aaron Schulz
  24  */
  25
  26 /**
  27  * @brief Class for an OpenStack Swift based file backend.
  28  *
  29  * This requires the SwiftCloudFiles MediaWiki extension, which includes
  30  * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
  31  * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
  32  *
  33  * Status messages should avoid mentioning the Swift account name.
  34  * Likewise, error suppression should be used to avoid path disclosure.
  35  *
  36  * @ingroup FileBackend
  37  * @since 1.19
  38  */
  39 class SwiftFileBackend extends FileBackendStore {
  40         /** @var CF_Authentication */
  41         protected $auth; // Swift authentication handler
  42         protected $authTTL; // integer seconds
  43         protected $swiftTempUrlKey; // string; shared secret value for making temp urls
  44         protected $swiftAnonUser; // string; username to handle unauthenticated requests
  45         protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
  46         protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
  47         protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
  48
  49         // Rados Gateway specific options
  50         protected $rgwS3AccessKey; // string; S3 access key
  51         protected $rgwS3SecretKey; // string; S3 authentication key
  52
  53         /** @var CF_Connection */
  54         protected $conn; // Swift connection handle
  55         protected $sessionStarted = 0; // integer UNIX timestamp
  56
  57         /** @var CloudFilesException */
  58         protected $connException;
  59         protected $connErrorTime = 0; // UNIX timestamp
  60
  61         /** @var BagOStuff */
  62         protected $srvCache;
  63
  64         /** @var ProcessCacheLRU */
  65         protected $connContainerCache; // container object cache
  66
  67         /**
  68          * @see FileBackendStore::__construct()
  69          * Additional $config params include:
  70          *   - swiftAuthUrl       : Swift authentication server URL
  71          *   - swiftUser          : Swift user used by MediaWiki (account:username)
  72          *   - swiftKey           : Swift authentication key for the above user
  73          *   - swiftAuthTTL       : Swift authentication TTL (seconds)
  74          *   - swiftTempUrlKey    : Swift "X-Account-Meta-Temp-URL-Key" value on the account.
  75          *                          Do not set this until it has been set in the backend.
  76          *   - swiftAnonUser      : Swift user used for end-user requests (account:username).
  77          *                          If set, then views of public containers are assumed to go
  78          *                          through this user. If not set, then public containers are
  79          *                          accessible to unauthenticated requests via ".r:*" in the ACL.
  80          *   - swiftUseCDN        : Whether a Cloud Files Content Delivery Network is set up
  81          *   - swiftCDNExpiry     : How long (in seconds) to store content in the CDN.
  82          *                          If files may likely change, this should probably not exceed
  83          *                          a few days. For example, deletions may take this long to apply.
  84          *                          If object purging is enabled, however, this is not an issue.
  85          *   - swiftCDNPurgable   : Whether object purge requests are allowed by the CDN.
  86          *   - shardViaHashLevels : Map of container names to sharding config with:
  87          *                             - base   : base of hash characters, 16 or 36
  88          *                             - levels : the number of hash levels (and digits)
  89          *                             - repeat : hash subdirectories are prefixed with all the
  90          *                                        parent hash directory names (e.g. "a/ab/abc")
  91          *   - cacheAuthInfo      : Whether to cache authentication tokens in APC, XCache, ect.
  92          *                          If those are not available, then the main cache will be used.
  93          *                          This is probably insecure in shared hosting environments.
  94          *   - rgwS3AccessKey     : Ragos Gateway S3 "access key" value on the account.
  95          *                          Do not set this until it has been set in the backend.
  96          *                          This is used for generating expiring pre-authenticated URLs.
  97          *                          Only use this when using rgw and to work around
  98          *                          http://tracker.newdream.net/issues/3454.
  99          *   - rgwS3SecretKey     : Ragos Gateway S3 "secret key" value on the account.
 100          *                          Do not set this until it has been set in the backend.
 101          *                          This is used for generating expiring pre-authenticated URLs.
 102          *                          Only use this when using rgw and to work around
 103          *                          http://tracker.newdream.net/issues/3454.
 104          */
 105         public function __construct( array $config ) {
 106                 parent::__construct( $config );
 107                 if ( !MWInit::classExists( 'CF_Constants' ) ) {
 108                         throw new MWException( 'SwiftCloudFiles extension not installed.' );
 109                 }
 110                 // Required settings
 111                 $this->auth = new CF_Authentication(
 112                         $config['swiftUser'],
 113                         $config['swiftKey'],
 114                         null, // account; unused
 115                         $config['swiftAuthUrl']
 116                 );
 117                 // Optional settings
 118                 $this->authTTL = isset( $config['swiftAuthTTL'] )
 119                         ? $config['swiftAuthTTL']
 120                         : 5 * 60; // some sane number
 121                 $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
 122                         ? $config['swiftAnonUser']
 123                         : '';
 124                 $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] )
 125                         ? $config['swiftTempUrlKey']
 126                         : '';
 127                 $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
 128                         ? $config['shardViaHashLevels']
 129                         : '';
 130                 $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
 131                         ? $config['swiftUseCDN']
 132                         : false;
 133                 $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
 134                         ? $config['swiftCDNExpiry']
 135                         : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
 136                 $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
 137                         ? $config['swiftCDNPurgable']
 138                         : true;
 139                 $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] )
 140                         ? $config['rgwS3AccessKey']
 141                         : '';
 142                 $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] )
 143                         ? $config['rgwS3SecretKey']
 144                         : '';
 145                 // Cache container information to mask latency
 146                 $this->memCache = wfGetMainCache();
 147                 // Process cache for container info
 148                 $this->connContainerCache = new ProcessCacheLRU( 300 );
 149                 // Cache auth token information to avoid RTTs
 150                 if ( !empty( $config['cacheAuthInfo'] ) ) {
 151                         if ( PHP_SAPI === 'cli' ) {
 152                                 $this->srvCache = wfGetMainCache(); // preferrably memcached
 153                         } else {
 154                                 try { // look for APC, XCache, WinCache, ect...
 155                                         $this->srvCache = ObjectCache::newAccelerator( array() );
 156                                 } catch ( Exception $e ) {}
 157                         }
 158                 }
 159                 $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
 160         }
 161
 162         /**
 163          * @see FileBackendStore::resolveContainerPath()
 164          * @return null
 165          */
 166         protected function resolveContainerPath( $container, $relStoragePath ) {
 167                 if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
 168                         return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
 169                 } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
 170                         return null; // too long for Swift
 171                 }
 172                 return $relStoragePath;
 173         }
 174
 175         /**
 176          * @see FileBackendStore::isPathUsableInternal()
 177          * @return bool
 178          */
 179         public function isPathUsableInternal( $storagePath ) {
 180                 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
 181                 if ( $rel === null ) {
 182                         return false; // invalid
 183                 }
 184
 185                 try {
 186                         $this->getContainer( $container );
 187                         return true; // container exists
 188                 } catch ( NoSuchContainerException $e ) {
 189                 } catch ( CloudFilesException $e ) { // some other exception?
 190                         $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
 191                 }
 192
 193                 return false;
 194         }
 195
 196         /**
 197          * @param $disposition string Content-Disposition header value
 198          * @return string Truncated Content-Disposition header value to meet Swift limits
 199          */
 200         protected function truncDisp( $disposition ) {
 201                 $res = '';
 202                 foreach ( explode( ';', $disposition ) as $part ) {
 203                         $part = trim( $part );
 204                         $new = ( $res === '' ) ? $part : "{$res};{$part}";
 205                         if ( strlen( $new ) <= 255 ) {
 206                                 $res = $new;
 207                         } else {
 208                                 break; // too long; sigh
 209                         }
 210                 }
 211                 return $res;
 212         }
 213
 214         /**
 215          * @see FileBackendStore::doCreateInternal()
 216          * @return Status
 217          */
 218         protected function doCreateInternal( array $params ) {
 219                 $status = Status::newGood();
 220
 221                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 222                 if ( $dstRel === null ) {
 223                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 224                         return $status;
 225                 }
 226
 227                 // (a) Check the destination container and object
 228                 try {
 229                         $dContObj = $this->getContainer( $dstCont );
 230                 } catch ( NoSuchContainerException $e ) {
 231                         $status->fatal( 'backend-fail-create', $params['dst'] );
 232                         return $status;
 233                 } catch ( CloudFilesException $e ) { // some other exception?
 234                         $this->handleException( $e, $status, __METHOD__, $params );
 235                         return $status;
 236                 }
 237
 238                 // (b) Get a SHA-1 hash of the object
 239                 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
 240
 241                 // (c) Actually create the object
 242                 try {
 243                         // Create a fresh CF_Object with no fields preloaded.
 244                         // We don't want to preserve headers, metadata, and such.
 245                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 246                         $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
 247                         // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
 248                         // The MD5 here will be checked within Swift against its own MD5.
 249                         $obj->set_etag( md5( $params['content'] ) );
 250                         // Use the same content type as StreamFile for security
 251                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 252                         if ( !strlen( $obj->content_type ) ) { // special case
 253                                 $obj->content_type = 'unknown/unknown';
 254                         }
 255                         // Set the Content-Disposition header if requested
 256                         if ( isset( $params['disposition'] ) ) {
 257                                 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 258                         }
 259                         // Set any other custom headers if requested
 260                         if ( isset( $params['headers'] ) ) {
 261                                 $obj->headers += $params['headers'];
 262                         }
 263                         if ( !empty( $params['async'] ) ) { // deferred
 264                                 $op = $obj->write_async( $params['content'] );
 265                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op );
 266                                 $status->value->affectedObjects[] = $obj;
 267                         } else { // actually write the object in Swift
 268                                 $obj->write( $params['content'] );
 269                                 $this->purgeCDNCache( array( $obj ) );
 270                         }
 271                 } catch ( CDNNotEnabledException $e ) {
 272                         // CDN not enabled; nothing to see here
 273                 } catch ( BadContentTypeException $e ) {
 274                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 275                 } catch ( CloudFilesException $e ) { // some other exception?
 276                         $this->handleException( $e, $status, __METHOD__, $params );
 277                 }
 278
 279                 return $status;
 280         }
 281
 282         /**
 283          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 284          */
 285         protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
 286                 try {
 287                         $cfOp->getLastResponse();
 288                 } catch ( BadContentTypeException $e ) {
 289                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 290                 }
 291         }
 292
 293         /**
 294          * @see FileBackendStore::doStoreInternal()
 295          * @return Status
 296          */
 297         protected function doStoreInternal( array $params ) {
 298                 $status = Status::newGood();
 299
 300                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 301                 if ( $dstRel === null ) {
 302                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 303                         return $status;
 304                 }
 305
 306                 // (a) Check the destination container and object
 307                 try {
 308                         $dContObj = $this->getContainer( $dstCont );
 309                 } catch ( NoSuchContainerException $e ) {
 310                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 311                         return $status;
 312                 } catch ( CloudFilesException $e ) { // some other exception?
 313                         $this->handleException( $e, $status, __METHOD__, $params );
 314                         return $status;
 315                 }
 316
 317                 // (b) Get a SHA-1 hash of the object
 318                 wfSuppressWarnings();
 319                 $sha1Hash = sha1_file( $params['src'] );
 320                 wfRestoreWarnings();
 321                 if ( $sha1Hash === false ) { // source doesn't exist?
 322                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 323                         return $status;
 324                 }
 325                 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
 326
 327                 // (c) Actually store the object
 328                 try {
 329                         // Create a fresh CF_Object with no fields preloaded.
 330                         // We don't want to preserve headers, metadata, and such.
 331                         $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 332                         $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
 333                         // The MD5 here will be checked within Swift against its own MD5.
 334                         $obj->set_etag( md5_file( $params['src'] ) );
 335                         // Use the same content type as StreamFile for security
 336                         $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] );
 337                         if ( !strlen( $obj->content_type ) ) { // special case
 338                                 $obj->content_type = 'unknown/unknown';
 339                         }
 340                         // Set the Content-Disposition header if requested
 341                         if ( isset( $params['disposition'] ) ) {
 342                                 $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 343                         }
 344                         // Set any other custom headers if requested
 345                         if ( isset( $params['headers'] ) ) {
 346                                 $obj->headers += $params['headers'];
 347                         }
 348                         if ( !empty( $params['async'] ) ) { // deferred
 349                                 wfSuppressWarnings();
 350                                 $fp = fopen( $params['src'], 'rb' );
 351                                 wfRestoreWarnings();
 352                                 if ( !$fp ) {
 353                                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 354                                 } else {
 355                                         $op = $obj->write_async( $fp, filesize( $params['src'] ), true );
 356                                         $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op );
 357                                         $status->value->resourcesToClose[] = $fp;
 358                                         $status->value->affectedObjects[] = $obj;
 359                                 }
 360                         } else { // actually write the object in Swift
 361                                 $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
 362                                 $this->purgeCDNCache( array( $obj ) );
 363                         }
 364                 } catch ( CDNNotEnabledException $e ) {
 365                         // CDN not enabled; nothing to see here
 366                 } catch ( BadContentTypeException $e ) {
 367                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 368                 } catch ( IOException $e ) {
 369                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 370                 } catch ( CloudFilesException $e ) { // some other exception?
 371                         $this->handleException( $e, $status, __METHOD__, $params );
 372                 }
 373
 374                 return $status;
 375         }
 376
 377         /**
 378          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 379          */
 380         protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
 381                 try {
 382                         $cfOp->getLastResponse();
 383                 } catch ( BadContentTypeException $e ) {
 384                         $status->fatal( 'backend-fail-contenttype', $params['dst'] );
 385                 } catch ( IOException $e ) {
 386                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 387                 }
 388         }
 389
 390         /**
 391          * @see FileBackendStore::doCopyInternal()
 392          * @return Status
 393          */
 394         protected function doCopyInternal( array $params ) {
 395                 $status = Status::newGood();
 396
 397                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 398                 if ( $srcRel === null ) {
 399                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 400                         return $status;
 401                 }
 402
 403                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 404                 if ( $dstRel === null ) {
 405                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 406                         return $status;
 407                 }
 408
 409                 // (a) Check the source/destination containers and destination object
 410                 try {
 411                         $sContObj = $this->getContainer( $srcCont );
 412                         $dContObj = $this->getContainer( $dstCont );
 413                 } catch ( NoSuchContainerException $e ) {
 414                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 415                         return $status;
 416                 } catch ( CloudFilesException $e ) { // some other exception?
 417                         $this->handleException( $e, $status, __METHOD__, $params );
 418                         return $status;
 419                 }
 420
 421                 // (b) Actually copy the file to the destination
 422                 try {
 423                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 424                         $hdrs = array(); // source file headers to override with new values
 425                         if ( isset( $params['disposition'] ) ) {
 426                                 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 427                         }
 428                         if ( !empty( $params['async'] ) ) { // deferred
 429                                 $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
 430                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op );
 431                                 $status->value->affectedObjects[] = $dstObj;
 432                         } else { // actually write the object in Swift
 433                                 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
 434                                 $this->purgeCDNCache( array( $dstObj ) );
 435                         }
 436                 } catch ( CDNNotEnabledException $e ) {
 437                         // CDN not enabled; nothing to see here
 438                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 439                         if ( empty( $params['ignoreMissingSource'] ) ) {
 440                                 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 441                         }
 442                 } catch ( CloudFilesException $e ) { // some other exception?
 443                         $this->handleException( $e, $status, __METHOD__, $params );
 444                 }
 445
 446                 return $status;
 447         }
 448
 449         /**
 450          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 451          */
 452         protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
 453                 try {
 454                         $cfOp->getLastResponse();
 455                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 456                         $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
 457                 }
 458         }
 459
 460         /**
 461          * @see FileBackendStore::doMoveInternal()
 462          * @return Status
 463          */
 464         protected function doMoveInternal( array $params ) {
 465                 $status = Status::newGood();
 466
 467                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 468                 if ( $srcRel === null ) {
 469                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 470                         return $status;
 471                 }
 472
 473                 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
 474                 if ( $dstRel === null ) {
 475                         $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
 476                         return $status;
 477                 }
 478
 479                 // (a) Check the source/destination containers and destination object
 480                 try {
 481                         $sContObj = $this->getContainer( $srcCont );
 482                         $dContObj = $this->getContainer( $dstCont );
 483                 } catch ( NoSuchContainerException $e ) {
 484                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 485                         return $status;
 486                 } catch ( CloudFilesException $e ) { // some other exception?
 487                         $this->handleException( $e, $status, __METHOD__, $params );
 488                         return $status;
 489                 }
 490
 491                 // (b) Actually move the file to the destination
 492                 try {
 493                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 494                         $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
 495                         $hdrs = array(); // source file headers to override with new values
 496                         if ( isset( $params['disposition'] ) ) {
 497                                 $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 498                         }
 499                         if ( !empty( $params['async'] ) ) { // deferred
 500                                 $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
 501                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op );
 502                                 $status->value->affectedObjects[] = $srcObj;
 503                                 $status->value->affectedObjects[] = $dstObj;
 504                         } else { // actually write the object in Swift
 505                                 $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
 506                                 $this->purgeCDNCache( array( $srcObj ) );
 507                                 $this->purgeCDNCache( array( $dstObj ) );
 508                         }
 509                 } catch ( CDNNotEnabledException $e ) {
 510                         // CDN not enabled; nothing to see here
 511                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 512                         if ( empty( $params['ignoreMissingSource'] ) ) {
 513                                 $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 514                         }
 515                 } catch ( CloudFilesException $e ) { // some other exception?
 516                         $this->handleException( $e, $status, __METHOD__, $params );
 517                 }
 518
 519                 return $status;
 520         }
 521
 522         /**
 523          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 524          */
 525         protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
 526                 try {
 527                         $cfOp->getLastResponse();
 528                 } catch ( NoSuchObjectException $e ) { // source object does not exist
 529                         $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
 530                 }
 531         }
 532
 533         /**
 534          * @see FileBackendStore::doDeleteInternal()
 535          * @return Status
 536          */
 537         protected function doDeleteInternal( array $params ) {
 538                 $status = Status::newGood();
 539
 540                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 541                 if ( $srcRel === null ) {
 542                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 543                         return $status;
 544                 }
 545
 546                 try {
 547                         $sContObj = $this->getContainer( $srcCont );
 548                         $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 549                         if ( !empty( $params['async'] ) ) { // deferred
 550                                 $op = $sContObj->delete_object_async( $srcRel );
 551                                 $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $op );
 552                                 $status->value->affectedObjects[] = $srcObj;
 553                         } else { // actually write the object in Swift
 554                                 $sContObj->delete_object( $srcRel );
 555                                 $this->purgeCDNCache( array( $srcObj ) );
 556                         }
 557                 } catch ( CDNNotEnabledException $e ) {
 558                         // CDN not enabled; nothing to see here
 559                 } catch ( NoSuchContainerException $e ) {
 560                         $status->fatal( 'backend-fail-delete', $params['src'] );
 561                 } catch ( NoSuchObjectException $e ) {
 562                         if ( empty( $params['ignoreMissingSource'] ) ) {
 563                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 564                         }
 565                 } catch ( CloudFilesException $e ) { // some other exception?
 566                         $this->handleException( $e, $status, __METHOD__, $params );
 567                 }
 568
 569                 return $status;
 570         }
 571
 572         /**
 573          * @see SwiftFileBackend::doExecuteOpHandlesInternal()
 574          */
 575         protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
 576                 try {
 577                         $cfOp->getLastResponse();
 578                 } catch ( NoSuchContainerException $e ) {
 579                         $status->fatal( 'backend-fail-delete', $params['src'] );
 580                 } catch ( NoSuchObjectException $e ) {
 581                         if ( empty( $params['ignoreMissingSource'] ) ) {
 582                                 $status->fatal( 'backend-fail-delete', $params['src'] );
 583                         }
 584                 }
 585         }
 586
 587         /**
 588          * @see FileBackendStore::doDescribeInternal()
 589          * @return Status
 590          */
 591         protected function doDescribeInternal( array $params ) {
 592                 $status = Status::newGood();
 593
 594                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 595                 if ( $srcRel === null ) {
 596                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
 597                         return $status;
 598                 }
 599
 600                 $hdrs = isset( $params['headers'] ) ? $params['headers'] : array();
 601                 // Set the Content-Disposition header if requested
 602                 if ( isset( $params['disposition'] ) ) {
 603                         $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
 604                 }
 605
 606                 try {
 607                         $sContObj = $this->getContainer( $srcCont );
 608                         // Get the latest version of the current metadata
 609                         $srcObj = $sContObj->get_object( $srcRel,
 610                                 $this->headersFromParams( array( 'latest' => true ) ) );
 611                         // Merge in the metadata updates...
 612                         $srcObj->headers = $hdrs + $srcObj->headers;
 613                         $srcObj->sync_metadata(); // save to Swift
 614                         $this->purgeCDNCache( array( $srcObj ) );
 615                 } catch ( CDNNotEnabledException $e ) {
 616                         // CDN not enabled; nothing to see here
 617                 } catch ( NoSuchContainerException $e ) {
 618                         $status->fatal( 'backend-fail-describe', $params['src'] );
 619                 } catch ( NoSuchObjectException $e ) {
 620                         $status->fatal( 'backend-fail-describe', $params['src'] );
 621                 } catch ( CloudFilesException $e ) { // some other exception?
 622                         $this->handleException( $e, $status, __METHOD__, $params );
 623                 }
 624
 625                 return $status;
 626         }
 627
 628         /**
 629          * @see FileBackendStore::doPrepareInternal()
 630          * @return Status
 631          */
 632         protected function doPrepareInternal( $fullCont, $dir, array $params ) {
 633                 $status = Status::newGood();
 634
 635                 // (a) Check if container already exists
 636                 try {
 637                         $this->getContainer( $fullCont );
 638                         // NoSuchContainerException not thrown: container must exist
 639                         return $status; // already exists
 640                 } catch ( NoSuchContainerException $e ) {
 641                         // NoSuchContainerException thrown: container does not exist
 642                 } catch ( CloudFilesException $e ) { // some other exception?
 643                         $this->handleException( $e, $status, __METHOD__, $params );
 644                         return $status;
 645                 }
 646
 647                 // (b) Create container as needed
 648                 try {
 649                         $contObj = $this->createContainer( $fullCont );
 650                         if ( !empty( $params['noAccess'] ) ) {
 651                                 // Make container private to end-users...
 652                                 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 653                         } else {
 654                                 // Make container public to end-users...
 655                                 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 656                         }
 657                         if ( $this->swiftUseCDN ) { // Rackspace style CDN
 658                                 $contObj->make_public( $this->swiftCDNExpiry );
 659                         }
 660                 } catch ( CDNNotEnabledException $e ) {
 661                         // CDN not enabled; nothing to see here
 662                 } catch ( CloudFilesException $e ) { // some other exception?
 663                         $this->handleException( $e, $status, __METHOD__, $params );
 664                         return $status;
 665                 }
 666
 667                 return $status;
 668         }
 669
 670         /**
 671          * @see FileBackendStore::doSecureInternal()
 672          * @return Status
 673          */
 674         protected function doSecureInternal( $fullCont, $dir, array $params ) {
 675                 $status = Status::newGood();
 676                 if ( empty( $params['noAccess'] ) ) {
 677                         return $status; // nothing to do
 678                 }
 679
 680                 // Restrict container from end-users...
 681                 try {
 682                         // doPrepareInternal() should have been called,
 683                         // so the Swift container should already exist...
 684                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 685                         // NoSuchContainerException not thrown: container must exist
 686
 687                         // Make container private to end-users...
 688                         $status->merge( $this->setContainerAccess(
 689                                 $contObj,
 690                                 array( $this->auth->username ), // read
 691                                 array( $this->auth->username ) // write
 692                         ) );
 693                         if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
 694                                 $contObj->make_private();
 695                         }
 696                 } catch ( CDNNotEnabledException $e ) {
 697                         // CDN not enabled; nothing to see here
 698                 } catch ( CloudFilesException $e ) { // some other exception?
 699                         $this->handleException( $e, $status, __METHOD__, $params );
 700                 }
 701
 702                 return $status;
 703         }
 704
 705         /**
 706          * @see FileBackendStore::doPublishInternal()
 707          * @return Status
 708          */
 709         protected function doPublishInternal( $fullCont, $dir, array $params ) {
 710                 $status = Status::newGood();
 711
 712                 // Unrestrict container from end-users...
 713                 try {
 714                         // doPrepareInternal() should have been called,
 715                         // so the Swift container should already exist...
 716                         $contObj = $this->getContainer( $fullCont ); // normally a cache hit
 717                         // NoSuchContainerException not thrown: container must exist
 718
 719                         // Make container public to end-users...
 720                         if ( $this->swiftAnonUser != '' ) {
 721                                 $status->merge( $this->setContainerAccess(
 722                                         $contObj,
 723                                         array( $this->auth->username, $this->swiftAnonUser ), // read
 724                                         array( $this->auth->username, $this->swiftAnonUser ) // write
 725                                 ) );
 726                         } else {
 727                                 $status->merge( $this->setContainerAccess(
 728                                         $contObj,
 729                                         array( $this->auth->username, '.r:*' ), // read
 730                                         array( $this->auth->username ) // write
 731                                 ) );
 732                         }
 733                         if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
 734                                 $contObj->make_public();
 735                         }
 736                 } catch ( CDNNotEnabledException $e ) {
 737                         // CDN not enabled; nothing to see here
 738                 } catch ( CloudFilesException $e ) { // some other exception?
 739                         $this->handleException( $e, $status, __METHOD__, $params );
 740                 }
 741
 742                 return $status;
 743         }
 744
 745         /**
 746          * @see FileBackendStore::doCleanInternal()
 747          * @return Status
 748          */
 749         protected function doCleanInternal( $fullCont, $dir, array $params ) {
 750                 $status = Status::newGood();
 751
 752                 // Only containers themselves can be removed, all else is virtual
 753                 if ( $dir != '' ) {
 754                         return $status; // nothing to do
 755                 }
 756
 757                 // (a) Check the container
 758                 try {
 759                         $contObj = $this->getContainer( $fullCont, true );
 760                 } catch ( NoSuchContainerException $e ) {
 761                         return $status; // ok, nothing to do
 762                 } catch ( CloudFilesException $e ) { // some other exception?
 763                         $this->handleException( $e, $status, __METHOD__, $params );
 764                         return $status;
 765                 }
 766
 767                 // (b) Delete the container if empty
 768                 if ( $contObj->object_count == 0 ) {
 769                         try {
 770                                 $this->deleteContainer( $fullCont );
 771                         } catch ( NoSuchContainerException $e ) {
 772                                 return $status; // race?
 773                         } catch ( NonEmptyContainerException $e ) {
 774                                 return $status; // race? consistency delay?
 775                         } catch ( CloudFilesException $e ) { // some other exception?
 776                                 $this->handleException( $e, $status, __METHOD__, $params );
 777                                 return $status;
 778                         }
 779                 }
 780
 781                 return $status;
 782         }
 783
 784         /**
 785          * @see FileBackendStore::doFileExists()
 786          * @return array|bool|null
 787          */
 788         protected function doGetFileStat( array $params ) {
 789                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
 790                 if ( $srcRel === null ) {
 791                         return false; // invalid storage path
 792                 }
 793
 794                 $stat = false;
 795                 try {
 796                         $contObj = $this->getContainer( $srcCont );
 797                         $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
 798                         $this->addMissingMetadata( $srcObj, $params['src'] );
 799                         $stat = array(
 800                                 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 801                                 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
 802                                 'size'  => (int)$srcObj->content_length,
 803                                 'sha1'  => $srcObj->getMetadataValue( 'Sha1base36' )
 804                         );
 805                 } catch ( NoSuchContainerException $e ) {
 806                 } catch ( NoSuchObjectException $e ) {
 807                 } catch ( CloudFilesException $e ) { // some other exception?
 808                         $stat = null;
 809                         $this->handleException( $e, null, __METHOD__, $params );
 810                 }
 811
 812                 return $stat;
 813         }
 814
 815         /**
 816          * Fill in any missing object metadata and save it to Swift
 817          *
 818          * @param $obj CF_Object
 819          * @param $path string Storage path to object
 820          * @return bool Success
 821          * @throws Exception cloudfiles exceptions
 822          */
 823         protected function addMissingMetadata( CF_Object $obj, $path ) {
 824                 if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) {
 825                         return true; // nothing to do
 826                 }
 827                 wfProfileIn( __METHOD__ );
 828                 trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING );
 829                 $status = Status::newGood();
 830                 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
 831                 if ( $status->isOK() ) {
 832                         $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
 833                         if ( $tmpFile ) {
 834                                 $hash = $tmpFile->getSha1Base36();
 835                                 if ( $hash !== false ) {
 836                                         $obj->setMetadataValues( array( 'Sha1base36' => $hash ) );
 837                                         $obj->sync_metadata(); // save to Swift
 838                                         wfProfileOut( __METHOD__ );
 839                                         return true; // success
 840                                 }
 841                         }
 842                 }
 843                 trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING );
 844                 $obj->setMetadataValues( array( 'Sha1base36' => false ) );
 845                 wfProfileOut( __METHOD__ );
 846                 return false; // failed
 847         }
 848
 849         /**
 850          * @see FileBackendStore::doGetFileContentsMulti()
 851          * @return Array
 852          */
 853         protected function doGetFileContentsMulti( array $params ) {
 854                 $contents = array();
 855
 856                 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
 857                 // Blindly create tmp files and stream to them, catching any exception if the file does
 858                 // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
 859                 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
 860                         $cfOps = array(); // (path => CF_Async_Op)
 861
 862                         foreach ( $pathBatch as $path ) { // each path in this concurrent batch
 863                                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
 864                                 if ( $srcRel === null ) {
 865                                         $contents[$path] = false;
 866                                         continue;
 867                                 }
 868                                 $data = false;
 869                                 try {
 870                                         $sContObj = $this->getContainer( $srcCont );
 871                                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
 872                                         // Create a new temporary memory file...
 873                                         $handle = fopen( 'php://temp', 'wb' );
 874                                         if ( $handle ) {
 875                                                 $headers = $this->headersFromParams( $params );
 876                                                 if ( count( $pathBatch ) > 1 ) {
 877                                                         $cfOps[$path] = $obj->stream_async( $handle, $headers );
 878                                                         $cfOps[$path]->_file_handle = $handle; // close this later
 879                                                 } else {
 880                                                         $obj->stream( $handle, $headers );
 881                                                         rewind( $handle ); // start from the beginning
 882                                                         $data = stream_get_contents( $handle );
 883                                                         fclose( $handle );
 884                                                 }
 885                                         } else {
 886                                                 $data = false;
 887                                         }
 888                                 } catch ( NoSuchContainerException $e ) {
 889                                         $data = false;
 890                                 } catch ( NoSuchObjectException $e ) {
 891                                         $data = false;
 892                                 } catch ( CloudFilesException $e ) { // some other exception?
 893                                         $data = false;
 894                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
 895                                 }
 896                                 $contents[$path] = $data;
 897                         }
 898
 899                         $batch = new CF_Async_Op_Batch( $cfOps );
 900                         $cfOps = $batch->execute();
 901                         foreach ( $cfOps as $path => $cfOp ) {
 902                                 try {
 903                                         $cfOp->getLastResponse();
 904                                         rewind( $cfOp->_file_handle ); // start from the beginning
 905                                         $contents[$path] = stream_get_contents( $cfOp->_file_handle );
 906                                 } catch ( NoSuchContainerException $e ) {
 907                                         $contents[$path] = false;
 908                                 } catch ( NoSuchObjectException $e ) {
 909                                         $contents[$path] = false;
 910                                 } catch ( CloudFilesException $e ) { // some other exception?
 911                                         $contents[$path] = false;
 912                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
 913                                 }
 914                                 fclose( $cfOp->_file_handle ); // close open handle
 915                         }
 916                 }
 917
 918                 return $contents;
 919         }
 920
 921         /**
 922          * @see FileBackendStore::doDirectoryExists()
 923          * @return bool|null
 924          */
 925         protected function doDirectoryExists( $fullCont, $dir, array $params ) {
 926                 try {
 927                         $container = $this->getContainer( $fullCont );
 928                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 929                         return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
 930                 } catch ( NoSuchContainerException $e ) {
 931                         return false;
 932                 } catch ( CloudFilesException $e ) { // some other exception?
 933                         $this->handleException( $e, null, __METHOD__,
 934                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
 935                 }
 936
 937                 return null; // error
 938         }
 939
 940         /**
 941          * @see FileBackendStore::getDirectoryListInternal()
 942          * @return SwiftFileBackendDirList
 943          */
 944         public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
 945                 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
 946         }
 947
 948         /**
 949          * @see FileBackendStore::getFileListInternal()
 950          * @return SwiftFileBackendFileList
 951          */
 952         public function getFileListInternal( $fullCont, $dir, array $params ) {
 953                 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
 954         }
 955
 956         /**
 957          * Do not call this function outside of SwiftFileBackendFileList
 958          *
 959          * @param $fullCont string Resolved container name
 960          * @param $dir string Resolved storage directory with no trailing slash
 961          * @param $after string|null Storage path of file to list items after
 962          * @param $limit integer Max number of items to list
 963          * @param $params Array Includes flag for 'topOnly'
 964          * @return Array List of relative paths of dirs directly under $dir
 965          */
 966         public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
 967                 $dirs = array();
 968                 if ( $after === INF ) {
 969                         return $dirs; // nothing more
 970                 }
 971                 wfProfileIn( __METHOD__ . '-' . $this->name );
 972
 973                 try {
 974                         $container = $this->getContainer( $fullCont );
 975                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
 976                         // Non-recursive: only list dirs right under $dir
 977                         if ( !empty( $params['topOnly'] ) ) {
 978                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
 979                                 foreach ( $objects as $object ) { // files and dirs
 980                                         if ( substr( $object, -1 ) === '/' ) {
 981                                                 $dirs[] = $object; // directories end in '/'
 982                                         }
 983                                 }
 984                         // Recursive: list all dirs under $dir and its subdirs
 985                         } else {
 986                                 // Get directory from last item of prior page
 987                                 $lastDir = $this->getParentDir( $after ); // must be first page
 988                                 $objects = $container->list_objects( $limit, $after, $prefix );
 989                                 foreach ( $objects as $object ) { // files
 990                                         $objectDir = $this->getParentDir( $object ); // directory of object
 991                                         if ( $objectDir !== false && $objectDir !== $dir ) {
 992                                                 // Swift stores paths in UTF-8, using binary sorting.
 993                                                 // See function "create_container_table" in common/db.py.
 994                                                 // If a directory is not "greater" than the last one,
 995                                                 // then it was already listed by the calling iterator.
 996                                                 if ( strcmp( $objectDir, $lastDir ) > 0 ) {
 997                                                         $pDir = $objectDir;
 998                                                         do { // add dir and all its parent dirs
 999                                                                 $dirs[] = "{$pDir}/";
1000                                                                 $pDir = $this->getParentDir( $pDir );
1001                                                         } while ( $pDir !== false // sanity
1002                                                                 && strcmp( $pDir, $lastDir ) > 0 // not done already
1003                                                                 && strlen( $pDir ) > strlen( $dir ) // within $dir
1004                                                         );
1005                                                 }
1006                                                 $lastDir = $objectDir;
1007                                         }
1008                                 }
1009                         }
1010                         if ( count( $objects ) < $limit ) {
1011                                 $after = INF; // avoid a second RTT
1012                         } else {
1013                                 $after = end( $objects ); // update last item
1014                         }
1015                 } catch ( NoSuchContainerException $e ) {
1016                 } catch ( CloudFilesException $e ) { // some other exception?
1017                         $this->handleException( $e, null, __METHOD__,
1018                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
1019                 }
1020
1021                 wfProfileOut( __METHOD__ . '-' . $this->name );
1022                 return $dirs;
1023         }
1024
1025         protected function getParentDir( $path ) {
1026                 return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
1027         }
1028
1029         /**
1030          * Do not call this function outside of SwiftFileBackendFileList
1031          *
1032          * @param $fullCont string Resolved container name
1033          * @param $dir string Resolved storage directory with no trailing slash
1034          * @param $after string|null Storage path of file to list items after
1035          * @param $limit integer Max number of items to list
1036          * @param $params Array Includes flag for 'topOnly'
1037          * @return Array List of relative paths of files under $dir
1038          */
1039         public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
1040                 $files = array();
1041                 if ( $after === INF ) {
1042                         return $files; // nothing more
1043                 }
1044                 wfProfileIn( __METHOD__ . '-' . $this->name );
1045
1046                 try {
1047                         $container = $this->getContainer( $fullCont );
1048                         $prefix = ( $dir == '' ) ? null : "{$dir}/";
1049                         // Non-recursive: only list files right under $dir
1050                         if ( !empty( $params['topOnly'] ) ) { // files and dirs
1051                                 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
1052                                 foreach ( $objects as $object ) {
1053                                         if ( substr( $object, -1 ) !== '/' ) {
1054                                                 $files[] = $object; // directories end in '/'
1055                                         }
1056                                 }
1057                         // Recursive: list all files under $dir and its subdirs
1058                         } else { // files
1059                                 $objects = $container->list_objects( $limit, $after, $prefix );
1060                                 $files = $objects;
1061                         }
1062                         if ( count( $objects ) < $limit ) {
1063                                 $after = INF; // avoid a second RTT
1064                         } else {
1065                                 $after = end( $objects ); // update last item
1066                         }
1067                 } catch ( NoSuchContainerException $e ) {
1068                 } catch ( CloudFilesException $e ) { // some other exception?
1069                         $this->handleException( $e, null, __METHOD__,
1070                                 array( 'cont' => $fullCont, 'dir' => $dir ) );
1071                 }
1072
1073                 wfProfileOut( __METHOD__ . '-' . $this->name );
1074                 return $files;
1075         }
1076
1077         /**
1078          * @see FileBackendStore::doGetFileSha1base36()
1079          * @return bool
1080          */
1081         protected function doGetFileSha1base36( array $params ) {
1082                 $stat = $this->getFileStat( $params );
1083                 if ( $stat ) {
1084                         return $stat['sha1'];
1085                 } else {
1086                         return false;
1087                 }
1088         }
1089
1090         /**
1091          * @see FileBackendStore::doStreamFile()
1092          * @return Status
1093          */
1094         protected function doStreamFile( array $params ) {
1095                 $status = Status::newGood();
1096
1097                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1098                 if ( $srcRel === null ) {
1099                         $status->fatal( 'backend-fail-invalidpath', $params['src'] );
1100                 }
1101
1102                 try {
1103                         $cont = $this->getContainer( $srcCont );
1104                 } catch ( NoSuchContainerException $e ) {
1105                         $status->fatal( 'backend-fail-stream', $params['src'] );
1106                         return $status;
1107                 } catch ( CloudFilesException $e ) { // some other exception?
1108                         $this->handleException( $e, $status, __METHOD__, $params );
1109                         return $status;
1110                 }
1111
1112                 try {
1113                         $output = fopen( 'php://output', 'wb' );
1114                         $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
1115                         $obj->stream( $output, $this->headersFromParams( $params ) );
1116                 } catch ( NoSuchObjectException $e ) {
1117                         $status->fatal( 'backend-fail-stream', $params['src'] );
1118                 } catch ( CloudFilesException $e ) { // some other exception?
1119                         $this->handleException( $e, $status, __METHOD__, $params );
1120                 }
1121
1122                 return $status;
1123         }
1124
1125         /**
1126          * @see FileBackendStore::doGetLocalCopyMulti()
1127          * @return null|TempFSFile
1128          */
1129         protected function doGetLocalCopyMulti( array $params ) {
1130                 $tmpFiles = array();
1131
1132                 $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
1133                 // Blindly create tmp files and stream to them, catching any exception if the file does
1134                 // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
1135                 foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
1136                         $cfOps = array(); // (path => CF_Async_Op)
1137
1138                         foreach ( $pathBatch as $path ) { // each path in this concurrent batch
1139                                 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
1140                                 if ( $srcRel === null ) {
1141                                         $tmpFiles[$path] = null;
1142                                         continue;
1143                                 }
1144                                 $tmpFile = null;
1145                                 try {
1146                                         $sContObj = $this->getContainer( $srcCont );
1147                                         $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
1148                                         // Get source file extension
1149                                         $ext = FileBackend::extensionFromPath( $path );
1150                                         // Create a new temporary file...
1151                                         $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
1152                                         if ( $tmpFile ) {
1153                                                 $handle = fopen( $tmpFile->getPath(), 'wb' );
1154                                                 if ( $handle ) {
1155                                                         $headers = $this->headersFromParams( $params );
1156                                                         if ( count( $pathBatch ) > 1 ) {
1157                                                                 $cfOps[$path] = $obj->stream_async( $handle, $headers );
1158                                                                 $cfOps[$path]->_file_handle = $handle; // close this later
1159                                                         } else {
1160                                                                 $obj->stream( $handle, $headers );
1161                                                                 fclose( $handle );
1162                                                         }
1163                                                 } else {
1164                                                         $tmpFile = null;
1165                                                 }
1166                                         }
1167                                 } catch ( NoSuchContainerException $e ) {
1168                                         $tmpFile = null;
1169                                 } catch ( NoSuchObjectException $e ) {
1170                                         $tmpFile = null;
1171                                 } catch ( CloudFilesException $e ) { // some other exception?
1172                                         $tmpFile = null;
1173                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
1174                                 }
1175                                 $tmpFiles[$path] = $tmpFile;
1176                         }
1177
1178                         $batch = new CF_Async_Op_Batch( $cfOps );
1179                         $cfOps = $batch->execute();
1180                         foreach ( $cfOps as $path => $cfOp ) {
1181                                 try {
1182                                         $cfOp->getLastResponse();
1183                                 } catch ( NoSuchContainerException $e ) {
1184                                         $tmpFiles[$path] = null;
1185                                 } catch ( NoSuchObjectException $e ) {
1186                                         $tmpFiles[$path] = null;
1187                                 } catch ( CloudFilesException $e ) { // some other exception?
1188                                         $tmpFiles[$path] = null;
1189                                         $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
1190                                 }
1191                                 fclose( $cfOp->_file_handle ); // close open handle
1192                         }
1193                 }
1194
1195                 return $tmpFiles;
1196         }
1197
1198         /**
1199          * @see FileBackendStore::getFileHttpUrl()
1200          * @return string|null
1201          */
1202         public function getFileHttpUrl( array $params ) {
1203                 if ( $this->swiftTempUrlKey != '' ||
1204                         ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) )
1205                 {
1206                         list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
1207                         if ( $srcRel === null ) {
1208                                 return null; // invalid path
1209                         }
1210                         try {
1211                                 $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400;
1212                                 $sContObj = $this->getContainer( $srcCont );
1213                                 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
1214                                 if ( $this->swiftTempUrlKey != '' ) {
1215                                         return $obj->get_temp_url( $this->swiftTempUrlKey, $ttl, "GET" );
1216                                 } else { // give S3 API URL for rgw
1217                                         $expires = time() + $ttl;
1218                                         // Path for signature starts with the bucket
1219                                         $spath = '/' . rawurlencode( $srcCont ) . '/' .
1220                                                 str_replace( '%2F', '/', rawurlencode( $srcRel ) );
1221                                         // Calculate the hash
1222                                         $signature = base64_encode( hash_hmac(
1223                                                 'sha1',
1224                                                 "GET\n\n\n{$expires}\n{$spath}",
1225                                                 $this->rgwS3SecretKey,
1226                                                 true // raw
1227                                         ) );
1228                                         // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
1229                                         // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
1230                                         return wfAppendQuery(
1231                                                 str_replace( '/swift/v1', '', // S3 API is the rgw default
1232                                                         $sContObj->cfs_http->getStorageUrl() . $spath ),
1233                                                 array(
1234                                                         'Signature'      => $signature,
1235                                                         'Expires'        => $expires,
1236                                                         'AWSAccessKeyId' => $this->rgwS3AccessKey )
1237                                         );
1238                                 }
1239                         } catch ( NoSuchContainerException $e ) {
1240                         } catch ( CloudFilesException $e ) { // some other exception?
1241                                 $this->handleException( $e, null, __METHOD__, $params );
1242                         }
1243                 }
1244                 return null;
1245         }
1246
1247         /**
1248          * @see FileBackendStore::directoriesAreVirtual()
1249          * @return bool
1250          */
1251         protected function directoriesAreVirtual() {
1252                 return true;
1253         }
1254
1255         /**
1256          * Get headers to send to Swift when reading a file based
1257          * on a FileBackend params array, e.g. that of getLocalCopy().
1258          * $params is currently only checked for a 'latest' flag.
1259          *
1260          * @param $params Array
1261          * @return Array
1262          */
1263         protected function headersFromParams( array $params ) {
1264                 $hdrs = array();
1265                 if ( !empty( $params['latest'] ) ) {
1266                         $hdrs[] = 'X-Newest: true';
1267                 }
1268                 return $hdrs;
1269         }
1270
1271         /**
1272          * @see FileBackendStore::doExecuteOpHandlesInternal()
1273          * @return Array List of corresponding Status objects
1274          */
1275         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1276                 $statuses = array();
1277
1278                 $cfOps = array(); // list of CF_Async_Op objects
1279                 foreach ( $fileOpHandles as $index => $fileOpHandle ) {
1280                         $cfOps[$index] = $fileOpHandle->cfOp;
1281                 }
1282                 $batch = new CF_Async_Op_Batch( $cfOps );
1283
1284                 $cfOps = $batch->execute();
1285                 foreach ( $cfOps as $index => $cfOp ) {
1286                         $status = Status::newGood();
1287                         $function = '_getResponse' . $fileOpHandles[$index]->call;
1288                         try { // catch exceptions; update status
1289                                 $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
1290                                 $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
1291                         } catch ( CloudFilesException $e ) { // some other exception?
1292                                 $this->handleException( $e, $status,
1293                                         __CLASS__ . ":$function", $fileOpHandles[$index]->params );
1294                         }
1295                         $statuses[$index] = $status;
1296                 }
1297
1298                 return $statuses;
1299         }
1300
1301         /**
1302          * Set read/write permissions for a Swift container.
1303          *
1304          * $readGrps is a list of the possible criteria for a request to have
1305          * access to read a container. Each item is one of the following formats:
1306          *   - account:user        : Grants access if the request is by the given user
1307          *   - ".r:<regex>"        : Grants access if the request is from a referrer host that
1308          *                           matches the expression and the request is not for a listing.
1309          *                           Setting this to '*' effectively makes a container public.
1310          *   -".rlistings:<regex>" : Grants access if the request is from a referrer host that
1311          *                           matches the expression and the request for a listing.
1312          *
1313          * $writeGrps is a list of the possible criteria for a request to have
1314          * access to write to a container. Each item is of the following format:
1315          *   - account:user       : Grants access if the request is by the given user
1316          *
1317          * @see http://swift.openstack.org/misc.html#acls
1318          *
1319          * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
1320          * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
1321          *
1322          * @param $contObj CF_Container Swift container
1323          * @param $readGrps Array List of read access routes
1324          * @param $writeGrps Array List of write access routes
1325          * @return Status
1326          */
1327         protected function setContainerAccess(
1328                 CF_Container $contObj, array $readGrps, array $writeGrps
1329         ) {
1330                 $creds = $contObj->cfs_auth->export_credentials();
1331
1332                 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
1333
1334                 // Note: 10 second timeout consistent with php-cloudfiles
1335                 $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
1336                 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
1337                 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
1338                 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
1339
1340                 return $req->execute(); // should return 204
1341         }
1342
1343         /**
1344          * Purge the CDN cache of affected objects if CDN caching is enabled.
1345          * This is for Rackspace/Akamai CDNs.
1346          *
1347          * @param $objects Array List of CF_Object items
1348          * @return void
1349          */
1350         public function purgeCDNCache( array $objects ) {
1351                 if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
1352                         foreach ( $objects as $object ) {
1353                                 try {
1354                                         $object->purge_from_cdn();
1355                                 } catch ( CDNNotEnabledException $e ) {
1356                                         // CDN not enabled; nothing to see here
1357                                 } catch ( CloudFilesException $e ) {
1358                                         $this->handleException( $e, null, __METHOD__,
1359                                                 array( 'cont' => $object->container->name, 'obj' => $object->name ) );
1360                                 }
1361                         }
1362                 }
1363         }
1364
1365         /**
1366          * Get an authenticated connection handle to the Swift proxy
1367          *
1368          * @throws CloudFilesException
1369          * @throws CloudFilesException|Exception
1370          * @return CF_Connection|bool False on failure
1371          */
1372         protected function getConnection() {
1373                 if ( $this->connException instanceof CloudFilesException ) {
1374                         if ( ( time() - $this->connErrorTime ) < 60 ) {
1375                                 throw $this->connException; // failed last attempt; don't bother
1376                         } else { // actually retry this time
1377                                 $this->connException = null;
1378                                 $this->connErrorTime = 0;
1379                         }
1380                 }
1381                 // Session keys expire after a while, so we renew them periodically
1382                 $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL );
1383                 // Authenticate with proxy and get a session key...
1384                 if ( !$this->conn || $reAuth ) {
1385                         $this->sessionStarted = 0;
1386                         $this->connContainerCache->clear();
1387                         $cacheKey = $this->getCredsCacheKey( $this->auth->username );
1388                         $creds = $this->srvCache->get( $cacheKey ); // credentials
1389                         if ( is_array( $creds ) ) { // cache hit
1390                                 $this->auth->load_cached_credentials(
1391                                         $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] );
1392                                 $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case
1393                         } else { // cache miss
1394                                 try {
1395                                         $this->auth->authenticate();
1396                                         $creds = $this->auth->export_credentials();
1397                                         $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache
1398                                         $this->sessionStarted = time();
1399                                 } catch ( CloudFilesException $e ) {
1400                                         $this->connException = $e; // don't keep re-trying
1401                                         $this->connErrorTime = time();
1402                                         throw $e; // throw it back
1403                                 }
1404                         }
1405                         if ( $this->conn ) { // re-authorizing?
1406                                 $this->conn->close(); // close active cURL handles in CF_Http object
1407                         }
1408                         $this->conn = new CF_Connection( $this->auth );
1409                 }
1410                 return $this->conn;
1411         }
1412
1413         /**
1414          * Close the connection to the Swift proxy
1415          *
1416          * @return void
1417          */
1418         protected function closeConnection() {
1419                 if ( $this->conn ) {
1420                         $this->conn->close(); // close active cURL handles in CF_Http object
1421                         $this->conn = null;
1422                         $this->sessionStarted = 0;
1423                         $this->connContainerCache->clear();
1424                 }
1425         }
1426
1427         /**
1428          * Get the cache key for a container
1429          *
1430          * @param $username string
1431          * @return string
1432          */
1433         private function getCredsCacheKey( $username ) {
1434                 return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username );
1435         }
1436
1437         /**
1438          * Get a Swift container object, possibly from process cache.
1439          * Use $reCache if the file count or byte count is needed.
1440          *
1441          * @param $container string Container name
1442          * @param $bypassCache bool Bypass all caches and load from Swift
1443          * @return CF_Container
1444          * @throws CloudFilesException
1445          */
1446         protected function getContainer( $container, $bypassCache = false ) {
1447                 $conn = $this->getConnection(); // Swift proxy connection
1448                 if ( $bypassCache ) { // purge cache
1449                         $this->connContainerCache->clear( $container );
1450                 } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1451                         $this->primeContainerCache( array( $container ) ); // check persistent cache
1452                 }
1453                 if ( !$this->connContainerCache->has( $container, 'obj' ) ) {
1454                         $contObj = $conn->get_container( $container );
1455                         // NoSuchContainerException not thrown: container must exist
1456                         $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it
1457                         if ( !$bypassCache ) {
1458                                 $this->setContainerCache( $container, // update persistent cache
1459                                         array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
1460                                 );
1461                         }
1462                 }
1463                 return $this->connContainerCache->get( $container, 'obj' );
1464         }
1465
1466         /**
1467          * Create a Swift container
1468          *
1469          * @param $container string Container name
1470          * @return CF_Container
1471          * @throws CloudFilesException
1472          */
1473         protected function createContainer( $container ) {
1474                 $conn = $this->getConnection(); // Swift proxy connection
1475                 $contObj = $conn->create_container( $container );
1476                 $this->connContainerCache->set( $container, 'obj', $contObj ); // cache
1477                 return $contObj;
1478         }
1479
1480         /**
1481          * Delete a Swift container
1482          *
1483          * @param $container string Container name
1484          * @return void
1485          * @throws CloudFilesException
1486          */
1487         protected function deleteContainer( $container ) {
1488                 $conn = $this->getConnection(); // Swift proxy connection
1489                 $this->connContainerCache->clear( $container ); // purge
1490                 $conn->delete_container( $container );
1491         }
1492
1493         /**
1494          * @see FileBackendStore::doPrimeContainerCache()
1495          * @return void
1496          */
1497         protected function doPrimeContainerCache( array $containerInfo ) {
1498                 try {
1499                         $conn = $this->getConnection(); // Swift proxy connection
1500                         foreach ( $containerInfo as $container => $info ) {
1501                                 $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http,
1502                                         $container, $info['count'], $info['bytes'] );
1503                                 $this->connContainerCache->set( $container, 'obj', $contObj );
1504                         }
1505                 } catch ( CloudFilesException $e ) { // some other exception?
1506                         $this->handleException( $e, null, __METHOD__, array() );
1507                 }
1508         }
1509
1510         /**
1511          * Log an unexpected exception for this backend.
1512          * This also sets the Status object to have a fatal error.
1513          *
1514          * @param $e Exception
1515          * @param $status Status|null
1516          * @param $func string
1517          * @param $params Array
1518          * @return void
1519          */
1520         protected function handleException( Exception $e, $status, $func, array $params ) {
1521                 if ( $status instanceof Status ) {
1522                         if ( $e instanceof AuthenticationException ) {
1523                                 $status->fatal( 'backend-fail-connect', $this->name );
1524                         } else {
1525                                 $status->fatal( 'backend-fail-internal', $this->name );
1526                         }
1527                 }
1528                 if ( $e->getMessage() ) {
1529                         trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
1530                 }
1531                 if ( $e instanceof InvalidResponseException ) { // possibly a stale token
1532                         $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) );
1533                         $this->closeConnection(); // force a re-connect and re-auth next time
1534                 }
1535                 wfDebugLog( 'SwiftBackend',
1536                         get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
1537                         ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
1538                 );
1539         }
1540 }
1541
1542 /**
1543  * @see FileBackendStoreOpHandle
1544  */
1545 class SwiftFileOpHandle extends FileBackendStoreOpHandle {
1546         /** @var CF_Async_Op */
1547         public $cfOp;
1548         /** @var Array */
1549         public $affectedObjects = array();
1550
1551         public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) {
1552                 $this->backend = $backend;
1553                 $this->params = $params;
1554                 $this->call = $call;
1555                 $this->cfOp = $cfOp;
1556         }
1557 }
1558
1559 /**
1560  * SwiftFileBackend helper class to page through listings.
1561  * Swift also has a listing limit of 10,000 objects for sanity.
1562  * Do not use this class from places outside SwiftFileBackend.
1563  *
1564  * @ingroup FileBackend
1565  */
1566 abstract class SwiftFileBackendList implements Iterator {
1567         /** @var Array */
1568         protected $bufferIter = array();
1569         protected $bufferAfter = null; // string; list items *after* this path
1570         protected $pos = 0; // integer
1571         /** @var Array */
1572         protected $params = array();
1573
1574         /** @var SwiftFileBackend */
1575         protected $backend;
1576         protected $container; // string; container name
1577         protected $dir; // string; storage directory
1578         protected $suffixStart; // integer
1579
1580         const PAGE_SIZE = 9000; // file listing buffer size
1581
1582         /**
1583          * @param $backend SwiftFileBackend
1584          * @param $fullCont string Resolved container name
1585          * @param $dir string Resolved directory relative to container
1586          * @param $params Array
1587          */
1588         public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) {
1589                 $this->backend = $backend;
1590                 $this->container = $fullCont;
1591                 $this->dir = $dir;
1592                 if ( substr( $this->dir, -1 ) === '/' ) {
1593                         $this->dir = substr( $this->dir, 0, -1 ); // remove trailing slash
1594                 }
1595                 if ( $this->dir == '' ) { // whole container
1596                         $this->suffixStart = 0;
1597                 } else { // dir within container
1598                         $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/"
1599                 }
1600                 $this->params = $params;
1601         }
1602
1603         /**
1604          * @see Iterator::key()
1605          * @return integer
1606          */
1607         public function key() {
1608                 return $this->pos;
1609         }
1610
1611         /**
1612          * @see Iterator::next()
1613          * @return void
1614          */
1615         public function next() {
1616                 // Advance to the next file in the page
1617                 next( $this->bufferIter );
1618                 ++$this->pos;
1619                 // Check if there are no files left in this page and
1620                 // advance to the next page if this page was not empty.
1621                 if ( !$this->valid() && count( $this->bufferIter ) ) {
1622                         $this->bufferIter = $this->pageFromList(
1623                                 $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1624                         ); // updates $this->bufferAfter
1625                 }
1626         }
1627
1628         /**
1629          * @see Iterator::rewind()
1630          * @return void
1631          */
1632         public function rewind() {
1633                 $this->pos = 0;
1634                 $this->bufferAfter = null;
1635                 $this->bufferIter = $this->pageFromList(
1636                         $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params
1637                 ); // updates $this->bufferAfter
1638         }
1639
1640         /**
1641          * @see Iterator::valid()
1642          * @return bool
1643          */
1644         public function valid() {
1645                 if ( $this->bufferIter === null ) {
1646                         return false; // some failure?
1647                 } else {
1648                         return ( current( $this->bufferIter ) !== false ); // no paths can have this value
1649                 }
1650         }
1651
1652         /**
1653          * Get the given list portion (page)
1654          *
1655          * @param $container string Resolved container name
1656          * @param $dir string Resolved path relative to container
1657          * @param $after string|null
1658          * @param $limit integer
1659          * @param $params Array
1660          * @return Traversable|Array|null Returns null on failure
1661          */
1662         abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1663 }
1664
1665 /**
1666  * Iterator for listing directories
1667  */
1668 class SwiftFileBackendDirList extends SwiftFileBackendList {
1669         /**
1670          * @see Iterator::current()
1671          * @return string|bool String (relative path) or false
1672          */
1673         public function current() {
1674                 return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
1675         }
1676
1677         /**
1678          * @see SwiftFileBackendList::pageFromList()
1679          * @return Array|null
1680          */
1681         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1682                 return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1683         }
1684 }
1685
1686 /**
1687  * Iterator for listing regular files
1688  */
1689 class SwiftFileBackendFileList extends SwiftFileBackendList {
1690         /**
1691          * @see Iterator::current()
1692          * @return string|bool String (relative path) or false
1693          */
1694         public function current() {
1695                 return substr( current( $this->bufferIter ), $this->suffixStart );
1696         }
1697
1698         /**
1699          * @see SwiftFileBackendList::pageFromList()
1700          * @return Array|null
1701          */
1702         protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1703                 return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
1704         }
1705 }