includes/filebackend/FileBackendStore.php

   1 <?php
   2 /**
   3  * Base class for all backends using particular storage medium.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Aaron Schulz
  23  */
  24
  25 /**
  26  * @brief Base class for all backends using particular storage medium.
  27  *
  28  * This class defines the methods as abstract that subclasses must implement.
  29  * Outside callers should *not* use functions with "Internal" in the name.
  30  *
  31  * The FileBackend operations are implemented using basic functions
  32  * such as storeInternal(), copyInternal(), deleteInternal() and the like.
  33  * This class is also responsible for path resolution and sanitization.
  34  *
  35  * @ingroup FileBackend
  36  * @since 1.19
  37  */
  38 abstract class FileBackendStore extends FileBackend {
  39         /** @var BagOStuff */
  40         protected $memCache;
  41         /** @var ProcessCacheLRU Map of paths to small (RAM/disk) cache items */
  42         protected $cheapCache;
  43         /** @var ProcessCacheLRU Map of paths to large (RAM/disk) cache items */
  44         protected $expensiveCache;
  45
  46         /** @var Array Map of container names to sharding config */
  47         protected $shardViaHashLevels = array();
  48
  49         /** @var callback Method to get the MIME type of files */
  50         protected $mimeCallback;
  51
  52         protected $maxFileSize = 4294967296; // integer bytes (4GiB)
  53
  54         const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
  55         const CACHE_CHEAP_SIZE = 300; // integer; max entries in "cheap cache"
  56         const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
  57
  58         /**
  59          * @see FileBackend::__construct()
  60          * Additional $config params include:
  61          *   - mimeCallback : Callback that takes (storage path, content, file system path) and
  62          *                    returns the MIME type of the file or 'unknown/unknown'. The file
  63          *                    system path parameter should be used if the content one is null.
  64          *
  65          * @param array $config
  66          */
  67         public function __construct( array $config ) {
  68                 parent::__construct( $config );
  69                 $this->mimeCallback = isset( $config['mimeCallback'] )
  70                         ? $config['mimeCallback']
  71                         : function( $storagePath, $content, $fsPath ) {
  72                                 // @TODO: handle the case of extension-less files using the contents
  73                                 return StreamFile::contentTypeFromPath( $storagePath ) ?: 'unknown/unknown';
  74                         };
  75                 $this->memCache = new EmptyBagOStuff(); // disabled by default
  76                 $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE );
  77                 $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE );
  78         }
  79
  80         /**
  81          * Get the maximum allowable file size given backend
  82          * medium restrictions and basic performance constraints.
  83          * Do not call this function from places outside FileBackend and FileOp.
  84          *
  85          * @return integer Bytes
  86          */
  87         final public function maxFileSizeInternal() {
  88                 return $this->maxFileSize;
  89         }
  90
  91         /**
  92          * Check if a file can be created or changed at a given storage path.
  93          * FS backends should check if the parent directory exists, files can be
  94          * written under it, and that any file already there is writable.
  95          * Backends using key/value stores should check if the container exists.
  96          *
  97          * @param string $storagePath
  98          * @return bool
  99          */
 100         abstract public function isPathUsableInternal( $storagePath );
 101
 102         /**
 103          * Create a file in the backend with the given contents.
 104          * This will overwrite any file that exists at the destination.
 105          * Do not call this function from places outside FileBackend and FileOp.
 106          *
 107          * $params include:
 108          *   - content     : the raw file contents
 109          *   - dst         : destination storage path
 110          *   - headers     : HTTP header name/value map
 111          *   - async       : Status will be returned immediately if supported.
 112          *                   If the status is OK, then its value field will be
 113          *                   set to a FileBackendStoreOpHandle object.
 114          *   - dstExists   : Whether a file exists at the destination (optimization).
 115          *                   Callers can use "false" if no existing file is being changed.
 116          *
 117          * @param array $params
 118          * @return Status
 119          */
 120         final public function createInternal( array $params ) {
 121                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 122                 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
 123                         $status = Status::newFatal( 'backend-fail-maxsize',
 124                                 $params['dst'], $this->maxFileSizeInternal() );
 125                 } else {
 126                         $status = $this->doCreateInternal( $params );
 127                         $this->clearCache( array( $params['dst'] ) );
 128                         if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
 129                                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 130                         }
 131                 }
 132                 return $status;
 133         }
 134
 135         /**
 136          * @see FileBackendStore::createInternal()
 137          * @return Status
 138          */
 139         abstract protected function doCreateInternal( array $params );
 140
 141         /**
 142          * Store a file into the backend from a file on disk.
 143          * This will overwrite any file that exists at the destination.
 144          * Do not call this function from places outside FileBackend and FileOp.
 145          *
 146          * $params include:
 147          *   - src         : source path on disk
 148          *   - dst         : destination storage path
 149          *   - headers     : HTTP header name/value map
 150          *   - async       : Status will be returned immediately if supported.
 151          *                   If the status is OK, then its value field will be
 152          *                   set to a FileBackendStoreOpHandle object.
 153          *   - dstExists   : Whether a file exists at the destination (optimization).
 154          *                   Callers can use "false" if no existing file is being changed.
 155          *
 156          * @param array $params
 157          * @return Status
 158          */
 159         final public function storeInternal( array $params ) {
 160                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 161                 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
 162                         $status = Status::newFatal( 'backend-fail-maxsize',
 163                                 $params['dst'], $this->maxFileSizeInternal() );
 164                 } else {
 165                         $status = $this->doStoreInternal( $params );
 166                         $this->clearCache( array( $params['dst'] ) );
 167                         if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
 168                                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 169                         }
 170                 }
 171                 return $status;
 172         }
 173
 174         /**
 175          * @see FileBackendStore::storeInternal()
 176          * @return Status
 177          */
 178         abstract protected function doStoreInternal( array $params );
 179
 180         /**
 181          * Copy a file from one storage path to another in the backend.
 182          * This will overwrite any file that exists at the destination.
 183          * Do not call this function from places outside FileBackend and FileOp.
 184          *
 185          * $params include:
 186          *   - src                 : source storage path
 187          *   - dst                 : destination storage path
 188          *   - ignoreMissingSource : do nothing if the source file does not exist
 189          *   - headers             : HTTP header name/value map
 190          *   - async               : Status will be returned immediately if supported.
 191          *                           If the status is OK, then its value field will be
 192          *                           set to a FileBackendStoreOpHandle object.
 193          *   - dstExists           : Whether a file exists at the destination (optimization).
 194          *                           Callers can use "false" if no existing file is being changed.
 195          *
 196          * @param array $params
 197          * @return Status
 198          */
 199         final public function copyInternal( array $params ) {
 200                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 201                 $status = $this->doCopyInternal( $params );
 202                 $this->clearCache( array( $params['dst'] ) );
 203                 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
 204                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 205                 }
 206                 return $status;
 207         }
 208
 209         /**
 210          * @see FileBackendStore::copyInternal()
 211          * @return Status
 212          */
 213         abstract protected function doCopyInternal( array $params );
 214
 215         /**
 216          * Delete a file at the storage path.
 217          * Do not call this function from places outside FileBackend and FileOp.
 218          *
 219          * $params include:
 220          *   - src                 : source storage path
 221          *   - ignoreMissingSource : do nothing if the source file does not exist
 222          *   - async               : Status will be returned immediately if supported.
 223          *                           If the status is OK, then its value field will be
 224          *                           set to a FileBackendStoreOpHandle object.
 225          *
 226          * @param array $params
 227          * @return Status
 228          */
 229         final public function deleteInternal( array $params ) {
 230                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 231                 $status = $this->doDeleteInternal( $params );
 232                 $this->clearCache( array( $params['src'] ) );
 233                 $this->deleteFileCache( $params['src'] ); // persistent cache
 234                 return $status;
 235         }
 236
 237         /**
 238          * @see FileBackendStore::deleteInternal()
 239          * @return Status
 240          */
 241         abstract protected function doDeleteInternal( array $params );
 242
 243         /**
 244          * Move a file from one storage path to another in the backend.
 245          * This will overwrite any file that exists at the destination.
 246          * Do not call this function from places outside FileBackend and FileOp.
 247          *
 248          * $params include:
 249          *   - src                 : source storage path
 250          *   - dst                 : destination storage path
 251          *   - ignoreMissingSource : do nothing if the source file does not exist
 252          *   - headers             : HTTP header name/value map
 253          *   - async               : Status will be returned immediately if supported.
 254          *                           If the status is OK, then its value field will be
 255          *                           set to a FileBackendStoreOpHandle object.
 256          *   - dstExists           : Whether a file exists at the destination (optimization).
 257          *                           Callers can use "false" if no existing file is being changed.
 258          *
 259          * @param array $params
 260          * @return Status
 261          */
 262         final public function moveInternal( array $params ) {
 263                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 264                 $status = $this->doMoveInternal( $params );
 265                 $this->clearCache( array( $params['src'], $params['dst'] ) );
 266                 $this->deleteFileCache( $params['src'] ); // persistent cache
 267                 if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
 268                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 269                 }
 270                 return $status;
 271         }
 272
 273         /**
 274          * @see FileBackendStore::moveInternal()
 275          * @return Status
 276          */
 277         protected function doMoveInternal( array $params ) {
 278                 unset( $params['async'] ); // two steps, won't work here :)
 279                 $nsrc = FileBackend::normalizeStoragePath( $params['src'] );
 280                 $ndst = FileBackend::normalizeStoragePath( $params['dst'] );
 281                 // Copy source to dest
 282                 $status = $this->copyInternal( $params );
 283                 if ( $nsrc !== $ndst && $status->isOK() ) {
 284                         // Delete source (only fails due to races or network problems)
 285                         $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
 286                         $status->setResult( true, $status->value ); // ignore delete() errors
 287                 }
 288                 return $status;
 289         }
 290
 291         /**
 292          * Alter metadata for a file at the storage path.
 293          * Do not call this function from places outside FileBackend and FileOp.
 294          *
 295          * $params include:
 296          *   - src           : source storage path
 297          *   - headers       : HTTP header name/value map
 298          *   - async         : Status will be returned immediately if supported.
 299          *                     If the status is OK, then its value field will be
 300          *                     set to a FileBackendStoreOpHandle object.
 301          *
 302          * @param array $params
 303          * @return Status
 304          */
 305         final public function describeInternal( array $params ) {
 306                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 307                 if ( count( $params['headers'] ) ) {
 308                         $status = $this->doDescribeInternal( $params );
 309                         $this->clearCache( array( $params['src'] ) );
 310                         $this->deleteFileCache( $params['src'] ); // persistent cache
 311                 } else {
 312                         $status = Status::newGood(); // nothing to do
 313                 }
 314                 return $status;
 315         }
 316
 317         /**
 318          * @see FileBackendStore::describeInternal()
 319          * @return Status
 320          */
 321         protected function doDescribeInternal( array $params ) {
 322                 return Status::newGood();
 323         }
 324
 325         /**
 326          * No-op file operation that does nothing.
 327          * Do not call this function from places outside FileBackend and FileOp.
 328          *
 329          * @param array $params
 330          * @return Status
 331          */
 332         final public function nullInternal( array $params ) {
 333                 return Status::newGood();
 334         }
 335
 336         final public function concatenate( array $params ) {
 337                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 338                 $status = Status::newGood();
 339
 340                 // Try to lock the source files for the scope of this function
 341                 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
 342                 if ( $status->isOK() ) {
 343                         // Actually do the file concatenation...
 344                         $start_time = microtime( true );
 345                         $status->merge( $this->doConcatenate( $params ) );
 346                         $sec = microtime( true ) - $start_time;
 347                         if ( !$status->isOK() ) {
 348                                 wfDebugLog( 'FileOperation', get_class( $this ) . " failed to concatenate " .
 349                                         count( $params['srcs'] ) . " file(s) [$sec sec]" );
 350                         }
 351                 }
 352
 353                 return $status;
 354         }
 355
 356         /**
 357          * @see FileBackendStore::concatenate()
 358          * @return Status
 359          */
 360         protected function doConcatenate( array $params ) {
 361                 $status = Status::newGood();
 362                 $tmpPath = $params['dst']; // convenience
 363                 unset( $params['latest'] ); // sanity
 364
 365                 // Check that the specified temp file is valid...
 366                 wfSuppressWarnings();
 367                 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
 368                 wfRestoreWarnings();
 369                 if ( !$ok ) { // not present or not empty
 370                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 371                         return $status;
 372                 }
 373
 374                 // Get local FS versions of the chunks needed for the concatenation...
 375                 $fsFiles = $this->getLocalReferenceMulti( $params );
 376                 foreach ( $fsFiles as $path => &$fsFile ) {
 377                         if ( !$fsFile ) { // chunk failed to download?
 378                                 $fsFile = $this->getLocalReference( array( 'src' => $path ) );
 379                                 if ( !$fsFile ) { // retry failed?
 380                                         $status->fatal( 'backend-fail-read', $path );
 381                                         return $status;
 382                                 }
 383                         }
 384                 }
 385                 unset( $fsFile ); // unset reference so we can reuse $fsFile
 386
 387                 // Get a handle for the destination temp file
 388                 $tmpHandle = fopen( $tmpPath, 'ab' );
 389                 if ( $tmpHandle === false ) {
 390                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 391                         return $status;
 392                 }
 393
 394                 // Build up the temp file using the source chunks (in order)...
 395                 foreach ( $fsFiles as $virtualSource => $fsFile ) {
 396                         // Get a handle to the local FS version
 397                         $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
 398                         if ( $sourceHandle === false ) {
 399                                 fclose( $tmpHandle );
 400                                 $status->fatal( 'backend-fail-read', $virtualSource );
 401                                 return $status;
 402                         }
 403                         // Append chunk to file (pass chunk size to avoid magic quotes)
 404                         if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
 405                                 fclose( $sourceHandle );
 406                                 fclose( $tmpHandle );
 407                                 $status->fatal( 'backend-fail-writetemp', $tmpPath );
 408                                 return $status;
 409                         }
 410                         fclose( $sourceHandle );
 411                 }
 412                 if ( !fclose( $tmpHandle ) ) {
 413                         $status->fatal( 'backend-fail-closetemp', $tmpPath );
 414                         return $status;
 415                 }
 416
 417                 clearstatcache(); // temp file changed
 418
 419                 return $status;
 420         }
 421
 422         final protected function doPrepare( array $params ) {
 423                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 424                 $status = Status::newGood();
 425
 426                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 427                 if ( $dir === null ) {
 428                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 429                         return $status; // invalid storage path
 430                 }
 431
 432                 if ( $shard !== null ) { // confined to a single container/shard
 433                         $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
 434                 } else { // directory is on several shards
 435                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 436                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 437                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 438                                 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 439                         }
 440                 }
 441
 442                 return $status;
 443         }
 444
 445         /**
 446          * @see FileBackendStore::doPrepare()
 447          * @return Status
 448          */
 449         protected function doPrepareInternal( $container, $dir, array $params ) {
 450                 return Status::newGood();
 451         }
 452
 453         final protected function doSecure( array $params ) {
 454                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 455                 $status = Status::newGood();
 456
 457                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 458                 if ( $dir === null ) {
 459                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 460                         return $status; // invalid storage path
 461                 }
 462
 463                 if ( $shard !== null ) { // confined to a single container/shard
 464                         $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 465                 } else { // directory is on several shards
 466                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 467                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 468                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 469                                 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 470                         }
 471                 }
 472
 473                 return $status;
 474         }
 475
 476         /**
 477          * @see FileBackendStore::doSecure()
 478          * @return Status
 479          */
 480         protected function doSecureInternal( $container, $dir, array $params ) {
 481                 return Status::newGood();
 482         }
 483
 484         final protected function doPublish( array $params ) {
 485                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 486                 $status = Status::newGood();
 487
 488                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 489                 if ( $dir === null ) {
 490                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 491                         return $status; // invalid storage path
 492                 }
 493
 494                 if ( $shard !== null ) { // confined to a single container/shard
 495                         $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
 496                 } else { // directory is on several shards
 497                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 498                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 499                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 500                                 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 501                         }
 502                 }
 503
 504                 return $status;
 505         }
 506
 507         /**
 508          * @see FileBackendStore::doPublish()
 509          * @return Status
 510          */
 511         protected function doPublishInternal( $container, $dir, array $params ) {
 512                 return Status::newGood();
 513         }
 514
 515         final protected function doClean( array $params ) {
 516                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 517                 $status = Status::newGood();
 518
 519                 // Recursive: first delete all empty subdirs recursively
 520                 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
 521                         $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) );
 522                         if ( $subDirsRel !== null ) { // no errors
 523                                 foreach ( $subDirsRel as $subDirRel ) {
 524                                         $subDir = $params['dir'] . "/{$subDirRel}"; // full path
 525                                         $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) );
 526                                 }
 527                                 unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
 528                         }
 529                 }
 530
 531                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 532                 if ( $dir === null ) {
 533                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 534                         return $status; // invalid storage path
 535                 }
 536
 537                 // Attempt to lock this directory...
 538                 $filesLockEx = array( $params['dir'] );
 539                 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 540                 if ( !$status->isOK() ) {
 541                         return $status; // abort
 542                 }
 543
 544                 if ( $shard !== null ) { // confined to a single container/shard
 545                         $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
 546                         $this->deleteContainerCache( $fullCont ); // purge cache
 547                 } else { // directory is on several shards
 548                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 549                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 550                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 551                                 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 552                                 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
 553                         }
 554                 }
 555
 556                 return $status;
 557         }
 558
 559         /**
 560          * @see FileBackendStore::doClean()
 561          * @return Status
 562          */
 563         protected function doCleanInternal( $container, $dir, array $params ) {
 564                 return Status::newGood();
 565         }
 566
 567         final public function fileExists( array $params ) {
 568                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 569                 $stat = $this->getFileStat( $params );
 570                 return ( $stat === null ) ? null : (bool)$stat; // null => failure
 571         }
 572
 573         final public function getFileTimestamp( array $params ) {
 574                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 575                 $stat = $this->getFileStat( $params );
 576                 return $stat ? $stat['mtime'] : false;
 577         }
 578
 579         final public function getFileSize( array $params ) {
 580                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 581                 $stat = $this->getFileStat( $params );
 582                 return $stat ? $stat['size'] : false;
 583         }
 584
 585         final public function getFileStat( array $params ) {
 586                 $path = self::normalizeStoragePath( $params['src'] );
 587                 if ( $path === null ) {
 588                         return false; // invalid storage path
 589                 }
 590                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 591                 $latest = !empty( $params['latest'] ); // use latest data?
 592                 if ( !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
 593                         $this->primeFileCache( array( $path ) ); // check persistent cache
 594                 }
 595                 if ( $this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) {
 596                         $stat = $this->cheapCache->get( $path, 'stat' );
 597                         // If we want the latest data, check that this cached
 598                         // value was in fact fetched with the latest available data.
 599                         if ( is_array( $stat ) ) {
 600                                 if ( !$latest || $stat['latest'] ) {
 601                                         return $stat;
 602                                 }
 603                         } elseif ( in_array( $stat, array( 'NOT_EXIST', 'NOT_EXIST_LATEST' ) ) ) {
 604                                 if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) {
 605                                         return false;
 606                                 }
 607                         }
 608                 }
 609                 wfProfileIn( __METHOD__ . '-miss' );
 610                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 611                 $stat = $this->doGetFileStat( $params );
 612                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 613                 wfProfileOut( __METHOD__ . '-miss' );
 614                 if ( is_array( $stat ) ) { // file exists
 615                         $stat['latest'] = $latest;
 616                         $this->cheapCache->set( $path, 'stat', $stat );
 617                         $this->setFileCache( $path, $stat ); // update persistent cache
 618                         if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
 619                                 $this->cheapCache->set( $path, 'sha1',
 620                                         array( 'hash' => $stat['sha1'], 'latest' => $latest ) );
 621                         }
 622                 } elseif ( $stat === false ) { // file does not exist
 623                         $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
 624                         $this->cheapCache->set( $path, 'sha1', // the SHA-1 must be false too
 625                                 array( 'hash' => false, 'latest' => $latest ) );
 626                         wfDebug( __METHOD__ . ": File $path does not exist.\n" );
 627                 } else { // an error occurred
 628                         wfDebug( __METHOD__ . ": Could not stat file $path.\n" );
 629                 }
 630                 return $stat;
 631         }
 632
 633         /**
 634          * @see FileBackendStore::getFileStat()
 635          */
 636         abstract protected function doGetFileStat( array $params );
 637
 638         public function getFileContentsMulti( array $params ) {
 639                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 640
 641                 $params = $this->setConcurrencyFlags( $params );
 642                 $contents = $this->doGetFileContentsMulti( $params );
 643
 644                 return $contents;
 645         }
 646
 647         /**
 648          * @see FileBackendStore::getFileContentsMulti()
 649          * @return Array
 650          */
 651         protected function doGetFileContentsMulti( array $params ) {
 652                 $contents = array();
 653                 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
 654                         wfSuppressWarnings();
 655                         $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false;
 656                         wfRestoreWarnings();
 657                 }
 658                 return $contents;
 659         }
 660
 661         final public function getFileSha1Base36( array $params ) {
 662                 $path = self::normalizeStoragePath( $params['src'] );
 663                 if ( $path === null ) {
 664                         return false; // invalid storage path
 665                 }
 666                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 667                 $latest = !empty( $params['latest'] ); // use latest data?
 668                 if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) {
 669                         $stat = $this->cheapCache->get( $path, 'sha1' );
 670                         // If we want the latest data, check that this cached
 671                         // value was in fact fetched with the latest available data.
 672                         if ( !$latest || $stat['latest'] ) {
 673                                 return $stat['hash'];
 674                         }
 675                 }
 676                 wfProfileIn( __METHOD__ . '-miss' );
 677                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 678                 $hash = $this->doGetFileSha1Base36( $params );
 679                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 680                 wfProfileOut( __METHOD__ . '-miss' );
 681                 $this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) );
 682                 return $hash;
 683         }
 684
 685         /**
 686          * @see FileBackendStore::getFileSha1Base36()
 687          * @return bool|string
 688          */
 689         protected function doGetFileSha1Base36( array $params ) {
 690                 $fsFile = $this->getLocalReference( $params );
 691                 if ( !$fsFile ) {
 692                         return false;
 693                 } else {
 694                         return $fsFile->getSha1Base36();
 695                 }
 696         }
 697
 698         final public function getFileProps( array $params ) {
 699                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 700                 $fsFile = $this->getLocalReference( $params );
 701                 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
 702                 return $props;
 703         }
 704
 705         final public function getLocalReferenceMulti( array $params ) {
 706                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 707
 708                 $params = $this->setConcurrencyFlags( $params );
 709
 710                 $fsFiles = array(); // (path => FSFile)
 711                 $latest = !empty( $params['latest'] ); // use latest data?
 712                 // Reuse any files already in process cache...
 713                 foreach ( $params['srcs'] as $src ) {
 714                         $path = self::normalizeStoragePath( $src );
 715                         if ( $path === null ) {
 716                                 $fsFiles[$src] = null; // invalid storage path
 717                         } elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) {
 718                                 $val = $this->expensiveCache->get( $path, 'localRef' );
 719                                 // If we want the latest data, check that this cached
 720                                 // value was in fact fetched with the latest available data.
 721                                 if ( !$latest || $val['latest'] ) {
 722                                         $fsFiles[$src] = $val['object'];
 723                                 }
 724                         }
 725                 }
 726                 // Fetch local references of any remaning files...
 727                 $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
 728                 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
 729                         $fsFiles[$path] = $fsFile;
 730                         if ( $fsFile ) { // update the process cache...
 731                                 $this->expensiveCache->set( $path, 'localRef',
 732                                         array( 'object' => $fsFile, 'latest' => $latest ) );
 733                         }
 734                 }
 735
 736                 return $fsFiles;
 737         }
 738
 739         /**
 740          * @see FileBackendStore::getLocalReferenceMulti()
 741          * @return Array
 742          */
 743         protected function doGetLocalReferenceMulti( array $params ) {
 744                 return $this->doGetLocalCopyMulti( $params );
 745         }
 746
 747         final public function getLocalCopyMulti( array $params ) {
 748                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 749
 750                 $params = $this->setConcurrencyFlags( $params );
 751                 $tmpFiles = $this->doGetLocalCopyMulti( $params );
 752
 753                 return $tmpFiles;
 754         }
 755
 756         /**
 757          * @see FileBackendStore::getLocalCopyMulti()
 758          * @return Array
 759          */
 760         abstract protected function doGetLocalCopyMulti( array $params );
 761
 762         /**
 763          * @see FileBackend::getFileHttpUrl()
 764          * @return string|null
 765          */
 766         public function getFileHttpUrl( array $params ) {
 767                 return null; // not supported
 768         }
 769
 770         final public function streamFile( array $params ) {
 771                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 772                 $status = Status::newGood();
 773
 774                 $info = $this->getFileStat( $params );
 775                 if ( !$info ) { // let StreamFile handle the 404
 776                         $status->fatal( 'backend-fail-notexists', $params['src'] );
 777                 }
 778
 779                 // Set output buffer and HTTP headers for stream
 780                 $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array();
 781                 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
 782                 if ( $res == StreamFile::NOT_MODIFIED ) {
 783                         // do nothing; client cache is up to date
 784                 } elseif ( $res == StreamFile::READY_STREAM ) {
 785                         wfProfileIn( __METHOD__ . '-send' );
 786                         wfProfileIn( __METHOD__ . '-send-' . $this->name );
 787                         $status = $this->doStreamFile( $params );
 788                         wfProfileOut( __METHOD__ . '-send-' . $this->name );
 789                         wfProfileOut( __METHOD__ . '-send' );
 790                         if ( !$status->isOK() ) {
 791                                 // Per bug 41113, nasty things can happen if bad cache entries get
 792                                 // stuck in cache. It's also possible that this error can come up
 793                                 // with simple race conditions. Clear out the stat cache to be safe.
 794                                 $this->clearCache( array( $params['src'] ) );
 795                                 $this->deleteFileCache( $params['src'] );
 796                                 trigger_error( "Bad stat cache or race condition for file {$params['src']}." );
 797                         }
 798                 } else {
 799                         $status->fatal( 'backend-fail-stream', $params['src'] );
 800                 }
 801
 802                 return $status;
 803         }
 804
 805         /**
 806          * @see FileBackendStore::streamFile()
 807          * @return Status
 808          */
 809         protected function doStreamFile( array $params ) {
 810                 $status = Status::newGood();
 811
 812                 $fsFile = $this->getLocalReference( $params );
 813                 if ( !$fsFile ) {
 814                         $status->fatal( 'backend-fail-stream', $params['src'] );
 815                 } elseif ( !readfile( $fsFile->getPath() ) ) {
 816                         $status->fatal( 'backend-fail-stream', $params['src'] );
 817                 }
 818
 819                 return $status;
 820         }
 821
 822         final public function directoryExists( array $params ) {
 823                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 824                 if ( $dir === null ) {
 825                         return false; // invalid storage path
 826                 }
 827                 if ( $shard !== null ) { // confined to a single container/shard
 828                         return $this->doDirectoryExists( $fullCont, $dir, $params );
 829                 } else { // directory is on several shards
 830                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 831                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 832                         $res = false; // response
 833                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 834                                 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
 835                                 if ( $exists ) {
 836                                         $res = true;
 837                                         break; // found one!
 838                                 } elseif ( $exists === null ) { // error?
 839                                         $res = null; // if we don't find anything, it is indeterminate
 840                                 }
 841                         }
 842                         return $res;
 843                 }
 844         }
 845
 846         /**
 847          * @see FileBackendStore::directoryExists()
 848          *
 849          * @param string $container Resolved container name
 850          * @param string $dir Resolved path relative to container
 851          * @param array $params
 852          * @return bool|null
 853          */
 854         abstract protected function doDirectoryExists( $container, $dir, array $params );
 855
 856         final public function getDirectoryList( array $params ) {
 857                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 858                 if ( $dir === null ) { // invalid storage path
 859                         return null;
 860                 }
 861                 if ( $shard !== null ) {
 862                         // File listing is confined to a single container/shard
 863                         return $this->getDirectoryListInternal( $fullCont, $dir, $params );
 864                 } else {
 865                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 866                         // File listing spans multiple containers/shards
 867                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 868                         return new FileBackendStoreShardDirIterator( $this,
 869                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 870                 }
 871         }
 872
 873         /**
 874          * Do not call this function from places outside FileBackend
 875          *
 876          * @see FileBackendStore::getDirectoryList()
 877          *
 878          * @param string $container Resolved container name
 879          * @param string $dir Resolved path relative to container
 880          * @param array $params
 881          * @return Traversable|Array|null Returns null on failure
 882          */
 883         abstract public function getDirectoryListInternal( $container, $dir, array $params );
 884
 885         final public function getFileList( array $params ) {
 886                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 887                 if ( $dir === null ) { // invalid storage path
 888                         return null;
 889                 }
 890                 if ( $shard !== null ) {
 891                         // File listing is confined to a single container/shard
 892                         return $this->getFileListInternal( $fullCont, $dir, $params );
 893                 } else {
 894                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 895                         // File listing spans multiple containers/shards
 896                         list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
 897                         return new FileBackendStoreShardFileIterator( $this,
 898                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 899                 }
 900         }
 901
 902         /**
 903          * Do not call this function from places outside FileBackend
 904          *
 905          * @see FileBackendStore::getFileList()
 906          *
 907          * @param string $container Resolved container name
 908          * @param string $dir Resolved path relative to container
 909          * @param array $params
 910          * @return Traversable|Array|null Returns null on failure
 911          */
 912         abstract public function getFileListInternal( $container, $dir, array $params );
 913
 914         /**
 915          * Return a list of FileOp objects from a list of operations.
 916          * Do not call this function from places outside FileBackend.
 917          *
 918          * The result must have the same number of items as the input.
 919          * An exception is thrown if an unsupported operation is requested.
 920          *
 921          * @param array $ops Same format as doOperations()
 922          * @return Array List of FileOp objects
 923          * @throws MWException
 924          */
 925         final public function getOperationsInternal( array $ops ) {
 926                 $supportedOps = array(
 927                         'store' => 'StoreFileOp',
 928                         'copy' => 'CopyFileOp',
 929                         'move' => 'MoveFileOp',
 930                         'delete' => 'DeleteFileOp',
 931                         'create' => 'CreateFileOp',
 932                         'describe' => 'DescribeFileOp',
 933                         'null' => 'NullFileOp'
 934                 );
 935
 936                 $performOps = array(); // array of FileOp objects
 937                 // Build up ordered array of FileOps...
 938                 foreach ( $ops as $operation ) {
 939                         $opName = $operation['op'];
 940                         if ( isset( $supportedOps[$opName] ) ) {
 941                                 $class = $supportedOps[$opName];
 942                                 // Get params for this operation
 943                                 $params = $operation;
 944                                 // Append the FileOp class
 945                                 $performOps[] = new $class( $this, $params );
 946                         } else {
 947                                 throw new MWException( "Operation '$opName' is not supported." );
 948                         }
 949                 }
 950
 951                 return $performOps;
 952         }
 953
 954         /**
 955          * Get a list of storage paths to lock for a list of operations
 956          * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
 957          * each corresponding to a list of storage paths to be locked.
 958          * All returned paths are normalized.
 959          *
 960          * @param array $performOps List of FileOp objects
 961          * @return Array ('sh' => list of paths, 'ex' => list of paths)
 962          */
 963         final public function getPathsToLockForOpsInternal( array $performOps ) {
 964                 // Build up a list of files to lock...
 965                 $paths = array( 'sh' => array(), 'ex' => array() );
 966                 foreach ( $performOps as $fileOp ) {
 967                         $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
 968                         $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
 969                 }
 970                 // Optimization: if doing an EX lock anyway, don't also set an SH one
 971                 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
 972                 // Get a shared lock on the parent directory of each path changed
 973                 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
 974
 975                 return $paths;
 976         }
 977
 978         public function getScopedLocksForOps( array $ops, Status $status ) {
 979                 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
 980                 return array(
 981                         $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ),
 982                         $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status )
 983                 );
 984         }
 985
 986         final protected function doOperationsInternal( array $ops, array $opts ) {
 987                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
 988                 $status = Status::newGood();
 989
 990                 // Fix up custom header name/value pairs...
 991                 $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops );
 992
 993                 // Build up a list of FileOps...
 994                 $performOps = $this->getOperationsInternal( $ops );
 995
 996                 // Acquire any locks as needed...
 997                 if ( empty( $opts['nonLocking'] ) ) {
 998                         // Build up a list of files to lock...
 999                         $paths = $this->getPathsToLockForOpsInternal( $performOps );
1000                         // Try to lock those files for the scope of this function...
1001                         $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
1002                         $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
1003                         if ( !$status->isOK() ) {
1004                                 return $status; // abort
1005                         }
1006                 }
1007
1008                 // Clear any file cache entries (after locks acquired)
1009                 if ( empty( $opts['preserveCache'] ) ) {
1010                         $this->clearCache();
1011                 }
1012
1013                 // Load from the persistent file and container caches
1014                 $this->primeFileCache( $performOps );
1015                 $this->primeContainerCache( $performOps );
1016
1017                 // Actually attempt the operation batch...
1018                 $opts = $this->setConcurrencyFlags( $opts );
1019                 $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
1020
1021                 // Merge errors into status fields
1022                 $status->merge( $subStatus );
1023                 $status->success = $subStatus->success; // not done in merge()
1024
1025                 return $status;
1026         }
1027
1028         final protected function doQuickOperationsInternal( array $ops ) {
1029                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
1030                 $status = Status::newGood();
1031
1032                 // Fix up custom header name/value pairs...
1033                 $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops );
1034
1035                 // Clear any file cache entries
1036                 $this->clearCache();
1037
1038                 $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' );
1039                 $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
1040                 $maxConcurrency = $this->concurrency; // throttle
1041
1042                 $statuses = array(); // array of (index => Status)
1043                 $fileOpHandles = array(); // list of (index => handle) arrays
1044                 $curFileOpHandles = array(); // current handle batch
1045                 // Perform the sync-only ops and build up op handles for the async ops...
1046                 foreach ( $ops as $index => $params ) {
1047                         if ( !in_array( $params['op'], $supportedOps ) ) {
1048                                 throw new MWException( "Operation '{$params['op']}' is not supported." );
1049                         }
1050                         $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
1051                         $subStatus = $this->$method( array( 'async' => $async ) + $params );
1052                         if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
1053                                 if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1054                                         $fileOpHandles[] = $curFileOpHandles; // push this batch
1055                                         $curFileOpHandles = array();
1056                                 }
1057                                 $curFileOpHandles[$index] = $subStatus->value; // keep index
1058                         } else { // error or completed
1059                                 $statuses[$index] = $subStatus; // keep index
1060                         }
1061                 }
1062                 if ( count( $curFileOpHandles ) ) {
1063                         $fileOpHandles[] = $curFileOpHandles; // last batch
1064                 }
1065                 // Do all the async ops that can be done concurrently...
1066                 foreach ( $fileOpHandles as $fileHandleBatch ) {
1067                         $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
1068                 }
1069                 // Marshall and merge all the responses...
1070                 foreach ( $statuses as $index => $subStatus ) {
1071                         $status->merge( $subStatus );
1072                         if ( $subStatus->isOK() ) {
1073                                 $status->success[$index] = true;
1074                                 ++$status->successCount;
1075                         } else {
1076                                 $status->success[$index] = false;
1077                                 ++$status->failCount;
1078                         }
1079                 }
1080
1081                 return $status;
1082         }
1083
1084         /**
1085          * Execute a list of FileBackendStoreOpHandle handles in parallel.
1086          * The resulting Status object fields will correspond
1087          * to the order in which the handles where given.
1088          *
1089          * @param array $handles List of FileBackendStoreOpHandle objects
1090          * @return Array Map of Status objects
1091          * @throws MWException
1092          */
1093         final public function executeOpHandlesInternal( array $fileOpHandles ) {
1094                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
1095                 foreach ( $fileOpHandles as $fileOpHandle ) {
1096                         if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1097                                 throw new MWException( "Given a non-FileBackendStoreOpHandle object." );
1098                         } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1099                                 throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." );
1100                         }
1101                 }
1102                 $res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1103                 foreach ( $fileOpHandles as $fileOpHandle ) {
1104                         $fileOpHandle->closeResources();
1105                 }
1106                 return $res;
1107         }
1108
1109         /**
1110          * @see FileBackendStore::executeOpHandlesInternal()
1111          * @param array $fileOpHandles
1112          * @throws MWException
1113          * @return Array List of corresponding Status objects
1114          */
1115         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1116                 foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty
1117                         throw new MWException( "This backend supports no asynchronous operations." );
1118                 }
1119                 return array();
1120         }
1121
1122         /**
1123          * Strip long HTTP headers from a file operation.
1124          * Most headers are just numbers, but some are allowed to be long.
1125          * This function is useful for cleaning up headers and avoiding backend
1126          * specific errors, especially in the middle of batch file operations.
1127          *
1128          * @param array $op Same format as doOperation()
1129          * @return Array
1130          */
1131         protected function stripInvalidHeadersFromOp( array $op ) {
1132                 static $longs = array( 'Content-Disposition' );
1133                 if ( isset( $op['headers'] ) ) { // op sets HTTP headers
1134                         foreach ( $op['headers'] as $name => $value ) {
1135                                 $maxHVLen = in_array( $name, $longs ) ? INF : 255;
1136                                 if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) {
1137                                         trigger_error( "Header '$name: $value' is too long." );
1138                                         unset( $op['headers'][$name] );
1139                                 } elseif ( !strlen( $value ) ) {
1140                                         $op['headers'][$name] = ''; // null/false => ""
1141                                 }
1142                         }
1143                 }
1144                 return $op;
1145         }
1146
1147         final public function preloadCache( array $paths ) {
1148                 $fullConts = array(); // full container names
1149                 foreach ( $paths as $path ) {
1150                         list( $fullCont, , ) = $this->resolveStoragePath( $path );
1151                         $fullConts[] = $fullCont;
1152                 }
1153                 // Load from the persistent file and container caches
1154                 $this->primeContainerCache( $fullConts );
1155                 $this->primeFileCache( $paths );
1156         }
1157
1158         final public function clearCache( array $paths = null ) {
1159                 if ( is_array( $paths ) ) {
1160                         $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1161                         $paths = array_filter( $paths, 'strlen' ); // remove nulls
1162                 }
1163                 if ( $paths === null ) {
1164                         $this->cheapCache->clear();
1165                         $this->expensiveCache->clear();
1166                 } else {
1167                         foreach ( $paths as $path ) {
1168                                 $this->cheapCache->clear( $path );
1169                                 $this->expensiveCache->clear( $path );
1170                         }
1171                 }
1172                 $this->doClearCache( $paths );
1173         }
1174
1175         /**
1176          * Clears any additional stat caches for storage paths
1177          *
1178          * @see FileBackend::clearCache()
1179          *
1180          * @param array $paths Storage paths (optional)
1181          * @return void
1182          */
1183         protected function doClearCache( array $paths = null ) {}
1184
1185         /**
1186          * Is this a key/value store where directories are just virtual?
1187          * Virtual directories exists in so much as files exists that are
1188          * prefixed with the directory path followed by a forward slash.
1189          *
1190          * @return bool
1191          */
1192         abstract protected function directoriesAreVirtual();
1193
1194         /**
1195          * Check if a container name is valid.
1196          * This checks for for length and illegal characters.
1197          *
1198          * @param string $container
1199          * @return bool
1200          */
1201         final protected static function isValidContainerName( $container ) {
1202                 // This accounts for Swift and S3 restrictions while leaving room
1203                 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1204                 // This disallows directory separators or traversal characters.
1205                 // Note that matching strings URL encode to the same string;
1206                 // in Swift, the length restriction is *after* URL encoding.
1207                 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1208         }
1209
1210         /**
1211          * Splits a storage path into an internal container name,
1212          * an internal relative file name, and a container shard suffix.
1213          * Any shard suffix is already appended to the internal container name.
1214          * This also checks that the storage path is valid and within this backend.
1215          *
1216          * If the container is sharded but a suffix could not be determined,
1217          * this means that the path can only refer to a directory and can only
1218          * be scanned by looking in all the container shards.
1219          *
1220          * @param string $storagePath
1221          * @return Array (container, path, container suffix) or (null, null, null) if invalid
1222          */
1223         final protected function resolveStoragePath( $storagePath ) {
1224                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1225                 if ( $backend === $this->name ) { // must be for this backend
1226                         $relPath = self::normalizeContainerPath( $relPath );
1227                         if ( $relPath !== null ) {
1228                                 // Get shard for the normalized path if this container is sharded
1229                                 $cShard = $this->getContainerShard( $container, $relPath );
1230                                 // Validate and sanitize the relative path (backend-specific)
1231                                 $relPath = $this->resolveContainerPath( $container, $relPath );
1232                                 if ( $relPath !== null ) {
1233                                         // Prepend any wiki ID prefix to the container name
1234                                         $container = $this->fullContainerName( $container );
1235                                         if ( self::isValidContainerName( $container ) ) {
1236                                                 // Validate and sanitize the container name (backend-specific)
1237                                                 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1238                                                 if ( $container !== null ) {
1239                                                         return array( $container, $relPath, $cShard );
1240                                                 }
1241                                         }
1242                                 }
1243                         }
1244                 }
1245                 return array( null, null, null );
1246         }
1247
1248         /**
1249          * Like resolveStoragePath() except null values are returned if
1250          * the container is sharded and the shard could not be determined
1251          * or if the path ends with '/'. The later case is illegal for FS
1252          * backends and can confuse listings for object store backends.
1253          *
1254          * This function is used when resolving paths that must be valid
1255          * locations for files. Directory and listing functions should
1256          * generally just use resolveStoragePath() instead.
1257          *
1258          * @see FileBackendStore::resolveStoragePath()
1259          *
1260          * @param string $storagePath
1261          * @return Array (container, path) or (null, null) if invalid
1262          */
1263         final protected function resolveStoragePathReal( $storagePath ) {
1264                 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1265                 if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
1266                         return array( $container, $relPath );
1267                 }
1268                 return array( null, null );
1269         }
1270
1271         /**
1272          * Get the container name shard suffix for a given path.
1273          * Any empty suffix means the container is not sharded.
1274          *
1275          * @param string $container Container name
1276          * @param string $relPath Storage path relative to the container
1277          * @return string|null Returns null if shard could not be determined
1278          */
1279         final protected function getContainerShard( $container, $relPath ) {
1280                 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1281                 if ( $levels == 1 || $levels == 2 ) {
1282                         // Hash characters are either base 16 or 36
1283                         $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1284                         // Get a regex that represents the shard portion of paths.
1285                         // The concatenation of the captures gives us the shard.
1286                         if ( $levels === 1 ) { // 16 or 36 shards per container
1287                                 $hashDirRegex = '(' . $char . ')';
1288                         } else { // 256 or 1296 shards per container
1289                                 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1290                                         $hashDirRegex = $char . '/(' . $char . '{2})';
1291                                 } else { // short hash dir format (e.g. "a/b/c")
1292                                         $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1293                                 }
1294                         }
1295                         // Allow certain directories to be above the hash dirs so as
1296                         // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1297                         // They must be 2+ chars to avoid any hash directory ambiguity.
1298                         $m = array();
1299                         if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1300                                 return '.' . implode( '', array_slice( $m, 1 ) );
1301                         }
1302                         return null; // failed to match
1303                 }
1304                 return ''; // no sharding
1305         }
1306
1307         /**
1308          * Check if a storage path maps to a single shard.
1309          * Container dirs like "a", where the container shards on "x/xy",
1310          * can reside on several shards. Such paths are tricky to handle.
1311          *
1312          * @param string $storagePath Storage path
1313          * @return bool
1314          */
1315         final public function isSingleShardPathInternal( $storagePath ) {
1316                 list( , , $shard ) = $this->resolveStoragePath( $storagePath );
1317                 return ( $shard !== null );
1318         }
1319
1320         /**
1321          * Get the sharding config for a container.
1322          * If greater than 0, then all file storage paths within
1323          * the container are required to be hashed accordingly.
1324          *
1325          * @param string $container
1326          * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
1327          */
1328         final protected function getContainerHashLevels( $container ) {
1329                 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1330                         $config = $this->shardViaHashLevels[$container];
1331                         $hashLevels = (int)$config['levels'];
1332                         if ( $hashLevels == 1 || $hashLevels == 2 ) {
1333                                 $hashBase = (int)$config['base'];
1334                                 if ( $hashBase == 16 || $hashBase == 36 ) {
1335                                         return array( $hashLevels, $hashBase, $config['repeat'] );
1336                                 }
1337                         }
1338                 }
1339                 return array( 0, 0, false ); // no sharding
1340         }
1341
1342         /**
1343          * Get a list of full container shard suffixes for a container
1344          *
1345          * @param string $container
1346          * @return Array
1347          */
1348         final protected function getContainerSuffixes( $container ) {
1349                 $shards = array();
1350                 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1351                 if ( $digits > 0 ) {
1352                         $numShards = pow( $base, $digits );
1353                         for ( $index = 0; $index < $numShards; $index++ ) {
1354                                 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
1355                         }
1356                 }
1357                 return $shards;
1358         }
1359
1360         /**
1361          * Get the full container name, including the wiki ID prefix
1362          *
1363          * @param string $container
1364          * @return string
1365          */
1366         final protected function fullContainerName( $container ) {
1367                 if ( $this->wikiId != '' ) {
1368                         return "{$this->wikiId}-$container";
1369                 } else {
1370                         return $container;
1371                 }
1372         }
1373
1374         /**
1375          * Resolve a container name, checking if it's allowed by the backend.
1376          * This is intended for internal use, such as encoding illegal chars.
1377          * Subclasses can override this to be more restrictive.
1378          *
1379          * @param string $container
1380          * @return string|null
1381          */
1382         protected function resolveContainerName( $container ) {
1383                 return $container;
1384         }
1385
1386         /**
1387          * Resolve a relative storage path, checking if it's allowed by the backend.
1388          * This is intended for internal use, such as encoding illegal chars or perhaps
1389          * getting absolute paths (e.g. FS based backends). Note that the relative path
1390          * may be the empty string (e.g. the path is simply to the container).
1391          *
1392          * @param string $container Container name
1393          * @param string $relStoragePath Storage path relative to the container
1394          * @return string|null Path or null if not valid
1395          */
1396         protected function resolveContainerPath( $container, $relStoragePath ) {
1397                 return $relStoragePath;
1398         }
1399
1400         /**
1401          * Get the cache key for a container
1402          *
1403          * @param string $container Resolved container name
1404          * @return string
1405          */
1406         private function containerCacheKey( $container ) {
1407                 return wfMemcKey( 'backend', $this->getName(), 'container', $container );
1408         }
1409
1410         /**
1411          * Set the cached info for a container
1412          *
1413          * @param string $container Resolved container name
1414          * @param array $val Information to cache
1415          * @return void
1416          */
1417         final protected function setContainerCache( $container, array $val ) {
1418                 $this->memCache->add( $this->containerCacheKey( $container ), $val, 14 * 86400 );
1419         }
1420
1421         /**
1422          * Delete the cached info for a container.
1423          * The cache key is salted for a while to prevent race conditions.
1424          *
1425          * @param string $container Resolved container name
1426          * @return void
1427          */
1428         final protected function deleteContainerCache( $container ) {
1429                 if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) {
1430                         trigger_error( "Unable to delete stat cache for container $container." );
1431                 }
1432         }
1433
1434         /**
1435          * Do a batch lookup from cache for container stats for all containers
1436          * used in a list of container names, storage paths, or FileOp objects.
1437          * This loads the persistent cache values into the process cache.
1438          *
1439          * @param Array $items
1440          * @return void
1441          */
1442         final protected function primeContainerCache( array $items ) {
1443                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
1444
1445                 $paths = array(); // list of storage paths
1446                 $contNames = array(); // (cache key => resolved container name)
1447                 // Get all the paths/containers from the items...
1448                 foreach ( $items as $item ) {
1449                         if ( $item instanceof FileOp ) {
1450                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1451                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1452                         } elseif ( self::isStoragePath( $item ) ) {
1453                                 $paths[] = $item;
1454                         } elseif ( is_string( $item ) ) { // full container name
1455                                 $contNames[$this->containerCacheKey( $item )] = $item;
1456                         }
1457                 }
1458                 // Get all the corresponding cache keys for paths...
1459                 foreach ( $paths as $path ) {
1460                         list( $fullCont, , ) = $this->resolveStoragePath( $path );
1461                         if ( $fullCont !== null ) { // valid path for this backend
1462                                 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1463                         }
1464                 }
1465
1466                 $contInfo = array(); // (resolved container name => cache value)
1467                 // Get all cache entries for these container cache keys...
1468                 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1469                 foreach ( $values as $cacheKey => $val ) {
1470                         $contInfo[$contNames[$cacheKey]] = $val;
1471                 }
1472
1473                 // Populate the container process cache for the backend...
1474                 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1475         }
1476
1477         /**
1478          * Fill the backend-specific process cache given an array of
1479          * resolved container names and their corresponding cached info.
1480          * Only containers that actually exist should appear in the map.
1481          *
1482          * @param array $containerInfo Map of resolved container names to cached info
1483          * @return void
1484          */
1485         protected function doPrimeContainerCache( array $containerInfo ) {}
1486
1487         /**
1488          * Get the cache key for a file path
1489          *
1490          * @param string $path Normalized storage path
1491          * @return string
1492          */
1493         private function fileCacheKey( $path ) {
1494                 return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
1495         }
1496
1497         /**
1498          * Set the cached stat info for a file path.
1499          * Negatives (404s) are not cached. By not caching negatives, we can skip cache
1500          * salting for the case when a file is created at a path were there was none before.
1501          *
1502          * @param string $path Storage path
1503          * @param array $val Stat information to cache
1504          * @return void
1505          */
1506         final protected function setFileCache( $path, array $val ) {
1507                 $path = FileBackend::normalizeStoragePath( $path );
1508                 if ( $path === null ) {
1509                         return; // invalid storage path
1510                 }
1511                 $age = time() - wfTimestamp( TS_UNIX, $val['mtime'] );
1512                 $ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) );
1513                 $this->memCache->add( $this->fileCacheKey( $path ), $val, $ttl );
1514         }
1515
1516         /**
1517          * Delete the cached stat info for a file path.
1518          * The cache key is salted for a while to prevent race conditions.
1519          * Since negatives (404s) are not cached, this does not need to be called when
1520          * a file is created at a path were there was none before.
1521          *
1522          * @param string $path Storage path
1523          * @return void
1524          */
1525         final protected function deleteFileCache( $path ) {
1526                 $path = FileBackend::normalizeStoragePath( $path );
1527                 if ( $path === null ) {
1528                         return; // invalid storage path
1529                 }
1530                 if ( !$this->memCache->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) {
1531                         trigger_error( "Unable to delete stat cache for file $path." );
1532                 }
1533         }
1534
1535         /**
1536          * Do a batch lookup from cache for file stats for all paths
1537          * used in a list of storage paths or FileOp objects.
1538          * This loads the persistent cache values into the process cache.
1539          *
1540          * @param array $items List of storage paths or FileOps
1541          * @return void
1542          */
1543         final protected function primeFileCache( array $items ) {
1544                 $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
1545
1546                 $paths = array(); // list of storage paths
1547                 $pathNames = array(); // (cache key => storage path)
1548                 // Get all the paths/containers from the items...
1549                 foreach ( $items as $item ) {
1550                         if ( $item instanceof FileOp ) {
1551                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1552                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1553                         } elseif ( self::isStoragePath( $item ) ) {
1554                                 $paths[] = FileBackend::normalizeStoragePath( $item );
1555                         }
1556                 }
1557                 // Get rid of any paths that failed normalization...
1558                 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1559                 // Get all the corresponding cache keys for paths...
1560                 foreach ( $paths as $path ) {
1561                         list( , $rel, ) = $this->resolveStoragePath( $path );
1562                         if ( $rel !== null ) { // valid path for this backend
1563                                 $pathNames[$this->fileCacheKey( $path )] = $path;
1564                         }
1565                 }
1566                 // Get all cache entries for these container cache keys...
1567                 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1568                 foreach ( $values as $cacheKey => $val ) {
1569                         if ( is_array( $val ) ) {
1570                                 $path = $pathNames[$cacheKey];
1571                                 $this->cheapCache->set( $path, 'stat', $val );
1572                                 if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
1573                                         $this->cheapCache->set( $path, 'sha1',
1574                                                 array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) );
1575                                 }
1576                         }
1577                 }
1578         }
1579
1580         /**
1581          * Set the 'concurrency' option from a list of operation options
1582          *
1583          * @param array $opts Map of operation options
1584          * @return Array
1585          */
1586         final protected function setConcurrencyFlags( array $opts ) {
1587                 $opts['concurrency'] = 1; // off
1588                 if ( $this->parallelize === 'implicit' ) {
1589                         if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) {
1590                                 $opts['concurrency'] = $this->concurrency;
1591                         }
1592                 } elseif ( $this->parallelize === 'explicit' ) {
1593                         if ( !empty( $opts['parallelize'] ) ) {
1594                                 $opts['concurrency'] = $this->concurrency;
1595                         }
1596                 }
1597                 return $opts;
1598         }
1599
1600         /**
1601          * Get the content type to use in HEAD/GET requests for a file
1602          *
1603          * @param string $storagePath
1604          * @param string|null $content File data
1605          * @param string|null $fsPath File system path
1606          * @return MIME type
1607          */
1608         protected function getContentType( $storagePath, $content, $fsPath ) {
1609                 return call_user_func_array( $this->mimeCallback, func_get_args() );
1610         }
1611 }
1612
1613 /**
1614  * FileBackendStore helper class for performing asynchronous file operations.
1615  *
1616  * For example, calling FileBackendStore::createInternal() with the "async"
1617  * param flag may result in a Status that contains this object as a value.
1618  * This class is largely backend-specific and is mostly just "magic" to be
1619  * passed to FileBackendStore::executeOpHandlesInternal().
1620  */
1621 abstract class FileBackendStoreOpHandle {
1622         /** @var Array */
1623         public $params = array(); // params to caller functions
1624         /** @var FileBackendStore */
1625         public $backend;
1626         /** @var Array */
1627         public $resourcesToClose = array();
1628
1629         public $call; // string; name that identifies the function called
1630
1631         /**
1632          * Close all open file handles
1633          *
1634          * @return void
1635          */
1636         public function closeResources() {
1637                 array_map( 'fclose', $this->resourcesToClose );
1638         }
1639 }
1640
1641 /**
1642  * FileBackendStore helper function to handle listings that span container shards.
1643  * Do not use this class from places outside of FileBackendStore.
1644  *
1645  * @ingroup FileBackend
1646  */
1647 abstract class FileBackendStoreShardListIterator extends FilterIterator {
1648         /** @var FileBackendStore */
1649         protected $backend;
1650         /** @var Array */
1651         protected $params;
1652
1653         protected $container; // string; full container name
1654         protected $directory; // string; resolved relative path
1655
1656         /** @var Array */
1657         protected $multiShardPaths = array(); // (rel path => 1)
1658
1659         /**
1660          * @param FileBackendStore $backend
1661          * @param string $container Full storage container name
1662          * @param string $dir Storage directory relative to container
1663          * @param array $suffixes List of container shard suffixes
1664          * @param array $params
1665          */
1666         public function __construct(
1667                 FileBackendStore $backend, $container, $dir, array $suffixes, array $params
1668         ) {
1669                 $this->backend = $backend;
1670                 $this->container = $container;
1671                 $this->directory = $dir;
1672                 $this->params = $params;
1673
1674                 $iter = new AppendIterator();
1675                 foreach ( $suffixes as $suffix ) {
1676                         $iter->append( $this->listFromShard( $this->container . $suffix ) );
1677                 }
1678
1679                 parent::__construct( $iter );
1680         }
1681
1682         public function accept() {
1683                 $rel = $this->getInnerIterator()->current(); // path relative to given directory
1684                 $path = $this->params['dir'] . "/{$rel}"; // full storage path
1685                 if ( $this->backend->isSingleShardPathInternal( $path ) ) {
1686                         return true; // path is only on one shard; no issue with duplicates
1687                 } elseif ( isset( $this->multiShardPaths[$rel] ) ) {
1688                         // Don't keep listing paths that are on multiple shards
1689                         return false;
1690                 } else {
1691                         $this->multiShardPaths[$rel] = 1;
1692                         return true;
1693                 }
1694         }
1695
1696         public function rewind() {
1697                 parent::rewind();
1698                 $this->multiShardPaths = array();
1699         }
1700
1701         /**
1702          * Get the list for a given container shard
1703          *
1704          * @param string $container Resolved container name
1705          * @return Iterator
1706          */
1707         abstract protected function listFromShard( $container );
1708 }
1709
1710 /**
1711  * Iterator for listing directories
1712  */
1713 class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator {
1714         protected function listFromShard( $container ) {
1715                 $list = $this->backend->getDirectoryListInternal(
1716                         $container, $this->directory, $this->params );
1717                 if ( $list === null ) {
1718                         return new ArrayIterator( array() );
1719                 } else {
1720                         return is_array( $list ) ? new ArrayIterator( $list ) : $list;
1721                 }
1722         }
1723 }
1724
1725 /**
1726  * Iterator for listing regular files
1727  */
1728 class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator {
1729         protected function listFromShard( $container ) {
1730                 $list = $this->backend->getFileListInternal(
1731                         $container, $this->directory, $this->params );
1732                 if ( $list === null ) {
1733                         return new ArrayIterator( array() );
1734                 } else {
1735                         return is_array( $list ) ? new ArrayIterator( $list ) : $list;
1736                 }
1737         }
1738 }