includes/filerepo/backend/FileBackendStore.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Aaron Schulz
   6  */
   7
   8 /**
   9  * @brief Base class for all backends using particular storage medium.
  10  *
  11  * This class defines the methods as abstract that subclasses must implement.
  12  * Outside callers should *not* use functions with "Internal" in the name.
  13  *
  14  * The FileBackend operations are implemented using basic functions
  15  * such as storeInternal(), copyInternal(), deleteInternal() and the like.
  16  * This class is also responsible for path resolution and sanitization.
  17  *
  18  * @ingroup FileBackend
  19  * @since 1.19
  20  */
  21 abstract class FileBackendStore extends FileBackend {
  22         /** @var Array Map of paths to small (RAM/disk) cache items */
  23         protected $cache = array(); // (storage path => key => value)
  24         protected $maxCacheSize = 100; // integer; max paths with entries
  25         /** @var Array Map of paths to large (RAM/disk) cache items */
  26         protected $expensiveCache = array(); // (storage path => key => value)
  27         protected $maxExpensiveCacheSize = 10; // integer; max paths with entries
  28
  29         /** @var Array Map of container names to sharding settings */
  30         protected $shardViaHashLevels = array(); // (container name => config array)
  31
  32         protected $maxFileSize = 4294967296; // integer bytes (4GiB)
  33
  34         /**
  35          * Get the maximum allowable file size given backend
  36          * medium restrictions and basic performance constraints.
  37          * Do not call this function from places outside FileBackend and FileOp.
  38          *
  39          * @return integer Bytes
  40          */
  41         final public function maxFileSizeInternal() {
  42                 return $this->maxFileSize;
  43         }
  44
  45         /**
  46          * Check if a file can be created at a given storage path.
  47          * FS backends should check if the parent directory exists and the file is writable.
  48          * Backends using key/value stores should check if the container exists.
  49          *
  50          * @param $storagePath string
  51          * @return bool
  52          */
  53         abstract public function isPathUsableInternal( $storagePath );
  54
  55         /**
  56          * Create a file in the backend with the given contents.
  57          * Do not call this function from places outside FileBackend and FileOp.
  58          *
  59          * $params include:
  60          *     content       : the raw file contents
  61          *     dst           : destination storage path
  62          *     overwrite     : overwrite any file that exists at the destination
  63          *
  64          * @param $params Array
  65          * @return Status
  66          */
  67         final public function createInternal( array $params ) {
  68                 wfProfileIn( __METHOD__ );
  69                 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
  70                         $status = Status::newFatal( 'backend-fail-maxsize',
  71                                 $params['dst'], $this->maxFileSizeInternal() );
  72                 } else {
  73                         $status = $this->doCreateInternal( $params );
  74                         $this->clearCache( array( $params['dst'] ) );
  75                 }
  76                 wfProfileOut( __METHOD__ );
  77                 return $status;
  78         }
  79
  80         /**
  81          * @see FileBackendStore::createInternal()
  82          */
  83         abstract protected function doCreateInternal( array $params );
  84
  85         /**
  86          * Store a file into the backend from a file on disk.
  87          * Do not call this function from places outside FileBackend and FileOp.
  88          *
  89          * $params include:
  90          *     src           : source path on disk
  91          *     dst           : destination storage path
  92          *     overwrite     : overwrite any file that exists at the destination
  93          *
  94          * @param $params Array
  95          * @return Status
  96          */
  97         final public function storeInternal( array $params ) {
  98                 wfProfileIn( __METHOD__ );
  99                 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
 100                         $status = Status::newFatal( 'backend-fail-store', $params['dst'] );
 101                 } else {
 102                         $status = $this->doStoreInternal( $params );
 103                         $this->clearCache( array( $params['dst'] ) );
 104                 }
 105                 wfProfileOut( __METHOD__ );
 106                 return $status;
 107         }
 108
 109         /**
 110          * @see FileBackendStore::storeInternal()
 111          */
 112         abstract protected function doStoreInternal( array $params );
 113
 114         /**
 115          * Copy a file from one storage path to another in the backend.
 116          * Do not call this function from places outside FileBackend and FileOp.
 117          *
 118          * $params include:
 119          *     src           : source storage path
 120          *     dst           : destination storage path
 121          *     overwrite     : overwrite any file that exists at the destination
 122          *
 123          * @param $params Array
 124          * @return Status
 125          */
 126         final public function copyInternal( array $params ) {
 127                 wfProfileIn( __METHOD__ );
 128                 $status = $this->doCopyInternal( $params );
 129                 $this->clearCache( array( $params['dst'] ) );
 130                 wfProfileOut( __METHOD__ );
 131                 return $status;
 132         }
 133
 134         /**
 135          * @see FileBackendStore::copyInternal()
 136          */
 137         abstract protected function doCopyInternal( array $params );
 138
 139         /**
 140          * Delete a file at the storage path.
 141          * Do not call this function from places outside FileBackend and FileOp.
 142          *
 143          * $params include:
 144          *     src                 : source storage path
 145          *     ignoreMissingSource : do nothing if the source file does not exist
 146          *
 147          * @param $params Array
 148          * @return Status
 149          */
 150         final public function deleteInternal( array $params ) {
 151                 wfProfileIn( __METHOD__ );
 152                 $status = $this->doDeleteInternal( $params );
 153                 $this->clearCache( array( $params['src'] ) );
 154                 wfProfileOut( __METHOD__ );
 155                 return $status;
 156         }
 157
 158         /**
 159          * @see FileBackendStore::deleteInternal()
 160          */
 161         abstract protected function doDeleteInternal( array $params );
 162
 163         /**
 164          * Move a file from one storage path to another in the backend.
 165          * Do not call this function from places outside FileBackend and FileOp.
 166          *
 167          * $params include:
 168          *     src           : source storage path
 169          *     dst           : destination storage path
 170          *     overwrite     : overwrite any file that exists at the destination
 171          *
 172          * @param $params Array
 173          * @return Status
 174          */
 175         final public function moveInternal( array $params ) {
 176                 wfProfileIn( __METHOD__ );
 177                 $status = $this->doMoveInternal( $params );
 178                 $this->clearCache( array( $params['src'], $params['dst'] ) );
 179                 wfProfileOut( __METHOD__ );
 180                 return $status;
 181         }
 182
 183         /**
 184          * @see FileBackendStore::moveInternal()
 185          * @return Status
 186          */
 187         protected function doMoveInternal( array $params ) {
 188                 // Copy source to dest
 189                 $status = $this->copyInternal( $params );
 190                 if ( $status->isOK() ) {
 191                         // Delete source (only fails due to races or medium going down)
 192                         $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
 193                         $status->setResult( true, $status->value ); // ignore delete() errors
 194                 }
 195                 return $status;
 196         }
 197
 198         /**
 199          * @see FileBackend::concatenate()
 200          * @return Status
 201          */
 202         final public function concatenate( array $params ) {
 203                 wfProfileIn( __METHOD__ );
 204                 $status = Status::newGood();
 205
 206                 // Try to lock the source files for the scope of this function
 207                 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
 208                 if ( $status->isOK() ) {
 209                         // Actually do the concatenation
 210                         $status->merge( $this->doConcatenate( $params ) );
 211                 }
 212
 213                 wfProfileOut( __METHOD__ );
 214                 return $status;
 215         }
 216
 217         /**
 218          * @see FileBackendStore::concatenate()
 219          * @return Status
 220          */
 221         protected function doConcatenate( array $params ) {
 222                 $status = Status::newGood();
 223                 $tmpPath = $params['dst']; // convenience
 224
 225                 // Check that the specified temp file is valid...
 226                 wfSuppressWarnings();
 227                 $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
 228                 wfRestoreWarnings();
 229                 if ( !$ok ) { // not present or not empty
 230                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 231                         return $status;
 232                 }
 233
 234                 // Build up the temp file using the source chunks (in order)...
 235                 $tmpHandle = fopen( $tmpPath, 'ab' );
 236                 if ( $tmpHandle === false ) {
 237                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 238                         return $status;
 239                 }
 240                 foreach ( $params['srcs'] as $virtualSource ) {
 241                         // Get a local FS version of the chunk
 242                         $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
 243                         if ( !$tmpFile ) {
 244                                 $status->fatal( 'backend-fail-read', $virtualSource );
 245                                 return $status;
 246                         }
 247                         // Get a handle to the local FS version
 248                         $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
 249                         if ( $sourceHandle === false ) {
 250                                 fclose( $tmpHandle );
 251                                 $status->fatal( 'backend-fail-read', $virtualSource );
 252                                 return $status;
 253                         }
 254                         // Append chunk to file (pass chunk size to avoid magic quotes)
 255                         if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
 256                                 fclose( $sourceHandle );
 257                                 fclose( $tmpHandle );
 258                                 $status->fatal( 'backend-fail-writetemp', $tmpPath );
 259                                 return $status;
 260                         }
 261                         fclose( $sourceHandle );
 262                 }
 263                 if ( !fclose( $tmpHandle ) ) {
 264                         $status->fatal( 'backend-fail-closetemp', $tmpPath );
 265                         return $status;
 266                 }
 267
 268                 clearstatcache(); // temp file changed
 269
 270                 return $status;
 271         }
 272
 273         /**
 274          * @see FileBackend::doPrepare()
 275          * @return Status
 276          */
 277         final protected function doPrepare( array $params ) {
 278                 wfProfileIn( __METHOD__ );
 279
 280                 $status = Status::newGood();
 281                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 282                 if ( $dir === null ) {
 283                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 284                         wfProfileOut( __METHOD__ );
 285                         return $status; // invalid storage path
 286                 }
 287
 288                 if ( $shard !== null ) { // confined to a single container/shard
 289                         $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
 290                 } else { // directory is on several shards
 291                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 292                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 293                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 294                                 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 295                         }
 296                 }
 297
 298                 wfProfileOut( __METHOD__ );
 299                 return $status;
 300         }
 301
 302         /**
 303          * @see FileBackendStore::doPrepare()
 304          * @return Status
 305          */
 306         protected function doPrepareInternal( $container, $dir, array $params ) {
 307                 return Status::newGood();
 308         }
 309
 310         /**
 311          * @see FileBackend::doSecure()
 312          * @return Status
 313          */
 314         final protected function doSecure( array $params ) {
 315                 wfProfileIn( __METHOD__ );
 316                 $status = Status::newGood();
 317
 318                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 319                 if ( $dir === null ) {
 320                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 321                         wfProfileOut( __METHOD__ );
 322                         return $status; // invalid storage path
 323                 }
 324
 325                 if ( $shard !== null ) { // confined to a single container/shard
 326                         $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 327                 } else { // directory is on several shards
 328                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 329                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 330                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 331                                 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 332                         }
 333                 }
 334
 335                 wfProfileOut( __METHOD__ );
 336                 return $status;
 337         }
 338
 339         /**
 340          * @see FileBackendStore::doSecure()
 341          * @return Status
 342          */
 343         protected function doSecureInternal( $container, $dir, array $params ) {
 344                 return Status::newGood();
 345         }
 346
 347         /**
 348          * @see FileBackend::doClean()
 349          * @return Status
 350          */
 351         final protected function doClean( array $params ) {
 352                 wfProfileIn( __METHOD__ );
 353                 $status = Status::newGood();
 354
 355                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 356                 if ( $dir === null ) {
 357                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 358                         wfProfileOut( __METHOD__ );
 359                         return $status; // invalid storage path
 360                 }
 361
 362                 // Attempt to lock this directory...
 363                 $filesLockEx = array( $params['dir'] );
 364                 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 365                 if ( !$status->isOK() ) {
 366                         wfProfileOut( __METHOD__ );
 367                         return $status; // abort
 368                 }
 369
 370                 if ( $shard !== null ) { // confined to a single container/shard
 371                         $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
 372                 } else { // directory is on several shards
 373                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 374                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 375                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 376                                 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 377                         }
 378                 }
 379
 380                 wfProfileOut( __METHOD__ );
 381                 return $status;
 382         }
 383
 384         /**
 385          * @see FileBackendStore::doClean()
 386          * @return Status
 387          */
 388         protected function doCleanInternal( $container, $dir, array $params ) {
 389                 return Status::newGood();
 390         }
 391
 392         /**
 393          * @see FileBackend::fileExists()
 394          * @return bool|null
 395          */
 396         final public function fileExists( array $params ) {
 397                 wfProfileIn( __METHOD__ );
 398                 $stat = $this->getFileStat( $params );
 399                 wfProfileOut( __METHOD__ );
 400                 return ( $stat === null ) ? null : (bool)$stat; // null => failure
 401         }
 402
 403         /**
 404          * @see FileBackend::getFileTimestamp()
 405          * @return bool
 406          */
 407         final public function getFileTimestamp( array $params ) {
 408                 wfProfileIn( __METHOD__ );
 409                 $stat = $this->getFileStat( $params );
 410                 wfProfileOut( __METHOD__ );
 411                 return $stat ? $stat['mtime'] : false;
 412         }
 413
 414         /**
 415          * @see FileBackend::getFileSize()
 416          * @return bool
 417          */
 418         final public function getFileSize( array $params ) {
 419                 wfProfileIn( __METHOD__ );
 420                 $stat = $this->getFileStat( $params );
 421                 wfProfileOut( __METHOD__ );
 422                 return $stat ? $stat['size'] : false;
 423         }
 424
 425         /**
 426          * @see FileBackend::getFileStat()
 427          * @return bool
 428          */
 429         final public function getFileStat( array $params ) {
 430                 wfProfileIn( __METHOD__ );
 431                 $path = self::normalizeStoragePath( $params['src'] );
 432                 if ( $path === null ) {
 433                         wfProfileOut( __METHOD__ );
 434                         return false; // invalid storage path
 435                 }
 436                 $latest = !empty( $params['latest'] );
 437                 if ( isset( $this->cache[$path]['stat'] ) ) {
 438                         // If we want the latest data, check that this cached
 439                         // value was in fact fetched with the latest available data.
 440                         if ( !$latest || $this->cache[$path]['stat']['latest'] ) {
 441                                 wfProfileOut( __METHOD__ );
 442                                 return $this->cache[$path]['stat'];
 443                         }
 444                 }
 445                 wfProfileIn( __METHOD__ . '-miss' );
 446                 $stat = $this->doGetFileStat( $params );
 447                 wfProfileOut( __METHOD__ . '-miss' );
 448                 if ( is_array( $stat ) ) { // don't cache negatives
 449                         $this->trimCache(); // limit memory
 450                         $this->cache[$path]['stat'] = $stat;
 451                         $this->cache[$path]['stat']['latest'] = $latest;
 452                 }
 453                 wfProfileOut( __METHOD__ );
 454                 return $stat;
 455         }
 456
 457         /**
 458          * @see FileBackendStore::getFileStat()
 459          */
 460         abstract protected function doGetFileStat( array $params );
 461
 462         /**
 463          * @see FileBackend::getFileContents()
 464          * @return bool|string
 465          */
 466         public function getFileContents( array $params ) {
 467                 wfProfileIn( __METHOD__ );
 468                 $tmpFile = $this->getLocalReference( $params );
 469                 if ( !$tmpFile ) {
 470                         wfProfileOut( __METHOD__ );
 471                         return false;
 472                 }
 473                 wfSuppressWarnings();
 474                 $data = file_get_contents( $tmpFile->getPath() );
 475                 wfRestoreWarnings();
 476                 wfProfileOut( __METHOD__ );
 477                 return $data;
 478         }
 479
 480         /**
 481          * @see FileBackend::getFileSha1Base36()
 482          * @return bool|string
 483          */
 484         final public function getFileSha1Base36( array $params ) {
 485                 wfProfileIn( __METHOD__ );
 486                 $path = $params['src'];
 487                 if ( isset( $this->cache[$path]['sha1'] ) ) {
 488                         wfProfileOut( __METHOD__ );
 489                         return $this->cache[$path]['sha1'];
 490                 }
 491                 wfProfileIn( __METHOD__ . '-miss' );
 492                 $hash = $this->doGetFileSha1Base36( $params );
 493                 wfProfileOut( __METHOD__ . '-miss' );
 494                 if ( $hash ) { // don't cache negatives
 495                         $this->trimCache(); // limit memory
 496                         $this->cache[$path]['sha1'] = $hash;
 497                 }
 498                 wfProfileOut( __METHOD__ );
 499                 return $hash;
 500         }
 501
 502         /**
 503          * @see FileBackendStore::getFileSha1Base36()
 504          * @return bool
 505          */
 506         protected function doGetFileSha1Base36( array $params ) {
 507                 $fsFile = $this->getLocalReference( $params );
 508                 if ( !$fsFile ) {
 509                         return false;
 510                 } else {
 511                         return $fsFile->getSha1Base36();
 512                 }
 513         }
 514
 515         /**
 516          * @see FileBackend::getFileProps()
 517          * @return Array
 518          */
 519         final public function getFileProps( array $params ) {
 520                 wfProfileIn( __METHOD__ );
 521                 $fsFile = $this->getLocalReference( $params );
 522                 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
 523                 wfProfileOut( __METHOD__ );
 524                 return $props;
 525         }
 526
 527         /**
 528          * @see FileBackend::getLocalReference()
 529          * @return TempFSFile|null
 530          */
 531         public function getLocalReference( array $params ) {
 532                 wfProfileIn( __METHOD__ );
 533                 $path = $params['src'];
 534                 if ( isset( $this->expensiveCache[$path]['localRef'] ) ) {
 535                         wfProfileOut( __METHOD__ );
 536                         return $this->expensiveCache[$path]['localRef'];
 537                 }
 538                 $tmpFile = $this->getLocalCopy( $params );
 539                 if ( $tmpFile ) { // don't cache negatives
 540                         $this->trimExpensiveCache(); // limit memory
 541                         $this->expensiveCache[$path]['localRef'] = $tmpFile;
 542                 }
 543                 wfProfileOut( __METHOD__ );
 544                 return $tmpFile;
 545         }
 546
 547         /**
 548          * @see FileBackend::streamFile()
 549          * @return Status
 550          */
 551         final public function streamFile( array $params ) {
 552                 wfProfileIn( __METHOD__ );
 553                 $status = Status::newGood();
 554
 555                 $info = $this->getFileStat( $params );
 556                 if ( !$info ) { // let StreamFile handle the 404
 557                         $status->fatal( 'backend-fail-notexists', $params['src'] );
 558                 }
 559
 560                 // Set output buffer and HTTP headers for stream
 561                 $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array();
 562                 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
 563                 if ( $res == StreamFile::NOT_MODIFIED ) {
 564                         // do nothing; client cache is up to date
 565                 } elseif ( $res == StreamFile::READY_STREAM ) {
 566                         wfProfileIn( __METHOD__ . '-send' );
 567                         $status = $this->doStreamFile( $params );
 568                         wfProfileOut( __METHOD__ . '-send' );
 569                 } else {
 570                         $status->fatal( 'backend-fail-stream', $params['src'] );
 571                 }
 572
 573                 wfProfileOut( __METHOD__ );
 574                 return $status;
 575         }
 576
 577         /**
 578          * @see FileBackendStore::streamFile()
 579          * @return Status
 580          */
 581         protected function doStreamFile( array $params ) {
 582                 $status = Status::newGood();
 583
 584                 $fsFile = $this->getLocalReference( $params );
 585                 if ( !$fsFile ) {
 586                         $status->fatal( 'backend-fail-stream', $params['src'] );
 587                 } elseif ( !readfile( $fsFile->getPath() ) ) {
 588                         $status->fatal( 'backend-fail-stream', $params['src'] );
 589                 }
 590
 591                 return $status;
 592         }
 593
 594         /**
 595          * @copydoc FileBackend::getFileList()
 596          * @return Array|null|Traversable
 597          */
 598         final public function getFileList( array $params ) {
 599                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 600                 if ( $dir === null ) { // invalid storage path
 601                         return null;
 602                 }
 603                 if ( $shard !== null ) {
 604                         // File listing is confined to a single container/shard
 605                         return $this->getFileListInternal( $fullCont, $dir, $params );
 606                 } else {
 607                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 608                         // File listing spans multiple containers/shards
 609                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 610                         return new FileBackendStoreShardListIterator( $this,
 611                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 612                 }
 613         }
 614
 615         /**
 616          * Do not call this function from places outside FileBackend
 617          *
 618          * @see FileBackendStore::getFileList()
 619          *
 620          * @param $container string Resolved container name
 621          * @param $dir string Resolved path relative to container
 622          * @param $params Array
 623          * @return Traversable|Array|null
 624          */
 625         abstract public function getFileListInternal( $container, $dir, array $params );
 626
 627         /**
 628          * Get the list of supported operations and their corresponding FileOp classes.
 629          *
 630          * @return Array
 631          */
 632         protected function supportedOperations() {
 633                 return array(
 634                         'store'       => 'StoreFileOp',
 635                         'copy'        => 'CopyFileOp',
 636                         'move'        => 'MoveFileOp',
 637                         'delete'      => 'DeleteFileOp',
 638                         'create'      => 'CreateFileOp',
 639                         'null'        => 'NullFileOp'
 640                 );
 641         }
 642
 643         /**
 644          * Return a list of FileOp objects from a list of operations.
 645          * Do not call this function from places outside FileBackend.
 646          *
 647          * The result must have the same number of items as the input.
 648          * An exception is thrown if an unsupported operation is requested.
 649          *
 650          * @param $ops Array Same format as doOperations()
 651          * @return Array List of FileOp objects
 652          * @throws MWException
 653          */
 654         final public function getOperations( array $ops ) {
 655                 $supportedOps = $this->supportedOperations();
 656
 657                 $performOps = array(); // array of FileOp objects
 658                 // Build up ordered array of FileOps...
 659                 foreach ( $ops as $operation ) {
 660                         $opName = $operation['op'];
 661                         if ( isset( $supportedOps[$opName] ) ) {
 662                                 $class = $supportedOps[$opName];
 663                                 // Get params for this operation
 664                                 $params = $operation;
 665                                 // Append the FileOp class
 666                                 $performOps[] = new $class( $this, $params );
 667                         } else {
 668                                 throw new MWException( "Operation `$opName` is not supported." );
 669                         }
 670                 }
 671
 672                 return $performOps;
 673         }
 674
 675         /**
 676          * @see FileBackend::doOperationsInternal()
 677          * @return Status
 678          */
 679         protected function doOperationsInternal( array $ops, array $opts ) {
 680                 wfProfileIn( __METHOD__ );
 681                 $status = Status::newGood();
 682
 683                 // Build up a list of FileOps...
 684                 $performOps = $this->getOperations( $ops );
 685
 686                 // Acquire any locks as needed...
 687                 if ( empty( $opts['nonLocking'] ) ) {
 688                         // Build up a list of files to lock...
 689                         $filesLockEx = $filesLockSh = array();
 690                         foreach ( $performOps as $fileOp ) {
 691                                 $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() );
 692                                 $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() );
 693                         }
 694                         // Optimization: if doing an EX lock anyway, don't also set an SH one
 695                         $filesLockSh = array_diff( $filesLockSh, $filesLockEx );
 696                         // Get a shared lock on the parent directory of each path changed
 697                         $filesLockSh = array_merge( $filesLockSh, array_map( 'dirname', $filesLockEx ) );
 698                         // Try to lock those files for the scope of this function...
 699                         $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
 700                         $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 701                         if ( !$status->isOK() ) {
 702                                 wfProfileOut( __METHOD__ );
 703                                 return $status; // abort
 704                         }
 705                 }
 706
 707                 // Clear any cache entries (after locks acquired)
 708                 $this->clearCache();
 709
 710                 // Actually attempt the operation batch...
 711                 $subStatus = FileOp::attemptBatch( $performOps, $opts, $this->fileJournal );
 712
 713                 // Merge errors into status fields
 714                 $status->merge( $subStatus );
 715                 $status->success = $subStatus->success; // not done in merge()
 716
 717                 wfProfileOut( __METHOD__ );
 718                 return $status;
 719         }
 720
 721         /**
 722          * @see FileBackend::clearCache()
 723          */
 724         final public function clearCache( array $paths = null ) {
 725                 if ( is_array( $paths ) ) {
 726                         $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
 727                         $paths = array_filter( $paths, 'strlen' ); // remove nulls
 728                 }
 729                 if ( $paths === null ) {
 730                         $this->cache = array();
 731                         $this->expensiveCache = array();
 732                 } else {
 733                         foreach ( $paths as $path ) {
 734                                 unset( $this->cache[$path] );
 735                                 unset( $this->expensiveCache[$path] );
 736                         }
 737                 }
 738                 $this->doClearCache( $paths );
 739         }
 740
 741         /**
 742          * Clears any additional stat caches for storage paths
 743          *
 744          * @see FileBackend::clearCache()
 745          *
 746          * @param $paths Array Storage paths (optional)
 747          * @return void
 748          */
 749         protected function doClearCache( array $paths = null ) {}
 750
 751         /**
 752          * Prune the inexpensive cache if it is too big to add an item
 753          *
 754          * @return void
 755          */
 756         protected function trimCache() {
 757                 if ( count( $this->cache ) >= $this->maxCacheSize ) {
 758                         reset( $this->cache );
 759                         unset( $this->cache[key( $this->cache )] );
 760                 }
 761         }
 762
 763         /**
 764          * Prune the expensive cache if it is too big to add an item
 765          *
 766          * @return void
 767          */
 768         protected function trimExpensiveCache() {
 769                 if ( count( $this->expensiveCache ) >= $this->maxExpensiveCacheSize ) {
 770                         reset( $this->expensiveCache );
 771                         unset( $this->expensiveCache[key( $this->expensiveCache )] );
 772                 }
 773         }
 774
 775         /**
 776          * Check if a container name is valid.
 777          * This checks for for length and illegal characters.
 778          *
 779          * @param $container string
 780          * @return bool
 781          */
 782         final protected static function isValidContainerName( $container ) {
 783                 // This accounts for Swift and S3 restrictions while leaving room
 784                 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
 785                 // This disallows directory separators or traversal characters.
 786                 // Note that matching strings URL encode to the same string;
 787                 // in Swift, the length restriction is *after* URL encoding.
 788                 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
 789         }
 790
 791         /**
 792          * Splits a storage path into an internal container name,
 793          * an internal relative file name, and a container shard suffix.
 794          * Any shard suffix is already appended to the internal container name.
 795          * This also checks that the storage path is valid and within this backend.
 796          *
 797          * If the container is sharded but a suffix could not be determined,
 798          * this means that the path can only refer to a directory and can only
 799          * be scanned by looking in all the container shards.
 800          *
 801          * @param $storagePath string
 802          * @return Array (container, path, container suffix) or (null, null, null) if invalid
 803          */
 804         final protected function resolveStoragePath( $storagePath ) {
 805                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
 806                 if ( $backend === $this->name ) { // must be for this backend
 807                         $relPath = self::normalizeContainerPath( $relPath );
 808                         if ( $relPath !== null ) {
 809                                 // Get shard for the normalized path if this container is sharded
 810                                 $cShard = $this->getContainerShard( $container, $relPath );
 811                                 // Validate and sanitize the relative path (backend-specific)
 812                                 $relPath = $this->resolveContainerPath( $container, $relPath );
 813                                 if ( $relPath !== null ) {
 814                                         // Prepend any wiki ID prefix to the container name
 815                                         $container = $this->fullContainerName( $container );
 816                                         if ( self::isValidContainerName( $container ) ) {
 817                                                 // Validate and sanitize the container name (backend-specific)
 818                                                 $container = $this->resolveContainerName( "{$container}{$cShard}" );
 819                                                 if ( $container !== null ) {
 820                                                         return array( $container, $relPath, $cShard );
 821                                                 }
 822                                         }
 823                                 }
 824                         }
 825                 }
 826                 return array( null, null, null );
 827         }
 828
 829         /**
 830          * Like resolveStoragePath() except null values are returned if
 831          * the container is sharded and the shard could not be determined.
 832          *
 833          * @see FileBackendStore::resolveStoragePath()
 834          *
 835          * @param $storagePath string
 836          * @return Array (container, path) or (null, null) if invalid
 837          */
 838         final protected function resolveStoragePathReal( $storagePath ) {
 839                 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
 840                 if ( $cShard !== null ) {
 841                         return array( $container, $relPath );
 842                 }
 843                 return array( null, null );
 844         }
 845
 846         /**
 847          * Get the container name shard suffix for a given path.
 848          * Any empty suffix means the container is not sharded.
 849          *
 850          * @param $container string Container name
 851          * @param $relStoragePath string Storage path relative to the container
 852          * @return string|null Returns null if shard could not be determined
 853          */
 854         final protected function getContainerShard( $container, $relPath ) {
 855                 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
 856                 if ( $levels == 1 || $levels == 2 ) {
 857                         // Hash characters are either base 16 or 36
 858                         $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
 859                         // Get a regex that represents the shard portion of paths.
 860                         // The concatenation of the captures gives us the shard.
 861                         if ( $levels === 1 ) { // 16 or 36 shards per container
 862                                 $hashDirRegex = '(' . $char . ')';
 863                         } else { // 256 or 1296 shards per container
 864                                 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
 865                                         $hashDirRegex = $char . '/(' . $char . '{2})';
 866                                 } else { // short hash dir format (e.g. "a/b/c")
 867                                         $hashDirRegex = '(' . $char . ')/(' . $char . ')';
 868                                 }
 869                         }
 870                         // Allow certain directories to be above the hash dirs so as
 871                         // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
 872                         // They must be 2+ chars to avoid any hash directory ambiguity.
 873                         $m = array();
 874                         if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
 875                                 return '.' . implode( '', array_slice( $m, 1 ) );
 876                         }
 877                         return null; // failed to match
 878                 }
 879                 return ''; // no sharding
 880         }
 881
 882         /**
 883          * Get the sharding config for a container.
 884          * If greater than 0, then all file storage paths within
 885          * the container are required to be hashed accordingly.
 886          *
 887          * @param $container string
 888          * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
 889          */
 890         final protected function getContainerHashLevels( $container ) {
 891                 if ( isset( $this->shardViaHashLevels[$container] ) ) {
 892                         $config = $this->shardViaHashLevels[$container];
 893                         $hashLevels = (int)$config['levels'];
 894                         if ( $hashLevels == 1 || $hashLevels == 2 ) {
 895                                 $hashBase = (int)$config['base'];
 896                                 if ( $hashBase == 16 || $hashBase == 36 ) {
 897                                         return array( $hashLevels, $hashBase, $config['repeat'] );
 898                                 }
 899                         }
 900                 }
 901                 return array( 0, 0, false ); // no sharding
 902         }
 903
 904         /**
 905          * Get a list of full container shard suffixes for a container
 906          *
 907          * @param $container string
 908          * @return Array
 909          */
 910         final protected function getContainerSuffixes( $container ) {
 911                 $shards = array();
 912                 list( $digits, $base ) = $this->getContainerHashLevels( $container );
 913                 if ( $digits > 0 ) {
 914                         $numShards = pow( $base, $digits );
 915                         for ( $index = 0; $index < $numShards; $index++ ) {
 916                                 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
 917                         }
 918                 }
 919                 return $shards;
 920         }
 921
 922         /**
 923          * Get the full container name, including the wiki ID prefix
 924          *
 925          * @param $container string
 926          * @return string
 927          */
 928         final protected function fullContainerName( $container ) {
 929                 if ( $this->wikiId != '' ) {
 930                         return "{$this->wikiId}-$container";
 931                 } else {
 932                         return $container;
 933                 }
 934         }
 935
 936         /**
 937          * Resolve a container name, checking if it's allowed by the backend.
 938          * This is intended for internal use, such as encoding illegal chars.
 939          * Subclasses can override this to be more restrictive.
 940          *
 941          * @param $container string
 942          * @return string|null
 943          */
 944         protected function resolveContainerName( $container ) {
 945                 return $container;
 946         }
 947
 948         /**
 949          * Resolve a relative storage path, checking if it's allowed by the backend.
 950          * This is intended for internal use, such as encoding illegal chars or perhaps
 951          * getting absolute paths (e.g. FS based backends). Note that the relative path
 952          * may be the empty string (e.g. the path is simply to the container).
 953          *
 954          * @param $container string Container name
 955          * @param $relStoragePath string Storage path relative to the container
 956          * @return string|null Path or null if not valid
 957          */
 958         protected function resolveContainerPath( $container, $relStoragePath ) {
 959                 return $relStoragePath;
 960         }
 961 }
 962
 963 /**
 964  * FileBackendStore helper function to handle file listings that span container shards.
 965  * Do not use this class from places outside of FileBackendStore.
 966  *
 967  * @ingroup FileBackend
 968  */
 969 class FileBackendStoreShardListIterator implements Iterator {
 970         /* @var FileBackendStore */
 971         protected $backend;
 972         /* @var Array */
 973         protected $params;
 974         /* @var Array */
 975         protected $shardSuffixes;
 976         protected $container; // string
 977         protected $directory; // string
 978
 979         /* @var Traversable */
 980         protected $iter;
 981         protected $curShard = 0; // integer
 982         protected $pos = 0; // integer
 983
 984         /**
 985          * @param $backend FileBackendStore
 986          * @param $container string Full storage container name
 987          * @param $dir string Storage directory relative to container
 988          * @param $suffixes Array List of container shard suffixes
 989          * @param $params Array
 990          */
 991         public function __construct(
 992                 FileBackendStore $backend, $container, $dir, array $suffixes, array $params
 993         ) {
 994                 $this->backend = $backend;
 995                 $this->container = $container;
 996                 $this->directory = $dir;
 997                 $this->shardSuffixes = $suffixes;
 998                 $this->params = $params;
 999         }
1000
1001         /**
1002          * @see Iterator::current()
1003          * @return string|bool String or false
1004          */
1005         public function current() {
1006                 if ( is_array( $this->iter ) ) {
1007                         return current( $this->iter );
1008                 } else {
1009                         return $this->iter->current();
1010                 }
1011         }
1012
1013         /**
1014          * @see Iterator::key()
1015          * @return integer
1016          */
1017         public function key() {
1018                 return $this->pos;
1019         }
1020
1021         /**
1022          * @see Iterator::next()
1023          * @return void
1024          */
1025         public function next() {
1026                 ++$this->pos;
1027                 if ( is_array( $this->iter ) ) {
1028                         next( $this->iter );
1029                 } else {
1030                         $this->iter->next();
1031                 }
1032                 // Find the next non-empty shard if no elements are left
1033                 $this->nextShardIteratorIfNotValid();
1034         }
1035
1036         /**
1037          * @see Iterator::rewind()
1038          * @return void
1039          */
1040         public function rewind() {
1041                 $this->pos = 0;
1042                 $this->curShard = 0;
1043                 $this->setIteratorFromCurrentShard();
1044                 // Find the next non-empty shard if this one has no elements
1045                 $this->nextShardIteratorIfNotValid();
1046         }
1047
1048         /**
1049          * @see Iterator::valid()
1050          * @return bool
1051          */
1052         public function valid() {
1053                 if ( $this->iter == null ) {
1054                         return false; // some failure?
1055                 } elseif ( is_array( $this->iter ) ) {
1056                         return ( current( $this->iter ) !== false ); // no paths can have this value
1057                 } else {
1058                         return $this->iter->valid();
1059                 }
1060         }
1061
1062         /**
1063          * If the list iterator for this container shard is out of items,
1064          * then move on to the next container that has items.
1065          * If there are none, then it advances to the last container.
1066          */
1067         protected function nextShardIteratorIfNotValid() {
1068                 while ( !$this->valid() ) {
1069                         if ( ++$this->curShard >= count( $this->shardSuffixes ) ) {
1070                                 break; // no more container shards
1071                         }
1072                         $this->setIteratorFromCurrentShard();
1073                 }
1074         }
1075
1076         /**
1077          * Set the list iterator to that of the current container shard
1078          */
1079         protected function setIteratorFromCurrentShard() {
1080                 $suffix = $this->shardSuffixes[$this->curShard];
1081                 $this->iter = $this->backend->getFileListInternal(
1082                         "{$this->container}{$suffix}", $this->directory, $this->params );
1083         }
1084 }