includes/filerepo/backend/FileBackendStore.php

   1 <?php
   2 /**
   3  * Base class for all backends using particular storage medium.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Aaron Schulz
  23  */
  24
  25 /**
  26  * @brief Base class for all backends using particular storage medium.
  27  *
  28  * This class defines the methods as abstract that subclasses must implement.
  29  * Outside callers should *not* use functions with "Internal" in the name.
  30  *
  31  * The FileBackend operations are implemented using basic functions
  32  * such as storeInternal(), copyInternal(), deleteInternal() and the like.
  33  * This class is also responsible for path resolution and sanitization.
  34  *
  35  * @ingroup FileBackend
  36  * @since 1.19
  37  */
  38 abstract class FileBackendStore extends FileBackend {
  39         /** @var BagOStuff */
  40         protected $memCache;
  41
  42         /** @var Array Map of paths to small (RAM/disk) cache items */
  43         protected $cache = array(); // (storage path => key => value)
  44         protected $maxCacheSize = 300; // integer; max paths with entries
  45         /** @var Array Map of paths to large (RAM/disk) cache items */
  46         protected $expensiveCache = array(); // (storage path => key => value)
  47         protected $maxExpensiveCacheSize = 5; // integer; max paths with entries
  48
  49         /** @var Array Map of container names to sharding settings */
  50         protected $shardViaHashLevels = array(); // (container name => config array)
  51
  52         protected $maxFileSize = 4294967296; // integer bytes (4GiB)
  53
  54         /**
  55          * @see FileBackend::__construct()
  56          *
  57          * @param $config Array
  58          */
  59         public function __construct( array $config ) {
  60                 parent::__construct( $config );
  61                 $this->memCache = new EmptyBagOStuff(); // disabled by default
  62         }
  63
  64         /**
  65          * Get the maximum allowable file size given backend
  66          * medium restrictions and basic performance constraints.
  67          * Do not call this function from places outside FileBackend and FileOp.
  68          *
  69          * @return integer Bytes
  70          */
  71         final public function maxFileSizeInternal() {
  72                 return $this->maxFileSize;
  73         }
  74
  75         /**
  76          * Check if a file can be created at a given storage path.
  77          * FS backends should check if the parent directory exists and the file is writable.
  78          * Backends using key/value stores should check if the container exists.
  79          *
  80          * @param $storagePath string
  81          * @return bool
  82          */
  83         abstract public function isPathUsableInternal( $storagePath );
  84
  85         /**
  86          * Create a file in the backend with the given contents.
  87          * Do not call this function from places outside FileBackend and FileOp.
  88          *
  89          * $params include:
  90          *     content       : the raw file contents
  91          *     dst           : destination storage path
  92          *     overwrite     : overwrite any file that exists at the destination
  93          *     async         : Status will be returned immediately if supported.
  94          *                     If the status is OK, then its value field will be
  95          *                     set to a FileBackendStoreOpHandle object.
  96          *
  97          * @param $params Array
  98          * @return Status
  99          */
 100         final public function createInternal( array $params ) {
 101                 wfProfileIn( __METHOD__ );
 102                 wfProfileIn( __METHOD__ . '-' . $this->name );
 103                 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
 104                         $status = Status::newFatal( 'backend-fail-maxsize',
 105                                 $params['dst'], $this->maxFileSizeInternal() );
 106                 } else {
 107                         $status = $this->doCreateInternal( $params );
 108                         $this->clearCache( array( $params['dst'] ) );
 109                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 110                 }
 111                 wfProfileOut( __METHOD__ . '-' . $this->name );
 112                 wfProfileOut( __METHOD__ );
 113                 return $status;
 114         }
 115
 116         /**
 117          * @see FileBackendStore::createInternal()
 118          */
 119         abstract protected function doCreateInternal( array $params );
 120
 121         /**
 122          * Store a file into the backend from a file on disk.
 123          * Do not call this function from places outside FileBackend and FileOp.
 124          *
 125          * $params include:
 126          *     src           : source path on disk
 127          *     dst           : destination storage path
 128          *     overwrite     : overwrite any file that exists at the destination
 129          *     async         : Status will be returned immediately if supported.
 130          *                     If the status is OK, then its value field will be
 131          *                     set to a FileBackendStoreOpHandle object.
 132          *
 133          * @param $params Array
 134          * @return Status
 135          */
 136         final public function storeInternal( array $params ) {
 137                 wfProfileIn( __METHOD__ );
 138                 wfProfileIn( __METHOD__ . '-' . $this->name );
 139                 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
 140                         $status = Status::newFatal( 'backend-fail-store', $params['dst'] );
 141                 } else {
 142                         $status = $this->doStoreInternal( $params );
 143                         $this->clearCache( array( $params['dst'] ) );
 144                         $this->deleteFileCache( $params['dst'] ); // persistent cache
 145                 }
 146                 wfProfileOut( __METHOD__ . '-' . $this->name );
 147                 wfProfileOut( __METHOD__ );
 148                 return $status;
 149         }
 150
 151         /**
 152          * @see FileBackendStore::storeInternal()
 153          */
 154         abstract protected function doStoreInternal( array $params );
 155
 156         /**
 157          * Copy a file from one storage path to another in the backend.
 158          * Do not call this function from places outside FileBackend and FileOp.
 159          *
 160          * $params include:
 161          *     src           : source storage path
 162          *     dst           : destination storage path
 163          *     overwrite     : overwrite any file that exists at the destination
 164          *     async         : Status will be returned immediately if supported.
 165          *                     If the status is OK, then its value field will be
 166          *                     set to a FileBackendStoreOpHandle object.
 167          *
 168          * @param $params Array
 169          * @return Status
 170          */
 171         final public function copyInternal( array $params ) {
 172                 wfProfileIn( __METHOD__ );
 173                 wfProfileIn( __METHOD__ . '-' . $this->name );
 174                 $status = $this->doCopyInternal( $params );
 175                 $this->clearCache( array( $params['dst'] ) );
 176                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 177                 wfProfileOut( __METHOD__ . '-' . $this->name );
 178                 wfProfileOut( __METHOD__ );
 179                 return $status;
 180         }
 181
 182         /**
 183          * @see FileBackendStore::copyInternal()
 184          */
 185         abstract protected function doCopyInternal( array $params );
 186
 187         /**
 188          * Delete a file at the storage path.
 189          * Do not call this function from places outside FileBackend and FileOp.
 190          *
 191          * $params include:
 192          *     src                 : source storage path
 193          *     ignoreMissingSource : do nothing if the source file does not exist
 194          *     async               : Status will be returned immediately if supported.
 195          *                           If the status is OK, then its value field will be
 196          *                           set to a FileBackendStoreOpHandle object.
 197          *
 198          * @param $params Array
 199          * @return Status
 200          */
 201         final public function deleteInternal( array $params ) {
 202                 wfProfileIn( __METHOD__ );
 203                 wfProfileIn( __METHOD__ . '-' . $this->name );
 204                 $status = $this->doDeleteInternal( $params );
 205                 $this->clearCache( array( $params['src'] ) );
 206                 $this->deleteFileCache( $params['src'] ); // persistent cache
 207                 wfProfileOut( __METHOD__ . '-' . $this->name );
 208                 wfProfileOut( __METHOD__ );
 209                 return $status;
 210         }
 211
 212         /**
 213          * @see FileBackendStore::deleteInternal()
 214          */
 215         abstract protected function doDeleteInternal( array $params );
 216
 217         /**
 218          * Move a file from one storage path to another in the backend.
 219          * Do not call this function from places outside FileBackend and FileOp.
 220          *
 221          * $params include:
 222          *     src           : source storage path
 223          *     dst           : destination storage path
 224          *     overwrite     : overwrite any file that exists at the destination
 225          *     async         : Status will be returned immediately if supported.
 226          *                     If the status is OK, then its value field will be
 227          *                     set to a FileBackendStoreOpHandle object.
 228          *
 229          * @param $params Array
 230          * @return Status
 231          */
 232         final public function moveInternal( array $params ) {
 233                 wfProfileIn( __METHOD__ );
 234                 wfProfileIn( __METHOD__ . '-' . $this->name );
 235                 $status = $this->doMoveInternal( $params );
 236                 $this->clearCache( array( $params['src'], $params['dst'] ) );
 237                 $this->deleteFileCache( $params['src'] ); // persistent cache
 238                 $this->deleteFileCache( $params['dst'] ); // persistent cache
 239                 wfProfileOut( __METHOD__ . '-' . $this->name );
 240                 wfProfileOut( __METHOD__ );
 241                 return $status;
 242         }
 243
 244         /**
 245          * @see FileBackendStore::moveInternal()
 246          * @return Status
 247          */
 248         protected function doMoveInternal( array $params ) {
 249                 unset( $params['async'] ); // two steps, won't work here :)
 250                 // Copy source to dest
 251                 $status = $this->copyInternal( $params );
 252                 if ( $status->isOK() ) {
 253                         // Delete source (only fails due to races or medium going down)
 254                         $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
 255                         $status->setResult( true, $status->value ); // ignore delete() errors
 256                 }
 257                 return $status;
 258         }
 259
 260         /**
 261          * No-op file operation that does nothing.
 262          * Do not call this function from places outside FileBackend and FileOp.
 263          *
 264          * @param $params Array
 265          * @return Status
 266          */
 267         final public function nullInternal( array $params ) {
 268                 return Status::newGood();
 269         }
 270
 271         /**
 272          * @see FileBackend::concatenate()
 273          * @return Status
 274          */
 275         final public function concatenate( array $params ) {
 276                 wfProfileIn( __METHOD__ );
 277                 wfProfileIn( __METHOD__ . '-' . $this->name );
 278                 $status = Status::newGood();
 279
 280                 // Try to lock the source files for the scope of this function
 281                 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
 282                 if ( $status->isOK() ) {
 283                         // Actually do the concatenation
 284                         $status->merge( $this->doConcatenate( $params ) );
 285                 }
 286
 287                 wfProfileOut( __METHOD__ . '-' . $this->name );
 288                 wfProfileOut( __METHOD__ );
 289                 return $status;
 290         }
 291
 292         /**
 293          * @see FileBackendStore::concatenate()
 294          * @return Status
 295          */
 296         protected function doConcatenate( array $params ) {
 297                 $status = Status::newGood();
 298                 $tmpPath = $params['dst']; // convenience
 299
 300                 // Check that the specified temp file is valid...
 301                 wfSuppressWarnings();
 302                 $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
 303                 wfRestoreWarnings();
 304                 if ( !$ok ) { // not present or not empty
 305                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 306                         return $status;
 307                 }
 308
 309                 // Build up the temp file using the source chunks (in order)...
 310                 $tmpHandle = fopen( $tmpPath, 'ab' );
 311                 if ( $tmpHandle === false ) {
 312                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 313                         return $status;
 314                 }
 315                 foreach ( $params['srcs'] as $virtualSource ) {
 316                         // Get a local FS version of the chunk
 317                         $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
 318                         if ( !$tmpFile ) {
 319                                 $status->fatal( 'backend-fail-read', $virtualSource );
 320                                 return $status;
 321                         }
 322                         // Get a handle to the local FS version
 323                         $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
 324                         if ( $sourceHandle === false ) {
 325                                 fclose( $tmpHandle );
 326                                 $status->fatal( 'backend-fail-read', $virtualSource );
 327                                 return $status;
 328                         }
 329                         // Append chunk to file (pass chunk size to avoid magic quotes)
 330                         if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
 331                                 fclose( $sourceHandle );
 332                                 fclose( $tmpHandle );
 333                                 $status->fatal( 'backend-fail-writetemp', $tmpPath );
 334                                 return $status;
 335                         }
 336                         fclose( $sourceHandle );
 337                 }
 338                 if ( !fclose( $tmpHandle ) ) {
 339                         $status->fatal( 'backend-fail-closetemp', $tmpPath );
 340                         return $status;
 341                 }
 342
 343                 clearstatcache(); // temp file changed
 344
 345                 return $status;
 346         }
 347
 348         /**
 349          * @see FileBackend::doPrepare()
 350          * @return Status
 351          */
 352         final protected function doPrepare( array $params ) {
 353                 wfProfileIn( __METHOD__ );
 354                 wfProfileIn( __METHOD__ . '-' . $this->name );
 355
 356                 $status = Status::newGood();
 357                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 358                 if ( $dir === null ) {
 359                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 360                         wfProfileOut( __METHOD__ . '-' . $this->name );
 361                         wfProfileOut( __METHOD__ );
 362                         return $status; // invalid storage path
 363                 }
 364
 365                 if ( $shard !== null ) { // confined to a single container/shard
 366                         $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
 367                 } else { // directory is on several shards
 368                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 369                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 370                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 371                                 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 372                         }
 373                 }
 374
 375                 wfProfileOut( __METHOD__ . '-' . $this->name );
 376                 wfProfileOut( __METHOD__ );
 377                 return $status;
 378         }
 379
 380         /**
 381          * @see FileBackendStore::doPrepare()
 382          * @return Status
 383          */
 384         protected function doPrepareInternal( $container, $dir, array $params ) {
 385                 return Status::newGood();
 386         }
 387
 388         /**
 389          * @see FileBackend::doSecure()
 390          * @return Status
 391          */
 392         final protected function doSecure( array $params ) {
 393                 wfProfileIn( __METHOD__ );
 394                 wfProfileIn( __METHOD__ . '-' . $this->name );
 395                 $status = Status::newGood();
 396
 397                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 398                 if ( $dir === null ) {
 399                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 400                         wfProfileOut( __METHOD__ . '-' . $this->name );
 401                         wfProfileOut( __METHOD__ );
 402                         return $status; // invalid storage path
 403                 }
 404
 405                 if ( $shard !== null ) { // confined to a single container/shard
 406                         $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 407                 } else { // directory is on several shards
 408                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 409                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 410                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 411                                 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 412                         }
 413                 }
 414
 415                 wfProfileOut( __METHOD__ . '-' . $this->name );
 416                 wfProfileOut( __METHOD__ );
 417                 return $status;
 418         }
 419
 420         /**
 421          * @see FileBackendStore::doSecure()
 422          * @return Status
 423          */
 424         protected function doSecureInternal( $container, $dir, array $params ) {
 425                 return Status::newGood();
 426         }
 427
 428         /**
 429          * @see FileBackend::doClean()
 430          * @return Status
 431          */
 432         final protected function doClean( array $params ) {
 433                 wfProfileIn( __METHOD__ );
 434                 wfProfileIn( __METHOD__ . '-' . $this->name );
 435                 $status = Status::newGood();
 436
 437                 // Recursive: first delete all empty subdirs recursively
 438                 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
 439                         $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) );
 440                         if ( $subDirsRel !== null ) { // no errors
 441                                 foreach ( $subDirsRel as $subDirRel ) {
 442                                         $subDir = $params['dir'] . "/{$subDirRel}"; // full path
 443                                         $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) );
 444                                 }
 445                         }
 446                 }
 447
 448                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 449                 if ( $dir === null ) {
 450                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 451                         wfProfileOut( __METHOD__ . '-' . $this->name );
 452                         wfProfileOut( __METHOD__ );
 453                         return $status; // invalid storage path
 454                 }
 455
 456                 // Attempt to lock this directory...
 457                 $filesLockEx = array( $params['dir'] );
 458                 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 459                 if ( !$status->isOK() ) {
 460                         wfProfileOut( __METHOD__ . '-' . $this->name );
 461                         wfProfileOut( __METHOD__ );
 462                         return $status; // abort
 463                 }
 464
 465                 if ( $shard !== null ) { // confined to a single container/shard
 466                         $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
 467                         $this->deleteContainerCache( $fullCont ); // purge cache
 468                 } else { // directory is on several shards
 469                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 470                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 471                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 472                                 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 473                                 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
 474                         }
 475                 }
 476
 477                 wfProfileOut( __METHOD__ . '-' . $this->name );
 478                 wfProfileOut( __METHOD__ );
 479                 return $status;
 480         }
 481
 482         /**
 483          * @see FileBackendStore::doClean()
 484          * @return Status
 485          */
 486         protected function doCleanInternal( $container, $dir, array $params ) {
 487                 return Status::newGood();
 488         }
 489
 490         /**
 491          * @see FileBackend::fileExists()
 492          * @return bool|null
 493          */
 494         final public function fileExists( array $params ) {
 495                 wfProfileIn( __METHOD__ );
 496                 wfProfileIn( __METHOD__ . '-' . $this->name );
 497                 $stat = $this->getFileStat( $params );
 498                 wfProfileOut( __METHOD__ . '-' . $this->name );
 499                 wfProfileOut( __METHOD__ );
 500                 return ( $stat === null ) ? null : (bool)$stat; // null => failure
 501         }
 502
 503         /**
 504          * @see FileBackend::getFileTimestamp()
 505          * @return bool
 506          */
 507         final public function getFileTimestamp( array $params ) {
 508                 wfProfileIn( __METHOD__ );
 509                 wfProfileIn( __METHOD__ . '-' . $this->name );
 510                 $stat = $this->getFileStat( $params );
 511                 wfProfileOut( __METHOD__ . '-' . $this->name );
 512                 wfProfileOut( __METHOD__ );
 513                 return $stat ? $stat['mtime'] : false;
 514         }
 515
 516         /**
 517          * @see FileBackend::getFileSize()
 518          * @return bool
 519          */
 520         final public function getFileSize( array $params ) {
 521                 wfProfileIn( __METHOD__ );
 522                 wfProfileIn( __METHOD__ . '-' . $this->name );
 523                 $stat = $this->getFileStat( $params );
 524                 wfProfileOut( __METHOD__ . '-' . $this->name );
 525                 wfProfileOut( __METHOD__ );
 526                 return $stat ? $stat['size'] : false;
 527         }
 528
 529         /**
 530          * @see FileBackend::getFileStat()
 531          * @return bool
 532          */
 533         final public function getFileStat( array $params ) {
 534                 wfProfileIn( __METHOD__ );
 535                 wfProfileIn( __METHOD__ . '-' . $this->name );
 536                 $path = self::normalizeStoragePath( $params['src'] );
 537                 if ( $path === null ) {
 538                         wfProfileOut( __METHOD__ . '-' . $this->name );
 539                         wfProfileOut( __METHOD__ );
 540                         return false; // invalid storage path
 541                 }
 542                 $latest = !empty( $params['latest'] ); // use latest data?
 543                 if ( !isset( $this->cache[$path]['stat'] ) ) {
 544                         $this->primeFileCache( array( $path ) ); // check persistent cache
 545                 }
 546                 if ( isset( $this->cache[$path]['stat'] ) ) {
 547                         // If we want the latest data, check that this cached
 548                         // value was in fact fetched with the latest available data.
 549                         if ( !$latest || $this->cache[$path]['stat']['latest'] ) {
 550                                 $this->pingCache( $path ); // LRU
 551                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 552                                 wfProfileOut( __METHOD__ );
 553                                 return $this->cache[$path]['stat'];
 554                         }
 555                 }
 556                 wfProfileIn( __METHOD__ . '-miss' );
 557                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 558                 $stat = $this->doGetFileStat( $params );
 559                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 560                 wfProfileOut( __METHOD__ . '-miss' );
 561                 if ( is_array( $stat ) ) { // don't cache negatives
 562                         $stat['latest'] = $latest;
 563                         $this->trimCache(); // limit memory
 564                         $this->cache[$path]['stat'] = $stat;
 565                         $this->setFileCache( $path, $stat ); // update persistent cache
 566                 }
 567                 wfProfileOut( __METHOD__ . '-' . $this->name );
 568                 wfProfileOut( __METHOD__ );
 569                 return $stat;
 570         }
 571
 572         /**
 573          * @see FileBackendStore::getFileStat()
 574          */
 575         abstract protected function doGetFileStat( array $params );
 576
 577         /**
 578          * @see FileBackend::getFileContents()
 579          * @return bool|string
 580          */
 581         public function getFileContents( array $params ) {
 582                 wfProfileIn( __METHOD__ );
 583                 wfProfileIn( __METHOD__ . '-' . $this->name );
 584                 $tmpFile = $this->getLocalReference( $params );
 585                 if ( !$tmpFile ) {
 586                         wfProfileOut( __METHOD__ . '-' . $this->name );
 587                         wfProfileOut( __METHOD__ );
 588                         return false;
 589                 }
 590                 wfSuppressWarnings();
 591                 $data = file_get_contents( $tmpFile->getPath() );
 592                 wfRestoreWarnings();
 593                 wfProfileOut( __METHOD__ . '-' . $this->name );
 594                 wfProfileOut( __METHOD__ );
 595                 return $data;
 596         }
 597
 598         /**
 599          * @see FileBackend::getFileSha1Base36()
 600          * @return bool|string
 601          */
 602         final public function getFileSha1Base36( array $params ) {
 603                 wfProfileIn( __METHOD__ );
 604                 wfProfileIn( __METHOD__ . '-' . $this->name );
 605                 $path = $params['src'];
 606                 if ( isset( $this->cache[$path]['sha1'] ) ) {
 607                         $this->pingCache( $path ); // LRU
 608                         wfProfileOut( __METHOD__ . '-' . $this->name );
 609                         wfProfileOut( __METHOD__ );
 610                         return $this->cache[$path]['sha1'];
 611                 }
 612                 wfProfileIn( __METHOD__ . '-miss' );
 613                 wfProfileIn( __METHOD__ . '-miss-' . $this->name );
 614                 $hash = $this->doGetFileSha1Base36( $params );
 615                 wfProfileOut( __METHOD__ . '-miss-' . $this->name );
 616                 wfProfileOut( __METHOD__ . '-miss' );
 617                 if ( $hash ) { // don't cache negatives
 618                         $this->trimCache(); // limit memory
 619                         $this->cache[$path]['sha1'] = $hash;
 620                 }
 621                 wfProfileOut( __METHOD__ . '-' . $this->name );
 622                 wfProfileOut( __METHOD__ );
 623                 return $hash;
 624         }
 625
 626         /**
 627          * @see FileBackendStore::getFileSha1Base36()
 628          * @return bool
 629          */
 630         protected function doGetFileSha1Base36( array $params ) {
 631                 $fsFile = $this->getLocalReference( $params );
 632                 if ( !$fsFile ) {
 633                         return false;
 634                 } else {
 635                         return $fsFile->getSha1Base36();
 636                 }
 637         }
 638
 639         /**
 640          * @see FileBackend::getFileProps()
 641          * @return Array
 642          */
 643         final public function getFileProps( array $params ) {
 644                 wfProfileIn( __METHOD__ );
 645                 wfProfileIn( __METHOD__ . '-' . $this->name );
 646                 $fsFile = $this->getLocalReference( $params );
 647                 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
 648                 wfProfileOut( __METHOD__ . '-' . $this->name );
 649                 wfProfileOut( __METHOD__ );
 650                 return $props;
 651         }
 652
 653         /**
 654          * @see FileBackend::getLocalReference()
 655          * @return TempFSFile|null
 656          */
 657         public function getLocalReference( array $params ) {
 658                 wfProfileIn( __METHOD__ );
 659                 wfProfileIn( __METHOD__ . '-' . $this->name );
 660                 $path = $params['src'];
 661                 if ( isset( $this->expensiveCache[$path]['localRef'] ) ) {
 662                         $this->pingExpensiveCache( $path );
 663                         wfProfileOut( __METHOD__ . '-' . $this->name );
 664                         wfProfileOut( __METHOD__ );
 665                         return $this->expensiveCache[$path]['localRef'];
 666                 }
 667                 $tmpFile = $this->getLocalCopy( $params );
 668                 if ( $tmpFile ) { // don't cache negatives
 669                         $this->trimExpensiveCache(); // limit memory
 670                         $this->expensiveCache[$path]['localRef'] = $tmpFile;
 671                 }
 672                 wfProfileOut( __METHOD__ . '-' . $this->name );
 673                 wfProfileOut( __METHOD__ );
 674                 return $tmpFile;
 675         }
 676
 677         /**
 678          * @see FileBackend::streamFile()
 679          * @return Status
 680          */
 681         final public function streamFile( array $params ) {
 682                 wfProfileIn( __METHOD__ );
 683                 wfProfileIn( __METHOD__ . '-' . $this->name );
 684                 $status = Status::newGood();
 685
 686                 $info = $this->getFileStat( $params );
 687                 if ( !$info ) { // let StreamFile handle the 404
 688                         $status->fatal( 'backend-fail-notexists', $params['src'] );
 689                 }
 690
 691                 // Set output buffer and HTTP headers for stream
 692                 $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array();
 693                 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
 694                 if ( $res == StreamFile::NOT_MODIFIED ) {
 695                         // do nothing; client cache is up to date
 696                 } elseif ( $res == StreamFile::READY_STREAM ) {
 697                         wfProfileIn( __METHOD__ . '-send' );
 698                         wfProfileIn( __METHOD__ . '-send-' . $this->name );
 699                         $status = $this->doStreamFile( $params );
 700                         wfProfileOut( __METHOD__ . '-send-' . $this->name );
 701                         wfProfileOut( __METHOD__ . '-send' );
 702                 } else {
 703                         $status->fatal( 'backend-fail-stream', $params['src'] );
 704                 }
 705
 706                 wfProfileOut( __METHOD__ . '-' . $this->name );
 707                 wfProfileOut( __METHOD__ );
 708                 return $status;
 709         }
 710
 711         /**
 712          * @see FileBackendStore::streamFile()
 713          * @return Status
 714          */
 715         protected function doStreamFile( array $params ) {
 716                 $status = Status::newGood();
 717
 718                 $fsFile = $this->getLocalReference( $params );
 719                 if ( !$fsFile ) {
 720                         $status->fatal( 'backend-fail-stream', $params['src'] );
 721                 } elseif ( !readfile( $fsFile->getPath() ) ) {
 722                         $status->fatal( 'backend-fail-stream', $params['src'] );
 723                 }
 724
 725                 return $status;
 726         }
 727
 728         /**
 729          * @see FileBackend::directoryExists()
 730          * @return bool|null
 731          */
 732         final public function directoryExists( array $params ) {
 733                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 734                 if ( $dir === null ) {
 735                         return false; // invalid storage path
 736                 }
 737                 if ( $shard !== null ) { // confined to a single container/shard
 738                         return $this->doDirectoryExists( $fullCont, $dir, $params );
 739                 } else { // directory is on several shards
 740                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 741                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 742                         $res = false; // response
 743                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 744                                 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
 745                                 if ( $exists ) {
 746                                         $res = true;
 747                                         break; // found one!
 748                                 } elseif ( $exists === null ) { // error?
 749                                         $res = null; // if we don't find anything, it is indeterminate
 750                                 }
 751                         }
 752                         return $res;
 753                 }
 754         }
 755
 756         /**
 757          * @see FileBackendStore::directoryExists()
 758          *
 759          * @param $container string Resolved container name
 760          * @param $dir string Resolved path relative to container
 761          * @param $params Array
 762          * @return bool|null
 763          */
 764         abstract protected function doDirectoryExists( $container, $dir, array $params );
 765
 766         /**
 767          * @see FileBackend::getDirectoryList()
 768          * @return Traversable|Array|null Returns null on failure
 769          */
 770         final public function getDirectoryList( array $params ) {
 771                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 772                 if ( $dir === null ) { // invalid storage path
 773                         return null;
 774                 }
 775                 if ( $shard !== null ) {
 776                         // File listing is confined to a single container/shard
 777                         return $this->getDirectoryListInternal( $fullCont, $dir, $params );
 778                 } else {
 779                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 780                         // File listing spans multiple containers/shards
 781                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 782                         return new FileBackendStoreShardDirIterator( $this,
 783                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 784                 }
 785         }
 786
 787         /**
 788          * Do not call this function from places outside FileBackend
 789          *
 790          * @see FileBackendStore::getDirectoryList()
 791          *
 792          * @param $container string Resolved container name
 793          * @param $dir string Resolved path relative to container
 794          * @param $params Array
 795          * @return Traversable|Array|null Returns null on failure
 796          */
 797         abstract public function getDirectoryListInternal( $container, $dir, array $params );
 798
 799         /**
 800          * @see FileBackend::getFileList()
 801          * @return Traversable|Array|null Returns null on failure
 802          */
 803         final public function getFileList( array $params ) {
 804                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 805                 if ( $dir === null ) { // invalid storage path
 806                         return null;
 807                 }
 808                 if ( $shard !== null ) {
 809                         // File listing is confined to a single container/shard
 810                         return $this->getFileListInternal( $fullCont, $dir, $params );
 811                 } else {
 812                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 813                         // File listing spans multiple containers/shards
 814                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 815                         return new FileBackendStoreShardFileIterator( $this,
 816                                 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
 817                 }
 818         }
 819
 820         /**
 821          * Do not call this function from places outside FileBackend
 822          *
 823          * @see FileBackendStore::getFileList()
 824          *
 825          * @param $container string Resolved container name
 826          * @param $dir string Resolved path relative to container
 827          * @param $params Array
 828          * @return Traversable|Array|null Returns null on failure
 829          */
 830         abstract public function getFileListInternal( $container, $dir, array $params );
 831
 832         /**
 833          * Return a list of FileOp objects from a list of operations.
 834          * Do not call this function from places outside FileBackend.
 835          *
 836          * The result must have the same number of items as the input.
 837          * An exception is thrown if an unsupported operation is requested.
 838          *
 839          * @param $ops Array Same format as doOperations()
 840          * @return Array List of FileOp objects
 841          * @throws MWException
 842          */
 843         final public function getOperationsInternal( array $ops ) {
 844                 $supportedOps = array(
 845                         'store'       => 'StoreFileOp',
 846                         'copy'        => 'CopyFileOp',
 847                         'move'        => 'MoveFileOp',
 848                         'delete'      => 'DeleteFileOp',
 849                         'create'      => 'CreateFileOp',
 850                         'null'        => 'NullFileOp'
 851                 );
 852
 853                 $performOps = array(); // array of FileOp objects
 854                 // Build up ordered array of FileOps...
 855                 foreach ( $ops as $operation ) {
 856                         $opName = $operation['op'];
 857                         if ( isset( $supportedOps[$opName] ) ) {
 858                                 $class = $supportedOps[$opName];
 859                                 // Get params for this operation
 860                                 $params = $operation;
 861                                 // Append the FileOp class
 862                                 $performOps[] = new $class( $this, $params );
 863                         } else {
 864                                 throw new MWException( "Operation '$opName' is not supported." );
 865                         }
 866                 }
 867
 868                 return $performOps;
 869         }
 870
 871         /**
 872          * Get a list of storage paths to lock for a list of operations
 873          * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
 874          * each corresponding to a list of storage paths to be locked.
 875          *
 876          * @param $performOps Array List of FileOp objects
 877          * @return Array ('sh' => list of paths, 'ex' => list of paths)
 878          */
 879         final public function getPathsToLockForOpsInternal( array $performOps ) {
 880                 // Build up a list of files to lock...
 881                 $paths = array( 'sh' => array(), 'ex' => array() );
 882                 foreach ( $performOps as $fileOp ) {
 883                         $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
 884                         $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
 885                 }
 886                 // Optimization: if doing an EX lock anyway, don't also set an SH one
 887                 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
 888                 // Get a shared lock on the parent directory of each path changed
 889                 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
 890
 891                 return $paths;
 892         }
 893
 894         /**
 895          * @see FileBackend::doOperationsInternal()
 896          * @return Status
 897          */
 898         protected function doOperationsInternal( array $ops, array $opts ) {
 899                 wfProfileIn( __METHOD__ );
 900                 wfProfileIn( __METHOD__ . '-' . $this->name );
 901                 $status = Status::newGood();
 902
 903                 // Build up a list of FileOps...
 904                 $performOps = $this->getOperationsInternal( $ops );
 905
 906                 // Acquire any locks as needed...
 907                 if ( empty( $opts['nonLocking'] ) ) {
 908                         // Build up a list of files to lock...
 909                         $paths = $this->getPathsToLockForOpsInternal( $performOps );
 910                         // Try to lock those files for the scope of this function...
 911                         $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
 912                         $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
 913                         if ( !$status->isOK() ) {
 914                                 wfProfileOut( __METHOD__ . '-' . $this->name );
 915                                 wfProfileOut( __METHOD__ );
 916                                 return $status; // abort
 917                         }
 918                 }
 919
 920                 // Clear any file cache entries (after locks acquired)
 921                 $this->clearCache();
 922
 923                 // Load from the persistent file and container caches
 924                 $this->primeFileCache( $performOps );
 925                 $this->primeContainerCache( $performOps );
 926
 927                 // Actually attempt the operation batch...
 928                 $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
 929
 930                 // Merge errors into status fields
 931                 $status->merge( $subStatus );
 932                 $status->success = $subStatus->success; // not done in merge()
 933
 934                 wfProfileOut( __METHOD__ . '-' . $this->name );
 935                 wfProfileOut( __METHOD__ );
 936                 return $status;
 937         }
 938
 939         /**
 940          * @see FileBackend::doQuickOperationsInternal()
 941          * @return Status
 942          * @throws MWException
 943          */
 944         final protected function doQuickOperationsInternal( array $ops ) {
 945                 $status = Status::newGood();
 946
 947                 $async = $this->parallelize;
 948                 $maxConcurrency = $this->concurrency; // throttle
 949
 950                 $statuses = array(); // array of (index => Status)
 951                 $fileOpHandles = array(); // list of (index => handle) arrays
 952                 $curFileOpHandles = array(); // current handle batch
 953                 // Perform the sync-only ops and build up op handles for the async ops...
 954                 foreach ( $ops as $index => $params ) {
 955                         $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
 956                         if ( !MWInit::methodExists( __CLASS__, $method ) ) {
 957                                 throw new MWException( "Operation '{$params['op']}' is not supported." );
 958                         }
 959                         $subStatus = $this->$method( array( 'async' => $async ) + $params );
 960                         if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
 961                                 if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
 962                                         $fileOpHandles[] = $curFileOpHandles; // push this batch
 963                                         $curFileOpHandles = array();
 964                                 }
 965                                 $curFileOpHandles[$index] = $subStatus->value; // keep index
 966                         } else { // error or completed
 967                                 $statuses[$index] = $subStatus; // keep index
 968                         }
 969                 }
 970                 if ( count( $curFileOpHandles ) ) {
 971                         $fileOpHandles[] = $curFileOpHandles; // last batch
 972                 }
 973                 // Do all the async ops that can be done concurrently...
 974                 foreach ( $fileOpHandles as $fileHandleBatch ) {
 975                         $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
 976                 }
 977                 // Marshall and merge all the responses...
 978                 foreach ( $statuses as $index => $subStatus ) {
 979                         $status->merge( $subStatus );
 980                         if ( $subStatus->isOK() ) {
 981                                 $status->success[$index] = true;
 982                                 ++$status->successCount;
 983                         } else {
 984                                 $status->success[$index] = false;
 985                                 ++$status->failCount;
 986                         }
 987                 }
 988
 989                 return $status;
 990         }
 991
 992         /**
 993          * Execute a list of FileBackendStoreOpHandle handles in parallel.
 994          * The resulting Status object fields will correspond
 995          * to the order in which the handles where given.
 996          *
 997          * @param $handles Array List of FileBackendStoreOpHandle objects
 998          * @return Array Map of Status objects
 999          * @throws MWException
1000          */
1001         final public function executeOpHandlesInternal( array $fileOpHandles ) {
1002                 wfProfileIn( __METHOD__ );
1003                 wfProfileIn( __METHOD__ . '-' . $this->name );
1004                 foreach ( $fileOpHandles as $fileOpHandle ) {
1005                         if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
1006                                 throw new MWException( "Given a non-FileBackendStoreOpHandle object." );
1007                         } elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
1008                                 throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." );
1009                         }
1010                 }
1011                 $res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1012                 wfProfileOut( __METHOD__ . '-' . $this->name );
1013                 wfProfileOut( __METHOD__ );
1014                 return $res;
1015         }
1016
1017         /**
1018          * @see FileBackendStore::executeOpHandlesInternal()
1019          * @return Array List of corresponding Status objects
1020          * @throws MWException
1021          */
1022         protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1023                 foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty
1024                         throw new MWException( "This backend supports no asynchronous operations." );
1025                 }
1026                 return array();
1027         }
1028
1029         /**
1030          * @see FileBackend::clearCache()
1031          */
1032         final public function clearCache( array $paths = null ) {
1033                 if ( is_array( $paths ) ) {
1034                         $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1035                         $paths = array_filter( $paths, 'strlen' ); // remove nulls
1036                 }
1037                 if ( $paths === null ) {
1038                         $this->cache = array();
1039                         $this->expensiveCache = array();
1040                 } else {
1041                         foreach ( $paths as $path ) {
1042                                 unset( $this->cache[$path] );
1043                                 unset( $this->expensiveCache[$path] );
1044                         }
1045                 }
1046                 $this->doClearCache( $paths );
1047         }
1048
1049         /**
1050          * Clears any additional stat caches for storage paths
1051          *
1052          * @see FileBackend::clearCache()
1053          *
1054          * @param $paths Array Storage paths (optional)
1055          * @return void
1056          */
1057         protected function doClearCache( array $paths = null ) {}
1058
1059         /**
1060          * Is this a key/value store where directories are just virtual?
1061          * Virtual directories exists in so much as files exists that are
1062          * prefixed with the directory path followed by a forward slash.
1063          *
1064          * @return bool
1065          */
1066         abstract protected function directoriesAreVirtual();
1067
1068         /**
1069          * Move a cache entry to the top (such as when accessed)
1070          *
1071          * @param $path string Storage path
1072          * @return void
1073          */
1074         protected function pingCache( $path ) {
1075                 if ( isset( $this->cache[$path] ) ) {
1076                         $tmp = $this->cache[$path];
1077                         unset( $this->cache[$path] );
1078                         $this->cache[$path] = $tmp;
1079                 }
1080         }
1081
1082         /**
1083          * Prune the inexpensive cache if it is too big to add an item
1084          *
1085          * @return void
1086          */
1087         protected function trimCache() {
1088                 if ( count( $this->cache ) >= $this->maxCacheSize ) {
1089                         reset( $this->cache );
1090                         unset( $this->cache[key( $this->cache )] );
1091                 }
1092         }
1093
1094         /**
1095          * Move a cache entry to the top (such as when accessed)
1096          *
1097          * @param $path string Storage path
1098          * @return void
1099          */
1100         protected function pingExpensiveCache( $path ) {
1101                 if ( isset( $this->expensiveCache[$path] ) ) {
1102                         $tmp = $this->expensiveCache[$path];
1103                         unset( $this->expensiveCache[$path] );
1104                         $this->expensiveCache[$path] = $tmp;
1105                 }
1106         }
1107
1108         /**
1109          * Prune the expensive cache if it is too big to add an item
1110          *
1111          * @return void
1112          */
1113         protected function trimExpensiveCache() {
1114                 if ( count( $this->expensiveCache ) >= $this->maxExpensiveCacheSize ) {
1115                         reset( $this->expensiveCache );
1116                         unset( $this->expensiveCache[key( $this->expensiveCache )] );
1117                 }
1118         }
1119
1120         /**
1121          * Check if a container name is valid.
1122          * This checks for for length and illegal characters.
1123          *
1124          * @param $container string
1125          * @return bool
1126          */
1127         final protected static function isValidContainerName( $container ) {
1128                 // This accounts for Swift and S3 restrictions while leaving room
1129                 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1130                 // This disallows directory separators or traversal characters.
1131                 // Note that matching strings URL encode to the same string;
1132                 // in Swift, the length restriction is *after* URL encoding.
1133                 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1134         }
1135
1136         /**
1137          * Splits a storage path into an internal container name,
1138          * an internal relative file name, and a container shard suffix.
1139          * Any shard suffix is already appended to the internal container name.
1140          * This also checks that the storage path is valid and within this backend.
1141          *
1142          * If the container is sharded but a suffix could not be determined,
1143          * this means that the path can only refer to a directory and can only
1144          * be scanned by looking in all the container shards.
1145          *
1146          * @param $storagePath string
1147          * @return Array (container, path, container suffix) or (null, null, null) if invalid
1148          */
1149         final protected function resolveStoragePath( $storagePath ) {
1150                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1151                 if ( $backend === $this->name ) { // must be for this backend
1152                         $relPath = self::normalizeContainerPath( $relPath );
1153                         if ( $relPath !== null ) {
1154                                 // Get shard for the normalized path if this container is sharded
1155                                 $cShard = $this->getContainerShard( $container, $relPath );
1156                                 // Validate and sanitize the relative path (backend-specific)
1157                                 $relPath = $this->resolveContainerPath( $container, $relPath );
1158                                 if ( $relPath !== null ) {
1159                                         // Prepend any wiki ID prefix to the container name
1160                                         $container = $this->fullContainerName( $container );
1161                                         if ( self::isValidContainerName( $container ) ) {
1162                                                 // Validate and sanitize the container name (backend-specific)
1163                                                 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1164                                                 if ( $container !== null ) {
1165                                                         return array( $container, $relPath, $cShard );
1166                                                 }
1167                                         }
1168                                 }
1169                         }
1170                 }
1171                 return array( null, null, null );
1172         }
1173
1174         /**
1175          * Like resolveStoragePath() except null values are returned if
1176          * the container is sharded and the shard could not be determined.
1177          *
1178          * @see FileBackendStore::resolveStoragePath()
1179          *
1180          * @param $storagePath string
1181          * @return Array (container, path) or (null, null) if invalid
1182          */
1183         final protected function resolveStoragePathReal( $storagePath ) {
1184                 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1185                 if ( $cShard !== null ) {
1186                         return array( $container, $relPath );
1187                 }
1188                 return array( null, null );
1189         }
1190
1191         /**
1192          * Get the container name shard suffix for a given path.
1193          * Any empty suffix means the container is not sharded.
1194          *
1195          * @param $container string Container name
1196          * @param $relStoragePath string Storage path relative to the container
1197          * @return string|null Returns null if shard could not be determined
1198          */
1199         final protected function getContainerShard( $container, $relPath ) {
1200                 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1201                 if ( $levels == 1 || $levels == 2 ) {
1202                         // Hash characters are either base 16 or 36
1203                         $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]';
1204                         // Get a regex that represents the shard portion of paths.
1205                         // The concatenation of the captures gives us the shard.
1206                         if ( $levels === 1 ) { // 16 or 36 shards per container
1207                                 $hashDirRegex = '(' . $char . ')';
1208                         } else { // 256 or 1296 shards per container
1209                                 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1210                                         $hashDirRegex = $char . '/(' . $char . '{2})';
1211                                 } else { // short hash dir format (e.g. "a/b/c")
1212                                         $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1213                                 }
1214                         }
1215                         // Allow certain directories to be above the hash dirs so as
1216                         // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1217                         // They must be 2+ chars to avoid any hash directory ambiguity.
1218                         $m = array();
1219                         if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1220                                 return '.' . implode( '', array_slice( $m, 1 ) );
1221                         }
1222                         return null; // failed to match
1223                 }
1224                 return ''; // no sharding
1225         }
1226
1227         /**
1228          * Check if a storage path maps to a single shard.
1229          * Container dirs like "a", where the container shards on "x/xy",
1230          * can reside on several shards. Such paths are tricky to handle.
1231          *
1232          * @param $storagePath string Storage path
1233          * @return bool
1234          */
1235         final public function isSingleShardPathInternal( $storagePath ) {
1236                 list( $c, $r, $shard ) = $this->resolveStoragePath( $storagePath );
1237                 return ( $shard !== null );
1238         }
1239
1240         /**
1241          * Get the sharding config for a container.
1242          * If greater than 0, then all file storage paths within
1243          * the container are required to be hashed accordingly.
1244          *
1245          * @param $container string
1246          * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
1247          */
1248         final protected function getContainerHashLevels( $container ) {
1249                 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1250                         $config = $this->shardViaHashLevels[$container];
1251                         $hashLevels = (int)$config['levels'];
1252                         if ( $hashLevels == 1 || $hashLevels == 2 ) {
1253                                 $hashBase = (int)$config['base'];
1254                                 if ( $hashBase == 16 || $hashBase == 36 ) {
1255                                         return array( $hashLevels, $hashBase, $config['repeat'] );
1256                                 }
1257                         }
1258                 }
1259                 return array( 0, 0, false ); // no sharding
1260         }
1261
1262         /**
1263          * Get a list of full container shard suffixes for a container
1264          *
1265          * @param $container string
1266          * @return Array
1267          */
1268         final protected function getContainerSuffixes( $container ) {
1269                 $shards = array();
1270                 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1271                 if ( $digits > 0 ) {
1272                         $numShards = pow( $base, $digits );
1273                         for ( $index = 0; $index < $numShards; $index++ ) {
1274                                 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
1275                         }
1276                 }
1277                 return $shards;
1278         }
1279
1280         /**
1281          * Get the full container name, including the wiki ID prefix
1282          *
1283          * @param $container string
1284          * @return string
1285          */
1286         final protected function fullContainerName( $container ) {
1287                 if ( $this->wikiId != '' ) {
1288                         return "{$this->wikiId}-$container";
1289                 } else {
1290                         return $container;
1291                 }
1292         }
1293
1294         /**
1295          * Resolve a container name, checking if it's allowed by the backend.
1296          * This is intended for internal use, such as encoding illegal chars.
1297          * Subclasses can override this to be more restrictive.
1298          *
1299          * @param $container string
1300          * @return string|null
1301          */
1302         protected function resolveContainerName( $container ) {
1303                 return $container;
1304         }
1305
1306         /**
1307          * Resolve a relative storage path, checking if it's allowed by the backend.
1308          * This is intended for internal use, such as encoding illegal chars or perhaps
1309          * getting absolute paths (e.g. FS based backends). Note that the relative path
1310          * may be the empty string (e.g. the path is simply to the container).
1311          *
1312          * @param $container string Container name
1313          * @param $relStoragePath string Storage path relative to the container
1314          * @return string|null Path or null if not valid
1315          */
1316         protected function resolveContainerPath( $container, $relStoragePath ) {
1317                 return $relStoragePath;
1318         }
1319
1320         /**
1321          * Get the cache key for a container
1322          *
1323          * @param $container string Resolved container name
1324          * @return string
1325          */
1326         private function containerCacheKey( $container ) {
1327                 return wfMemcKey( 'backend', $this->getName(), 'container', $container );
1328         }
1329
1330         /**
1331          * Set the cached info for a container
1332          *
1333          * @param $container string Resolved container name
1334          * @param $val mixed Information to cache
1335          * @return void
1336          */
1337         final protected function setContainerCache( $container, $val ) {
1338                 $this->memCache->set( $this->containerCacheKey( $container ), $val, 14*86400 );
1339         }
1340
1341         /**
1342          * Delete the cached info for a container
1343          *
1344          * @param $containers string Resolved container name
1345          * @return void
1346          */
1347         final protected function deleteContainerCache( $container ) {
1348                 if ( !$this->memCache->delete( $this->containerCacheKey( $container ) ) ) {
1349                         trigger_error( "Unable to delete stat cache for container $container." );
1350                 }
1351         }
1352
1353         /**
1354          * Do a batch lookup from cache for container stats for all containers
1355          * used in a list of container names, storage paths, or FileOp objects.
1356          *
1357          * @param $items Array
1358          * @return void
1359          */
1360         final protected function primeContainerCache( array $items ) {
1361                 wfProfileIn( __METHOD__ );
1362                 wfProfileIn( __METHOD__ . '-' . $this->name );
1363
1364                 $paths = array(); // list of storage paths
1365                 $contNames = array(); // (cache key => resolved container name)
1366                 // Get all the paths/containers from the items...
1367                 foreach ( $items as $item ) {
1368                         if ( $item instanceof FileOp ) {
1369                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1370                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1371                         } elseif ( self::isStoragePath( $item ) ) {
1372                                 $paths[] = $item;
1373                         } elseif ( is_string( $item ) ) { // full container name
1374                                 $contNames[$this->containerCacheKey( $item )] = $item;
1375                         }
1376                 }
1377                 // Get all the corresponding cache keys for paths...
1378                 foreach ( $paths as $path ) {
1379                         list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path );
1380                         if ( $fullCont !== null ) { // valid path for this backend
1381                                 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1382                         }
1383                 }
1384
1385                 $contInfo = array(); // (resolved container name => cache value)
1386                 // Get all cache entries for these container cache keys...
1387                 $values = $this->memCache->getMulti( array_keys( $contNames ) );
1388                 foreach ( $values as $cacheKey => $val ) {
1389                         $contInfo[$contNames[$cacheKey]] = $val;
1390                 }
1391
1392                 // Populate the container process cache for the backend...
1393                 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1394
1395                 wfProfileOut( __METHOD__ . '-' . $this->name );
1396                 wfProfileOut( __METHOD__ );
1397         }
1398
1399         /**
1400          * Fill the backend-specific process cache given an array of
1401          * resolved container names and their corresponding cached info.
1402          * Only containers that actually exist should appear in the map.
1403          *
1404          * @param $containerInfo Array Map of resolved container names to cached info
1405          * @return void
1406          */
1407         protected function doPrimeContainerCache( array $containerInfo ) {}
1408
1409         /**
1410          * Get the cache key for a file path
1411          *
1412          * @param $path string Storage path
1413          * @return string
1414          */
1415         private function fileCacheKey( $path ) {
1416                 return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
1417         }
1418
1419         /**
1420          * Set the cached stat info for a file path
1421          *
1422          * @param $path string Storage path
1423          * @param $val mixed Information to cache
1424          * @return void
1425          */
1426         final protected function setFileCache( $path, $val ) {
1427                 $this->memCache->set( $this->fileCacheKey( $path ), $val, 7*86400 );
1428         }
1429
1430         /**
1431          * Delete the cached stat info for a file path
1432          *
1433          * @param $path string Storage path
1434          * @return void
1435          */
1436         final protected function deleteFileCache( $path ) {
1437                 if ( !$this->memCache->delete( $this->fileCacheKey( $path ) ) ) {
1438                         trigger_error( "Unable to delete stat cache for file $path." );
1439                 }
1440         }
1441
1442         /**
1443          * Do a batch lookup from cache for file stats for all paths
1444          * used in a list of storage paths or FileOp objects.
1445          *
1446          * @param $items Array List of storage paths or FileOps
1447          * @return void
1448          */
1449         final protected function primeFileCache( array $items ) {
1450                 wfProfileIn( __METHOD__ );
1451                 wfProfileIn( __METHOD__ . '-' . $this->name );
1452
1453                 $paths = array(); // list of storage paths
1454                 $pathNames = array(); // (cache key => storage path)
1455                 // Get all the paths/containers from the items...
1456                 foreach ( $items as $item ) {
1457                         if ( $item instanceof FileOp ) {
1458                                 $paths = array_merge( $paths, $item->storagePathsRead() );
1459                                 $paths = array_merge( $paths, $item->storagePathsChanged() );
1460                         } elseif ( self::isStoragePath( $item ) ) {
1461                                 $paths[] = $item;
1462                         }
1463                 }
1464                 // Get all the corresponding cache keys for paths...
1465                 foreach ( $paths as $path ) {
1466                         list( $cont, $rel, $s ) = $this->resolveStoragePath( $path );
1467                         if ( $rel !== null ) { // valid path for this backend
1468                                 $pathNames[$this->fileCacheKey( $path )] = $path;
1469                         }
1470                 }
1471                 // Get all cache entries for these container cache keys...
1472                 $values = $this->memCache->getMulti( array_keys( $pathNames ) );
1473                 foreach ( $values as $cacheKey => $val ) {
1474                         if ( is_array( $val ) ) {
1475                                 $this->trimCache(); // limit memory
1476                                 $this->cache[$pathNames[$cacheKey]]['stat'] = $val;
1477                         }
1478                 }
1479
1480                 wfProfileOut( __METHOD__ . '-' . $this->name );
1481                 wfProfileOut( __METHOD__ );
1482         }
1483 }
1484
1485 /**
1486  * FileBackendStore helper class for performing asynchronous file operations.
1487  *
1488  * For example, calling FileBackendStore::createInternal() with the "async"
1489  * param flag may result in a Status that contains this object as a value.
1490  * This class is largely backend-specific and is mostly just "magic" to be
1491  * passed to FileBackendStore::executeOpHandlesInternal().
1492  */
1493 abstract class FileBackendStoreOpHandle {
1494         /** @var Array */
1495         public $params = array(); // params to caller functions
1496         /** @var FileBackendStore */
1497         public $backend;
1498         /** @var Array */
1499         public $resourcesToClose = array();
1500
1501         public $call; // string; name that identifies the function called
1502
1503         /**
1504          * Close all open file handles
1505          *
1506          * @return void
1507          */
1508         public function closeResources() {
1509                 array_map( 'fclose', $this->resourcesToClose );
1510         }
1511 }
1512
1513 /**
1514  * FileBackendStore helper function to handle listings that span container shards.
1515  * Do not use this class from places outside of FileBackendStore.
1516  *
1517  * @ingroup FileBackend
1518  */
1519 abstract class FileBackendStoreShardListIterator implements Iterator {
1520         /** @var FileBackendStore */
1521         protected $backend;
1522         /** @var Array */
1523         protected $params;
1524         /** @var Array */
1525         protected $shardSuffixes;
1526         protected $container; // string; full container name
1527         protected $directory; // string; resolved relative path
1528
1529         /** @var Traversable */
1530         protected $iter;
1531         protected $curShard = 0; // integer
1532         protected $pos = 0; // integer
1533
1534         /** @var Array */
1535         protected $multiShardPaths = array(); // (rel path => 1)
1536
1537         /**
1538          * @param $backend FileBackendStore
1539          * @param $container string Full storage container name
1540          * @param $dir string Storage directory relative to container
1541          * @param $suffixes Array List of container shard suffixes
1542          * @param $params Array
1543          */
1544         public function __construct(
1545                 FileBackendStore $backend, $container, $dir, array $suffixes, array $params
1546         ) {
1547                 $this->backend = $backend;
1548                 $this->container = $container;
1549                 $this->directory = $dir;
1550                 $this->shardSuffixes = $suffixes;
1551                 $this->params = $params;
1552         }
1553
1554         /**
1555          * @see Iterator::current()
1556          * @return string|bool String or false
1557          */
1558         public function current() {
1559                 if ( is_array( $this->iter ) ) {
1560                         return current( $this->iter );
1561                 } else {
1562                         return $this->iter->current();
1563                 }
1564         }
1565
1566         /**
1567          * @see Iterator::key()
1568          * @return integer
1569          */
1570         public function key() {
1571                 return $this->pos;
1572         }
1573
1574         /**
1575          * @see Iterator::next()
1576          * @return void
1577          */
1578         public function next() {
1579                 ++$this->pos;
1580                 if ( is_array( $this->iter ) ) {
1581                         next( $this->iter );
1582                 } else {
1583                         $this->iter->next();
1584                 }
1585                 // Filter out items that we already listed
1586                 $this->filterViaNext();
1587                 // Find the next non-empty shard if no elements are left
1588                 $this->nextShardIteratorIfNotValid();
1589         }
1590
1591         /**
1592          * @see Iterator::rewind()
1593          * @return void
1594          */
1595         public function rewind() {
1596                 $this->pos = 0;
1597                 $this->curShard = 0;
1598                 $this->setIteratorFromCurrentShard();
1599                 // Filter out items that we already listed
1600                 $this->filterViaNext();
1601                 // Find the next non-empty shard if this one has no elements
1602                 $this->nextShardIteratorIfNotValid();
1603         }
1604
1605         /**
1606          * @see Iterator::valid()
1607          * @return bool
1608          */
1609         public function valid() {
1610                 if ( $this->iter === null ) {
1611                         return false; // some failure?
1612                 } elseif ( is_array( $this->iter ) ) {
1613                         return ( current( $this->iter ) !== false ); // no paths can have this value
1614                 } else {
1615                         return $this->iter->valid();
1616                 }
1617         }
1618
1619         /**
1620          * Filter out duplicate items by advancing to the next ones
1621          */
1622         protected function filterViaNext() {
1623                 while ( $this->iter->valid() ) {
1624                         $rel = $this->iter->current(); // path relative to given directory
1625                         $path = $this->params['dir'] . "/{$rel}"; // full storage path
1626                         if ( !$this->backend->isSingleShardPathInternal( $path ) ) {
1627                                 // Don't keep listing paths that are on multiple shards
1628                                 if ( isset( $this->multiShardPaths[$rel] ) ) {
1629                                         $this->iter->next(); // we already listed this path
1630                                 } else {
1631                                         $this->multiShardPaths[$rel] = 1;
1632                                         break;
1633                                 }
1634                         }
1635                 }
1636         }
1637
1638         /**
1639          * If the list iterator for this container shard is out of items,
1640          * then move on to the next container that has items.
1641          * If there are none, then it advances to the last container.
1642          */
1643         protected function nextShardIteratorIfNotValid() {
1644                 while ( !$this->valid() ) {
1645                         if ( ++$this->curShard >= count( $this->shardSuffixes ) ) {
1646                                 break; // no more container shards
1647                         }
1648                         $this->setIteratorFromCurrentShard();
1649                 }
1650         }
1651
1652         /**
1653          * Set the list iterator to that of the current container shard
1654          */
1655         protected function setIteratorFromCurrentShard() {
1656                 $suffix = $this->shardSuffixes[$this->curShard];
1657                 $this->iter = $this->listFromShard(
1658                         "{$this->container}{$suffix}", $this->directory, $this->params );
1659         }
1660
1661         /**
1662          * Get the list for a given container shard
1663          *
1664          * @param $container string Resolved container name
1665          * @param $dir string Resolved path relative to container
1666          * @param $params Array
1667          * @return Traversable|Array|null
1668          */
1669         abstract protected function listFromShard( $container, $dir, array $params );
1670 }
1671
1672 /**
1673  * Iterator for listing directories
1674  */
1675 class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator {
1676         protected function listFromShard( $container, $dir, array $params ) {
1677                 return $this->backend->getDirectoryListInternal( $container, $dir, $params );
1678         }
1679 }
1680
1681 /**
1682  * Iterator for listing regular files
1683  */
1684 class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator {
1685         protected function listFromShard( $container, $dir, array $params ) {
1686                 return $this->backend->getFileListInternal( $container, $dir, $params );
1687         }
1688 }