includes/filerepo/backend/FileBackend.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Aaron Schulz
   6  */
   7
   8 /**
   9  * Base class for all file backend classes (including multi-write backends).
  10  * This class defines the methods as abstract that subclasses must implement.
  11  * Outside callers can assume that all backends will have these functions.
  12  *
  13  * All "storage paths" are of the format "mwstore://backend/container/path".
  14  * The paths use UNIX file system (FS) notation, though any particular backend may
  15  * not actually be using a local filesystem. Therefore, the paths are only virtual.
  16  *
  17  * Backend contents are stored under wiki-specific container names by default.
  18  * For legacy reasons, this has no effect for the FS backend class, and per-wiki
  19  * segregation must be done by setting the container paths appropriately.
  20  *
  21  * FS-based backends are somewhat more restrictive due to the existence of real
  22  * directory files; a regular file cannot have the same name as a directory. Other
  23  * backends with virtual directories may not have this limitation. Callers should
  24  * store files in such a way that no files and directories are under the same path.
  25  *
  26  * Methods should avoid throwing exceptions at all costs.
  27  * As a corollary, external dependencies should be kept to a minimum.
  28  *
  29  * @ingroup FileBackend
  30  * @since 1.19
  31  */
  32 abstract class FileBackendBase {
  33         protected $name; // unique backend name
  34         protected $wikiId; // unique wiki name
  35         protected $readOnly; // string
  36         /** @var LockManager */
  37         protected $lockManager;
  38
  39         /**
  40          * Create a new backend instance from configuration.
  41          * This should only be called from within FileBackendGroup.
  42          *
  43          * $config includes:
  44          *     'name'        : The unique name of this backend.
  45          *     'wikiId'      : Prefix to container names that is unique to this wiki.
  46          *                     This should consist of alphanumberic, '-', and '_' chars.
  47          *     'lockManager' : Registered name of a file lock manager to use.
  48          *     'readOnly'    : Write operations are disallowed if this is a non-empty string.
  49          *                     It should be an explanation for the backend being read-only.
  50          *
  51          * @param $config Array
  52          */
  53         public function __construct( array $config ) {
  54                 $defaultConfig = array(
  55                         'lockManager' => null,
  56                         'wikiId' => wfWikiID(),
  57                 );
  58                 $config = array_merge(  $defaultConfig, $config );
  59                 $this->name = $config['name'];
  60                 $this->wikiId = $this->resolveWikiId( $config['wikiId'] );
  61                 $this->lockManager = LockManagerGroup::singleton()->get( $config['lockManager'] );
  62                 $this->readOnly = isset( $config['readOnly'] )
  63                         ? (string)$config['readOnly']
  64                         : '';
  65         }
  66
  67         /**
  68          * Normalize a wiki ID by replacing characters that are
  69          * not supported by the backend as part of container names.
  70          *
  71          * @param $wikiId string
  72          * @return string
  73          */
  74         protected function resolveWikiId( $wikiId ) {
  75                 return $wikiId;
  76         }
  77
  78         /**
  79          * Get the unique backend name.
  80          * We may have multiple different backends of the same type.
  81          * For example, we can have two Swift backends using different proxies.
  82          *
  83          * @return string
  84          */
  85         final public function getName() {
  86                 return $this->name;
  87         }
  88
  89         /**
  90          * This is the main entry point into the backend for write operations.
  91          * Callers supply an ordered list of operations to perform as a transaction.
  92          * If any serious errors occur, all attempted operations will be rolled back.
  93          *
  94          * $ops is an array of arrays. The outer array holds a list of operations.
  95          * Each inner array is a set of key value pairs that specify an operation.
  96          *
  97          * Supported operations and their parameters:
  98          * a) Create a new file in storage with the contents of a string
  99          *     array(
 100          *         'op'                  => 'create',
 101          *         'dst'                 => <storage path>,
 102          *         'content'             => <string of new file contents>,
 103          *         'overwrite'           => <boolean>,
 104          *         'overwriteSame'       => <boolean>
 105          *     )
 106          * b) Copy a file system file into storage
 107          *     array(
 108          *         'op'                  => 'store',
 109          *         'src'                 => <file system path>,
 110          *         'dst'                 => <storage path>,
 111          *         'overwrite'           => <boolean>,
 112          *         'overwriteSame'       => <boolean>
 113          *     )
 114          * c) Copy a file within storage
 115          *     array(
 116          *         'op'                  => 'copy',
 117          *         'src'                 => <storage path>,
 118          *         'dst'                 => <storage path>,
 119          *         'overwrite'           => <boolean>,
 120          *         'overwriteSame'       => <boolean>
 121          *     )
 122          * d) Move a file within storage
 123          *     array(
 124          *         'op'                  => 'move',
 125          *         'src'                 => <storage path>,
 126          *         'dst'                 => <storage path>,
 127          *         'overwrite'           => <boolean>,
 128          *         'overwriteSame'       => <boolean>
 129          *     )
 130          * e) Delete a file within storage
 131          *     array(
 132          *         'op'                  => 'delete',
 133          *         'src'                 => <storage path>,
 134          *         'ignoreMissingSource' => <boolean>
 135          *     )
 136          * f) Do nothing (no-op)
 137          *     array(
 138          *         'op'                  => 'null',
 139          *     )
 140          *
 141          * Boolean flags for operations (operation-specific):
 142          * 'ignoreMissingSource' : The operation will simply succeed and do
 143          *                         nothing if the source file does not exist.
 144          * 'overwrite'           : Any destination file will be overwritten.
 145          * 'overwriteSame'       : An error will not be given if a file already
 146          *                         exists at the destination that has the same
 147          *                         contents as the new contents to be written there.
 148          *
 149          * $opts is an associative of boolean flags, including:
 150          * 'force'               : Errors that would normally cause a rollback do not.
 151          *                         The remaining operations are still attempted if any fail.
 152          * 'nonLocking'          : No locks are acquired for the operations.
 153          *                         This can increase performance for non-critical writes.
 154          *                         This has no effect unless the 'force' flag is set.
 155          * 'allowStale'          : Don't require the latest available data.
 156          *                         This can increase performance for non-critical writes.
 157          *                         This has no effect unless the 'force' flag is set.
 158          *
 159          * Remarks:
 160          * File system paths given to operations should refer to files that are
 161          * either locked or otherwise safe from modification from other processes.
 162          * Normally these files will be new temp files, which should be adequate.
 163          *
 164          * Return value:
 165          * This returns a Status, which contains all warnings and fatals that occured
 166          * during the operation. The 'failCount', 'successCount', and 'success' members
 167          * will reflect each operation attempted. The status will be "OK" unless any
 168          * of the operations failed and the 'force' parameter was not set.
 169          *
 170          * @param $ops Array List of operations to execute in order
 171          * @param $opts Array Batch operation options
 172          * @return Status
 173          */
 174         final public function doOperations( array $ops, array $opts = array() ) {
 175                 if ( $this->readOnly != '' ) {
 176                         return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
 177                 }
 178                 if ( empty( $opts['force'] ) ) { // sanity
 179                         unset( $opts['nonLocking'] );
 180                         unset( $opts['allowStale'] );
 181                 }
 182                 return $this->doOperationsInternal( $ops, $opts );
 183         }
 184
 185         /**
 186          * @see FileBackendBase::doOperations()
 187          */
 188         abstract protected function doOperationsInternal( array $ops, array $opts );
 189
 190         /**
 191          * Same as doOperations() except it takes a single operation.
 192          * If you are doing a batch of operations that should either
 193          * all succeed or all fail, then use that function instead.
 194          *
 195          * @see FileBackendBase::doOperations()
 196          *
 197          * @param $op Array Operation
 198          * @param $opts Array Operation options
 199          * @return Status
 200          */
 201         final public function doOperation( array $op, array $opts = array() ) {
 202                 return $this->doOperations( array( $op ), $opts );
 203         }
 204
 205         /**
 206          * Performs a single create operation.
 207          * This sets $params['op'] to 'create' and passes it to doOperation().
 208          *
 209          * @see FileBackendBase::doOperation()
 210          *
 211          * @param $params Array Operation parameters
 212          * @param $opts Array Operation options
 213          * @return Status
 214          */
 215         final public function create( array $params, array $opts = array() ) {
 216                 $params['op'] = 'create';
 217                 return $this->doOperation( $params, $opts );
 218         }
 219
 220         /**
 221          * Performs a single store operation.
 222          * This sets $params['op'] to 'store' and passes it to doOperation().
 223          *
 224          * @see FileBackendBase::doOperation()
 225          *
 226          * @param $params Array Operation parameters
 227          * @param $opts Array Operation options
 228          * @return Status
 229          */
 230         final public function store( array $params, array $opts = array() ) {
 231                 $params['op'] = 'store';
 232                 return $this->doOperation( $params, $opts );
 233         }
 234
 235         /**
 236          * Performs a single copy operation.
 237          * This sets $params['op'] to 'copy' and passes it to doOperation().
 238          *
 239          * @see FileBackendBase::doOperation()
 240          *
 241          * @param $params Array Operation parameters
 242          * @param $opts Array Operation options
 243          * @return Status
 244          */
 245         final public function copy( array $params, array $opts = array() ) {
 246                 $params['op'] = 'copy';
 247                 return $this->doOperation( $params, $opts );
 248         }
 249
 250         /**
 251          * Performs a single move operation.
 252          * This sets $params['op'] to 'move' and passes it to doOperation().
 253          *
 254          * @see FileBackendBase::doOperation()
 255          *
 256          * @param $params Array Operation parameters
 257          * @param $opts Array Operation options
 258          * @return Status
 259          */
 260         final public function move( array $params, array $opts = array() ) {
 261                 $params['op'] = 'move';
 262                 return $this->doOperation( $params, $opts );
 263         }
 264
 265         /**
 266          * Performs a single delete operation.
 267          * This sets $params['op'] to 'delete' and passes it to doOperation().
 268          *
 269          * @see FileBackendBase::doOperation()
 270          *
 271          * @param $params Array Operation parameters
 272          * @param $opts Array Operation options
 273          * @return Status
 274          */
 275         final public function delete( array $params, array $opts = array() ) {
 276                 $params['op'] = 'delete';
 277                 return $this->doOperation( $params, $opts );
 278         }
 279
 280         /**
 281          * Concatenate a list of storage files into a single file on the file system
 282          * $params include:
 283          *     srcs          : ordered source storage paths (e.g. chunk1, chunk2, ...)
 284          *     dst           : file system path to 0-byte temp file
 285          *
 286          * @param $params Array Operation parameters
 287          * @return Status
 288          */
 289         abstract public function concatenate( array $params );
 290
 291         /**
 292          * Prepare a storage directory for usage.
 293          * This will create any required containers and parent directories.
 294          * Backends using key/value stores only need to create the container.
 295          *
 296          * $params include:
 297          *     dir : storage directory
 298          *
 299          * @param $params Array
 300          * @return Status
 301          */
 302         final public function prepare( array $params ) {
 303                 if ( $this->readOnly != '' ) {
 304                         return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
 305                 }
 306                 return $this->doPrepare( $params );
 307         }
 308
 309         /**
 310          * @see FileBackendBase::prepare()
 311          */
 312         abstract protected function doPrepare( array $params );
 313
 314         /**
 315          * Take measures to block web access to a storage directory and
 316          * the container it belongs to. FS backends might add .htaccess
 317          * files whereas key/value store backends might restrict container
 318          * access to the auth user that represents end-users in web request.
 319          * This is not guaranteed to actually do anything.
 320          *
 321          * $params include:
 322          *     dir       : storage directory
 323          *     noAccess  : try to deny file access
 324          *     noListing : try to deny file listing
 325          *
 326          * @param $params Array
 327          * @return Status
 328          */
 329         final public function secure( array $params ) {
 330                 if ( $this->readOnly != '' ) {
 331                         return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
 332                 }
 333                 $status = $this->doPrepare( $params ); // dir must exist to restrict it
 334                 if ( $status->isOK() ) {
 335                         $status->merge( $this->doSecure( $params ) );
 336                 }
 337                 return $status;
 338         }
 339
 340         /**
 341          * @see FileBackendBase::secure()
 342          */
 343         abstract protected function doSecure( array $params );
 344
 345         /**
 346          * Delete a storage directory if it is empty.
 347          * Backends using key/value stores may do nothing unless the directory
 348          * is that of an empty container, in which case it should be deleted.
 349          *
 350          * $params include:
 351          *     dir : storage directory
 352          *
 353          * @param $params Array
 354          * @return Status
 355          */
 356         final public function clean( array $params ) {
 357                 if ( $this->readOnly != '' ) {
 358                         return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
 359                 }
 360                 return $this->doClean( $params );
 361         }
 362
 363         /**
 364          * @see FileBackendBase::clean()
 365          */
 366         abstract protected function doClean( array $params );
 367
 368         /**
 369          * Check if a file exists at a storage path in the backend.
 370          * This returns false if only a directory exists at the path.
 371          *
 372          * $params include:
 373          *     src    : source storage path
 374          *     latest : use the latest available data
 375          *
 376          * @param $params Array
 377          * @return bool|null Returns null on failure
 378          */
 379         abstract public function fileExists( array $params );
 380
 381         /**
 382          * Get the last-modified timestamp of the file at a storage path.
 383          *
 384          * $params include:
 385          *     src    : source storage path
 386          *     latest : use the latest available data
 387          *
 388          * @param $params Array
 389          * @return string|false TS_MW timestamp or false on failure
 390          */
 391         abstract public function getFileTimestamp( array $params );
 392
 393         /**
 394          * Get the contents of a file at a storage path in the backend.
 395          * This should be avoided for potentially large files.
 396          *
 397          * $params include:
 398          *     src    : source storage path
 399          *     latest : use the latest available data
 400          *
 401          * @param $params Array
 402          * @return string|false Returns false on failure
 403          */
 404         abstract public function getFileContents( array $params );
 405
 406         /**
 407          * Get the size (bytes) of a file at a storage path in the backend.
 408          *
 409          * $params include:
 410          *     src    : source storage path
 411          *     latest : use the latest available data
 412          *
 413          * @param $params Array
 414          * @return integer|false Returns false on failure
 415          */
 416         abstract public function getFileSize( array $params );
 417
 418         /**
 419          * Get quick information about a file at a storage path in the backend.
 420          * If the file does not exist, then this returns false.
 421          * Otherwise, the result is an associative array that includes:
 422          *     mtime  : the last-modified timestamp (TS_MW)
 423          *     size   : the file size (bytes)
 424          * Additional values may be included for internal use only.
 425          *
 426          * $params include:
 427          *     src    : source storage path
 428          *     latest : use the latest available data
 429          *
 430          * @param $params Array
 431          * @return Array|false|null Returns null on failure
 432          */
 433         abstract public function getFileStat( array $params );
 434
 435         /**
 436          * Get a SHA-1 hash of the file at a storage path in the backend.
 437          *
 438          * $params include:
 439          *     src    : source storage path
 440          *     latest : use the latest available data
 441          *
 442          * @param $params Array
 443          * @return string|false Hash string or false on failure
 444          */
 445         abstract public function getFileSha1Base36( array $params );
 446
 447         /**
 448          * Get the properties of the file at a storage path in the backend.
 449          * Returns FSFile::placeholderProps() on failure.
 450          *
 451          * $params include:
 452          *     src    : source storage path
 453          *     latest : use the latest available data
 454          *
 455          * @param $params Array
 456          * @return Array
 457          */
 458         abstract public function getFileProps( array $params );
 459
 460         /**
 461          * Stream the file at a storage path in the backend.
 462          * If the file does not exists, a 404 error will be given.
 463          * Appropriate HTTP headers (Status, Content-Type, Content-Length)
 464          * must be sent if streaming began, while none should be sent otherwise.
 465          * Implementations should flush the output buffer before sending data.
 466          *
 467          * $params include:
 468          *     src     : source storage path
 469          *     headers : additional HTTP headers to send on success
 470          *     latest  : use the latest available data
 471          *
 472          * @param $params Array
 473          * @return Status
 474          */
 475         abstract public function streamFile( array $params );
 476
 477         /**
 478          * Returns a file system file, identical to the file at a storage path.
 479          * The file returned is either:
 480          * a) A local copy of the file at a storage path in the backend.
 481          *    The temporary copy will have the same extension as the source.
 482          * b) An original of the file at a storage path in the backend.
 483          * Temporary files may be purged when the file object falls out of scope.
 484          *
 485          * Write operations should *never* be done on this file as some backends
 486          * may do internal tracking or may be instances of FileBackendMultiWrite.
 487          * In that later case, there are copies of the file that must stay in sync.
 488          *
 489          * $params include:
 490          *     src    : source storage path
 491          *     latest : use the latest available data
 492          *
 493          * @param $params Array
 494          * @return FSFile|null Returns null on failure
 495          */
 496         abstract public function getLocalReference( array $params );
 497
 498         /**
 499          * Get a local copy on disk of the file at a storage path in the backend.
 500          * The temporary copy will have the same file extension as the source.
 501          * Temporary files may be purged when the file object falls out of scope.
 502          *
 503          * $params include:
 504          *     src    : source storage path
 505          *     latest : use the latest available data
 506          *
 507          * @param $params Array
 508          * @return TempFSFile|null Returns null on failure
 509          */
 510         abstract public function getLocalCopy( array $params );
 511
 512         /**
 513          * Get an iterator to list out all stored files under a storage directory.
 514          * If the directory is of the form "mwstore://container", then all items in
 515          * the container should be listed. If of the form "mwstore://container/dir",
 516          * then all items under that container directory should be listed.
 517          * Results should be storage paths relative to the given directory.
 518          *
 519          * $params include:
 520          *     dir : storage path directory
 521          *
 522          * @return Traversable|Array|null Returns null on failure
 523          */
 524         abstract public function getFileList( array $params );
 525
 526         /**
 527          * Invalidate any in-process file existence and property cache.
 528          * If $paths is given, then only the cache for those files will be cleared.
 529          *
 530          * @param $paths Array Storage paths (optional)
 531          * @return void
 532          */
 533         abstract public function clearCache( array $paths = null );
 534
 535         /**
 536          * Lock the files at the given storage paths in the backend.
 537          * This will either lock all the files or none (on failure).
 538          *
 539          * Callers should consider using getScopedFileLocks() instead.
 540          *
 541          * @param $paths Array Storage paths
 542          * @param $type integer LockManager::LOCK_* constant
 543          * @return Status
 544          */
 545         final public function lockFiles( array $paths, $type ) {
 546                 return $this->lockManager->lock( $paths, $type );
 547         }
 548
 549         /**
 550          * Unlock the files at the given storage paths in the backend.
 551          *
 552          * @param $paths Array Storage paths
 553          * @param $type integer LockManager::LOCK_* constant
 554          * @return Status
 555          */
 556         final public function unlockFiles( array $paths, $type ) {
 557                 return $this->lockManager->unlock( $paths, $type );
 558         }
 559
 560         /**
 561          * Lock the files at the given storage paths in the backend.
 562          * This will either lock all the files or none (on failure).
 563          * On failure, the status object will be updated with errors.
 564          *
 565          * Once the return value goes out scope, the locks will be released and
 566          * the status updated. Unlock fatals will not change the status "OK" value.
 567          *
 568          * @param $paths Array Storage paths
 569          * @param $type integer LockManager::LOCK_* constant
 570          * @param $status Status Status to update on lock/unlock
 571          * @return ScopedLock|null Returns null on failure
 572          */
 573         final public function getScopedFileLocks( array $paths, $type, Status $status ) {
 574                 return ScopedLock::factory( $this->lockManager, $paths, $type, $status );
 575         }
 576 }
 577
 578 /**
 579  * Base class for all single-write backends.
 580  * This class defines the methods as abstract that subclasses must implement.
 581  * Callers outside of FileBackend and its helper classes, such as FileOp,
 582  * should only call functions that are present in FileBackendBase.
 583  *
 584  * The FileBackendBase operations are implemented using primitive functions
 585  * such as storeInternal(), copyInternal(), deleteInternal() and the like.
 586  * This class is also responsible for path resolution and sanitization.
 587  *
 588  * @ingroup FileBackend
 589  * @since 1.19
 590  */
 591 abstract class FileBackend extends FileBackendBase {
 592         /** @var Array */
 593         protected $cache = array(); // (storage path => key => value)
 594         protected $maxCacheSize = 75; // integer; max paths with entries
 595         /** @var Array */
 596         protected $shardViaHashLevels = array(); // (container name => integer)
 597
 598         protected $maxFileSize = 1000000000; // integer bytes (1GB)
 599
 600         /**
 601          * Get the maximum allowable file size given backend
 602          * medium restrictions and basic performance constraints.
 603          * Do not call this function from places outside FileBackend and FileOp.
 604          *
 605          * @return integer Bytes
 606          */
 607         final public function maxFileSizeInternal() {
 608                 return $this->maxFileSize;
 609         }
 610
 611         /**
 612          * Check if a file can be created at a given storage path.
 613          * FS backends should check if the parent directory exists and the file is writable.
 614          * Backends using key/value stores should check if the container exists.
 615          *
 616          * @param $storagePath string
 617          * @return bool
 618          */
 619         abstract public function isPathUsableInternal( $storagePath );
 620
 621         /**
 622          * Create a file in the backend with the given contents.
 623          * Do not call this function from places outside FileBackend and FileOp.
 624          *
 625          * $params include:
 626          *     content       : the raw file contents
 627          *     dst           : destination storage path
 628          *     overwrite     : overwrite any file that exists at the destination
 629          *
 630          * @param $params Array
 631          * @return Status
 632          */
 633         final public function createInternal( array $params ) {
 634                 wfProfileIn( __METHOD__ );
 635                 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
 636                         $status = Status::newFatal( 'backend-fail-create', $params['dst'] );
 637                 } else {
 638                         $status = $this->doCreateInternal( $params );
 639                         $this->clearCache( array( $params['dst'] ) );
 640                 }
 641                 wfProfileOut( __METHOD__ );
 642                 return $status;
 643         }
 644
 645         /**
 646          * @see FileBackend::createInternal()
 647          */
 648         abstract protected function doCreateInternal( array $params );
 649
 650         /**
 651          * Store a file into the backend from a file on disk.
 652          * Do not call this function from places outside FileBackend and FileOp.
 653          *
 654          * $params include:
 655          *     src           : source path on disk
 656          *     dst           : destination storage path
 657          *     overwrite     : overwrite any file that exists at the destination
 658          *
 659          * @param $params Array
 660          * @return Status
 661          */
 662         final public function storeInternal( array $params ) {
 663                 wfProfileIn( __METHOD__ );
 664                 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
 665                         $status = Status::newFatal( 'backend-fail-store', $params['dst'] );
 666                 } else {
 667                         $status = $this->doStoreInternal( $params );
 668                         $this->clearCache( array( $params['dst'] ) );
 669                 }
 670                 wfProfileOut( __METHOD__ );
 671                 return $status;
 672         }
 673
 674         /**
 675          * @see FileBackend::storeInternal()
 676          */
 677         abstract protected function doStoreInternal( array $params );
 678
 679         /**
 680          * Copy a file from one storage path to another in the backend.
 681          * Do not call this function from places outside FileBackend and FileOp.
 682          *
 683          * $params include:
 684          *     src           : source storage path
 685          *     dst           : destination storage path
 686          *     overwrite     : overwrite any file that exists at the destination
 687          *
 688          * @param $params Array
 689          * @return Status
 690          */
 691         final public function copyInternal( array $params ) {
 692                 wfProfileIn( __METHOD__ );
 693                 $status = $this->doCopyInternal( $params );
 694                 $this->clearCache( array( $params['dst'] ) );
 695                 wfProfileOut( __METHOD__ );
 696                 return $status;
 697         }
 698
 699         /**
 700          * @see FileBackend::copyInternal()
 701          */
 702         abstract protected function doCopyInternal( array $params );
 703
 704         /**
 705          * Delete a file at the storage path.
 706          * Do not call this function from places outside FileBackend and FileOp.
 707          *
 708          * $params include:
 709          *     src                 : source storage path
 710          *     ignoreMissingSource : do nothing if the source file does not exist
 711          *
 712          * @param $params Array
 713          * @return Status
 714          */
 715         final public function deleteInternal( array $params ) {
 716                 wfProfileIn( __METHOD__ );
 717                 $status = $this->doDeleteInternal( $params );
 718                 $this->clearCache( array( $params['src'] ) );
 719                 wfProfileOut( __METHOD__ );
 720                 return $status;
 721         }
 722
 723         /**
 724          * @see FileBackend::deleteInternal()
 725          */
 726         abstract protected function doDeleteInternal( array $params );
 727
 728         /**
 729          * Move a file from one storage path to another in the backend.
 730          * Do not call this function from places outside FileBackend and FileOp.
 731          *
 732          * $params include:
 733          *     src           : source storage path
 734          *     dst           : destination storage path
 735          *     overwrite     : overwrite any file that exists at the destination
 736          *
 737          * @param $params Array
 738          * @return Status
 739          */
 740         final public function moveInternal( array $params ) {
 741                 wfProfileIn( __METHOD__ );
 742                 $status = $this->doMoveInternal( $params );
 743                 $this->clearCache( array( $params['src'], $params['dst'] ) );
 744                 wfProfileOut( __METHOD__ );
 745                 return $status;
 746         }
 747
 748         /**
 749          * @see FileBackend::moveInternal()
 750          */
 751         protected function doMoveInternal( array $params ) {
 752                 // Copy source to dest
 753                 $status = $this->copyInternal( $params );
 754                 if ( $status->isOK() ) {
 755                         // Delete source (only fails due to races or medium going down)
 756                         $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
 757                         $status->setResult( true, $status->value ); // ignore delete() errors
 758                 }
 759                 return $status;
 760         }
 761
 762         /**
 763          * @see FileBackendBase::concatenate()
 764          */
 765         final public function concatenate( array $params ) {
 766                 wfProfileIn( __METHOD__ );
 767                 $status = Status::newGood();
 768
 769                 // Try to lock the source files for the scope of this function
 770                 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
 771                 if ( $status->isOK() ) {
 772                         // Actually do the concatenation
 773                         $status->merge( $this->doConcatenate( $params ) );
 774                 }
 775
 776                 wfProfileOut( __METHOD__ );
 777                 return $status;
 778         }
 779
 780         /**
 781          * @see FileBackend::concatenate()
 782          */
 783         protected function doConcatenate( array $params ) {
 784                 $status = Status::newGood();
 785                 $tmpPath = $params['dst']; // convenience
 786
 787                 // Check that the specified temp file is valid...
 788                 wfSuppressWarnings();
 789                 $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
 790                 wfRestoreWarnings();
 791                 if ( !$ok ) { // not present or not empty
 792                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 793                         return $status;
 794                 }
 795
 796                 // Build up the temp file using the source chunks (in order)...
 797                 $tmpHandle = fopen( $tmpPath, 'a' );
 798                 if ( $tmpHandle === false ) {
 799                         $status->fatal( 'backend-fail-opentemp', $tmpPath );
 800                         return $status;
 801                 }
 802                 foreach ( $params['srcs'] as $virtualSource ) {
 803                         // Get a local FS version of the chunk
 804                         $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
 805                         if ( !$tmpFile ) {
 806                                 $status->fatal( 'backend-fail-read', $virtualSource );
 807                                 return $status;
 808                         }
 809                         // Get a handle to the local FS version
 810                         $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
 811                         if ( $sourceHandle === false ) {
 812                                 fclose( $tmpHandle );
 813                                 $status->fatal( 'backend-fail-read', $virtualSource );
 814                                 return $status;
 815                         }
 816                         // Append chunk to file (pass chunk size to avoid magic quotes)
 817                         if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
 818                                 fclose( $sourceHandle );
 819                                 fclose( $tmpHandle );
 820                                 $status->fatal( 'backend-fail-writetemp', $tmpPath );
 821                                 return $status;
 822                         }
 823                         fclose( $sourceHandle );
 824                 }
 825                 if ( !fclose( $tmpHandle ) ) {
 826                         $status->fatal( 'backend-fail-closetemp', $tmpPath );
 827                         return $status;
 828                 }
 829
 830                 clearstatcache(); // temp file changed
 831
 832                 return $status;
 833         }
 834
 835         /**
 836          * @see FileBackendBase::doPrepare()
 837          */
 838         final protected function doPrepare( array $params ) {
 839                 wfProfileIn( __METHOD__ );
 840
 841                 $status = Status::newGood();
 842                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 843                 if ( $dir === null ) {
 844                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 845                         wfProfileOut( __METHOD__ );
 846                         return $status; // invalid storage path
 847                 }
 848
 849                 if ( $shard !== null ) { // confined to a single container/shard
 850                         $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
 851                 } else { // directory is on several shards
 852                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 853                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 854                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 855                                 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 856                         }
 857                 }
 858
 859                 wfProfileOut( __METHOD__ );
 860                 return $status;
 861         }
 862
 863         /**
 864          * @see FileBackend::doPrepare()
 865          */
 866         protected function doPrepareInternal( $container, $dir, array $params ) {
 867                 return Status::newGood();
 868         }
 869
 870         /**
 871          * @see FileBackendBase::doSecure()
 872          */
 873         final protected function doSecure( array $params ) {
 874                 wfProfileIn( __METHOD__ );
 875                 $status = Status::newGood();
 876
 877                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 878                 if ( $dir === null ) {
 879                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 880                         wfProfileOut( __METHOD__ );
 881                         return $status; // invalid storage path
 882                 }
 883
 884                 if ( $shard !== null ) { // confined to a single container/shard
 885                         $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
 886                 } else { // directory is on several shards
 887                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 888                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 889                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 890                                 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 891                         }
 892                 }
 893
 894                 wfProfileOut( __METHOD__ );
 895                 return $status;
 896         }
 897
 898         /**
 899          * @see FileBackend::doSecure()
 900          */
 901         protected function doSecureInternal( $container, $dir, array $params ) {
 902                 return Status::newGood();
 903         }
 904
 905         /**
 906          * @see FileBackendBase::doClean()
 907          */
 908         final protected function doClean( array $params ) {
 909                 wfProfileIn( __METHOD__ );
 910                 $status = Status::newGood();
 911
 912                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
 913                 if ( $dir === null ) {
 914                         $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 915                         wfProfileOut( __METHOD__ );
 916                         return $status; // invalid storage path
 917                 }
 918
 919                 // Attempt to lock this directory...
 920                 $filesLockEx = array( $params['dir'] );
 921                 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 922                 if ( !$status->isOK() ) {
 923                         wfProfileOut( __METHOD__ );
 924                         return $status; // abort
 925                 }
 926
 927                 if ( $shard !== null ) { // confined to a single container/shard
 928                         $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
 929                 } else { // directory is on several shards
 930                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
 931                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
 932                         foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
 933                                 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
 934                         }
 935                 }
 936
 937                 wfProfileOut( __METHOD__ );
 938                 return $status;
 939         }
 940
 941         /**
 942          * @see FileBackend::doClean()
 943          */
 944         protected function doCleanInternal( $container, $dir, array $params ) {
 945                 return Status::newGood();
 946         }
 947
 948         /**
 949          * @see FileBackendBase::fileExists()
 950          */
 951         final public function fileExists( array $params ) {
 952                 wfProfileIn( __METHOD__ );
 953                 $stat = $this->getFileStat( $params );
 954                 wfProfileOut( __METHOD__ );
 955                 return ( $stat === null ) ? null : (bool)$stat; // null => failure
 956         }
 957
 958         /**
 959          * @see FileBackendBase::getFileTimestamp()
 960          */
 961         final public function getFileTimestamp( array $params ) {
 962                 wfProfileIn( __METHOD__ );
 963                 $stat = $this->getFileStat( $params );
 964                 wfProfileOut( __METHOD__ );
 965                 return $stat ? $stat['mtime'] : false;
 966         }
 967
 968         /**
 969          * @see FileBackendBase::getFileSize()
 970          */
 971         final public function getFileSize( array $params ) {
 972                 wfProfileIn( __METHOD__ );
 973                 $stat = $this->getFileStat( $params );
 974                 wfProfileOut( __METHOD__ );
 975                 return $stat ? $stat['size'] : false;
 976         }
 977
 978         /**
 979          * @see FileBackendBase::getFileStat()
 980          */
 981         final public function getFileStat( array $params ) {
 982                 wfProfileIn( __METHOD__ );
 983                 $path = $params['src'];
 984                 $latest = !empty( $params['latest'] );
 985                 if ( isset( $this->cache[$path]['stat'] ) ) {
 986                         // If we want the latest data, check that this cached
 987                         // value was in fact fetched with the latest available data.
 988                         if ( !$latest || $this->cache[$path]['stat']['latest'] ) {
 989                                 wfProfileOut( __METHOD__ );
 990                                 return $this->cache[$path]['stat'];
 991                         }
 992                 }
 993                 $stat = $this->doGetFileStat( $params );
 994                 if ( is_array( $stat ) ) { // don't cache negatives
 995                         $this->trimCache(); // limit memory
 996                         $this->cache[$path]['stat'] = $stat;
 997                         $this->cache[$path]['stat']['latest'] = $latest;
 998                 }
 999                 wfProfileOut( __METHOD__ );
1000                 return $stat;
1001         }
1002
1003         /**
1004          * @see FileBackend::getFileStat()
1005          */
1006         abstract protected function doGetFileStat( array $params );
1007
1008         /**
1009          * @see FileBackendBase::getFileContents()
1010          */
1011         public function getFileContents( array $params ) {
1012                 wfProfileIn( __METHOD__ );
1013                 $tmpFile = $this->getLocalReference( $params );
1014                 if ( !$tmpFile ) {
1015                         wfProfileOut( __METHOD__ );
1016                         return false;
1017                 }
1018                 wfSuppressWarnings();
1019                 $data = file_get_contents( $tmpFile->getPath() );
1020                 wfRestoreWarnings();
1021                 wfProfileOut( __METHOD__ );
1022                 return $data;
1023         }
1024
1025         /**
1026          * @see FileBackendBase::getFileSha1Base36()
1027          */
1028         final public function getFileSha1Base36( array $params ) {
1029                 wfProfileIn( __METHOD__ );
1030                 $path = $params['src'];
1031                 if ( isset( $this->cache[$path]['sha1'] ) ) {
1032                         wfProfileOut( __METHOD__ );
1033                         return $this->cache[$path]['sha1'];
1034                 }
1035                 $hash = $this->doGetFileSha1Base36( $params );
1036                 if ( $hash ) { // don't cache negatives
1037                         $this->trimCache(); // limit memory
1038                         $this->cache[$path]['sha1'] = $hash;
1039                 }
1040                 wfProfileOut( __METHOD__ );
1041                 return $hash;
1042         }
1043
1044         /**
1045          * @see FileBackend::getFileSha1Base36()
1046          */
1047         protected function doGetFileSha1Base36( array $params ) {
1048                 $fsFile = $this->getLocalReference( $params );
1049                 if ( !$fsFile ) {
1050                         return false;
1051                 } else {
1052                         return $fsFile->getSha1Base36();
1053                 }
1054         }
1055
1056         /**
1057          * @see FileBackendBase::getFileProps()
1058          */
1059         final public function getFileProps( array $params ) {
1060                 wfProfileIn( __METHOD__ );
1061                 $fsFile = $this->getLocalReference( $params );
1062                 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
1063                 wfProfileOut( __METHOD__ );
1064                 return $props;
1065         }
1066
1067         /**
1068          * @see FileBackendBase::getLocalReference()
1069          */
1070         public function getLocalReference( array $params ) {
1071                 wfProfileIn( __METHOD__ );
1072                 $path = $params['src'];
1073                 if ( isset( $this->cache[$path]['localRef'] ) ) {
1074                         wfProfileOut( __METHOD__ );
1075                         return $this->cache[$path]['localRef'];
1076                 }
1077                 $tmpFile = $this->getLocalCopy( $params );
1078                 if ( $tmpFile ) { // don't cache negatives
1079                         $this->trimCache(); // limit memory
1080                         $this->cache[$path]['localRef'] = $tmpFile;
1081                 }
1082                 wfProfileOut( __METHOD__ );
1083                 return $tmpFile;
1084         }
1085
1086         /**
1087          * @see FileBackendBase::streamFile()
1088          */
1089         final public function streamFile( array $params ) {
1090                 wfProfileIn( __METHOD__ );
1091                 $status = Status::newGood();
1092
1093                 $info = $this->getFileStat( $params );
1094                 if ( !$info ) { // let StreamFile handle the 404
1095                         $status->fatal( 'backend-fail-notexists', $params['src'] );
1096                 }
1097
1098                 // Set output buffer and HTTP headers for stream
1099                 $extraHeaders = $params['headers'] ? $params['headers'] : array();
1100                 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
1101                 if ( $res == StreamFile::NOT_MODIFIED ) {
1102                         // do nothing; client cache is up to date
1103                 } elseif ( $res == StreamFile::READY_STREAM ) {
1104                         $status = $this->doStreamFile( $params );
1105                 } else {
1106                         $status->fatal( 'backend-fail-stream', $params['src'] );
1107                 }
1108
1109                 wfProfileOut( __METHOD__ );
1110                 return $status;
1111         }
1112
1113         /**
1114          * @see FileBackend::streamFile()
1115          */
1116         protected function doStreamFile( array $params ) {
1117                 $status = Status::newGood();
1118
1119                 $fsFile = $this->getLocalReference( $params );
1120                 if ( !$fsFile ) {
1121                         $status->fatal( 'backend-fail-stream', $params['src'] );
1122                 } elseif ( !readfile( $fsFile->getPath() ) ) {
1123                         $status->fatal( 'backend-fail-stream', $params['src'] );
1124                 }
1125
1126                 return $status;
1127         }
1128
1129         /**
1130          * @see FileBackendBase::getFileList()
1131          */
1132         final public function getFileList( array $params ) {
1133                 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1134                 if ( $dir === null ) { // invalid storage path
1135                         return null;
1136                 }
1137                 if ( $shard !== null ) {
1138                         // File listing is confined to a single container/shard
1139                         return $this->getFileListInternal( $fullCont, $dir, $params );
1140                 } else {
1141                         wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
1142                         // File listing spans multiple containers/shards
1143                         list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
1144                         return new FileBackendShardListIterator( $this,
1145                                 $fullCont, $this->getContainerSuffixes( $shortCont ), $params );
1146                 }
1147         }
1148
1149         /**
1150          * Do not call this function from places outside FileBackend and ContainerFileListIterator
1151          *
1152          * @param $container string Resolved container name
1153          * @param $dir string Resolved path relative to container
1154          * @param $params Array
1155          * @see FileBackend::getFileList()
1156          */
1157         abstract public function getFileListInternal( $container, $dir, array $params );
1158
1159         /**
1160          * Get the list of supported operations and their corresponding FileOp classes.
1161          *
1162          * @return Array
1163          */
1164         protected function supportedOperations() {
1165                 return array(
1166                         'store'       => 'StoreFileOp',
1167                         'copy'        => 'CopyFileOp',
1168                         'move'        => 'MoveFileOp',
1169                         'delete'      => 'DeleteFileOp',
1170                         'create'      => 'CreateFileOp',
1171                         'null'        => 'NullFileOp'
1172                 );
1173         }
1174
1175         /**
1176          * Return a list of FileOp objects from a list of operations.
1177          * Do not call this function from places outside FileBackend.
1178          *
1179          * The result must have the same number of items as the input.
1180          * An exception is thrown if an unsupported operation is requested.
1181          *
1182          * @param $ops Array Same format as doOperations()
1183          * @return Array List of FileOp objects
1184          * @throws MWException
1185          */
1186         final public function getOperations( array $ops ) {
1187                 $supportedOps = $this->supportedOperations();
1188
1189                 $performOps = array(); // array of FileOp objects
1190                 // Build up ordered array of FileOps...
1191                 foreach ( $ops as $operation ) {
1192                         $opName = $operation['op'];
1193                         if ( isset( $supportedOps[$opName] ) ) {
1194                                 $class = $supportedOps[$opName];
1195                                 // Get params for this operation
1196                                 $params = $operation;
1197                                 // Append the FileOp class
1198                                 $performOps[] = new $class( $this, $params );
1199                         } else {
1200                                 throw new MWException( "Operation `$opName` is not supported." );
1201                         }
1202                 }
1203
1204                 return $performOps;
1205         }
1206
1207         /**
1208          * @see FileBackendBase::doOperationsInternal()
1209          */
1210         protected function doOperationsInternal( array $ops, array $opts ) {
1211                 wfProfileIn( __METHOD__ );
1212                 $status = Status::newGood();
1213
1214                 // Build up a list of FileOps...
1215                 $performOps = $this->getOperations( $ops );
1216
1217                 // Acquire any locks as needed...
1218                 if ( empty( $opts['nonLocking'] ) ) {
1219                         // Build up a list of files to lock...
1220                         $filesLockEx = $filesLockSh = array();
1221                         foreach ( $performOps as $fileOp ) {
1222                                 $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() );
1223                                 $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() );
1224                         }
1225                         // Optimization: if doing an EX lock anyway, don't also set an SH one
1226                         $filesLockSh = array_diff( $filesLockSh, $filesLockEx );
1227                         // Get a shared lock on the parent directory of each path changed
1228                         $filesLockSh = array_merge( $filesLockSh, array_map( 'dirname', $filesLockEx ) );
1229                         // Try to lock those files for the scope of this function...
1230                         $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
1231                         $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
1232                         if ( !$status->isOK() ) {
1233                                 wfProfileOut( __METHOD__ );
1234                                 return $status; // abort
1235                         }
1236                 }
1237
1238                 // Clear any cache entries (after locks acquired)
1239                 $this->clearCache();
1240
1241                 // Actually attempt the operation batch...
1242                 $subStatus = FileOp::attemptBatch( $performOps, $opts );
1243
1244                 // Merge errors into status fields
1245                 $status->merge( $subStatus );
1246                 $status->success = $subStatus->success; // not done in merge()
1247
1248                 wfProfileOut( __METHOD__ );
1249                 return $status;
1250         }
1251
1252         /**
1253          * @see FileBackendBase::clearCache()
1254          */
1255         final public function clearCache( array $paths = null ) {
1256                 if ( $paths === null ) {
1257                         $this->cache = array();
1258                 } else {
1259                         foreach ( $paths as $path ) {
1260                                 unset( $this->cache[$path] );
1261                         }
1262                 }
1263                 $this->doClearCache( $paths );
1264         }
1265
1266         /**
1267          * Clears any additional stat caches for storage paths
1268          *
1269          * @see FileBackendBase::clearCache()
1270          *
1271          * @param $paths Array Storage paths (optional)
1272          * @return void
1273          */
1274         protected function doClearCache( array $paths = null ) {}
1275
1276         /**
1277          * Prune the cache if it is too big to add an item
1278          *
1279          * @return void
1280          */
1281         protected function trimCache() {
1282                 if ( count( $this->cache ) >= $this->maxCacheSize ) {
1283                         reset( $this->cache );
1284                         $key = key( $this->cache );
1285                         unset( $this->cache[$key] );
1286                 }
1287         }
1288
1289         /**
1290          * Get the parent storage directory of a storage path.
1291          * This returns a path like "mwstore://backend/container",
1292          * "mwstore://backend/container/...", or null if there is no parent.
1293          *
1294          * @param $storagePath string
1295          * @return string|null
1296          */
1297         final public static function parentStoragePath( $storagePath ) {
1298                 $storagePath = dirname( $storagePath );
1299                 list( $b, $cont, $rel ) = self::splitStoragePath( $storagePath );
1300                 return ( $rel === null ) ? null : $storagePath;
1301         }
1302
1303         /**
1304          * Check if a given path is a mwstore:// path.
1305          * This does not do any actual validation or existence checks.
1306          *
1307          * @param $path string
1308          * @return bool
1309          */
1310         final public static function isStoragePath( $path ) {
1311                 return ( strpos( $path, 'mwstore://' ) === 0 );
1312         }
1313
1314         /**
1315          * Split a storage path (e.g. "mwstore://backend/container/path/to/object")
1316          * into a backend name, a container name, and a relative object path.
1317          *
1318          * @param $storagePath string
1319          * @return Array (backend, container, rel object) or (null, null, null)
1320          */
1321         final public static function splitStoragePath( $storagePath ) {
1322                 if ( self::isStoragePath( $storagePath ) ) {
1323                         // Note: strlen( 'mwstore://' ) = 10
1324                         $parts = explode( '/', substr( $storagePath, 10 ), 3 );
1325                         if ( count( $parts ) == 3 ) {
1326                                 return $parts; // e.g. "backend/container/path"
1327                         } elseif ( count( $parts ) == 2 ) {
1328                                 return array( $parts[0], $parts[1], '' ); // e.g. "backend/container"
1329                         }
1330                 }
1331                 return array( null, null, null );
1332         }
1333
1334         /**
1335          * Check if a container name is valid.
1336          * This checks for for length and illegal characters.
1337          *
1338          * @param $container string
1339          * @return bool
1340          */
1341         final protected static function isValidContainerName( $container ) {
1342                 // This accounts for Swift and S3 restrictions while leaving room
1343                 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1344                 // Note that matching strings URL encode to the same string;
1345                 // in Swift, the length restriction is *after* URL encoding.
1346                 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1347         }
1348
1349         /**
1350          * Validate and normalize a relative storage path.
1351          * Null is returned if the path involves directory traversal.
1352          * Traversal is insecure for FS backends and broken for others.
1353          *
1354          * @param $path string Storage path relative to a container
1355          * @return string|null
1356          */
1357         final protected static function normalizeContainerPath( $path ) {
1358                 // Normalize directory separators
1359                 $path = strtr( $path, '\\', '/' );
1360                 // Collapse consecutive directory separators
1361                 $path = preg_replace( '![/]{2,}!', '/', $path );
1362                 // Use the same traversal protection as Title::secureAndSplit()
1363                 if ( strpos( $path, '.' ) !== false ) {
1364                         if (
1365                                 $path === '.' ||
1366                                 $path === '..' ||
1367                                 strpos( $path, './' ) === 0 ||
1368                                 strpos( $path, '../' ) === 0 ||
1369                                 strpos( $path, '/./' ) !== false ||
1370                                 strpos( $path, '/../' ) !== false
1371                         ) {
1372                                 return null;
1373                         }
1374                 }
1375                 return $path;
1376         }
1377
1378         /**
1379          * Splits a storage path into an internal container name,
1380          * an internal relative file name, and a container shard suffix.
1381          * Any shard suffix is already appended to the internal container name.
1382          * This also checks that the storage path is valid and within this backend.
1383          *
1384          * If the container is sharded but a suffix could not be determined,
1385          * this means that the path can only refer to a directory and can only
1386          * be scanned by looking in all the container shards.
1387          *
1388          * @param $storagePath string
1389          * @return Array (container, path, container suffix) or (null, null, null) if invalid
1390          */
1391         final protected function resolveStoragePath( $storagePath ) {
1392                 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1393                 if ( $backend === $this->name ) { // must be for this backend
1394                         $relPath = self::normalizeContainerPath( $relPath );
1395                         if ( $relPath !== null ) {
1396                                 // Get shard for the normalized path if this container is sharded
1397                                 $cShard = $this->getContainerShard( $container, $relPath );
1398                                 // Validate and sanitize the relative path (backend-specific)
1399                                 $relPath = $this->resolveContainerPath( $container, $relPath );
1400                                 if ( $relPath !== null ) {
1401                                         // Prepend any wiki ID prefix to the container name
1402                                         $container = $this->fullContainerName( $container );
1403                                         if ( self::isValidContainerName( $container ) ) {
1404                                                 // Validate and sanitize the container name (backend-specific)
1405                                                 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1406                                                 if ( $container !== null ) {
1407                                                         return array( $container, $relPath, $cShard );
1408                                                 }
1409                                         }
1410                                 }
1411                         }
1412                 }
1413                 return array( null, null, null );
1414         }
1415
1416         /**
1417          * Like resolveStoragePath() except null values are returned if
1418          * the container is sharded and the shard could not be determined.
1419          *
1420          * @see FileBackend::resolveStoragePath()
1421          *
1422          * @param $storagePath string
1423          * @return Array (container, path) or (null, null) if invalid
1424          */
1425         final protected function resolveStoragePathReal( $storagePath ) {
1426                 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1427                 if ( $cShard !== null ) {
1428                         return array( $container, $relPath );
1429                 }
1430                 return array( null, null );
1431         }
1432
1433         /**
1434          * Get the container name shard suffix for a given path.
1435          * Any empty suffix means the container is not sharded.
1436          *
1437          * @param $container string Container name
1438          * @param $relStoragePath string Storage path relative to the container
1439          * @return string|null Returns null if shard could not be determined
1440          */
1441         final protected function getContainerShard( $container, $relPath ) {
1442                 $hashLevels = $this->getContainerHashLevels( $container );
1443                 if ( $hashLevels === 1 ) { // 16 shards per container
1444                         $hashDirRegex = '(?P<shard>[0-9a-f])';
1445                 } elseif ( $hashLevels === 2 ) { // 256 shards per container
1446                         $hashDirRegex = '[0-9a-f]/(?P<shard>[0-9a-f]{2})';
1447                 } else {
1448                         return ''; // no sharding
1449                 }
1450                 // Allow certain directories to be above the hash dirs so as
1451                 // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1452                 // They must be 2+ chars to avoid any hash directory ambiguity.
1453                 if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1454                         return '.' . str_pad( $m['shard'], $hashLevels, '0', STR_PAD_LEFT );
1455                 }
1456                 return null; // failed to match
1457         }
1458
1459         /**
1460          * Get the number of hash levels for a container.
1461          * If greater than 0, then all file storage paths within
1462          * the container are required to be hashed accordingly.
1463          *
1464          * @param $container string
1465          * @return integer
1466          */
1467         final protected function getContainerHashLevels( $container ) {
1468                 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1469                         $hashLevels = (int)$this->shardViaHashLevels[$container];
1470                         if ( $hashLevels >= 0 && $hashLevels <= 2 ) {
1471                                 return $hashLevels;
1472                         }
1473                 }
1474                 return 0; // no sharding
1475         }
1476
1477         /**
1478          * Get a list of full container shard suffixes for a container
1479          *
1480          * @param $container string
1481          * @return Array
1482          */
1483         final protected function getContainerSuffixes( $container ) {
1484                 $shards = array();
1485                 $digits = $this->getContainerHashLevels( $container );
1486                 if ( $digits > 0 ) {
1487                         $numShards = 1 << ( $digits * 4 );
1488                         for ( $index = 0; $index < $numShards; $index++ ) {
1489                                 $shards[] = '.' . str_pad( dechex( $index ), $digits, '0', STR_PAD_LEFT );
1490                         }
1491                 }
1492                 return $shards;
1493         }
1494
1495         /**
1496          * Get the full container name, including the wiki ID prefix
1497          *
1498          * @param $container string
1499          * @return string
1500          */
1501         final protected function fullContainerName( $container ) {
1502                 if ( $this->wikiId != '' ) {
1503                         return "{$this->wikiId}-$container";
1504                 } else {
1505                         return $container;
1506                 }
1507         }
1508
1509         /**
1510          * Resolve a container name, checking if it's allowed by the backend.
1511          * This is intended for internal use, such as encoding illegal chars.
1512          * Subclasses can override this to be more restrictive.
1513          *
1514          * @param $container string
1515          * @return string|null
1516          */
1517         protected function resolveContainerName( $container ) {
1518                 return $container;
1519         }
1520
1521         /**
1522          * Resolve a relative storage path, checking if it's allowed by the backend.
1523          * This is intended for internal use, such as encoding illegal chars or perhaps
1524          * getting absolute paths (e.g. FS based backends). Note that the relative path
1525          * may be the empty string (e.g. the path is simply to the container).
1526          *
1527          * @param $container string Container name
1528          * @param $relStoragePath string Storage path relative to the container
1529          * @return string|null Path or null if not valid
1530          */
1531         protected function resolveContainerPath( $container, $relStoragePath ) {
1532                 return $relStoragePath;
1533         }
1534
1535         /**
1536          * Get the final extension from a storage or FS path
1537          *
1538          * @param $path string
1539          * @return string
1540          */
1541         final public static function extensionFromPath( $path ) {
1542                 $i = strrpos( $path, '.' );
1543                 return strtolower( $i ? substr( $path, $i + 1 ) : '' );
1544         }
1545 }
1546
1547 /**
1548  * FileBackend helper function to handle file listings that span container shards.
1549  * Do not use this class from places outside of FileBackend.
1550  *
1551  * @ingroup FileBackend
1552  */
1553 class FileBackendShardListIterator implements Iterator {
1554         /* @var FileBackend */
1555         protected $backend;
1556         /* @var Array */
1557         protected $params;
1558         /* @var Array */
1559         protected $shardSuffixes;
1560         protected $container; // string
1561         protected $directory; // string
1562
1563         /* @var Traversable */
1564         protected $iter;
1565         protected $curShard = 0; // integer
1566         protected $pos = 0; // integer
1567
1568         /**
1569          * @param $backend FileBackend
1570          * @param $container string Full storage container name
1571          * @param $dir string Storage directory relative to container
1572          * @param $suffixes Array List of container shard suffixes
1573          * @param $params Array
1574          */
1575         public function __construct(
1576                 FileBackend $backend, $container, $dir, array $suffixes, array $params
1577         ) {
1578                 $this->backend = $backend;
1579                 $this->container = $container;
1580                 $this->directory = $dir;
1581                 $this->shardSuffixes = $suffixes;
1582                 $this->params = $params;
1583         }
1584
1585         public function current() {
1586                 if ( is_array( $this->iter ) ) {
1587                         return current( $this->iter );
1588                 } else {
1589                         return $this->iter->current();
1590                 }
1591         }
1592
1593         public function key() {
1594                 return $this->pos;
1595         }
1596
1597         public function next() {
1598                 ++$this->pos;
1599                 if ( is_array( $this->iter ) ) {
1600                         next( $this->iter );
1601                 } else {
1602                         $this->iter->next();
1603                 }
1604                 // Find the next non-empty shard if no elements are left
1605                 $this->nextShardIteratorIfNotValid();
1606         }
1607
1608         /**
1609          * If the iterator for this container shard is out of items,
1610          * then move on to the next container that has items.
1611          * If there are none, then it advances to the last container.
1612          */
1613         protected function nextShardIteratorIfNotValid() {
1614                 while ( !$this->valid() ) {
1615                         if ( ++$this->curShard >= count( $this->shardSuffixes ) ) {
1616                                 break; // no more container shards
1617                         }
1618                         $this->setIteratorFromCurrentShard();
1619                 }
1620         }
1621
1622         protected function setIteratorFromCurrentShard() {
1623                 $suffix = $this->shardSuffixes[$this->curShard];
1624                 $this->iter = $this->backend->getFileListInternal(
1625                         "{$this->container}{$suffix}", $this->directory, $this->params );
1626         }
1627
1628         public function rewind() {
1629                 $this->pos = 0;
1630                 $this->curShard = 0;
1631                 $this->setIteratorFromCurrentShard();
1632                 // Find the next non-empty shard if this one has no elements
1633                 $this->nextShardIteratorIfNotValid();
1634         }
1635
1636         public function valid() {
1637                 if ( $this->iter == null ) {
1638                         return false; // some failure?
1639                 } elseif ( is_array( $this->iter ) ) {
1640                         return ( current( $this->iter ) !== false ); // no paths can have this value
1641                 } else {
1642                         return $this->iter->valid();
1643                 }
1644         }
1645 }