From: jenkins-bot Date: Fri, 23 Sep 2016 21:28:52 +0000 (+0000) Subject: Merge "Add HTTPFileStreamer class" X-Git-Tag: 1.31.0-rc.0~5346 X-Git-Url: https://git.cyclocoop.org/%27.WWW_URL.%27admin/?a=commitdiff_plain;h=5c529ac835abee3f760713bf1bab4867d03e73bb;hp=24aa72de84af6be14e556ede018548dcc8400e09;p=lhc%2Fweb%2Fwiklou.git Merge "Add HTTPFileStreamer class" --- diff --git a/autoload.php b/autoload.php index 117e660cba..9bd6c26bd0 100644 --- a/autoload.php +++ b/autoload.php @@ -458,7 +458,7 @@ $wgAutoloadLocalClasses = [ 'FileBackendDBRepoWrapper' => __DIR__ . '/includes/filerepo/FileBackendDBRepoWrapper.php', 'FileBackendError' => __DIR__ . '/includes/libs/filebackend/FileBackendError.php', 'FileBackendGroup' => __DIR__ . '/includes/filebackend/FileBackendGroup.php', - 'FileBackendMultiWrite' => __DIR__ . '/includes/filebackend/FileBackendMultiWrite.php', + 'FileBackendMultiWrite' => __DIR__ . '/includes/libs/filebackend/FileBackendMultiWrite.php', 'FileBackendStore' => __DIR__ . '/includes/filebackend/FileBackendStore.php', 'FileBackendStoreOpHandle' => __DIR__ . '/includes/filebackend/FileBackendStore.php', 'FileBackendStoreShardDirIterator' => __DIR__ . '/includes/filebackend/FileBackendStore.php', diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index 8180443347..7facf2fcf7 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -616,6 +616,11 @@ $wgUploadDialog = [ * Additional parameters are specific to the file backend class used. * These settings should be global to all wikis when possible. * + * FileBackendMultiWrite::__construct() is augmented with a 'template' option that + * can be used in any of the values of the 'backends' array. Its value is the name of + * another backend in $wgFileBackends. When set, it pre-fills the array with all of the + * configuration of the named backend. Explicitly set values in the array take precedence. + * * There are two particularly important aspects about each backend: * - a) Whether it is fully qualified or wiki-relative. * By default, the paths of files are relative to the current wiki, diff --git a/includes/filebackend/FileBackendGroup.php b/includes/filebackend/FileBackendGroup.php index 18a7cca59d..c8a68d2d26 100644 --- a/includes/filebackend/FileBackendGroup.php +++ b/includes/filebackend/FileBackendGroup.php @@ -174,6 +174,15 @@ class FileBackendGroup { $config['tmpDirectory'] = wfTempDir(); $config['logger'] = LoggerFactory::getInstance( 'FileOperation' ); $config['profiler'] = Profiler::instance(); + if ( $class === 'FileBackendMultiWrite' ) { + foreach ( $config['backends'] as $index => $beConfig ) { + if ( isset( $beConfig['template'] ) ) { + // Config is just a modified version of a registered backend's. + // This should only be used when that config is used only by this backend. + $config['backends'][$index] += $this->config( $beConfig['template'] ); + } + } + } $this->backends[$name]['instance'] = new $class( $config ); } diff --git a/includes/filebackend/FileBackendMultiWrite.php b/includes/filebackend/FileBackendMultiWrite.php deleted file mode 100644 index db79155533..0000000000 --- a/includes/filebackend/FileBackendMultiWrite.php +++ /dev/null @@ -1,761 +0,0 @@ -syncChecks = isset( $config['syncChecks'] ) - ? $config['syncChecks'] - : self::CHECK_SIZE; - $this->autoResync = isset( $config['autoResync'] ) - ? $config['autoResync'] - : false; - $this->asyncWrites = isset( $config['replication'] ) && $config['replication'] === 'async'; - // Construct backends here rather than via registration - // to keep these backends hidden from outside the proxy. - $namesUsed = []; - foreach ( $config['backends'] as $index => $config ) { - if ( isset( $config['template'] ) ) { - // Config is just a modified version of a registered backend's. - // This should only be used when that config is used only by this backend. - $config = $config + FileBackendGroup::singleton()->config( $config['template'] ); - } - $name = $config['name']; - if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates - throw new LogicException( "Two or more backends defined with the name $name." ); - } - $namesUsed[$name] = 1; - // Alter certain sub-backend settings for sanity - unset( $config['readOnly'] ); // use proxy backend setting - unset( $config['fileJournal'] ); // use proxy backend journal - unset( $config['lockManager'] ); // lock under proxy backend - $config['domainId'] = $this->domainId; // use the proxy backend wiki ID - if ( !empty( $config['isMultiMaster'] ) ) { - if ( $this->masterIndex >= 0 ) { - throw new LogicException( 'More than one master backend defined.' ); - } - $this->masterIndex = $index; // this is the "master" - $config['fileJournal'] = $this->fileJournal; // log under proxy backend - } - if ( !empty( $config['readAffinity'] ) ) { - $this->readIndex = $index; // prefer this for reads - } - // Create sub-backend object - if ( !isset( $config['class'] ) ) { - throw new InvalidArgumentException( 'No class given for a backend config.' ); - } - $class = $config['class']; - $this->backends[$index] = new $class( $config ); - } - if ( $this->masterIndex < 0 ) { // need backends and must have a master - throw new LogicException( 'No master backend defined.' ); - } - if ( $this->readIndex < 0 ) { - $this->readIndex = $this->masterIndex; // default - } - } - - final protected function doOperationsInternal( array $ops, array $opts ) { - $status = $this->newStatus(); - - $mbe = $this->backends[$this->masterIndex]; // convenience - - // Try to lock those files for the scope of this function... - $scopeLock = null; - if ( empty( $opts['nonLocking'] ) ) { - // Try to lock those files for the scope of this function... - /** @noinspection PhpUnusedLocalVariableInspection */ - $scopeLock = $this->getScopedLocksForOps( $ops, $status ); - if ( !$status->isOK() ) { - return $status; // abort - } - } - // Clear any cache entries (after locks acquired) - $this->clearCache(); - $opts['preserveCache'] = true; // only locked files are cached - // Get the list of paths to read/write... - $relevantPaths = $this->fileStoragePathsForOps( $ops ); - // Check if the paths are valid and accessible on all backends... - $status->merge( $this->accessibilityCheck( $relevantPaths ) ); - if ( !$status->isOK() ) { - return $status; // abort - } - // Do a consistency check to see if the backends are consistent... - $syncStatus = $this->consistencyCheck( $relevantPaths ); - if ( !$syncStatus->isOK() ) { - wfDebugLog( 'FileOperation', get_class( $this ) . - " failed sync check: " . FormatJson::encode( $relevantPaths ) ); - // Try to resync the clone backends to the master on the spot... - if ( $this->autoResync === false - || !$this->resyncFiles( $relevantPaths, $this->autoResync )->isOK() - ) { - $status->merge( $syncStatus ); - - return $status; // abort - } - } - // Actually attempt the operation batch on the master backend... - $realOps = $this->substOpBatchPaths( $ops, $mbe ); - $masterStatus = $mbe->doOperations( $realOps, $opts ); - $status->merge( $masterStatus ); - // Propagate the operations to the clone backends if there were no unexpected errors - // and if there were either no expected errors or if the 'force' option was used. - // However, if nothing succeeded at all, then don't replicate any of the operations. - // If $ops only had one operation, this might avoid backend sync inconsistencies. - if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) { - foreach ( $this->backends as $index => $backend ) { - if ( $index === $this->masterIndex ) { - continue; // done already - } - - $realOps = $this->substOpBatchPaths( $ops, $backend ); - if ( $this->asyncWrites && !$this->hasVolatileSources( $ops ) ) { - // Bind $scopeLock to the callback to preserve locks - DeferredUpdates::addCallableUpdate( - function() use ( $backend, $realOps, $opts, $scopeLock, $relevantPaths ) { - wfDebugLog( 'FileOperationReplication', - "'{$backend->getName()}' async replication; paths: " . - FormatJson::encode( $relevantPaths ) ); - $backend->doOperations( $realOps, $opts ); - } - ); - } else { - wfDebugLog( 'FileOperationReplication', - "'{$backend->getName()}' sync replication; paths: " . - FormatJson::encode( $relevantPaths ) ); - $status->merge( $backend->doOperations( $realOps, $opts ) ); - } - } - } - // Make 'success', 'successCount', and 'failCount' fields reflect - // the overall operation, rather than all the batches for each backend. - // Do this by only using success values from the master backend's batch. - $status->success = $masterStatus->success; - $status->successCount = $masterStatus->successCount; - $status->failCount = $masterStatus->failCount; - - return $status; - } - - /** - * Check that a set of files are consistent across all internal backends - * - * @param array $paths List of storage paths - * @return StatusValue - */ - public function consistencyCheck( array $paths ) { - $status = $this->newStatus(); - if ( $this->syncChecks == 0 || count( $this->backends ) <= 1 ) { - return $status; // skip checks - } - - // Preload all of the stat info in as few round trips as possible... - foreach ( $this->backends as $backend ) { - $realPaths = $this->substPaths( $paths, $backend ); - $backend->preloadFileStat( [ 'srcs' => $realPaths, 'latest' => true ] ); - } - - $mBackend = $this->backends[$this->masterIndex]; - foreach ( $paths as $path ) { - $params = [ 'src' => $path, 'latest' => true ]; - $mParams = $this->substOpPaths( $params, $mBackend ); - // Stat the file on the 'master' backend - $mStat = $mBackend->getFileStat( $mParams ); - if ( $this->syncChecks & self::CHECK_SHA1 ) { - $mSha1 = $mBackend->getFileSha1Base36( $mParams ); - } else { - $mSha1 = false; - } - // Check if all clone backends agree with the master... - foreach ( $this->backends as $index => $cBackend ) { - if ( $index === $this->masterIndex ) { - continue; // master - } - $cParams = $this->substOpPaths( $params, $cBackend ); - $cStat = $cBackend->getFileStat( $cParams ); - if ( $mStat ) { // file is in master - if ( !$cStat ) { // file should exist - $status->fatal( 'backend-fail-synced', $path ); - continue; - } - if ( $this->syncChecks & self::CHECK_SIZE ) { - if ( $cStat['size'] != $mStat['size'] ) { // wrong size - $status->fatal( 'backend-fail-synced', $path ); - continue; - } - } - if ( $this->syncChecks & self::CHECK_TIME ) { - $mTs = wfTimestamp( TS_UNIX, $mStat['mtime'] ); - $cTs = wfTimestamp( TS_UNIX, $cStat['mtime'] ); - if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere - $status->fatal( 'backend-fail-synced', $path ); - continue; - } - } - if ( $this->syncChecks & self::CHECK_SHA1 ) { - if ( $cBackend->getFileSha1Base36( $cParams ) !== $mSha1 ) { // wrong SHA1 - $status->fatal( 'backend-fail-synced', $path ); - continue; - } - } - } else { // file is not in master - if ( $cStat ) { // file should not exist - $status->fatal( 'backend-fail-synced', $path ); - } - } - } - } - - return $status; - } - - /** - * Check that a set of file paths are usable across all internal backends - * - * @param array $paths List of storage paths - * @return StatusValue - */ - public function accessibilityCheck( array $paths ) { - $status = $this->newStatus(); - if ( count( $this->backends ) <= 1 ) { - return $status; // skip checks - } - - foreach ( $paths as $path ) { - foreach ( $this->backends as $backend ) { - $realPath = $this->substPaths( $path, $backend ); - if ( !$backend->isPathUsableInternal( $realPath ) ) { - $status->fatal( 'backend-fail-usable', $path ); - } - } - } - - return $status; - } - - /** - * Check that a set of files are consistent across all internal backends - * and re-synchronize those files against the "multi master" if needed. - * - * @param array $paths List of storage paths - * @param string|bool $resyncMode False, True, or "conservative"; see __construct() - * @return StatusValue - */ - public function resyncFiles( array $paths, $resyncMode = true ) { - $status = $this->newStatus(); - - $mBackend = $this->backends[$this->masterIndex]; - foreach ( $paths as $path ) { - $mPath = $this->substPaths( $path, $mBackend ); - $mSha1 = $mBackend->getFileSha1Base36( [ 'src' => $mPath, 'latest' => true ] ); - $mStat = $mBackend->getFileStat( [ 'src' => $mPath, 'latest' => true ] ); - if ( $mStat === null || ( $mSha1 !== false && !$mStat ) ) { // sanity - $status->fatal( 'backend-fail-internal', $this->name ); - wfDebugLog( 'FileOperation', __METHOD__ - . ': File is not available on the master backend' ); - continue; // file is not available on the master backend... - } - // Check of all clone backends agree with the master... - foreach ( $this->backends as $index => $cBackend ) { - if ( $index === $this->masterIndex ) { - continue; // master - } - $cPath = $this->substPaths( $path, $cBackend ); - $cSha1 = $cBackend->getFileSha1Base36( [ 'src' => $cPath, 'latest' => true ] ); - $cStat = $cBackend->getFileStat( [ 'src' => $cPath, 'latest' => true ] ); - if ( $cStat === null || ( $cSha1 !== false && !$cStat ) ) { // sanity - $status->fatal( 'backend-fail-internal', $cBackend->getName() ); - wfDebugLog( 'FileOperation', __METHOD__ . - ': File is not available on the clone backend' ); - continue; // file is not available on the clone backend... - } - if ( $mSha1 === $cSha1 ) { - // already synced; nothing to do - } elseif ( $mSha1 !== false ) { // file is in master - if ( $resyncMode === 'conservative' - && $cStat && $cStat['mtime'] > $mStat['mtime'] - ) { - $status->fatal( 'backend-fail-synced', $path ); - continue; // don't rollback data - } - $fsFile = $mBackend->getLocalReference( - [ 'src' => $mPath, 'latest' => true ] ); - $status->merge( $cBackend->quickStore( - [ 'src' => $fsFile->getPath(), 'dst' => $cPath ] - ) ); - } elseif ( $mStat === false ) { // file is not in master - if ( $resyncMode === 'conservative' ) { - $status->fatal( 'backend-fail-synced', $path ); - continue; // don't delete data - } - $status->merge( $cBackend->quickDelete( [ 'src' => $cPath ] ) ); - } - } - } - - if ( !$status->isOK() ) { - wfDebugLog( 'FileOperation', get_class( $this ) . - " failed to resync: " . FormatJson::encode( $paths ) ); - } - - return $status; - } - - /** - * Get a list of file storage paths to read or write for a list of operations - * - * @param array $ops Same format as doOperations() - * @return array List of storage paths to files (does not include directories) - */ - protected function fileStoragePathsForOps( array $ops ) { - $paths = []; - foreach ( $ops as $op ) { - if ( isset( $op['src'] ) ) { - // For things like copy/move/delete with "ignoreMissingSource" and there - // is no source file, nothing should happen and there should be no errors. - if ( empty( $op['ignoreMissingSource'] ) - || $this->fileExists( [ 'src' => $op['src'] ] ) - ) { - $paths[] = $op['src']; - } - } - if ( isset( $op['srcs'] ) ) { - $paths = array_merge( $paths, $op['srcs'] ); - } - if ( isset( $op['dst'] ) ) { - $paths[] = $op['dst']; - } - } - - return array_values( array_unique( array_filter( $paths, 'FileBackend::isStoragePath' ) ) ); - } - - /** - * Substitute the backend name in storage path parameters - * for a set of operations with that of a given internal backend. - * - * @param array $ops List of file operation arrays - * @param FileBackendStore $backend - * @return array - */ - protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) { - $newOps = []; // operations - foreach ( $ops as $op ) { - $newOp = $op; // operation - foreach ( [ 'src', 'srcs', 'dst', 'dir' ] as $par ) { - if ( isset( $newOp[$par] ) ) { // string or array - $newOp[$par] = $this->substPaths( $newOp[$par], $backend ); - } - } - $newOps[] = $newOp; - } - - return $newOps; - } - - /** - * Same as substOpBatchPaths() but for a single operation - * - * @param array $ops File operation array - * @param FileBackendStore $backend - * @return array - */ - protected function substOpPaths( array $ops, FileBackendStore $backend ) { - $newOps = $this->substOpBatchPaths( [ $ops ], $backend ); - - return $newOps[0]; - } - - /** - * Substitute the backend of storage paths with an internal backend's name - * - * @param array|string $paths List of paths or single string path - * @param FileBackendStore $backend - * @return array|string - */ - protected function substPaths( $paths, FileBackendStore $backend ) { - return preg_replace( - '!^mwstore://' . preg_quote( $this->name, '!' ) . '/!', - StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ), - $paths // string or array - ); - } - - /** - * Substitute the backend of internal storage paths with the proxy backend's name - * - * @param array|string $paths List of paths or single string path - * @return array|string - */ - protected function unsubstPaths( $paths ) { - return preg_replace( - '!^mwstore://([^/]+)!', - StringUtils::escapeRegexReplacement( "mwstore://{$this->name}" ), - $paths // string or array - ); - } - - /** - * @param array $ops File operations for FileBackend::doOperations() - * @return bool Whether there are file path sources with outside lifetime/ownership - */ - protected function hasVolatileSources( array $ops ) { - foreach ( $ops as $op ) { - if ( $op['op'] === 'store' && !isset( $op['srcRef'] ) ) { - return true; // source file might be deleted anytime after do*Operations() - } - } - - return false; - } - - protected function doQuickOperationsInternal( array $ops ) { - $status = $this->newStatus(); - // Do the operations on the master backend; setting StatusValue fields... - $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] ); - $masterStatus = $this->backends[$this->masterIndex]->doQuickOperations( $realOps ); - $status->merge( $masterStatus ); - // Propagate the operations to the clone backends... - foreach ( $this->backends as $index => $backend ) { - if ( $index === $this->masterIndex ) { - continue; // done already - } - - $realOps = $this->substOpBatchPaths( $ops, $backend ); - if ( $this->asyncWrites && !$this->hasVolatileSources( $ops ) ) { - DeferredUpdates::addCallableUpdate( - function() use ( $backend, $realOps ) { - $backend->doQuickOperations( $realOps ); - } - ); - } else { - $status->merge( $backend->doQuickOperations( $realOps ) ); - } - } - // Make 'success', 'successCount', and 'failCount' fields reflect - // the overall operation, rather than all the batches for each backend. - // Do this by only using success values from the master backend's batch. - $status->success = $masterStatus->success; - $status->successCount = $masterStatus->successCount; - $status->failCount = $masterStatus->failCount; - - return $status; - } - - protected function doPrepare( array $params ) { - return $this->doDirectoryOp( 'prepare', $params ); - } - - protected function doSecure( array $params ) { - return $this->doDirectoryOp( 'secure', $params ); - } - - protected function doPublish( array $params ) { - return $this->doDirectoryOp( 'publish', $params ); - } - - protected function doClean( array $params ) { - return $this->doDirectoryOp( 'clean', $params ); - } - - /** - * @param string $method One of (doPrepare,doSecure,doPublish,doClean) - * @param array $params Method arguments - * @return StatusValue - */ - protected function doDirectoryOp( $method, array $params ) { - $status = $this->newStatus(); - - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - $masterStatus = $this->backends[$this->masterIndex]->$method( $realParams ); - $status->merge( $masterStatus ); - - foreach ( $this->backends as $index => $backend ) { - if ( $index === $this->masterIndex ) { - continue; // already done - } - - $realParams = $this->substOpPaths( $params, $backend ); - if ( $this->asyncWrites ) { - DeferredUpdates::addCallableUpdate( - function() use ( $backend, $method, $realParams ) { - $backend->$method( $realParams ); - } - ); - } else { - $status->merge( $backend->$method( $realParams ) ); - } - } - - return $status; - } - - public function concatenate( array $params ) { - // We are writing to an FS file, so we don't need to do this per-backend - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->concatenate( $realParams ); - } - - public function fileExists( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->fileExists( $realParams ); - } - - public function getFileTimestamp( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileTimestamp( $realParams ); - } - - public function getFileSize( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileSize( $realParams ); - } - - public function getFileStat( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileStat( $realParams ); - } - - public function getFileXAttributes( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileXAttributes( $realParams ); - } - - public function getFileContentsMulti( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - $contentsM = $this->backends[$index]->getFileContentsMulti( $realParams ); - - $contents = []; // (path => FSFile) mapping using the proxy backend's name - foreach ( $contentsM as $path => $data ) { - $contents[$this->unsubstPaths( $path )] = $data; - } - - return $contents; - } - - public function getFileSha1Base36( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileSha1Base36( $realParams ); - } - - public function getFileProps( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileProps( $realParams ); - } - - public function streamFile( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->streamFile( $realParams ); - } - - public function getLocalReferenceMulti( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - $fsFilesM = $this->backends[$index]->getLocalReferenceMulti( $realParams ); - - $fsFiles = []; // (path => FSFile) mapping using the proxy backend's name - foreach ( $fsFilesM as $path => $fsFile ) { - $fsFiles[$this->unsubstPaths( $path )] = $fsFile; - } - - return $fsFiles; - } - - public function getLocalCopyMulti( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - $tempFilesM = $this->backends[$index]->getLocalCopyMulti( $realParams ); - - $tempFiles = []; // (path => TempFSFile) mapping using the proxy backend's name - foreach ( $tempFilesM as $path => $tempFile ) { - $tempFiles[$this->unsubstPaths( $path )] = $tempFile; - } - - return $tempFiles; - } - - public function getFileHttpUrl( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->getFileHttpUrl( $realParams ); - } - - public function directoryExists( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - - return $this->backends[$this->masterIndex]->directoryExists( $realParams ); - } - - public function getDirectoryList( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - - return $this->backends[$this->masterIndex]->getDirectoryList( $realParams ); - } - - public function getFileList( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - - return $this->backends[$this->masterIndex]->getFileList( $realParams ); - } - - public function getFeatures() { - return $this->backends[$this->masterIndex]->getFeatures(); - } - - public function clearCache( array $paths = null ) { - foreach ( $this->backends as $backend ) { - $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null; - $backend->clearCache( $realPaths ); - } - } - - public function preloadCache( array $paths ) { - $realPaths = $this->substPaths( $paths, $this->backends[$this->readIndex] ); - $this->backends[$this->readIndex]->preloadCache( $realPaths ); - } - - public function preloadFileStat( array $params ) { - $index = $this->getReadIndexFromParams( $params ); - $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - - return $this->backends[$index]->preloadFileStat( $realParams ); - } - - public function getScopedLocksForOps( array $ops, StatusValue $status ) { - $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] ); - $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $realOps ); - // Get the paths to lock from the master backend - $paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps ); - // Get the paths under the proxy backend's name - $pbPaths = [ - LockManager::LOCK_UW => $this->unsubstPaths( $paths[LockManager::LOCK_UW] ), - LockManager::LOCK_EX => $this->unsubstPaths( $paths[LockManager::LOCK_EX] ) - ]; - - // Actually acquire the locks - return $this->getScopedFileLocks( $pbPaths, 'mixed', $status ); - } - - /** - * @param array $params - * @return int The master or read affinity backend index, based on $params['latest'] - */ - protected function getReadIndexFromParams( array $params ) { - return !empty( $params['latest'] ) ? $this->masterIndex : $this->readIndex; - } -} diff --git a/includes/libs/filebackend/FileBackendMultiWrite.php b/includes/libs/filebackend/FileBackendMultiWrite.php new file mode 100644 index 0000000000..7c32d02e86 --- /dev/null +++ b/includes/libs/filebackend/FileBackendMultiWrite.php @@ -0,0 +1,753 @@ +syncChecks = isset( $config['syncChecks'] ) + ? $config['syncChecks'] + : self::CHECK_SIZE; + $this->autoResync = isset( $config['autoResync'] ) + ? $config['autoResync'] + : false; + $this->asyncWrites = isset( $config['replication'] ) && $config['replication'] === 'async'; + // Construct backends here rather than via registration + // to keep these backends hidden from outside the proxy. + $namesUsed = []; + foreach ( $config['backends'] as $index => $config ) { + $name = $config['name']; + if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates + throw new LogicException( "Two or more backends defined with the name $name." ); + } + $namesUsed[$name] = 1; + // Alter certain sub-backend settings for sanity + unset( $config['readOnly'] ); // use proxy backend setting + unset( $config['fileJournal'] ); // use proxy backend journal + unset( $config['lockManager'] ); // lock under proxy backend + $config['domainId'] = $this->domainId; // use the proxy backend wiki ID + if ( !empty( $config['isMultiMaster'] ) ) { + if ( $this->masterIndex >= 0 ) { + throw new LogicException( 'More than one master backend defined.' ); + } + $this->masterIndex = $index; // this is the "master" + $config['fileJournal'] = $this->fileJournal; // log under proxy backend + } + if ( !empty( $config['readAffinity'] ) ) { + $this->readIndex = $index; // prefer this for reads + } + // Create sub-backend object + if ( !isset( $config['class'] ) ) { + throw new InvalidArgumentException( 'No class given for a backend config.' ); + } + $class = $config['class']; + $this->backends[$index] = new $class( $config ); + } + if ( $this->masterIndex < 0 ) { // need backends and must have a master + throw new LogicException( 'No master backend defined.' ); + } + if ( $this->readIndex < 0 ) { + $this->readIndex = $this->masterIndex; // default + } + } + + final protected function doOperationsInternal( array $ops, array $opts ) { + $status = $this->newStatus(); + + $mbe = $this->backends[$this->masterIndex]; // convenience + + // Try to lock those files for the scope of this function... + $scopeLock = null; + if ( empty( $opts['nonLocking'] ) ) { + // Try to lock those files for the scope of this function... + /** @noinspection PhpUnusedLocalVariableInspection */ + $scopeLock = $this->getScopedLocksForOps( $ops, $status ); + if ( !$status->isOK() ) { + return $status; // abort + } + } + // Clear any cache entries (after locks acquired) + $this->clearCache(); + $opts['preserveCache'] = true; // only locked files are cached + // Get the list of paths to read/write... + $relevantPaths = $this->fileStoragePathsForOps( $ops ); + // Check if the paths are valid and accessible on all backends... + $status->merge( $this->accessibilityCheck( $relevantPaths ) ); + if ( !$status->isOK() ) { + return $status; // abort + } + // Do a consistency check to see if the backends are consistent... + $syncStatus = $this->consistencyCheck( $relevantPaths ); + if ( !$syncStatus->isOK() ) { + wfDebugLog( 'FileOperation', get_class( $this ) . + " failed sync check: " . FormatJson::encode( $relevantPaths ) ); + // Try to resync the clone backends to the master on the spot... + if ( $this->autoResync === false + || !$this->resyncFiles( $relevantPaths, $this->autoResync )->isOK() + ) { + $status->merge( $syncStatus ); + + return $status; // abort + } + } + // Actually attempt the operation batch on the master backend... + $realOps = $this->substOpBatchPaths( $ops, $mbe ); + $masterStatus = $mbe->doOperations( $realOps, $opts ); + $status->merge( $masterStatus ); + // Propagate the operations to the clone backends if there were no unexpected errors + // and if there were either no expected errors or if the 'force' option was used. + // However, if nothing succeeded at all, then don't replicate any of the operations. + // If $ops only had one operation, this might avoid backend sync inconsistencies. + if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) { + foreach ( $this->backends as $index => $backend ) { + if ( $index === $this->masterIndex ) { + continue; // done already + } + + $realOps = $this->substOpBatchPaths( $ops, $backend ); + if ( $this->asyncWrites && !$this->hasVolatileSources( $ops ) ) { + // Bind $scopeLock to the callback to preserve locks + DeferredUpdates::addCallableUpdate( + function() use ( $backend, $realOps, $opts, $scopeLock, $relevantPaths ) { + wfDebugLog( 'FileOperationReplication', + "'{$backend->getName()}' async replication; paths: " . + FormatJson::encode( $relevantPaths ) ); + $backend->doOperations( $realOps, $opts ); + } + ); + } else { + wfDebugLog( 'FileOperationReplication', + "'{$backend->getName()}' sync replication; paths: " . + FormatJson::encode( $relevantPaths ) ); + $status->merge( $backend->doOperations( $realOps, $opts ) ); + } + } + } + // Make 'success', 'successCount', and 'failCount' fields reflect + // the overall operation, rather than all the batches for each backend. + // Do this by only using success values from the master backend's batch. + $status->success = $masterStatus->success; + $status->successCount = $masterStatus->successCount; + $status->failCount = $masterStatus->failCount; + + return $status; + } + + /** + * Check that a set of files are consistent across all internal backends + * + * @param array $paths List of storage paths + * @return StatusValue + */ + public function consistencyCheck( array $paths ) { + $status = $this->newStatus(); + if ( $this->syncChecks == 0 || count( $this->backends ) <= 1 ) { + return $status; // skip checks + } + + // Preload all of the stat info in as few round trips as possible... + foreach ( $this->backends as $backend ) { + $realPaths = $this->substPaths( $paths, $backend ); + $backend->preloadFileStat( [ 'srcs' => $realPaths, 'latest' => true ] ); + } + + $mBackend = $this->backends[$this->masterIndex]; + foreach ( $paths as $path ) { + $params = [ 'src' => $path, 'latest' => true ]; + $mParams = $this->substOpPaths( $params, $mBackend ); + // Stat the file on the 'master' backend + $mStat = $mBackend->getFileStat( $mParams ); + if ( $this->syncChecks & self::CHECK_SHA1 ) { + $mSha1 = $mBackend->getFileSha1Base36( $mParams ); + } else { + $mSha1 = false; + } + // Check if all clone backends agree with the master... + foreach ( $this->backends as $index => $cBackend ) { + if ( $index === $this->masterIndex ) { + continue; // master + } + $cParams = $this->substOpPaths( $params, $cBackend ); + $cStat = $cBackend->getFileStat( $cParams ); + if ( $mStat ) { // file is in master + if ( !$cStat ) { // file should exist + $status->fatal( 'backend-fail-synced', $path ); + continue; + } + if ( $this->syncChecks & self::CHECK_SIZE ) { + if ( $cStat['size'] != $mStat['size'] ) { // wrong size + $status->fatal( 'backend-fail-synced', $path ); + continue; + } + } + if ( $this->syncChecks & self::CHECK_TIME ) { + $mTs = wfTimestamp( TS_UNIX, $mStat['mtime'] ); + $cTs = wfTimestamp( TS_UNIX, $cStat['mtime'] ); + if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere + $status->fatal( 'backend-fail-synced', $path ); + continue; + } + } + if ( $this->syncChecks & self::CHECK_SHA1 ) { + if ( $cBackend->getFileSha1Base36( $cParams ) !== $mSha1 ) { // wrong SHA1 + $status->fatal( 'backend-fail-synced', $path ); + continue; + } + } + } else { // file is not in master + if ( $cStat ) { // file should not exist + $status->fatal( 'backend-fail-synced', $path ); + } + } + } + } + + return $status; + } + + /** + * Check that a set of file paths are usable across all internal backends + * + * @param array $paths List of storage paths + * @return StatusValue + */ + public function accessibilityCheck( array $paths ) { + $status = $this->newStatus(); + if ( count( $this->backends ) <= 1 ) { + return $status; // skip checks + } + + foreach ( $paths as $path ) { + foreach ( $this->backends as $backend ) { + $realPath = $this->substPaths( $path, $backend ); + if ( !$backend->isPathUsableInternal( $realPath ) ) { + $status->fatal( 'backend-fail-usable', $path ); + } + } + } + + return $status; + } + + /** + * Check that a set of files are consistent across all internal backends + * and re-synchronize those files against the "multi master" if needed. + * + * @param array $paths List of storage paths + * @param string|bool $resyncMode False, True, or "conservative"; see __construct() + * @return StatusValue + */ + public function resyncFiles( array $paths, $resyncMode = true ) { + $status = $this->newStatus(); + + $mBackend = $this->backends[$this->masterIndex]; + foreach ( $paths as $path ) { + $mPath = $this->substPaths( $path, $mBackend ); + $mSha1 = $mBackend->getFileSha1Base36( [ 'src' => $mPath, 'latest' => true ] ); + $mStat = $mBackend->getFileStat( [ 'src' => $mPath, 'latest' => true ] ); + if ( $mStat === null || ( $mSha1 !== false && !$mStat ) ) { // sanity + $status->fatal( 'backend-fail-internal', $this->name ); + wfDebugLog( 'FileOperation', __METHOD__ + . ': File is not available on the master backend' ); + continue; // file is not available on the master backend... + } + // Check of all clone backends agree with the master... + foreach ( $this->backends as $index => $cBackend ) { + if ( $index === $this->masterIndex ) { + continue; // master + } + $cPath = $this->substPaths( $path, $cBackend ); + $cSha1 = $cBackend->getFileSha1Base36( [ 'src' => $cPath, 'latest' => true ] ); + $cStat = $cBackend->getFileStat( [ 'src' => $cPath, 'latest' => true ] ); + if ( $cStat === null || ( $cSha1 !== false && !$cStat ) ) { // sanity + $status->fatal( 'backend-fail-internal', $cBackend->getName() ); + wfDebugLog( 'FileOperation', __METHOD__ . + ': File is not available on the clone backend' ); + continue; // file is not available on the clone backend... + } + if ( $mSha1 === $cSha1 ) { + // already synced; nothing to do + } elseif ( $mSha1 !== false ) { // file is in master + if ( $resyncMode === 'conservative' + && $cStat && $cStat['mtime'] > $mStat['mtime'] + ) { + $status->fatal( 'backend-fail-synced', $path ); + continue; // don't rollback data + } + $fsFile = $mBackend->getLocalReference( + [ 'src' => $mPath, 'latest' => true ] ); + $status->merge( $cBackend->quickStore( + [ 'src' => $fsFile->getPath(), 'dst' => $cPath ] + ) ); + } elseif ( $mStat === false ) { // file is not in master + if ( $resyncMode === 'conservative' ) { + $status->fatal( 'backend-fail-synced', $path ); + continue; // don't delete data + } + $status->merge( $cBackend->quickDelete( [ 'src' => $cPath ] ) ); + } + } + } + + if ( !$status->isOK() ) { + wfDebugLog( 'FileOperation', get_class( $this ) . + " failed to resync: " . FormatJson::encode( $paths ) ); + } + + return $status; + } + + /** + * Get a list of file storage paths to read or write for a list of operations + * + * @param array $ops Same format as doOperations() + * @return array List of storage paths to files (does not include directories) + */ + protected function fileStoragePathsForOps( array $ops ) { + $paths = []; + foreach ( $ops as $op ) { + if ( isset( $op['src'] ) ) { + // For things like copy/move/delete with "ignoreMissingSource" and there + // is no source file, nothing should happen and there should be no errors. + if ( empty( $op['ignoreMissingSource'] ) + || $this->fileExists( [ 'src' => $op['src'] ] ) + ) { + $paths[] = $op['src']; + } + } + if ( isset( $op['srcs'] ) ) { + $paths = array_merge( $paths, $op['srcs'] ); + } + if ( isset( $op['dst'] ) ) { + $paths[] = $op['dst']; + } + } + + return array_values( array_unique( array_filter( $paths, 'FileBackend::isStoragePath' ) ) ); + } + + /** + * Substitute the backend name in storage path parameters + * for a set of operations with that of a given internal backend. + * + * @param array $ops List of file operation arrays + * @param FileBackendStore $backend + * @return array + */ + protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) { + $newOps = []; // operations + foreach ( $ops as $op ) { + $newOp = $op; // operation + foreach ( [ 'src', 'srcs', 'dst', 'dir' ] as $par ) { + if ( isset( $newOp[$par] ) ) { // string or array + $newOp[$par] = $this->substPaths( $newOp[$par], $backend ); + } + } + $newOps[] = $newOp; + } + + return $newOps; + } + + /** + * Same as substOpBatchPaths() but for a single operation + * + * @param array $ops File operation array + * @param FileBackendStore $backend + * @return array + */ + protected function substOpPaths( array $ops, FileBackendStore $backend ) { + $newOps = $this->substOpBatchPaths( [ $ops ], $backend ); + + return $newOps[0]; + } + + /** + * Substitute the backend of storage paths with an internal backend's name + * + * @param array|string $paths List of paths or single string path + * @param FileBackendStore $backend + * @return array|string + */ + protected function substPaths( $paths, FileBackendStore $backend ) { + return preg_replace( + '!^mwstore://' . preg_quote( $this->name, '!' ) . '/!', + StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ), + $paths // string or array + ); + } + + /** + * Substitute the backend of internal storage paths with the proxy backend's name + * + * @param array|string $paths List of paths or single string path + * @return array|string + */ + protected function unsubstPaths( $paths ) { + return preg_replace( + '!^mwstore://([^/]+)!', + StringUtils::escapeRegexReplacement( "mwstore://{$this->name}" ), + $paths // string or array + ); + } + + /** + * @param array $ops File operations for FileBackend::doOperations() + * @return bool Whether there are file path sources with outside lifetime/ownership + */ + protected function hasVolatileSources( array $ops ) { + foreach ( $ops as $op ) { + if ( $op['op'] === 'store' && !isset( $op['srcRef'] ) ) { + return true; // source file might be deleted anytime after do*Operations() + } + } + + return false; + } + + protected function doQuickOperationsInternal( array $ops ) { + $status = $this->newStatus(); + // Do the operations on the master backend; setting StatusValue fields... + $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] ); + $masterStatus = $this->backends[$this->masterIndex]->doQuickOperations( $realOps ); + $status->merge( $masterStatus ); + // Propagate the operations to the clone backends... + foreach ( $this->backends as $index => $backend ) { + if ( $index === $this->masterIndex ) { + continue; // done already + } + + $realOps = $this->substOpBatchPaths( $ops, $backend ); + if ( $this->asyncWrites && !$this->hasVolatileSources( $ops ) ) { + DeferredUpdates::addCallableUpdate( + function() use ( $backend, $realOps ) { + $backend->doQuickOperations( $realOps ); + } + ); + } else { + $status->merge( $backend->doQuickOperations( $realOps ) ); + } + } + // Make 'success', 'successCount', and 'failCount' fields reflect + // the overall operation, rather than all the batches for each backend. + // Do this by only using success values from the master backend's batch. + $status->success = $masterStatus->success; + $status->successCount = $masterStatus->successCount; + $status->failCount = $masterStatus->failCount; + + return $status; + } + + protected function doPrepare( array $params ) { + return $this->doDirectoryOp( 'prepare', $params ); + } + + protected function doSecure( array $params ) { + return $this->doDirectoryOp( 'secure', $params ); + } + + protected function doPublish( array $params ) { + return $this->doDirectoryOp( 'publish', $params ); + } + + protected function doClean( array $params ) { + return $this->doDirectoryOp( 'clean', $params ); + } + + /** + * @param string $method One of (doPrepare,doSecure,doPublish,doClean) + * @param array $params Method arguments + * @return StatusValue + */ + protected function doDirectoryOp( $method, array $params ) { + $status = $this->newStatus(); + + $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + $masterStatus = $this->backends[$this->masterIndex]->$method( $realParams ); + $status->merge( $masterStatus ); + + foreach ( $this->backends as $index => $backend ) { + if ( $index === $this->masterIndex ) { + continue; // already done + } + + $realParams = $this->substOpPaths( $params, $backend ); + if ( $this->asyncWrites ) { + DeferredUpdates::addCallableUpdate( + function() use ( $backend, $method, $realParams ) { + $backend->$method( $realParams ); + } + ); + } else { + $status->merge( $backend->$method( $realParams ) ); + } + } + + return $status; + } + + public function concatenate( array $params ) { + // We are writing to an FS file, so we don't need to do this per-backend + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + return $this->backends[$index]->concatenate( $realParams ); + } + + public function fileExists( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + return $this->backends[$index]->fileExists( $realParams ); + } + + public function getFileTimestamp( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + return $this->backends[$index]->getFileTimestamp( $realParams ); + } + + public function getFileSize( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + return $this->backends[$index]->getFileSize( $realParams ); + } + + public function getFileStat( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + return $this->backends[$index]->getFileStat( $realParams ); + } + + public function getFileXAttributes( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + return $this->backends[$index]->getFileXAttributes( $realParams ); + } + + public function getFileContentsMulti( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + $contentsM = $this->backends[$index]->getFileContentsMulti( $realParams ); + + $contents = []; // (path => FSFile) mapping using the proxy backend's name + foreach ( $contentsM as $path => $data ) { + $contents[$this->unsubstPaths( $path )] = $data; + } + + return $contents; + } + + public function getFileSha1Base36( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + return $this->backends[$index]->getFileSha1Base36( $realParams ); + } + + public function getFileProps( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + return $this->backends[$index]->getFileProps( $realParams ); + } + + public function streamFile( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + return $this->backends[$index]->streamFile( $realParams ); + } + + public function getLocalReferenceMulti( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + $fsFilesM = $this->backends[$index]->getLocalReferenceMulti( $realParams ); + + $fsFiles = []; // (path => FSFile) mapping using the proxy backend's name + foreach ( $fsFilesM as $path => $fsFile ) { + $fsFiles[$this->unsubstPaths( $path )] = $fsFile; + } + + return $fsFiles; + } + + public function getLocalCopyMulti( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + $tempFilesM = $this->backends[$index]->getLocalCopyMulti( $realParams ); + + $tempFiles = []; // (path => TempFSFile) mapping using the proxy backend's name + foreach ( $tempFilesM as $path => $tempFile ) { + $tempFiles[$this->unsubstPaths( $path )] = $tempFile; + } + + return $tempFiles; + } + + public function getFileHttpUrl( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + return $this->backends[$index]->getFileHttpUrl( $realParams ); + } + + public function directoryExists( array $params ) { + $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + + return $this->backends[$this->masterIndex]->directoryExists( $realParams ); + } + + public function getDirectoryList( array $params ) { + $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + + return $this->backends[$this->masterIndex]->getDirectoryList( $realParams ); + } + + public function getFileList( array $params ) { + $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + + return $this->backends[$this->masterIndex]->getFileList( $realParams ); + } + + public function getFeatures() { + return $this->backends[$this->masterIndex]->getFeatures(); + } + + public function clearCache( array $paths = null ) { + foreach ( $this->backends as $backend ) { + $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null; + $backend->clearCache( $realPaths ); + } + } + + public function preloadCache( array $paths ) { + $realPaths = $this->substPaths( $paths, $this->backends[$this->readIndex] ); + $this->backends[$this->readIndex]->preloadCache( $realPaths ); + } + + public function preloadFileStat( array $params ) { + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + return $this->backends[$index]->preloadFileStat( $realParams ); + } + + public function getScopedLocksForOps( array $ops, StatusValue $status ) { + $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] ); + $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $realOps ); + // Get the paths to lock from the master backend + $paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps ); + // Get the paths under the proxy backend's name + $pbPaths = [ + LockManager::LOCK_UW => $this->unsubstPaths( $paths[LockManager::LOCK_UW] ), + LockManager::LOCK_EX => $this->unsubstPaths( $paths[LockManager::LOCK_EX] ) + ]; + + // Actually acquire the locks + return $this->getScopedFileLocks( $pbPaths, 'mixed', $status ); + } + + /** + * @param array $params + * @return int The master or read affinity backend index, based on $params['latest'] + */ + protected function getReadIndexFromParams( array $params ) { + return !empty( $params['latest'] ) ? $this->masterIndex : $this->readIndex; + } +} diff --git a/resources/src/mediawiki/mediawiki.js b/resources/src/mediawiki/mediawiki.js index 2c7343f81d..3122d42c4f 100644 --- a/resources/src/mediawiki/mediawiki.js +++ b/resources/src/mediawiki/mediawiki.js @@ -11,7 +11,7 @@ ( function ( $ ) { 'use strict'; - var mw, StringSet, + var mw, StringSet, log, hasOwn = Object.prototype.hasOwnProperty, slice = Array.prototype.slice, trackCallbacks = $.Callbacks( 'memory' ), @@ -433,6 +433,86 @@ } }; + log = ( function () { + // Also update the restoration of methods in mediawiki.log.js + // when adding or removing methods here. + var log = function () {}, + console = window.console; + + /** + * @class mw.log + * @singleton + */ + + /** + * Write a message to the console's warning channel. + * Actions not supported by the browser console are silently ignored. + * + * @param {...string} msg Messages to output to console + */ + log.warn = console && console.warn && Function.prototype.bind ? + Function.prototype.bind.call( console.warn, console ) : + $.noop; + + /** + * Write a message to the console's error channel. + * + * Most browsers provide a stacktrace by default if the argument + * is a caught Error object. + * + * @since 1.26 + * @param {Error|...string} msg Messages to output to console + */ + log.error = console && console.error && Function.prototype.bind ? + Function.prototype.bind.call( console.error, console ) : + $.noop; + + /** + * Create a property in a host object that, when accessed, will produce + * a deprecation warning in the console. + * + * @param {Object} obj Host object of deprecated property + * @param {string} key Name of property to create in `obj` + * @param {Mixed} val The value this property should return when accessed + * @param {string} [msg] Optional text to include in the deprecation message + */ + log.deprecate = !Object.defineProperty ? function ( obj, key, val ) { + obj[ key ] = val; + } : function ( obj, key, val, msg ) { + msg = 'Use of "' + key + '" is deprecated.' + ( msg ? ( ' ' + msg ) : '' ); + var logged = new StringSet(); + function uniqueTrace() { + var trace = new Error().stack; + if ( logged.has( trace ) ) { + return false; + } + logged.add( trace ); + return true; + } + Object.defineProperty( obj, key, { + configurable: true, + enumerable: true, + get: function () { + if ( uniqueTrace() ) { + mw.track( 'mw.deprecate', key ); + mw.log.warn( msg ); + } + return val; + }, + set: function ( newVal ) { + if ( uniqueTrace() ) { + mw.track( 'mw.deprecate', key ); + mw.log.warn( msg ); + } + val = newVal; + } + } ); + + }; + + return log; + }() ); + /** * @class mw */ @@ -627,89 +707,11 @@ }, /** - * Dummy placeholder for {@link mw.log} + * No-op dummy placeholder for {@link mw.log} in debug mode. * * @method */ - log: ( function () { - // Also update the restoration of methods in mediawiki.log.js - // when adding or removing methods here. - var log = function () {}, - console = window.console; - - /** - * @class mw.log - * @singleton - */ - - /** - * Write a message to the console's warning channel. - * Actions not supported by the browser console are silently ignored. - * - * @param {...string} msg Messages to output to console - */ - log.warn = console && console.warn && Function.prototype.bind ? - Function.prototype.bind.call( console.warn, console ) : - $.noop; - - /** - * Write a message to the console's error channel. - * - * Most browsers provide a stacktrace by default if the argument - * is a caught Error object. - * - * @since 1.26 - * @param {Error|...string} msg Messages to output to console - */ - log.error = console && console.error && Function.prototype.bind ? - Function.prototype.bind.call( console.error, console ) : - $.noop; - - /** - * Create a property in a host object that, when accessed, will produce - * a deprecation warning in the console with backtrace. - * - * @param {Object} obj Host object of deprecated property - * @param {string} key Name of property to create in `obj` - * @param {Mixed} val The value this property should return when accessed - * @param {string} [msg] Optional text to include in the deprecation message - */ - log.deprecate = !Object.defineProperty ? function ( obj, key, val ) { - obj[ key ] = val; - } : function ( obj, key, val, msg ) { - msg = 'Use of "' + key + '" is deprecated.' + ( msg ? ( ' ' + msg ) : '' ); - var logged = new StringSet(); - function uniqueTrace() { - var trace = new Error().stack; - if ( logged.has( trace ) ) { - return false; - } - logged.add( trace ); - return true; - } - Object.defineProperty( obj, key, { - configurable: true, - enumerable: true, - get: function () { - if ( uniqueTrace() ) { - mw.track( 'mw.deprecate', key ); - mw.log.warn( msg ); - } - return val; - }, - set: function ( newVal ) { - if ( uniqueTrace() ) { - mw.track( 'mw.deprecate', key ); - mw.log.warn( msg ); - } - val = newVal; - } - } ); - - }; - - return log; - }() ), + log: log, /** * Client for ResourceLoader server end point. @@ -1338,8 +1340,8 @@ markModuleReady(); } } catch ( e ) { - // This needs to NOT use mw.log because these errors are common in production mode - // and not in debug mode, such as when a symbol that should be global isn't exported + // Use mw.track instead of mw.log because these errors are common in production mode + // (e.g. undefined variable), and mw.log is only enabled in debug mode. registry[ module ].state = 'error'; mw.track( 'resourceloader.exception', { exception: e, module: module, source: 'module-execute' } ); handlePending( module ); @@ -1670,34 +1672,6 @@ } } - /** - * Evaluate a batch of load.php responses retrieved from mw.loader.store. - * - * @private - * @param {string[]} implementations Array containing pieces of JavaScript code in the - * form of calls to mw.loader#implement(). - * @param {Function} cb Callback in case of failure - * @param {Error} cb.err - */ - function batchEval( implementations, cb ) { - if ( !implementations.length ) { - return; - } - mw.requestIdleCallback( function iterate( deadline ) { - while ( implementations[ 0 ] && deadline.timeRemaining() > 5 ) { - try { - $.globalEval( implementations.shift() ); - } catch ( err ) { - cb( err ); - return; - } - } - if ( implementations[ 0 ] ) { - mw.requestIdleCallback( iterate ); - } - } ); - } - /* Public Members */ return { /** @@ -1722,7 +1696,7 @@ * @protected */ work: function () { - var q, batch, implementations, sourceModules; + var q, batch, concatSource, origBatch; batch = []; @@ -1752,35 +1726,39 @@ mw.loader.store.init(); if ( mw.loader.store.enabled ) { - implementations = []; - sourceModules = []; + concatSource = []; + origBatch = batch; batch = $.grep( batch, function ( module ) { - var implementation = mw.loader.store.get( module ); - if ( implementation ) { - implementations.push( implementation ); - sourceModules.push( module ); + var source = mw.loader.store.get( module ); + if ( source ) { + concatSource.push( source ); return false; } return true; } ); - batchEval( implementations, function ( err ) { + try { + $.globalEval( concatSource.join( ';' ) ); + } catch ( err ) { // Not good, the cached mw.loader.implement calls failed! This should // never happen, barring ResourceLoader bugs, browser bugs and PEBKACs. // Depending on how corrupt the string is, it is likely that some // modules' implement() succeeded while the ones after the error will // never run and leave their modules in the 'loading' state forever. + // Since this is an error not caused by an individual module but by // something that infected the implement call itself, don't take any // risks and clear everything in this cache. mw.loader.store.clear(); + // Re-add the ones still pending back to the batch and let the server + // repopulate these modules to the cache. + // This means that at most one module will be useless (the one that had + // the error) instead of all of them. mw.track( 'resourceloader.exception', { exception: err, source: 'store-eval' } ); - - // Re-add the failed ones that are still pending back to the batch - var failed = $.grep( sourceModules, function ( module ) { + origBatch = $.grep( origBatch, function ( module ) { return registry[ module ].state === 'loading'; } ); - batchRequest( failed ); - } ); + batch = batch.concat( origBatch ); + } } batchRequest( batch ); @@ -2679,14 +2657,13 @@ * reference, so that debugging tools loaded later are supported (e.g. Firebug Lite in IE). * * @private - * @method log_ * @param {string} topic Stream name passed by mw.track * @param {Object} data Data passed by mw.track * @param {Error} [data.exception] * @param {string} data.source Error source * @param {string} [data.module] Name of module which caused the error */ - function log( topic, data ) { + function logError( topic, data ) { var msg, e = data.exception, source = data.source, @@ -2711,8 +2688,8 @@ } // Subscribe to error streams - mw.trackSubscribe( 'resourceloader.exception', log ); - mw.trackSubscribe( 'resourceloader.assert', log ); + mw.trackSubscribe( 'resourceloader.exception', logError ); + mw.trackSubscribe( 'resourceloader.assert', logError ); /** * Fired when all modules associated with the page have finished loading. diff --git a/resources/src/mediawiki/mediawiki.log.js b/resources/src/mediawiki/mediawiki.log.js index 93fb470aa9..c886817799 100644 --- a/resources/src/mediawiki/mediawiki.log.js +++ b/resources/src/mediawiki/mediawiki.log.js @@ -8,9 +8,8 @@ ( function ( mw, $ ) { - // Reference to dummy - // We don't need the dummy, but it has other methods on it - // that we need to restore afterwards. + // Keep reference to the dummy placeholder from mediawiki.js + // The root is replaced below, but it has other methods that we need to restore. var original = mw.log, slice = Array.prototype.slice;