From bd6ec3bc78d6321e1a248eaee70b1ac541ed957a Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Sat, 19 Sep 2015 16:05:37 -0700 Subject: [PATCH] Added readAffinity flag to FileBackendMultiWrite * A backend with this set (normally 1) will be used for non-latest reads. This can be used to prefer a local, replicated, backend instead of one farther away (for multi-DC setups) * Note that listings still come from the master always. Bug: T112708 Change-Id: Ic4bf4ba5a2c9ef78abd11dbd0d4b48c73cad6923 --- .../filebackend/FileBackendMultiWrite.php | 95 +++++++++++++------ 1 file changed, 65 insertions(+), 30 deletions(-) diff --git a/includes/filebackend/FileBackendMultiWrite.php b/includes/filebackend/FileBackendMultiWrite.php index 6c6a90b375..c182959b4c 100644 --- a/includes/filebackend/FileBackendMultiWrite.php +++ b/includes/filebackend/FileBackendMultiWrite.php @@ -45,6 +45,8 @@ class FileBackendMultiWrite extends FileBackend { /** @var int Index of master backend */ protected $masterIndex = -1; + /** @var int Index of read affinity backend */ + protected $readIndex = -1; /** @var int Bitfield */ protected $syncChecks = 0; @@ -73,6 +75,7 @@ class FileBackendMultiWrite extends FileBackend { * FileBackendStore class, but with these additional settings: * - class : The name of the backend class * - isMultiMaster : This must be set for one backend. + * - readAffinity : Use this for reads without 'latest' set. * - template: : If given a backend name, this will use * the config of that backend as a template. * Values specified here take precedence. @@ -132,6 +135,9 @@ class FileBackendMultiWrite extends FileBackend { $this->masterIndex = $index; // this is the "master" $config['fileJournal'] = $this->fileJournal; // log under proxy backend } + if ( !empty( $config['readAffinity'] ) ) { + $this->readIndex = $index; // prefer this for reads + } // Create sub-backend object if ( !isset( $config['class'] ) ) { throw new FileBackendError( 'No class given for a backend config.' ); @@ -142,6 +148,9 @@ class FileBackendMultiWrite extends FileBackend { if ( $this->masterIndex < 0 ) { // need backends and must have a master throw new FileBackendError( 'No master backend defined.' ); } + if ( $this->readIndex < 0 ) { + $this->readIndex = $this->masterIndex; // default + } } final protected function doOperationsInternal( array $ops, array $opts ) { @@ -540,44 +549,52 @@ class FileBackendMultiWrite extends FileBackend { public function concatenate( array $params ) { // We are writing to an FS file, so we don't need to do this per-backend - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - return $this->backends[$this->masterIndex]->concatenate( $realParams ); + return $this->backends[$index]->concatenate( $realParams ); } public function fileExists( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - return $this->backends[$this->masterIndex]->fileExists( $realParams ); + return $this->backends[$index]->fileExists( $realParams ); } public function getFileTimestamp( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams ); + return $this->backends[$index]->getFileTimestamp( $realParams ); } public function getFileSize( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - return $this->backends[$this->masterIndex]->getFileSize( $realParams ); + return $this->backends[$index]->getFileSize( $realParams ); } public function getFileStat( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - return $this->backends[$this->masterIndex]->getFileStat( $realParams ); + return $this->backends[$index]->getFileStat( $realParams ); } public function getFileXAttributes( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - return $this->backends[$this->masterIndex]->getFileXAttributes( $realParams ); + return $this->backends[$index]->getFileXAttributes( $realParams ); } public function getFileContentsMulti( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - $contentsM = $this->backends[$this->masterIndex]->getFileContentsMulti( $realParams ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + $contentsM = $this->backends[$index]->getFileContentsMulti( $realParams ); $contents = array(); // (path => FSFile) mapping using the proxy backend's name foreach ( $contentsM as $path => $data ) { @@ -588,26 +605,31 @@ class FileBackendMultiWrite extends FileBackend { } public function getFileSha1Base36( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams ); + return $this->backends[$index]->getFileSha1Base36( $realParams ); } public function getFileProps( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - return $this->backends[$this->masterIndex]->getFileProps( $realParams ); + return $this->backends[$index]->getFileProps( $realParams ); } public function streamFile( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - return $this->backends[$this->masterIndex]->streamFile( $realParams ); + return $this->backends[$index]->streamFile( $realParams ); } public function getLocalReferenceMulti( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - $fsFilesM = $this->backends[$this->masterIndex]->getLocalReferenceMulti( $realParams ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + $fsFilesM = $this->backends[$index]->getLocalReferenceMulti( $realParams ); $fsFiles = array(); // (path => FSFile) mapping using the proxy backend's name foreach ( $fsFilesM as $path => $fsFile ) { @@ -618,8 +640,10 @@ class FileBackendMultiWrite extends FileBackend { } public function getLocalCopyMulti( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - $tempFilesM = $this->backends[$this->masterIndex]->getLocalCopyMulti( $realParams ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + $tempFilesM = $this->backends[$index]->getLocalCopyMulti( $realParams ); $tempFiles = array(); // (path => TempFSFile) mapping using the proxy backend's name foreach ( $tempFilesM as $path => $tempFile ) { @@ -630,9 +654,10 @@ class FileBackendMultiWrite extends FileBackend { } public function getFileHttpUrl( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); - return $this->backends[$this->masterIndex]->getFileHttpUrl( $realParams ); + return $this->backends[$index]->getFileHttpUrl( $realParams ); } public function directoryExists( array $params ) { @@ -665,13 +690,15 @@ class FileBackendMultiWrite extends FileBackend { } public function preloadCache( array $paths ) { - $realPaths = $this->substPaths( $paths, $this->backends[$this->masterIndex] ); - $this->backends[$this->masterIndex]->preloadCache( $realPaths ); + $realPaths = $this->substPaths( $paths, $this->backends[$this->readIndex] ); + $this->backends[$this->readIndex]->preloadCache( $realPaths ); } public function preloadFileStat( array $params ) { - $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - return $this->backends[$this->masterIndex]->preloadFileStat( $realParams ); + $index = $this->getReadIndexFromParams( $params ); + $realParams = $this->substOpPaths( $params, $this->backends[$index] ); + + return $this->backends[$index]->preloadFileStat( $realParams ); } public function getScopedLocksForOps( array $ops, Status $status ) { @@ -688,4 +715,12 @@ class FileBackendMultiWrite extends FileBackend { // Actually acquire the locks return $this->getScopedFileLocks( $pbPaths, 'mixed', $status ); } + + /** + * @param array $params + * @return int The master or read affinity backend index, based on $params['latest'] + */ + protected function getReadIndexFromParams( array $params ) { + return !empty( $params['latest'] ) ? $this->masterIndex : $this->readIndex; + } } -- 2.20.1