includes/filerepo/backend/FileBackendMultiWrite.php

   1 <?php
   2 /**
   3  * Proxy backend that mirrors writes to several internal backends.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup FileBackend
  22  * @author Aaron Schulz
  23  */
  24
  25 /**
  26  * @brief Proxy backend that mirrors writes to several internal backends.
  27  *
  28  * This class defines a multi-write backend. Multiple backends can be
  29  * registered to this proxy backend and it will act as a single backend.
  30  * Use this when all access to those backends is through this proxy backend.
  31  * At least one of the backends must be declared the "master" backend.
  32  *
  33  * Only use this class when transitioning from one storage system to another.
  34  *
  35  * Read operations are only done on the 'master' backend for consistency.
  36  * Write operations are performed on all backends, in the order defined.
  37  * If an operation fails on one backend it will be rolled back from the others.
  38  *
  39  * @ingroup FileBackend
  40  * @since 1.19
  41  */
  42 class FileBackendMultiWrite extends FileBackend {
  43         /** @var Array Prioritized list of FileBackendStore objects */
  44         protected $backends = array(); // array of (backend index => backends)
  45         protected $masterIndex = -1; // integer; index of master backend
  46         protected $syncChecks = 0; // integer bitfield
  47
  48         /* Possible internal backend consistency checks */
  49         const CHECK_SIZE = 1;
  50         const CHECK_TIME = 2;
  51
  52         /**
  53          * Construct a proxy backend that consists of several internal backends.
  54          * Additional $config params include:
  55          *     'backends'    : Array of backend config and multi-backend settings.
  56          *                     Each value is the config used in the constructor of a
  57          *                     FileBackendStore class, but with these additional settings:
  58          *                         'class'         : The name of the backend class
  59          *                         'isMultiMaster' : This must be set for one backend.
  60          *                         'template:      : If given a backend name, this will use
  61          *                                           the config of that backend as a template.
  62          *                                           Values specified here take precedence.
  63          *     'syncChecks'  : Integer bitfield of internal backend sync checks to perform.
  64          *                     Possible bits include self::CHECK_SIZE and self::CHECK_TIME.
  65          *                     The checks are done before allowing any file operations.
  66          * @param $config Array
  67          */
  68         public function __construct( array $config ) {
  69                 parent::__construct( $config );
  70                 $namesUsed = array();
  71                 // Construct backends here rather than via registration
  72                 // to keep these backends hidden from outside the proxy.
  73                 foreach ( $config['backends'] as $index => $config ) {
  74                         if ( isset( $config['template'] ) ) {
  75                                 // Config is just a modified version of a registered backend's.
  76                                 // This should only be used when that config is used only be this backend.
  77                                 $config = $config + FileBackendGroup::singleton()->config( $config['template'] );
  78                         }
  79                         $name = $config['name'];
  80                         if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
  81                                 throw new MWException( "Two or more backends defined with the name $name." );
  82                         }
  83                         $namesUsed[$name] = 1;
  84                         if ( !isset( $config['class'] ) ) {
  85                                 throw new MWException( 'No class given for a backend config.' );
  86                         }
  87                         $class = $config['class'];
  88                         $this->backends[$index] = new $class( $config );
  89                         if ( !empty( $config['isMultiMaster'] ) ) {
  90                                 if ( $this->masterIndex >= 0 ) {
  91                                         throw new MWException( 'More than one master backend defined.' );
  92                                 }
  93                                 $this->masterIndex = $index;
  94                         }
  95                 }
  96                 if ( $this->masterIndex < 0 ) { // need backends and must have a master
  97                         throw new MWException( 'No master backend defined.' );
  98                 }
  99                 $this->syncChecks = isset( $config['syncChecks'] )
 100                         ? $config['syncChecks']
 101                         : self::CHECK_SIZE;
 102         }
 103
 104         /**
 105          * @see FileBackend::doOperationsInternal()
 106          * @return Status
 107          */
 108         final protected function doOperationsInternal( array $ops, array $opts ) {
 109                 $status = Status::newGood();
 110
 111                 $performOps = array(); // list of FileOp objects
 112                 $paths = array(); // storage paths read from or written to
 113                 // Build up a list of FileOps. The list will have all the ops
 114                 // for one backend, then all the ops for the next, and so on.
 115                 // These batches of ops are all part of a continuous array.
 116                 // Also build up a list of files read/changed...
 117                 foreach ( $this->backends as $index => $backend ) {
 118                         $backendOps = $this->substOpBatchPaths( $ops, $backend );
 119                         // Add on the operation batch for this backend
 120                         $performOps = array_merge( $performOps,
 121                                 $backend->getOperationsInternal( $backendOps ) );
 122                         if ( $index == 0 ) { // first batch
 123                                 // Get the files used for these operations. Each backend has a batch of
 124                                 // the same operations, so we only need to get them from the first batch.
 125                                 $paths = $backend->getPathsToLockForOpsInternal( $performOps );
 126                                 // Get the paths under the proxy backend's name
 127                                 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
 128                                 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
 129                         }
 130                 }
 131
 132                 // Try to lock those files for the scope of this function...
 133                 if ( empty( $opts['nonLocking'] ) ) {
 134                         // Try to lock those files for the scope of this function...
 135                         $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
 136                         $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
 137                         if ( !$status->isOK() ) {
 138                                 return $status; // abort
 139                         }
 140                 }
 141
 142                 // Clear any cache entries (after locks acquired)
 143                 $this->clearCache();
 144
 145                 // Do a consistency check to see if the backends agree
 146                 if ( count( $this->backends ) > 1 ) {
 147                         $status->merge( $this->consistencyCheck( array_merge( $paths['sh'], $paths['ex'] ) ) );
 148                         if ( !$status->isOK() ) {
 149                                 return $status; // abort
 150                         }
 151                 }
 152
 153                 // Actually attempt the operation batch...
 154                 $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
 155
 156                 $success = array();
 157                 $failCount = 0;
 158                 $successCount = 0;
 159                 // Make 'success', 'successCount', and 'failCount' fields reflect
 160                 // the overall operation, rather than all the batches for each backend.
 161                 // Do this by only using success values from the master backend's batch.
 162                 $batchStart = $this->masterIndex * count( $ops );
 163                 $batchEnd = $batchStart + count( $ops ) - 1;
 164                 for ( $i = $batchStart; $i <= $batchEnd; $i++ ) {
 165                         if ( !isset( $subStatus->success[$i] ) ) {
 166                                 break; // failed out before trying this op
 167                         } elseif ( $subStatus->success[$i] ) {
 168                                 ++$successCount;
 169                         } else {
 170                                 ++$failCount;
 171                         }
 172                         $success[] = $subStatus->success[$i];
 173                 }
 174                 $subStatus->success = $success;
 175                 $subStatus->successCount = $successCount;
 176                 $subStatus->failCount = $failCount;
 177
 178                 // Merge errors into status fields
 179                 $status->merge( $subStatus );
 180                 $status->success = $subStatus->success; // not done in merge()
 181
 182                 return $status;
 183         }
 184
 185         /**
 186          * Check that a set of files are consistent across all internal backends
 187          *
 188          * @param $paths Array
 189          * @return Status
 190          */
 191         public function consistencyCheck( array $paths ) {
 192                 $status = Status::newGood();
 193                 if ( $this->syncChecks == 0 ) {
 194                         return $status; // skip checks
 195                 }
 196
 197                 $mBackend = $this->backends[$this->masterIndex];
 198                 foreach ( array_unique( $paths ) as $path ) {
 199                         $params = array( 'src' => $path, 'latest' => true );
 200                         // Stat the file on the 'master' backend
 201                         $mStat = $mBackend->getFileStat( $this->substOpPaths( $params, $mBackend ) );
 202                         // Check of all clone backends agree with the master...
 203                         foreach ( $this->backends as $index => $cBackend ) {
 204                                 if ( $index === $this->masterIndex ) {
 205                                         continue; // master
 206                                 }
 207                                 $cStat = $cBackend->getFileStat( $this->substOpPaths( $params, $cBackend ) );
 208                                 if ( $mStat ) { // file is in master
 209                                         if ( !$cStat ) { // file should exist
 210                                                 $status->fatal( 'backend-fail-synced', $path );
 211                                                 continue;
 212                                         }
 213                                         if ( $this->syncChecks & self::CHECK_SIZE ) {
 214                                                 if ( $cStat['size'] != $mStat['size'] ) { // wrong size
 215                                                         $status->fatal( 'backend-fail-synced', $path );
 216                                                         continue;
 217                                                 }
 218                                         }
 219                                         if ( $this->syncChecks & self::CHECK_TIME ) {
 220                                                 $mTs = wfTimestamp( TS_UNIX, $mStat['mtime'] );
 221                                                 $cTs = wfTimestamp( TS_UNIX, $cStat['mtime'] );
 222                                                 if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
 223                                                         $status->fatal( 'backend-fail-synced', $path );
 224                                                         continue;
 225                                                 }
 226                                         }
 227                                 } else { // file is not in master
 228                                         if ( $cStat ) { // file should not exist
 229                                                 $status->fatal( 'backend-fail-synced', $path );
 230                                         }
 231                                 }
 232                         }
 233                 }
 234
 235                 return $status;
 236         }
 237
 238         /**
 239          * Substitute the backend name in storage path parameters
 240          * for a set of operations with that of a given internal backend.
 241          *
 242          * @param $ops Array List of file operation arrays
 243          * @param $backend FileBackendStore
 244          * @return Array
 245          */
 246         protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
 247                 $newOps = array(); // operations
 248                 foreach ( $ops as $op ) {
 249                         $newOp = $op; // operation
 250                         foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
 251                                 if ( isset( $newOp[$par] ) ) { // string or array
 252                                         $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
 253                                 }
 254                         }
 255                         $newOps[] = $newOp;
 256                 }
 257                 return $newOps;
 258         }
 259
 260         /**
 261          * Same as substOpBatchPaths() but for a single operation
 262          *
 263          * @param $op File operation array
 264          * @param $backend FileBackendStore
 265          * @return Array
 266          */
 267         protected function substOpPaths( array $ops, FileBackendStore $backend ) {
 268                 $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
 269                 return $newOps[0];
 270         }
 271
 272         /**
 273          * Substitute the backend of storage paths with an internal backend's name
 274          *
 275          * @param $paths Array|string List of paths or single string path
 276          * @param $backend FileBackendStore
 277          * @return Array|string
 278          */
 279         protected function substPaths( $paths, FileBackendStore $backend ) {
 280                 return preg_replace(
 281                         '!^mwstore://' . preg_quote( $this->name ) . '/!',
 282                         StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
 283                         $paths // string or array
 284                 );
 285         }
 286
 287         /**
 288          * Substitute the backend of internal storage paths with the proxy backend's name
 289          *
 290          * @param $paths Array|string List of paths or single string path
 291          * @return Array|string
 292          */
 293         protected function unsubstPaths( $paths ) {
 294                 return preg_replace(
 295                         '!^mwstore://([^/]+)!',
 296                         StringUtils::escapeRegexReplacement( "mwstore://{$this->name}" ),
 297                         $paths // string or array
 298                 );
 299         }
 300
 301         /**
 302          * @see FileBackend::doPrepare()
 303          * @return Status
 304          */
 305         protected function doPrepare( array $params ) {
 306                 $status = Status::newGood();
 307                 foreach ( $this->backends as $backend ) {
 308                         $realParams = $this->substOpPaths( $params, $backend );
 309                         $status->merge( $backend->doPrepare( $realParams ) );
 310                 }
 311                 return $status;
 312         }
 313
 314         /**
 315          * @see FileBackend::doSecure()
 316          * @return Status
 317          */
 318         protected function doSecure( array $params ) {
 319                 $status = Status::newGood();
 320                 foreach ( $this->backends as $backend ) {
 321                         $realParams = $this->substOpPaths( $params, $backend );
 322                         $status->merge( $backend->doSecure( $realParams ) );
 323                 }
 324                 return $status;
 325         }
 326
 327         /**
 328          * @see FileBackend::doClean()
 329          * @return Status
 330          */
 331         protected function doClean( array $params ) {
 332                 $status = Status::newGood();
 333                 foreach ( $this->backends as $backend ) {
 334                         $realParams = $this->substOpPaths( $params, $backend );
 335                         $status->merge( $backend->doClean( $realParams ) );
 336                 }
 337                 return $status;
 338         }
 339
 340         /**
 341          * @see FileBackend::concatenate()
 342          */
 343         public function concatenate( array $params ) {
 344                 // We are writing to an FS file, so we don't need to do this per-backend
 345                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 346                 return $this->backends[$this->masterIndex]->concatenate( $realParams );
 347         }
 348
 349         /**
 350          * @see FileBackend::fileExists()
 351          */
 352         public function fileExists( array $params ) {
 353                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 354                 return $this->backends[$this->masterIndex]->fileExists( $realParams );
 355         }
 356
 357         /**
 358          * @see FileBackend::getFileTimestamp()
 359          */
 360         public function getFileTimestamp( array $params ) {
 361                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 362                 return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
 363         }
 364
 365         /**
 366          * @see FileBackend::getFileSize()
 367          */
 368         public function getFileSize( array $params ) {
 369                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 370                 return $this->backends[$this->masterIndex]->getFileSize( $realParams );
 371         }
 372
 373         /**
 374          * @see FileBackend::getFileStat()
 375          */
 376         public function getFileStat( array $params ) {
 377                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 378                 return $this->backends[$this->masterIndex]->getFileStat( $realParams );
 379         }
 380
 381         /**
 382          * @see FileBackend::getFileContents()
 383          */
 384         public function getFileContents( array $params ) {
 385                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 386                 return $this->backends[$this->masterIndex]->getFileContents( $realParams );
 387         }
 388
 389         /**
 390          * @see FileBackend::getFileSha1Base36()
 391          */
 392         public function getFileSha1Base36( array $params ) {
 393                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 394                 return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams );
 395         }
 396
 397         /**
 398          * @see FileBackend::getFileProps()
 399          */
 400         public function getFileProps( array $params ) {
 401                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 402                 return $this->backends[$this->masterIndex]->getFileProps( $realParams );
 403         }
 404
 405         /**
 406          * @see FileBackend::streamFile()
 407          */
 408         public function streamFile( array $params ) {
 409                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 410                 return $this->backends[$this->masterIndex]->streamFile( $realParams );
 411         }
 412
 413         /**
 414          * @see FileBackend::getLocalReference()
 415          */
 416         public function getLocalReference( array $params ) {
 417                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 418                 return $this->backends[$this->masterIndex]->getLocalReference( $realParams );
 419         }
 420
 421         /**
 422          * @see FileBackend::getLocalCopy()
 423          */
 424         public function getLocalCopy( array $params ) {
 425                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 426                 return $this->backends[$this->masterIndex]->getLocalCopy( $realParams );
 427         }
 428
 429         /**
 430          * @see FileBackend::directoryExists()
 431          */
 432         public function directoryExists( array $params ) {
 433                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 434                 return $this->backends[$this->masterIndex]->directoryExists( $realParams );
 435         }
 436
 437         /**
 438          * @see FileBackend::getSubdirectoryList()
 439          */
 440         public function getDirectoryList( array $params ) {
 441                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 442                 return $this->backends[$this->masterIndex]->getDirectoryList( $realParams );
 443         }
 444
 445         /**
 446          * @see FileBackend::getFileList()
 447          */
 448         public function getFileList( array $params ) {
 449                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 450                 return $this->backends[$this->masterIndex]->getFileList( $realParams );
 451         }
 452
 453         /**
 454          * @see FileBackend::clearCache()
 455          */
 456         public function clearCache( array $paths = null ) {
 457                 foreach ( $this->backends as $backend ) {
 458                         $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
 459                         $backend->clearCache( $realPaths );
 460                 }
 461         }
 462 }