includes/filerepo/backend/FileBackendMultiWrite.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Aaron Schulz
   6  */
   7
   8 /**
   9  * @brief Proxy backend that mirrors writes to several internal backends.
  10  *
  11  * This class defines a multi-write backend. Multiple backends can be
  12  * registered to this proxy backend and it will act as a single backend.
  13  * Use this when all access to those backends is through this proxy backend.
  14  * At least one of the backends must be declared the "master" backend.
  15  *
  16  * Only use this class when transitioning from one storage system to another.
  17  *
  18  * Read operations are only done on the 'master' backend for consistency.
  19  * Write operations are performed on all backends, in the order defined.
  20  * If an operation fails on one backend it will be rolled back from the others.
  21  *
  22  * @ingroup FileBackend
  23  * @since 1.19
  24  */
  25 class FileBackendMultiWrite extends FileBackend {
  26         /** @var Array Prioritized list of FileBackendStore objects */
  27         protected $backends = array(); // array of (backend index => backends)
  28         protected $masterIndex = -1; // integer; index of master backend
  29         protected $syncChecks = 0; // integer bitfield
  30
  31         /* Possible internal backend consistency checks */
  32         const CHECK_SIZE = 1;
  33         const CHECK_TIME = 2;
  34
  35         /**
  36          * Construct a proxy backend that consists of several internal backends.
  37          * Additional $config params include:
  38          *     'backends'    : Array of backend config and multi-backend settings.
  39          *                     Each value is the config used in the constructor of a
  40          *                     FileBackendStore class, but with these additional settings:
  41          *                         'class'         : The name of the backend class
  42          *                         'isMultiMaster' : This must be set for one backend.
  43          *     'syncChecks'  : Integer bitfield of internal backend sync checks to perform.
  44          *                     Possible bits include self::CHECK_SIZE and self::CHECK_TIME.
  45          *                     The checks are done before allowing any file operations.
  46          * @param $config Array
  47          */
  48         public function __construct( array $config ) {
  49                 parent::__construct( $config );
  50                 $namesUsed = array();
  51                 // Construct backends here rather than via registration
  52                 // to keep these backends hidden from outside the proxy.
  53                 foreach ( $config['backends'] as $index => $config ) {
  54                         $name = $config['name'];
  55                         if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
  56                                 throw new MWException( "Two or more backends defined with the name $name." );
  57                         }
  58                         $namesUsed[$name] = 1;
  59                         if ( !isset( $config['class'] ) ) {
  60                                 throw new MWException( 'No class given for a backend config.' );
  61                         }
  62                         $class = $config['class'];
  63                         $this->backends[$index] = new $class( $config );
  64                         if ( !empty( $config['isMultiMaster'] ) ) {
  65                                 if ( $this->masterIndex >= 0 ) {
  66                                         throw new MWException( 'More than one master backend defined.' );
  67                                 }
  68                                 $this->masterIndex = $index;
  69                         }
  70                 }
  71                 if ( $this->masterIndex < 0 ) { // need backends and must have a master
  72                         throw new MWException( 'No master backend defined.' );
  73                 }
  74                 $this->syncChecks = isset( $config['syncChecks'] )
  75                         ? $config['syncChecks']
  76                         : self::CHECK_SIZE;
  77         }
  78
  79         /**
  80          * @see FileBackend::doOperationsInternal()
  81          * @return Status
  82          */
  83         final protected function doOperationsInternal( array $ops, array $opts ) {
  84                 $status = Status::newGood();
  85
  86                 $performOps = array(); // list of FileOp objects
  87                 $paths = array(); // storage paths read from or written to
  88                 // Build up a list of FileOps. The list will have all the ops
  89                 // for one backend, then all the ops for the next, and so on.
  90                 // These batches of ops are all part of a continuous array.
  91                 // Also build up a list of files read/changed...
  92                 foreach ( $this->backends as $index => $backend ) {
  93                         $backendOps = $this->substOpBatchPaths( $ops, $backend );
  94                         // Add on the operation batch for this backend
  95                         $performOps = array_merge( $performOps,
  96                                 $backend->getOperationsInternal( $backendOps ) );
  97                         if ( $index == 0 ) { // first batch
  98                                 // Get the files used for these operations. Each backend has a batch of
  99                                 // the same operations, so we only need to get them from the first batch.
 100                                 $paths = $backend->getPathsToLockForOpsInternal( $performOps );
 101                                 // Get the paths under the proxy backend's name
 102                                 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
 103                                 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
 104                         }
 105                 }
 106
 107                 // Try to lock those files for the scope of this function...
 108                 if ( empty( $opts['nonLocking'] ) ) {
 109                         // Try to lock those files for the scope of this function...
 110                         $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status );
 111                         $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status );
 112                         if ( !$status->isOK() ) {
 113                                 return $status; // abort
 114                         }
 115                 }
 116
 117                 // Clear any cache entries (after locks acquired)
 118                 $this->clearCache();
 119
 120                 // Do a consistency check to see if the backends agree
 121                 if ( count( $this->backends ) > 1 ) {
 122                         $status->merge( $this->consistencyCheck( array_merge( $paths['sh'], $paths['ex'] ) ) );
 123                         if ( !$status->isOK() ) {
 124                                 return $status; // abort
 125                         }
 126                 }
 127
 128                 // Actually attempt the operation batch...
 129                 $subStatus = FileOp::attemptBatch( $performOps, $opts, $this->fileJournal );
 130
 131                 $success = array();
 132                 $failCount = 0;
 133                 $successCount = 0;
 134                 // Make 'success', 'successCount', and 'failCount' fields reflect
 135                 // the overall operation, rather than all the batches for each backend.
 136                 // Do this by only using success values from the master backend's batch.
 137                 $batchStart = $this->masterIndex * count( $ops );
 138                 $batchEnd = $batchStart + count( $ops ) - 1;
 139                 for ( $i = $batchStart; $i <= $batchEnd; $i++ ) {
 140                         if ( !isset( $subStatus->success[$i] ) ) {
 141                                 break; // failed out before trying this op
 142                         } elseif ( $subStatus->success[$i] ) {
 143                                 ++$successCount;
 144                         } else {
 145                                 ++$failCount;
 146                         }
 147                         $success[] = $subStatus->success[$i];
 148                 }
 149                 $subStatus->success = $success;
 150                 $subStatus->successCount = $successCount;
 151                 $subStatus->failCount = $failCount;
 152
 153                 // Merge errors into status fields
 154                 $status->merge( $subStatus );
 155                 $status->success = $subStatus->success; // not done in merge()
 156
 157                 return $status;
 158         }
 159
 160         /**
 161          * Check that a set of files are consistent across all internal backends
 162          *
 163          * @param $paths Array
 164          * @return Status
 165          */
 166         public function consistencyCheck( array $paths ) {
 167                 $status = Status::newGood();
 168                 if ( $this->syncChecks == 0 ) {
 169                         return $status; // skip checks
 170                 }
 171
 172                 $mBackend = $this->backends[$this->masterIndex];
 173                 foreach ( array_unique( $paths ) as $path ) {
 174                         $params = array( 'src' => $path, 'latest' => true );
 175                         // Stat the file on the 'master' backend
 176                         $mStat = $mBackend->getFileStat( $this->substOpPaths( $params, $mBackend ) );
 177                         // Check of all clone backends agree with the master...
 178                         foreach ( $this->backends as $index => $cBackend ) {
 179                                 if ( $index === $this->masterIndex ) {
 180                                         continue; // master
 181                                 }
 182                                 $cStat = $cBackend->getFileStat( $this->substOpPaths( $params, $cBackend ) );
 183                                 if ( $mStat ) { // file is in master
 184                                         if ( !$cStat ) { // file should exist
 185                                                 $status->fatal( 'backend-fail-synced', $path );
 186                                                 continue;
 187                                         }
 188                                         if ( $this->syncChecks & self::CHECK_SIZE ) {
 189                                                 if ( $cStat['size'] != $mStat['size'] ) { // wrong size
 190                                                         $status->fatal( 'backend-fail-synced', $path );
 191                                                         continue;
 192                                                 }
 193                                         }
 194                                         if ( $this->syncChecks & self::CHECK_TIME ) {
 195                                                 $mTs = wfTimestamp( TS_UNIX, $mStat['mtime'] );
 196                                                 $cTs = wfTimestamp( TS_UNIX, $cStat['mtime'] );
 197                                                 if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
 198                                                         $status->fatal( 'backend-fail-synced', $path );
 199                                                         continue;
 200                                                 }
 201                                         }
 202                                 } else { // file is not in master
 203                                         if ( $cStat ) { // file should not exist
 204                                                 $status->fatal( 'backend-fail-synced', $path );
 205                                         }
 206                                 }
 207                         }
 208                 }
 209
 210                 return $status;
 211         }
 212
 213         /**
 214          * Substitute the backend name in storage path parameters
 215          * for a set of operations with that of a given internal backend.
 216          *
 217          * @param $ops Array List of file operation arrays
 218          * @param $backend FileBackendStore
 219          * @return Array
 220          */
 221         protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
 222                 $newOps = array(); // operations
 223                 foreach ( $ops as $op ) {
 224                         $newOp = $op; // operation
 225                         foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
 226                                 if ( isset( $newOp[$par] ) ) { // string or array
 227                                         $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
 228                                 }
 229                         }
 230                         $newOps[] = $newOp;
 231                 }
 232                 return $newOps;
 233         }
 234
 235         /**
 236          * Same as substOpBatchPaths() but for a single operation
 237          *
 238          * @param $op File operation array
 239          * @param $backend FileBackendStore
 240          * @return Array
 241          */
 242         protected function substOpPaths( array $ops, FileBackendStore $backend ) {
 243                 $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
 244                 return $newOps[0];
 245         }
 246
 247         /**
 248          * Substitute the backend of storage paths with an internal backend's name
 249          *
 250          * @param $paths Array|string List of paths or single string path
 251          * @param $backend FileBackendStore
 252          * @return Array|string
 253          */
 254         protected function substPaths( $paths, FileBackendStore $backend ) {
 255                 return preg_replace(
 256                         '!^mwstore://' . preg_quote( $this->name ) . '/!',
 257                         StringUtils::escapeRegexReplacement( "mwstore://{$backend->getName()}/" ),
 258                         $paths // string or array
 259                 );
 260         }
 261
 262         /**
 263          * Substitute the backend of internal storage paths with the proxy backend's name
 264          *
 265          * @param $paths Array|string List of paths or single string path
 266          * @return Array|string
 267          */
 268         protected function unsubstPaths( $paths ) {
 269                 return preg_replace(
 270                         '!^mwstore://([^/]+)!',
 271                         StringUtils::escapeRegexReplacement( "mwstore://{$this->name}" ),
 272                         $paths // string or array
 273                 );
 274         }
 275
 276         /**
 277          * @see FileBackend::doPrepare()
 278          * @return Status
 279          */
 280         protected function doPrepare( array $params ) {
 281                 $status = Status::newGood();
 282                 foreach ( $this->backends as $backend ) {
 283                         $realParams = $this->substOpPaths( $params, $backend );
 284                         $status->merge( $backend->doPrepare( $realParams ) );
 285                 }
 286                 return $status;
 287         }
 288
 289         /**
 290          * @see FileBackend::doSecure()
 291          * @return Status
 292          */
 293         protected function doSecure( array $params ) {
 294                 $status = Status::newGood();
 295                 foreach ( $this->backends as $backend ) {
 296                         $realParams = $this->substOpPaths( $params, $backend );
 297                         $status->merge( $backend->doSecure( $realParams ) );
 298                 }
 299                 return $status;
 300         }
 301
 302         /**
 303          * @see FileBackend::doClean()
 304          * @return Status
 305          */
 306         protected function doClean( array $params ) {
 307                 $status = Status::newGood();
 308                 foreach ( $this->backends as $backend ) {
 309                         $realParams = $this->substOpPaths( $params, $backend );
 310                         $status->merge( $backend->doClean( $realParams ) );
 311                 }
 312                 return $status;
 313         }
 314
 315         /**
 316          * @see FileBackend::concatenate()
 317          */
 318         public function concatenate( array $params ) {
 319                 // We are writing to an FS file, so we don't need to do this per-backend
 320                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 321                 return $this->backends[$this->masterIndex]->concatenate( $realParams );
 322         }
 323
 324         /**
 325          * @see FileBackend::fileExists()
 326          */
 327         public function fileExists( array $params ) {
 328                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 329                 return $this->backends[$this->masterIndex]->fileExists( $realParams );
 330         }
 331
 332         /**
 333          * @see FileBackend::getFileTimestamp()
 334          */
 335         public function getFileTimestamp( array $params ) {
 336                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 337                 return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
 338         }
 339
 340         /**
 341          * @see FileBackend::getFileSize()
 342          */
 343         public function getFileSize( array $params ) {
 344                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 345                 return $this->backends[$this->masterIndex]->getFileSize( $realParams );
 346         }
 347
 348         /**
 349          * @see FileBackend::getFileStat()
 350          */
 351         public function getFileStat( array $params ) {
 352                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 353                 return $this->backends[$this->masterIndex]->getFileStat( $realParams );
 354         }
 355
 356         /**
 357          * @see FileBackend::getFileContents()
 358          */
 359         public function getFileContents( array $params ) {
 360                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 361                 return $this->backends[$this->masterIndex]->getFileContents( $realParams );
 362         }
 363
 364         /**
 365          * @see FileBackend::getFileSha1Base36()
 366          */
 367         public function getFileSha1Base36( array $params ) {
 368                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 369                 return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams );
 370         }
 371
 372         /**
 373          * @see FileBackend::getFileProps()
 374          */
 375         public function getFileProps( array $params ) {
 376                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 377                 return $this->backends[$this->masterIndex]->getFileProps( $realParams );
 378         }
 379
 380         /**
 381          * @see FileBackend::streamFile()
 382          */
 383         public function streamFile( array $params ) {
 384                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 385                 return $this->backends[$this->masterIndex]->streamFile( $realParams );
 386         }
 387
 388         /**
 389          * @see FileBackend::getLocalReference()
 390          */
 391         public function getLocalReference( array $params ) {
 392                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 393                 return $this->backends[$this->masterIndex]->getLocalReference( $realParams );
 394         }
 395
 396         /**
 397          * @see FileBackend::getLocalCopy()
 398          */
 399         public function getLocalCopy( array $params ) {
 400                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 401                 return $this->backends[$this->masterIndex]->getLocalCopy( $realParams );
 402         }
 403
 404         /**
 405          * @see FileBackend::directoryExists()
 406          */
 407         public function directoryExists( array $params ) {
 408                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 409                 return $this->backends[$this->masterIndex]->directoryExists( $realParams );
 410         }
 411
 412         /**
 413          * @see FileBackend::getSubdirectoryList()
 414          */
 415         public function getDirectoryList( array $params ) {
 416                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 417                 return $this->backends[$this->masterIndex]->getDirectoryList( $realParams );
 418         }
 419
 420         /**
 421          * @see FileBackend::getFileList()
 422          */
 423         public function getFileList( array $params ) {
 424                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 425                 return $this->backends[$this->masterIndex]->getFileList( $realParams );
 426         }
 427
 428         /**
 429          * @see FileBackend::clearCache()
 430          */
 431         public function clearCache( array $paths = null ) {
 432                 foreach ( $this->backends as $backend ) {
 433                         $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
 434                         $backend->clearCache( $realPaths );
 435                 }
 436         }
 437
 438         /**
 439          * @see FileBackend::getScopedLocksForOps()
 440          */
 441         public function getScopedLocksForOps( array $ops, Status $status ) {
 442                 $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $ops );
 443                 // Get the paths to lock from the master backend
 444                 $paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps );
 445                 // Get the paths under the proxy backend's name
 446                 $paths['sh'] = $this->unsubstPaths( $paths['sh'] );
 447                 $paths['ex'] = $this->unsubstPaths( $paths['ex'] );
 448                 return array(
 449                         $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ),
 450                         $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status )
 451                 );
 452         }
 453 }