includes/filerepo/backend/FileBackendMultiWrite.php

   1 <?php
   2 /**
   3  * @file
   4  * @ingroup FileBackend
   5  * @author Aaron Schulz
   6  */
   7
   8 /**
   9  * This class defines a multi-write backend. Multiple backends can be
  10  * registered to this proxy backend and it will act as a single backend.
  11  * Use this when all access to those backends is through this proxy backend.
  12  * At least one of the backends must be declared the "master" backend.
  13  *
  14  * Only use this class when transitioning from one storage system to another.
  15  *
  16  * Read operations are only done on the 'master' backend for consistency.
  17  * Write operations are performed on all backends, in the order defined.
  18  * If an operation fails on one backend it will be rolled back from the others.
  19  *
  20  * @ingroup FileBackend
  21  * @since 1.19
  22  */
  23 class FileBackendMultiWrite extends FileBackendBase {
  24         /** @var Array Prioritized list of FileBackend objects */
  25         protected $backends = array(); // array of (backend index => backends)
  26         protected $masterIndex = -1; // index of master backend
  27
  28         /**
  29          * Construct a proxy backend that consists of several internal backends.
  30          * Additional $config params include:
  31          *     'backends'    : Array of backend config and multi-backend settings.
  32          *                     Each value is the config used in the constructor of a
  33          *                     FileBackend class, but with these additional settings:
  34          *                         'class'         : The name of the backend class
  35          *                         'isMultiMaster' : This must be set for one backend.
  36          * @param $config Array
  37          */
  38         public function __construct( array $config ) {
  39                 parent::__construct( $config );
  40                 $namesUsed = array();
  41                 // Construct backends here rather than via registration
  42                 // to keep these backends hidden from outside the proxy.
  43                 foreach ( $config['backends'] as $index => $config ) {
  44                         $name = $config['name'];
  45                         if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
  46                                 throw new MWException( "Two or more backends defined with the name $name." );
  47                         }
  48                         $namesUsed[$name] = 1;
  49                         if ( !isset( $config['class'] ) ) {
  50                                 throw new MWException( 'No class given for a backend config.' );
  51                         }
  52                         $class = $config['class'];
  53                         $this->backends[$index] = new $class( $config );
  54                         if ( !empty( $config['isMultiMaster'] ) ) {
  55                                 if ( $this->masterIndex >= 0 ) {
  56                                         throw new MWException( 'More than one master backend defined.' );
  57                                 }
  58                                 $this->masterIndex = $index;
  59                         }
  60                 }
  61                 if ( $this->masterIndex < 0 ) { // need backends and must have a master
  62                         throw new MWException( 'No master backend defined.' );
  63                 }
  64         }
  65
  66         /**
  67          * @see FileBackendBase::doOperationsInternal()
  68          */
  69         final protected function doOperationsInternal( array $ops, array $opts ) {
  70                 $status = Status::newGood();
  71
  72                 $performOps = array(); // list of FileOp objects
  73                 $filesRead = $filesChanged = array(); // storage paths used
  74                 // Build up a list of FileOps. The list will have all the ops
  75                 // for one backend, then all the ops for the next, and so on.
  76                 // These batches of ops are all part of a continuous array.
  77                 // Also build up a list of files read/changed...
  78                 foreach ( $this->backends as $index => $backend ) {
  79                         $backendOps = $this->substOpBatchPaths( $ops, $backend );
  80                         // Add on the operation batch for this backend
  81                         $performOps = array_merge( $performOps, $backend->getOperations( $backendOps ) );
  82                         if ( $index == 0 ) { // first batch
  83                                 // Get the files used for these operations. Each backend has a batch of
  84                                 // the same operations, so we only need to get them from the first batch.
  85                                 foreach ( $performOps as $fileOp ) {
  86                                         $filesRead = array_merge( $filesRead, $fileOp->storagePathsRead() );
  87                                         $filesChanged = array_merge( $filesChanged, $fileOp->storagePathsChanged() );
  88                                 }
  89                                 // Get the paths under the proxy backend's name
  90                                 $filesRead = $this->unsubstPaths( $filesRead );
  91                                 $filesChanged = $this->unsubstPaths( $filesChanged );
  92                         }
  93                 }
  94
  95                 // Try to lock those files for the scope of this function...
  96                 if ( empty( $opts['nonLocking'] ) ) {
  97                         $filesLockSh = array_diff( $filesRead, $filesChanged ); // optimization
  98                         $filesLockEx = $filesChanged;
  99                         // Get a shared lock on the parent directory of each path changed
 100                         $filesLockSh = array_merge( $filesLockSh, array_map( 'dirname', $filesLockEx ) );
 101                         // Try to lock those files for the scope of this function...
 102                         $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
 103                         $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
 104                         if ( !$status->isOK() ) {
 105                                 return $status; // abort
 106                         }
 107                 }
 108
 109                 // Clear any cache entries (after locks acquired)
 110                 $this->clearCache();
 111
 112                 // Do a consistency check to see if the backends agree
 113                 if ( count( $this->backends ) > 1 ) {
 114                         $status->merge( $this->consistencyCheck( array_merge( $filesRead, $filesChanged ) ) );
 115                         if ( !$status->isOK() ) {
 116                                 return $status; // abort
 117                         }
 118                 }
 119
 120                 // Actually attempt the operation batch...
 121                 $subStatus = FileOp::attemptBatch( $performOps, $opts );
 122
 123                 $success = array();
 124                 $failCount = $successCount = 0;
 125                 // Make 'success', 'successCount', and 'failCount' fields reflect
 126                 // the overall operation, rather than all the batches for each backend.
 127                 // Do this by only using success values from the master backend's batch.
 128                 $batchStart = $this->masterIndex * count( $ops );
 129                 $batchEnd = $batchStart + count( $ops ) - 1;
 130                 for ( $i = $batchStart; $i <= $batchEnd; $i++ ) {
 131                         if ( !isset( $subStatus->success[$i] ) ) {
 132                                 break; // failed out before trying this op
 133                         } elseif ( $subStatus->success[$i] ) {
 134                                 ++$successCount;
 135                         } else {
 136                                 ++$failCount;
 137                         }
 138                         $success[] = $subStatus->success[$i];
 139                 }
 140                 $subStatus->success = $success;
 141                 $subStatus->successCount = $successCount;
 142                 $subStatus->failCount = $failCount;
 143
 144                 // Merge errors into status fields
 145                 $status->merge( $subStatus );
 146                 $status->success = $subStatus->success; // not done in merge()
 147
 148                 return $status;
 149         }
 150
 151         /**
 152          * Check that a set of files are consistent across all internal backends
 153          *
 154          * @param $paths Array
 155          * @return Status
 156          */
 157         public function consistencyCheck( array $paths ) {
 158                 $status = Status::newGood();
 159
 160                 $mBackend = $this->backends[$this->masterIndex];
 161                 foreach ( array_unique( $paths ) as $path ) {
 162                         $params = array( 'src' => $path, 'latest' => true );
 163                         // Stat the file on the 'master' backend
 164                         $mStat = $mBackend->getFileStat( $this->substOpPaths( $params, $mBackend ) );
 165                         // Check of all clone backends agree with the master...
 166                         foreach ( $this->backends as $index => $cBackend ) {
 167                                 if ( $index === $this->masterIndex ) {
 168                                         continue; // master
 169                                 }
 170                                 $cStat = $cBackend->getFileStat( $this->substOpPaths( $params, $cBackend ) );
 171                                 if ( $mStat ) { // file is in master
 172                                         if ( !$cStat ) { // file should exist
 173                                                 $status->fatal( 'backend-fail-synced', $path );
 174                                         } elseif ( $cStat['size'] != $mStat['size'] ) { // wrong size
 175                                                 $status->fatal( 'backend-fail-synced', $path );
 176                                         } else {
 177                                                 $mTs = wfTimestamp( TS_UNIX, $mStat['mtime'] );
 178                                                 $cTs = wfTimestamp( TS_UNIX, $cStat['mtime'] );
 179                                                 if ( abs( $mTs - $cTs ) > 30 ) { // outdated file somewhere
 180                                                         $status->fatal( 'backend-fail-synced', $path );
 181                                                 }
 182                                         }
 183                                 } else { // file is not in master
 184                                         if ( $cStat ) { // file should not exist
 185                                                 $status->fatal( 'backend-fail-synced', $path );
 186                                         }
 187                                 }
 188                         }
 189                 }
 190
 191                 return $status;
 192         }
 193
 194         /**
 195          * Substitute the backend name in storage path parameters
 196          * for a set of operations with that of a given internal backend.
 197          *
 198          * @param $ops Array List of file operation arrays
 199          * @param $backend FileBackend
 200          * @return Array
 201          */
 202         protected function substOpBatchPaths( array $ops, FileBackend $backend ) {
 203                 $newOps = array(); // operations
 204                 foreach ( $ops as $op ) {
 205                         $newOp = $op; // operation
 206                         foreach ( array( 'src', 'srcs', 'dst', 'dir' ) as $par ) {
 207                                 if ( isset( $newOp[$par] ) ) { // string or array
 208                                         $newOp[$par] = $this->substPaths( $newOp[$par], $backend );
 209                                 }
 210                         }
 211                         $newOps[] = $newOp;
 212                 }
 213                 return $newOps;
 214         }
 215
 216         /**
 217          * Same as substOpBatchPaths() but for a single operation
 218          *
 219          * @param $op File operation array
 220          * @param $backend FileBackend
 221          * @return Array
 222          */
 223         protected function substOpPaths( array $ops, FileBackend $backend ) {
 224                 $newOps = $this->substOpBatchPaths( array( $ops ), $backend );
 225                 return $newOps[0];
 226         }
 227
 228         /**
 229          * Substitute the backend of storage paths with an internal backend's name
 230          *
 231          * @param $paths Array|string List of paths or single string path
 232          * @param $backend FileBackend
 233          * @return Array|string
 234          */
 235         protected function substPaths( $paths, FileBackend $backend ) {
 236                 return preg_replace(
 237                         '!^mwstore://' . preg_quote( $this->name ) . '/!',
 238                         'mwstore://' . $backend->getName() . '/',
 239                         $paths // string or array
 240                 );
 241         }
 242
 243         /**
 244          * Substitute the backend of internal storage paths with the proxy backend's name
 245          *
 246          * @param $paths Array|string List of paths or single string path
 247          * @return Array|string
 248          */
 249         protected function unsubstPaths( $paths ) {
 250                 return preg_replace(
 251                         '!^mwstore://([^/]+)!',
 252                         "mwstore://{$this->name}",
 253                         $paths // string or array
 254                 );
 255         }
 256
 257         /**
 258          * @see FileBackendBase::doPrepare()
 259          */
 260         public function doPrepare( array $params ) {
 261                 $status = Status::newGood();
 262                 foreach ( $this->backends as $backend ) {
 263                         $realParams = $this->substOpPaths( $params, $backend );
 264                         $status->merge( $backend->doPrepare( $realParams ) );
 265                 }
 266                 return $status;
 267         }
 268
 269         /**
 270          * @see FileBackendBase::doSecure()
 271          */
 272         public function doSecure( array $params ) {
 273                 $status = Status::newGood();
 274                 foreach ( $this->backends as $backend ) {
 275                         $realParams = $this->substOpPaths( $params, $backend );
 276                         $status->merge( $backend->doSecure( $realParams ) );
 277                 }
 278                 return $status;
 279         }
 280
 281         /**
 282          * @see FileBackendBase::doClean()
 283          */
 284         public function doClean( array $params ) {
 285                 $status = Status::newGood();
 286                 foreach ( $this->backends as $backend ) {
 287                         $realParams = $this->substOpPaths( $params, $backend );
 288                         $status->merge( $backend->doClean( $realParams ) );
 289                 }
 290                 return $status;
 291         }
 292
 293         /**
 294          * @see FileBackendBase::getFileList()
 295          */
 296         public function concatenate( array $params ) {
 297                 // We are writing to an FS file, so we don't need to do this per-backend
 298                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 299                 return $this->backends[$this->masterIndex]->concatenate( $realParams );
 300         }
 301
 302         /**
 303          * @see FileBackendBase::fileExists()
 304          */
 305         public function fileExists( array $params ) {
 306                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 307                 return $this->backends[$this->masterIndex]->fileExists( $realParams );
 308         }
 309
 310         /**
 311          * @see FileBackendBase::getFileTimestamp()
 312          */
 313         public function getFileTimestamp( array $params ) {
 314                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 315                 return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
 316         }
 317
 318         /**
 319          * @see FileBackendBase::getFileSize()
 320          */
 321         public function getFileSize( array $params ) {
 322                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 323                 return $this->backends[$this->masterIndex]->getFileSize( $realParams );
 324         }
 325
 326         /**
 327          * @see FileBackendBase::getFileStat()
 328          */
 329         public function getFileStat( array $params ) {
 330                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 331                 return $this->backends[$this->masterIndex]->getFileStat( $realParams );
 332         }
 333
 334         /**
 335          * @see FileBackendBase::getFileContents()
 336          */
 337         public function getFileContents( array $params ) {
 338                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 339                 return $this->backends[$this->masterIndex]->getFileContents( $realParams );
 340         }
 341
 342         /**
 343          * @see FileBackendBase::getFileSha1Base36()
 344          */
 345         public function getFileSha1Base36( array $params ) {
 346                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 347                 return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams );
 348         }
 349
 350         /**
 351          * @see FileBackendBase::getFileProps()
 352          */
 353         public function getFileProps( array $params ) {
 354                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 355                 return $this->backends[$this->masterIndex]->getFileProps( $realParams );
 356         }
 357
 358         /**
 359          * @see FileBackendBase::streamFile()
 360          */
 361         public function streamFile( array $params ) {
 362                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 363                 return $this->backends[$this->masterIndex]->streamFile( $realParams );
 364         }
 365
 366         /**
 367          * @see FileBackendBase::getLocalReference()
 368          */
 369         public function getLocalReference( array $params ) {
 370                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 371                 return $this->backends[$this->masterIndex]->getLocalReference( $realParams );
 372         }
 373
 374         /**
 375          * @see FileBackendBase::getLocalCopy()
 376          */
 377         public function getLocalCopy( array $params ) {
 378                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 379                 return $this->backends[$this->masterIndex]->getLocalCopy( $realParams );
 380         }
 381
 382         /**
 383          * @see FileBackendBase::getFileList()
 384          */
 385         public function getFileList( array $params ) {
 386                 $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
 387                 return $this->backends[$this->masterIndex]->getFileList( $realParams );
 388         }
 389
 390         /**
 391          * @see FileBackendBase::clearCache()
 392          */
 393         public function clearCache( array $paths = null ) {
 394                 foreach ( $this->backends as $backend ) {
 395                         $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
 396                         $backend->clearCache( $realPaths );
 397                 }
 398         }
 399 }