c4bd7c9623cdc4429b2c2d1662e56f44c45db310
9 * Helper class for representing operations with transaction support.
10 * FileBackend::doOperations() will require these classes for supported operations.
12 * Use of large fields should be avoided as we want to be able to support
13 * potentially many FileOp classes in large arrays in memory.
15 * @ingroup FileBackend
18 abstract class FileOp
{
20 protected $params = array();
21 /** $var FileBackendBase */
23 /** @var TempFSFile|null */
24 protected $tmpSourceFile, $tmpDestFile;
26 protected $state = self
::STATE_NEW
; // integer
27 protected $failed = false; // boolean
28 protected $useBackups = true; // boolean
29 protected $useLatest = true; // boolean
30 protected $destSameAsSource = false; // boolean
31 protected $destAlreadyExists = false; // boolean
33 /* Object life-cycle */
35 const STATE_CHECKED
= 2;
36 const STATE_ATTEMPTED
= 3;
40 * Build a new file operation transaction
42 * @params $backend FileBackend
43 * @params $params Array
45 final public function __construct( FileBackendBase
$backend, array $params ) {
46 $this->backend
= $backend;
47 foreach ( $this->allowedParams() as $name ) {
48 if ( isset( $params[$name] ) ) {
49 $this->params
[$name] = $params[$name];
52 $this->params
= $params;
56 * Disable file backups for this operation
60 final protected function disableBackups() {
61 $this->useBackups
= false;
65 * Allow stale data for file reads and existence checks.
66 * If this is called, then disableBackups() should also be called
67 * unless the affected files are known to have not changed recently.
71 final protected function allowStaleReads() {
72 $this->useLatest
= false;
76 * Attempt a series of file operations.
77 * Callers are responsible for handling file locking.
79 * @param $performOps Array List of FileOp operations
80 * @param $opts Array Batch operation options
83 final public static function attemptBatch( array $performOps, array $opts ) {
84 $status = Status
::newGood();
86 $allowStale = isset( $opts['allowStale'] ) && $opts['allowStale'];
87 $ignoreErrors = isset( $opts['ignoreErrors'] ) && $opts['ignoreErrors'];
88 $predicates = FileOp
::newPredicates(); // account for previous op in prechecks
89 // Do pre-checks for each operation; abort on failure...
90 foreach ( $performOps as $index => $fileOp ) {
92 $fileOp->allowStaleReads(); // allow potentially stale reads
94 $status->merge( $fileOp->precheck( $predicates ) );
95 if ( !$status->isOK() ) { // operation failed?
96 if ( $ignoreErrors ) {
98 $status->success
[$index] = false;
105 // Attempt each operation; abort on failure...
106 foreach ( $performOps as $index => $fileOp ) {
107 if ( $fileOp->failed() ) {
108 continue; // nothing to do
109 } elseif ( $ignoreErrors ) {
110 $fileOp->disableBackups(); // no chance of revert() calls
112 $status->merge( $fileOp->attempt() );
113 if ( !$status->isOK() ) { // operation failed?
114 if ( $ignoreErrors ) {
115 ++
$status->failCount
;
116 $status->success
[$index] = false;
118 // Revert everything done so far and abort.
119 // Do this by reverting each previous operation in reverse order.
120 $pos = $index - 1; // last one failed; no need to revert()
121 while ( $pos >= 0 ) {
122 if ( !$performOps[$pos]->failed() ) {
123 $status->merge( $performOps[$pos]->revert() );
132 // Finish each operation...
133 foreach ( $performOps as $index => $fileOp ) {
134 if ( $fileOp->failed() ) {
135 continue; // nothing to do
137 $subStatus = $fileOp->finish();
138 if ( $subStatus->isOK() ) {
139 ++
$status->successCount
;
140 $status->success
[$index] = true;
142 ++
$status->failCount
;
143 $status->success
[$index] = false;
145 $status->merge( $subStatus );
148 // Make sure status is OK, despite any finish() fatals
149 $status->setResult( true, $status->value
);
155 * Get the value of the parameter with the given name.
156 * Returns null if the parameter is not set.
158 * @param $name string
161 final public function getParam( $name ) {
162 return isset( $this->params
[$name] ) ?
$this->params
[$name] : null;
166 * Check if this operation failed precheck() or attempt()
169 final public function failed() {
170 return $this->failed
;
174 * Get a new empty predicates array for precheck()
178 final public static function newPredicates() {
179 return array( 'exists' => array() );
183 * Check preconditions of the operation without writing anything
185 * @param $predicates Array
188 final public function precheck( array &$predicates ) {
189 if ( $this->state
!== self
::STATE_NEW
) {
190 return Status
::newFatal( 'fileop-fail-state', self
::STATE_NEW
, $this->state
);
192 $this->state
= self
::STATE_CHECKED
;
193 $status = $this->doPrecheck( $predicates );
194 if ( !$status->isOK() ) {
195 $this->failed
= true;
201 * Attempt the operation, backing up files as needed; this must be reversible
205 final public function attempt() {
206 if ( $this->state
!== self
::STATE_CHECKED
) {
207 return Status
::newFatal( 'fileop-fail-state', self
::STATE_CHECKED
, $this->state
);
208 } elseif ( $this->failed
) { // failed precheck
209 return Status
::newFatal( 'fileop-fail-attempt-precheck' );
211 $this->state
= self
::STATE_ATTEMPTED
;
212 $status = $this->doAttempt();
213 if ( !$status->isOK() ) {
214 $this->failed
= true;
215 $this->logFailure( 'attempt' );
221 * Revert the operation; affected files are restored
225 final public function revert() {
226 if ( $this->state
!== self
::STATE_ATTEMPTED
) {
227 return Status
::newFatal( 'fileop-fail-state', self
::STATE_ATTEMPTED
, $this->state
);
229 $this->state
= self
::STATE_DONE
;
230 if ( $this->failed
) {
231 $status = Status
::newGood(); // nothing to revert
233 $status = $this->doRevert();
234 if ( !$status->isOK() ) {
235 $this->logFailure( 'revert' );
242 * Finish the operation; this may be irreversible
246 final public function finish() {
247 if ( $this->state
!== self
::STATE_ATTEMPTED
) {
248 return Status
::newFatal( 'fileop-fail-state', self
::STATE_ATTEMPTED
, $this->state
);
250 $this->state
= self
::STATE_DONE
;
251 if ( $this->failed
) {
252 $status = Status
::newGood(); // nothing to finish
254 $status = $this->doFinish();
260 * Get a list of storage paths read from for this operation
264 public function storagePathsRead() {
269 * Get a list of storage paths written to for this operation
273 public function storagePathsChanged() {
278 * @return Array List of allowed parameters
280 protected function allowedParams() {
287 protected function doPrecheck( array &$predicates ) {
288 return Status
::newGood();
294 abstract protected function doAttempt();
299 abstract protected function doRevert();
304 protected function doFinish() {
305 return Status
::newGood();
309 * Check if the destination file exists and update the
310 * destAlreadyExists member variable. A bad status will
311 * be returned if there is no chance it can be overwritten.
313 * @param $predicates Array
316 protected function precheckDestExistence( array $predicates ) {
317 $status = Status
::newGood();
318 if ( $this->fileExists( $this->params
['dst'], $predicates ) ) {
319 $this->destAlreadyExists
= true;
320 if ( !$this->getParam( 'overwriteDest' ) && !$this->getParam( 'overwriteSame' ) ) {
321 $status->fatal( 'backend-fail-alreadyexists', $this->params
['dst'] );
325 $this->destAlreadyExists
= false;
331 * Backup any file at the source to a temporary file
335 protected function backupSource() {
336 $status = Status
::newGood();
337 if ( $this->useBackups
) {
338 // Check if a file already exists at the source...
339 $params = array( 'src' => $this->params
['src'], 'latest' => $this->useLatest
);
340 if ( $this->backend
->fileExists( $params ) ) {
341 // Create a temporary backup copy...
342 $this->tmpSourcePath
= $this->backend
->getLocalCopy( $params );
343 if ( $this->tmpSourcePath
=== null ) {
344 $status->fatal( 'backend-fail-backup', $this->params
['src'] );
353 * Backup the file at the destination to a temporary file.
354 * Don't bother backing it up unless we might overwrite the file.
355 * This assumes that the destination is in the backend and that
356 * the source is either in the backend or on the file system.
357 * This also handles the 'overwriteSame' check logic and updates
358 * the destSameAsSource member variable.
362 protected function checkAndBackupDest() {
363 $status = Status
::newGood();
364 $this->destSameAsSource
= false;
366 if ( $this->getParam( 'overwriteDest' ) ) {
367 if ( $this->useBackups
) {
368 // Create a temporary backup copy...
369 $params = array( 'src' => $this->params
['dst'], 'latest' => $this->useLatest
);
370 $this->tmpDestFile
= $this->backend
->getLocalCopy( $params );
371 if ( !$this->tmpDestFile
) {
372 $status->fatal( 'backend-fail-backup', $this->params
['dst'] );
376 } elseif ( $this->getParam( 'overwriteSame' ) ) {
377 $shash = $this->getSourceSha1Base36();
378 // If there is a single source, then we can do some checks already.
379 // For things like concatenate(), we would need to build a temp file
380 // first and thus don't support 'overwriteSame' ($shash is null).
381 if ( $shash !== null ) {
382 $dhash = $this->getFileSha1Base36( $this->params
['dst'] );
383 if ( !strlen( $shash ) ||
!strlen( $dhash ) ) {
384 $status->fatal( 'backend-fail-hashes' );
385 } elseif ( $shash !== $dhash ) {
386 // Give an error if the files are not identical
387 $status->fatal( 'backend-fail-notsame', $this->params
['dst'] );
389 $this->destSameAsSource
= true;
391 return $status; // do nothing; either OK or bad status
394 $status->fatal( 'backend-fail-alreadyexists', $this->params
['dst'] );
402 * checkAndBackupDest() helper function to get the source file Sha1.
403 * Returns false on failure and null if there is no single source.
405 * @return string|false|null
407 protected function getSourceSha1Base36() {
412 * checkAndBackupDest() helper function to get the Sha1 of a file.
414 * @return string|false False on failure
416 protected function getFileSha1Base36( $path ) {
417 // Source file is in backend
418 if ( FileBackend
::isStoragePath( $path ) ) {
419 // For some backends (e.g. Swift, Azure) we can get
420 // standard hashes to use for this types of comparisons.
421 $params = array( 'src' => $path, 'latest' => $this->useLatest
);
422 $hash = $this->backend
->getFileSha1Base36( $params );
423 // Source file is on file system
425 wfSuppressWarnings();
426 $hash = sha1_file( $path );
428 if ( $hash !== false ) {
429 $hash = wfBaseConvert( $hash, 16, 36, 31 );
436 * Restore any temporary source backup file
440 protected function restoreSource() {
441 $status = Status
::newGood();
442 // Restore any file that was at the destination
443 if ( $this->tmpSourcePath
!== null ) {
445 'src' => $this->tmpSourcePath
,
446 'dst' => $this->params
['src'],
447 'overwriteDest' => true
449 $status = $this->backend
->storeInternal( $params );
450 if ( !$status->isOK() ) {
458 * Restore any temporary destination backup file
462 protected function restoreDest() {
463 $status = Status
::newGood();
464 // Restore any file that was at the destination
465 if ( $this->tmpDestFile
) {
467 'src' => $this->tmpDestFile
->getPath(),
468 'dst' => $this->params
['dst'],
469 'overwriteDest' => true
471 $status = $this->backend
->storeInternal( $params );
472 if ( !$status->isOK() ) {
480 * Check if a file will exist in storage when this operation is attempted
482 * @param $source string Storage path
483 * @param $predicates Array
486 final protected function fileExists( $source, array $predicates ) {
487 if ( isset( $predicates['exists'][$source] ) ) {
488 return $predicates['exists'][$source]; // previous op assures this
490 $params = array( 'src' => $source, 'latest' => $this->useLatest
);
491 return $this->backend
->fileExists( $params );
496 * Log a file operation failure and preserve any temp files
498 * @param $fileOp FileOp
501 final protected function logFailure( $action ) {
502 $params = $this->params
;
503 $params['failedAction'] = $action;
504 // Preserve backup files just in case (for recovery)
505 if ( $this->tmpSourceFile
) {
506 $this->tmpSourceFile
->preserve(); // don't purge
507 $params['srcBackupPath'] = $this->tmpSourceFile
->getPath();
509 if ( $this->tmpDestFile
) {
510 $this->tmpDestFile
->preserve(); // don't purge
511 $params['dstBackupPath'] = $this->tmpDestFile
->getPath();
514 wfDebugLog( 'FileOperation',
515 get_class( $this ) . ' failed:' . serialize( $params ) );
516 } catch ( Exception
$e ) {
517 // bad config? debug log error?
523 * Store a file into the backend from a file on the file system.
524 * Parameters similar to FileBackend::storeInternal(), which include:
525 * src : source path on file system
526 * dst : destination storage path
527 * overwriteDest : do nothing and pass if an identical file exists at destination
528 * overwriteSame : override any existing file at destination
530 class StoreFileOp
extends FileOp
{
531 protected function allowedParams() {
532 return array( 'src', 'dst', 'overwriteDest', 'overwriteSame' );
535 protected function doPrecheck( array &$predicates ) {
536 $status = Status
::newGood();
537 // Check if destination file exists
538 $status->merge( $this->precheckDestExistence( $predicates ) );
539 if ( !$status->isOK() ) {
542 // Check if the source file exists on the file system
543 if ( !is_file( $this->params
['src'] ) ) {
544 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
547 // Update file existence predicates
548 $predicates['exists'][$this->params
['dst']] = true;
552 protected function doAttempt() {
553 $status = Status
::newGood();
554 // Create a destination backup copy as needed
555 if ( $this->destAlreadyExists
) {
556 $status->merge( $this->checkAndBackupDest() );
557 if ( !$status->isOK() ) {
561 // Store the file at the destination
562 if ( !$this->destSameAsSource
) {
563 $status->merge( $this->backend
->storeInternal( $this->params
) );
568 protected function doRevert() {
569 $status = Status
::newGood();
570 if ( !$this->destSameAsSource
) {
571 // Restore any file that was at the destination,
572 // overwritting what was put there in attempt()
573 $status->merge( $this->restoreDest() );
578 protected function getSourceSha1Base36() {
579 return $this->getFileSha1Base36( $this->params
['src'] );
582 public function storagePathsChanged() {
583 return array( $this->params
['dst'] );
588 * Create a file in the backend with the given content.
589 * Parameters similar to FileBackend::create(), which include:
590 * content : a string of raw file contents
591 * dst : destination storage path
592 * overwriteDest : do nothing and pass if an identical file exists at destination
593 * overwriteSame : override any existing file at destination
595 class CreateFileOp
extends FileOp
{
596 protected function allowedParams() {
597 return array( 'content', 'dst', 'overwriteDest', 'overwriteSame' );
600 protected function doPrecheck( array &$predicates ) {
601 $status = Status
::newGood();
602 // Check if destination file exists
603 $status->merge( $this->precheckDestExistence( $predicates ) );
604 if ( !$status->isOK() ) {
607 // Update file existence predicates
608 $predicates['exists'][$this->params
['dst']] = true;
612 protected function doAttempt() {
613 $status = Status
::newGood();
614 // Create a destination backup copy as needed
615 if ( $this->destAlreadyExists
) {
616 $status->merge( $this->checkAndBackupDest() );
617 if ( !$status->isOK() ) {
621 // Create the file at the destination
622 if ( !$this->destSameAsSource
) {
623 $status->merge( $this->backend
->createInternal( $this->params
) );
628 protected function doRevert() {
629 $status = Status
::newGood();
630 if ( !$this->destSameAsSource
) {
631 // Restore any file that was at the destination,
632 // overwritting what was put there in attempt()
633 $status->merge( $this->restoreDest() );
638 protected function getSourceSha1Base36() {
639 return wfBaseConvert( sha1( $this->params
['content'] ), 16, 36, 31 );
642 public function storagePathsChanged() {
643 return array( $this->params
['dst'] );
648 * Copy a file from one storage path to another in the backend.
649 * Parameters similar to FileBackend::copy(), which include:
650 * src : source storage path
651 * dst : destination storage path
652 * overwriteDest : do nothing and pass if an identical file exists at destination
653 * overwriteSame : override any existing file at destination
655 class CopyFileOp
extends FileOp
{
656 protected function allowedParams() {
657 return array( 'src', 'dst', 'overwriteDest', 'overwriteSame' );
660 protected function doPrecheck( array &$predicates ) {
661 $status = Status
::newGood();
662 // Check if destination file exists
663 $status->merge( $this->precheckDestExistence( $predicates ) );
664 if ( !$status->isOK() ) {
667 // Check if the source file exists
668 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
669 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
672 // Update file existence predicates
673 $predicates['exists'][$this->params
['dst']] = true;
677 protected function doAttempt() {
678 $status = Status
::newGood();
679 // Create a destination backup copy as needed
680 if ( $this->destAlreadyExists
) {
681 $status->merge( $this->checkAndBackupDest() );
682 if ( !$status->isOK() ) {
686 // Copy the file into the destination
687 if ( !$this->destSameAsSource
) {
688 $status->merge( $this->backend
->copyInternal( $this->params
) );
693 protected function doRevert() {
694 $status = Status
::newGood();
695 if ( !$this->destSameAsSource
) {
696 // Restore any file that was at the destination,
697 // overwritting what was put there in attempt()
698 $status->merge( $this->restoreDest() );
703 protected function getSourceSha1Base36() {
704 return $this->getFileSha1Base36( $this->params
['src'] );
707 public function storagePathsRead() {
708 return array( $this->params
['src'] );
711 public function storagePathsChanged() {
712 return array( $this->params
['dst'] );
717 * Move a file from one storage path to another in the backend.
718 * Parameters similar to FileBackend::move(), which include:
719 * src : source storage path
720 * dst : destination storage path
721 * overwriteDest : do nothing and pass if an identical file exists at destination
722 * overwriteSame : override any existing file at destination
724 class MoveFileOp
extends FileOp
{
725 protected function allowedParams() {
726 return array( 'src', 'dst', 'overwriteDest', 'overwriteSame' );
729 protected function doPrecheck( array &$predicates ) {
730 $status = Status
::newGood();
731 // Check if destination file exists
732 $status->merge( $this->precheckDestExistence( $predicates ) );
733 if ( !$status->isOK() ) {
736 // Check if the source file exists
737 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
738 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
741 // Update file existence predicates
742 $predicates['exists'][$this->params
['src']] = false;
743 $predicates['exists'][$this->params
['dst']] = true;
747 protected function doAttempt() {
748 $status = Status
::newGood();
749 // Create a destination backup copy as needed
750 if ( $this->destAlreadyExists
) {
751 $status->merge( $this->checkAndBackupDest() );
752 if ( !$status->isOK() ) {
756 if ( !$this->destSameAsSource
) {
757 // Move the file into the destination
758 $status->merge( $this->backend
->moveInternal( $this->params
) );
760 // Create a source backup copy as needed
761 $status->merge( $this->backupSource() );
762 if ( !$status->isOK() ) {
765 // Just delete source as the destination needs no changes
766 $params = array( 'src' => $this->params
['src'] );
767 $status->merge( $this->backend
->deleteInternal( $params ) );
768 if ( !$status->isOK() ) {
775 protected function doRevert() {
776 $status = Status
::newGood();
777 if ( !$this->destSameAsSource
) {
778 // Move the file back to the source
780 'src' => $this->params
['dst'],
781 'dst' => $this->params
['src']
783 $status->merge( $this->backend
->moveInternal( $params ) );
784 if ( !$status->isOK() ) {
785 return $status; // also can't restore any dest file
787 // Restore any file that was at the destination
788 $status->merge( $this->restoreDest() );
790 // Restore any source file
791 return $this->restoreSource();
797 protected function getSourceSha1Base36() {
798 return $this->getFileSha1Base36( $this->params
['src'] );
801 public function storagePathsRead() {
802 return array( $this->params
['src'] );
805 public function storagePathsChanged() {
806 return array( $this->params
['dst'] );
811 * Combines files from severals storage paths into a new file in the backend.
812 * Parameters similar to FileBackend::concatenate(), which include:
813 * srcs : ordered source storage paths (e.g. chunk1, chunk2, ...)
814 * dst : destination file system path to 0-byte temp file
816 class ConcatenateFileOp
extends FileOp
{
817 protected function allowedParams() {
818 return array( 'srcs', 'dst' );
821 protected function doPrecheck( array &$predicates ) {
822 $status = Status
::newGood();
823 // Check destination temp file
824 wfSuppressWarnings();
825 $ok = ( is_file( $this->params
['dst'] ) && !filesize( $this->params
['dst'] ) );
827 if ( !$ok ) { // not present or not empty
828 $status->fatal( 'backend-fail-opentemp', $this->params
['dst'] );
831 // Check that source files exists
832 foreach ( $this->params
['srcs'] as $source ) {
833 if ( !$this->fileExists( $source, $predicates ) ) {
834 $status->fatal( 'backend-fail-notexists', $source );
841 protected function doAttempt() {
842 $status = Status
::newGood();
843 // Concatenate the file at the destination
844 $status->merge( $this->backend
->concatenateInternal( $this->params
) );
848 protected function doRevert() {
849 $status = Status
::newGood();
850 // Clear out the temp file back to 0-bytes
851 wfSuppressWarnings();
852 $ok = file_put_contents( $this->params
['dst'], '' );
855 $status->fatal( 'backend-fail-writetemp', $this->params
['dst'] );
860 public function storagePathsRead() {
861 return $this->params
['srcs'];
866 * Delete a file at the storage path.
867 * Parameters similar to FileBackend::delete(), which include:
868 * src : source storage path
869 * ignoreMissingSource : don't return an error if the file does not exist
871 class DeleteFileOp
extends FileOp
{
872 protected $needsDelete = true;
874 protected function allowedParams() {
875 return array( 'src', 'ignoreMissingSource' );
878 protected function doPrecheck( array &$predicates ) {
879 $status = Status
::newGood();
880 // Check if the source file exists
881 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
882 if ( !$this->getParam( 'ignoreMissingSource' ) ) {
883 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
886 $this->needsDelete
= false;
888 // Update file existence predicates
889 $predicates['exists'][$this->params
['src']] = false;
893 protected function doAttempt() {
894 $status = Status
::newGood();
895 if ( $this->needsDelete
) {
896 // Create a source backup copy as needed
897 $status->merge( $this->backupSource() );
898 if ( !$status->isOK() ) {
901 // Delete the source file
902 $status->merge( $this->backend
->deleteInternal( $this->params
) );
903 if ( !$status->isOK() ) {
910 protected function doRevert() {
911 // Restore any source file that we deleted
912 return $this->restoreSource();
915 public function storagePathsChanged() {
916 return array( $this->params
['src'] );
921 * Placeholder operation that has no params and does nothing
923 class NullFileOp
extends FileOp
{
924 protected function doAttempt() {
925 return Status
::newGood();
928 protected function doRevert() {
929 return Status
::newGood();