9 * Helper class for representing operations with transaction support.
10 * Do not use this class from places outside FileBackend.
12 * Methods called from attemptBatch() should avoid throwing exceptions at all costs.
13 * FileOp objects should be lightweight in order to support large arrays in memory.
15 * @ingroup FileBackend
18 abstract class FileOp
{
20 protected $params = array();
21 /** @var FileBackendStore */
24 protected $state = self
::STATE_NEW
; // integer
25 protected $failed = false; // boolean
26 protected $useLatest = true; // boolean
28 protected $sourceSha1; // string
29 protected $destSameAsSource; // boolean
31 /* Object life-cycle */
33 const STATE_CHECKED
= 2;
34 const STATE_ATTEMPTED
= 3;
36 /* Timeout related parameters */
37 const MAX_BATCH_SIZE
= 1000;
38 const TIME_LIMIT_SEC
= 300; // 5 minutes
41 * Build a new file operation transaction
43 * @param $backend FileBackendStore
44 * @param $params Array
47 final public function __construct( FileBackendStore
$backend, array $params ) {
48 $this->backend
= $backend;
49 list( $required, $optional ) = $this->allowedParams();
50 foreach ( $required as $name ) {
51 if ( isset( $params[$name] ) ) {
52 $this->params
[$name] = $params[$name];
54 throw new MWException( "File operation missing parameter '$name'." );
57 foreach ( $optional as $name ) {
58 if ( isset( $params[$name] ) ) {
59 $this->params
[$name] = $params[$name];
62 $this->params
= $params;
66 * Allow stale data for file reads and existence checks
70 final protected function allowStaleReads() {
71 $this->useLatest
= false;
75 * Attempt a series of file operations.
76 * Callers are responsible for handling file locking.
78 * $opts is an array of options, including:
79 * 'force' : Errors that would normally cause a rollback do not.
80 * The remaining operations are still attempted if any fail.
81 * 'allowStale' : Don't require the latest available data.
82 * This can increase performance for non-critical writes.
83 * This has no effect unless the 'force' flag is set.
85 * The resulting Status will be "OK" unless:
86 * a) unexpected operation errors occurred (network partitions, disk full...)
87 * b) significant operation errors occured and 'force' was not set
89 * @param $performOps Array List of FileOp operations
90 * @param $opts Array Batch operation options
93 final public static function attemptBatch( array $performOps, array $opts ) {
94 $status = Status
::newGood();
96 $allowStale = !empty( $opts['allowStale'] );
97 $ignoreErrors = !empty( $opts['force'] );
99 $n = count( $performOps );
100 if ( $n > self
::MAX_BATCH_SIZE
) {
101 $status->fatal( 'backend-fail-batchsize', $n, self
::MAX_BATCH_SIZE
);
105 $predicates = FileOp
::newPredicates(); // account for previous op in prechecks
106 // Do pre-checks for each operation; abort on failure...
107 foreach ( $performOps as $index => $fileOp ) {
109 $fileOp->allowStaleReads(); // allow potentially stale reads
111 $subStatus = $fileOp->precheck( $predicates );
112 $status->merge( $subStatus );
113 if ( !$subStatus->isOK() ) { // operation failed?
114 $status->success
[$index] = false;
115 ++
$status->failCount
;
116 if ( !$ignoreErrors ) {
117 return $status; // abort
122 if ( $ignoreErrors ) {
123 # Treat all precheck() fatals as merely warnings
124 $status->setResult( true, $status->value
);
127 // Restart PHP's execution timer and set the timeout to safe amount.
128 // This handles cases where the operations take a long time or where we are
129 // already running low on time left. The old timeout is restored afterwards.
130 # @TODO: re-enable this for when the number of batches is high.
131 #$scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC );
133 // Attempt each operation...
134 foreach ( $performOps as $index => $fileOp ) {
135 if ( $fileOp->failed() ) {
136 continue; // nothing to do
138 $subStatus = $fileOp->attempt();
139 $status->merge( $subStatus );
140 if ( $subStatus->isOK() ) {
141 $status->success
[$index] = true;
142 ++
$status->successCount
;
144 $status->success
[$index] = false;
145 ++
$status->failCount
;
146 // We can't continue (even with $ignoreErrors) as $predicates is wrong.
147 // Log the remaining ops as failed for recovery...
148 for ( $i = ($index +
1); $i < count( $performOps ); $i++
) {
149 $performOps[$i]->logFailure( 'attempt_aborted' );
151 return $status; // bail out
159 * Get the value of the parameter with the given name
161 * @param $name string
162 * @return mixed Returns null if the parameter is not set
164 final public function getParam( $name ) {
165 return isset( $this->params
[$name] ) ?
$this->params
[$name] : null;
169 * Check if this operation failed precheck() or attempt()
173 final public function failed() {
174 return $this->failed
;
178 * Get a new empty predicates array for precheck()
182 final public static function newPredicates() {
183 return array( 'exists' => array(), 'sha1' => array() );
187 * Check preconditions of the operation without writing anything
189 * @param $predicates Array
192 final public function precheck( array &$predicates ) {
193 if ( $this->state
!== self
::STATE_NEW
) {
194 return Status
::newFatal( 'fileop-fail-state', self
::STATE_NEW
, $this->state
);
196 $this->state
= self
::STATE_CHECKED
;
197 $status = $this->doPrecheck( $predicates );
198 if ( !$status->isOK() ) {
199 $this->failed
= true;
205 * Attempt the operation, backing up files as needed; this must be reversible
209 final public function attempt() {
210 if ( $this->state
!== self
::STATE_CHECKED
) {
211 return Status
::newFatal( 'fileop-fail-state', self
::STATE_CHECKED
, $this->state
);
212 } elseif ( $this->failed
) { // failed precheck
213 return Status
::newFatal( 'fileop-fail-attempt-precheck' );
215 $this->state
= self
::STATE_ATTEMPTED
;
216 $status = $this->doAttempt();
217 if ( !$status->isOK() ) {
218 $this->failed
= true;
219 $this->logFailure( 'attempt' );
225 * Get the file operation parameters
227 * @return Array (required params list, optional params list)
229 protected function allowedParams() {
230 return array( array(), array() );
234 * Get a list of storage paths read from for this operation
238 public function storagePathsRead() {
243 * Get a list of storage paths written to for this operation
247 public function storagePathsChanged() {
254 protected function doPrecheck( array &$predicates ) {
255 return Status
::newGood();
261 protected function doAttempt() {
262 return Status
::newGood();
266 * Check for errors with regards to the destination file already existing.
267 * This also updates the destSameAsSource and sourceSha1 member variables.
268 * A bad status will be returned if there is no chance it can be overwritten.
270 * @param $predicates Array
273 protected function precheckDestExistence( array $predicates ) {
274 $status = Status
::newGood();
275 // Get hash of source file/string and the destination file
276 $this->sourceSha1
= $this->getSourceSha1Base36(); // FS file or data string
277 if ( $this->sourceSha1
=== null ) { // file in storage?
278 $this->sourceSha1
= $this->fileSha1( $this->params
['src'], $predicates );
280 $this->destSameAsSource
= false;
281 if ( $this->fileExists( $this->params
['dst'], $predicates ) ) {
282 if ( $this->getParam( 'overwrite' ) ) {
283 return $status; // OK
284 } elseif ( $this->getParam( 'overwriteSame' ) ) {
285 $dhash = $this->fileSha1( $this->params
['dst'], $predicates );
286 // Check if hashes are valid and match each other...
287 if ( !strlen( $this->sourceSha1
) ||
!strlen( $dhash ) ) {
288 $status->fatal( 'backend-fail-hashes' );
289 } elseif ( $this->sourceSha1
!== $dhash ) {
290 // Give an error if the files are not identical
291 $status->fatal( 'backend-fail-notsame', $this->params
['dst'] );
293 $this->destSameAsSource
= true; // OK
295 return $status; // do nothing; either OK or bad status
297 $status->fatal( 'backend-fail-alreadyexists', $this->params
['dst'] );
305 * precheckDestExistence() helper function to get the source file SHA-1.
306 * Subclasses should overwride this iff the source is not in storage.
308 * @return string|bool Returns false on failure
310 protected function getSourceSha1Base36() {
315 * Check if a file will exist in storage when this operation is attempted
317 * @param $source string Storage path
318 * @param $predicates Array
321 final protected function fileExists( $source, array $predicates ) {
322 if ( isset( $predicates['exists'][$source] ) ) {
323 return $predicates['exists'][$source]; // previous op assures this
325 $params = array( 'src' => $source, 'latest' => $this->useLatest
);
326 return $this->backend
->fileExists( $params );
331 * Get the SHA-1 of a file in storage when this operation is attempted
333 * @param $source string Storage path
334 * @param $predicates Array
335 * @return string|bool False on failure
337 final protected function fileSha1( $source, array $predicates ) {
338 if ( isset( $predicates['sha1'][$source] ) ) {
339 return $predicates['sha1'][$source]; // previous op assures this
341 $params = array( 'src' => $source, 'latest' => $this->useLatest
);
342 return $this->backend
->getFileSha1Base36( $params );
347 * Log a file operation failure and preserve any temp files
349 * @param $action string
352 final protected function logFailure( $action ) {
353 $params = $this->params
;
354 $params['failedAction'] = $action;
356 wfDebugLog( 'FileOperation',
357 get_class( $this ) . ' failed: ' . FormatJson
::encode( $params ) );
358 } catch ( Exception
$e ) {
359 // bad config? debug log error?
365 * FileOp helper class to expand PHP execution time for a function.
366 * On construction, set_time_limit() is called and set to $seconds.
367 * When the object goes out of scope, the timer is restarted, with
368 * the original time limit minus the time the object existed.
370 class FileOpScopedPHPTimeout
{
371 protected $startTime; // float; seconds
372 protected $oldTimeout; // integer; seconds
374 protected static $stackDepth = 0; // integer
375 protected static $totalCalls = 0; // integer
376 protected static $totalElapsed = 0; // float; seconds
378 /* Prevent callers in infinite loops from running forever */
379 const MAX_TOTAL_CALLS
= 1000000;
380 const MAX_TOTAL_TIME
= 300; // seconds
383 * @param $seconds integer
385 public function __construct( $seconds ) {
386 if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0
387 if ( self
::$totalCalls >= self
::MAX_TOTAL_CALLS
) {
388 trigger_error( "Maximum invocations of " . __CLASS__
. " exceeded." );
389 } elseif ( self
::$totalElapsed >= self
::MAX_TOTAL_TIME
) {
390 trigger_error( "Time limit within invocations of " . __CLASS__
. " exceeded." );
391 } elseif ( self
::$stackDepth > 0 ) { // recursion guard
392 trigger_error( "Resursive invocation of " . __CLASS__
. " attempted." );
394 $this->oldTimeout
= ini_set( 'max_execution_time', $seconds );
395 $this->startTime
= microtime( true );
397 ++self
::$totalCalls; // proof against < 1us scopes
403 * Restore the original timeout.
404 * This does not account for the timer value on __construct().
406 public function __destruct() {
407 if ( $this->oldTimeout
) {
408 $elapsed = microtime( true ) - $this->startTime
;
409 // Note: a limit of 0 is treated as "forever"
410 set_time_limit( max( 1, $this->oldTimeout
- (int)$elapsed ) );
411 // If each scoped timeout is for less than one second, we end up
412 // restoring the original timeout without any decrease in value.
413 // Thus web scripts in an infinite loop can run forever unless we
414 // take some measures to prevent this. Track total time and calls.
415 self
::$totalElapsed +
= $elapsed;
422 * Store a file into the backend from a file on the file system.
423 * Parameters similar to FileBackendStore::storeInternal(), which include:
424 * src : source path on file system
425 * dst : destination storage path
426 * overwrite : do nothing and pass if an identical file exists at destination
427 * overwriteSame : override any existing file at destination
429 class StoreFileOp
extends FileOp
{
430 protected function allowedParams() {
431 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
434 protected function doPrecheck( array &$predicates ) {
435 $status = Status
::newGood();
436 // Check if the source file exists on the file system
437 if ( !is_file( $this->params
['src'] ) ) {
438 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
440 // Check if the source file is too big
441 } elseif ( filesize( $this->params
['src'] ) > $this->backend
->maxFileSizeInternal() ) {
442 $status->fatal( 'backend-fail-store', $this->params
['src'], $this->params
['dst'] );
444 // Check if a file can be placed at the destination
445 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
446 $status->fatal( 'backend-fail-store', $this->params
['src'], $this->params
['dst'] );
449 // Check if destination file exists
450 $status->merge( $this->precheckDestExistence( $predicates ) );
451 if ( $status->isOK() ) {
452 // Update file existence predicates
453 $predicates['exists'][$this->params
['dst']] = true;
454 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
456 return $status; // safe to call attempt()
459 protected function doAttempt() {
460 $status = Status
::newGood();
461 // Store the file at the destination
462 if ( !$this->destSameAsSource
) {
463 $status->merge( $this->backend
->storeInternal( $this->params
) );
468 protected function getSourceSha1Base36() {
469 wfSuppressWarnings();
470 $hash = sha1_file( $this->params
['src'] );
472 if ( $hash !== false ) {
473 $hash = wfBaseConvert( $hash, 16, 36, 31 );
478 public function storagePathsChanged() {
479 return array( $this->params
['dst'] );
484 * Create a file in the backend with the given content.
485 * Parameters similar to FileBackendStore::createInternal(), which include:
486 * content : the raw file contents
487 * dst : destination storage path
488 * overwrite : do nothing and pass if an identical file exists at destination
489 * overwriteSame : override any existing file at destination
491 class CreateFileOp
extends FileOp
{
492 protected function allowedParams() {
493 return array( array( 'content', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
496 protected function doPrecheck( array &$predicates ) {
497 $status = Status
::newGood();
498 // Check if the source data is too big
499 if ( strlen( $this->getParam( 'content' ) ) > $this->backend
->maxFileSizeInternal() ) {
500 $status->fatal( 'backend-fail-create', $this->params
['dst'] );
502 // Check if a file can be placed at the destination
503 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
504 $status->fatal( 'backend-fail-create', $this->params
['dst'] );
507 // Check if destination file exists
508 $status->merge( $this->precheckDestExistence( $predicates ) );
509 if ( $status->isOK() ) {
510 // Update file existence predicates
511 $predicates['exists'][$this->params
['dst']] = true;
512 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
514 return $status; // safe to call attempt()
517 protected function doAttempt() {
518 $status = Status
::newGood();
519 // Create the file at the destination
520 if ( !$this->destSameAsSource
) {
521 $status->merge( $this->backend
->createInternal( $this->params
) );
526 protected function getSourceSha1Base36() {
527 return wfBaseConvert( sha1( $this->params
['content'] ), 16, 36, 31 );
530 public function storagePathsChanged() {
531 return array( $this->params
['dst'] );
536 * Copy a file from one storage path to another in the backend.
537 * Parameters similar to FileBackendStore::copyInternal(), which include:
538 * src : source storage path
539 * dst : destination storage path
540 * overwrite : do nothing and pass if an identical file exists at destination
541 * overwriteSame : override any existing file at destination
543 class CopyFileOp
extends FileOp
{
544 protected function allowedParams() {
545 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
548 protected function doPrecheck( array &$predicates ) {
549 $status = Status
::newGood();
550 // Check if the source file exists
551 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
552 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
554 // Check if a file can be placed at the destination
555 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
556 $status->fatal( 'backend-fail-copy', $this->params
['src'], $this->params
['dst'] );
559 // Check if destination file exists
560 $status->merge( $this->precheckDestExistence( $predicates ) );
561 if ( $status->isOK() ) {
562 // Update file existence predicates
563 $predicates['exists'][$this->params
['dst']] = true;
564 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
566 return $status; // safe to call attempt()
569 protected function doAttempt() {
570 $status = Status
::newGood();
571 // Do nothing if the src/dst paths are the same
572 if ( $this->params
['src'] !== $this->params
['dst'] ) {
573 // Copy the file into the destination
574 if ( !$this->destSameAsSource
) {
575 $status->merge( $this->backend
->copyInternal( $this->params
) );
581 public function storagePathsRead() {
582 return array( $this->params
['src'] );
585 public function storagePathsChanged() {
586 return array( $this->params
['dst'] );
591 * Move a file from one storage path to another in the backend.
592 * Parameters similar to FileBackendStore::moveInternal(), which include:
593 * src : source storage path
594 * dst : destination storage path
595 * overwrite : do nothing and pass if an identical file exists at destination
596 * overwriteSame : override any existing file at destination
598 class MoveFileOp
extends FileOp
{
599 protected function allowedParams() {
600 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
603 protected function doPrecheck( array &$predicates ) {
604 $status = Status
::newGood();
605 // Check if the source file exists
606 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
607 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
609 // Check if a file can be placed at the destination
610 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
611 $status->fatal( 'backend-fail-move', $this->params
['src'], $this->params
['dst'] );
614 // Check if destination file exists
615 $status->merge( $this->precheckDestExistence( $predicates ) );
616 if ( $status->isOK() ) {
617 // Update file existence predicates
618 $predicates['exists'][$this->params
['src']] = false;
619 $predicates['sha1'][$this->params
['src']] = false;
620 $predicates['exists'][$this->params
['dst']] = true;
621 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
623 return $status; // safe to call attempt()
626 protected function doAttempt() {
627 $status = Status
::newGood();
628 // Do nothing if the src/dst paths are the same
629 if ( $this->params
['src'] !== $this->params
['dst'] ) {
630 if ( !$this->destSameAsSource
) {
631 // Move the file into the destination
632 $status->merge( $this->backend
->moveInternal( $this->params
) );
634 // Just delete source as the destination needs no changes
635 $params = array( 'src' => $this->params
['src'] );
636 $status->merge( $this->backend
->deleteInternal( $params ) );
642 public function storagePathsRead() {
643 return array( $this->params
['src'] );
646 public function storagePathsChanged() {
647 return array( $this->params
['dst'] );
652 * Delete a file at the given storage path from the backend.
653 * Parameters similar to FileBackendStore::deleteInternal(), which include:
654 * src : source storage path
655 * ignoreMissingSource : don't return an error if the file does not exist
657 class DeleteFileOp
extends FileOp
{
658 protected function allowedParams() {
659 return array( array( 'src' ), array( 'ignoreMissingSource' ) );
662 protected $needsDelete = true;
664 protected function doPrecheck( array &$predicates ) {
665 $status = Status
::newGood();
666 // Check if the source file exists
667 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
668 if ( !$this->getParam( 'ignoreMissingSource' ) ) {
669 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
672 $this->needsDelete
= false;
674 // Update file existence predicates
675 $predicates['exists'][$this->params
['src']] = false;
676 $predicates['sha1'][$this->params
['src']] = false;
677 return $status; // safe to call attempt()
680 protected function doAttempt() {
681 $status = Status
::newGood();
682 if ( $this->needsDelete
) {
683 // Delete the source file
684 $status->merge( $this->backend
->deleteInternal( $this->params
) );
689 public function storagePathsChanged() {
690 return array( $this->params
['src'] );
695 * Placeholder operation that has no params and does nothing
697 class NullFileOp
extends FileOp
{}