9 * Helper class for representing operations with transaction support.
10 * Do not use this class from places outside FileBackend.
12 * Methods called from attemptBatch() should avoid throwing exceptions at all costs.
13 * FileOp objects should be lightweight in order to support large arrays in memory.
15 * @ingroup FileBackend
18 abstract class FileOp
{
20 protected $params = array();
21 /** @var FileBackendStore */
24 protected $state = self
::STATE_NEW
; // integer
25 protected $failed = false; // boolean
26 protected $useLatest = true; // boolean
28 protected $sourceSha1; // string
29 protected $destSameAsSource; // boolean
31 /* Object life-cycle */
33 const STATE_CHECKED
= 2;
34 const STATE_ATTEMPTED
= 3;
36 /* Timeout related parameters */
37 const MAX_BATCH_SIZE
= 1000;
38 const TIME_LIMIT_SEC
= 300; // 5 minutes
41 * Build a new file operation transaction
43 * @param $backend FileBackendStore
44 * @param $params Array
47 final public function __construct( FileBackendStore
$backend, array $params ) {
48 $this->backend
= $backend;
49 list( $required, $optional ) = $this->allowedParams();
50 foreach ( $required as $name ) {
51 if ( isset( $params[$name] ) ) {
52 $this->params
[$name] = $params[$name];
54 throw new MWException( "File operation missing parameter '$name'." );
57 foreach ( $optional as $name ) {
58 if ( isset( $params[$name] ) ) {
59 $this->params
[$name] = $params[$name];
62 $this->params
= $params;
66 * Allow stale data for file reads and existence checks
70 final protected function allowStaleReads() {
71 $this->useLatest
= false;
75 * Attempt a series of file operations.
76 * Callers are responsible for handling file locking.
78 * $opts is an array of options, including:
79 * 'force' : Errors that would normally cause a rollback do not.
80 * The remaining operations are still attempted if any fail.
81 * 'allowStale' : Don't require the latest available data.
82 * This can increase performance for non-critical writes.
83 * This has no effect unless the 'force' flag is set.
85 * @param $performOps Array List of FileOp operations
86 * @param $opts Array Batch operation options
89 final public static function attemptBatch( array $performOps, array $opts ) {
90 $status = Status
::newGood();
92 $allowStale = !empty( $opts['allowStale'] );
93 $ignoreErrors = !empty( $opts['force'] );
95 $n = count( $performOps );
96 if ( $n > self
::MAX_BATCH_SIZE
) {
97 $status->fatal( 'backend-fail-batchsize', $n, self
::MAX_BATCH_SIZE
);
101 $predicates = FileOp
::newPredicates(); // account for previous op in prechecks
102 // Do pre-checks for each operation; abort on failure...
103 foreach ( $performOps as $index => $fileOp ) {
105 $fileOp->allowStaleReads(); // allow potentially stale reads
107 $subStatus = $fileOp->precheck( $predicates );
108 $status->merge( $subStatus );
109 if ( !$subStatus->isOK() ) { // operation failed?
110 $status->success
[$index] = false;
111 ++
$status->failCount
;
112 if ( !$ignoreErrors ) {
113 return $status; // abort
118 // Restart PHP's execution timer and set the timeout to safe amount.
119 // This handles cases where the operations take a long time or where we are
120 // already running low on time left. The old timeout is restored afterwards.
121 # @TODO: re-enable this for when the number of batches is high.
122 #$scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC );
124 // Attempt each operation...
125 foreach ( $performOps as $index => $fileOp ) {
126 if ( $fileOp->failed() ) {
127 continue; // nothing to do
129 $subStatus = $fileOp->attempt();
130 $status->merge( $subStatus );
131 if ( $subStatus->isOK() ) {
132 $status->success
[$index] = true;
133 ++
$status->successCount
;
135 $status->success
[$index] = false;
136 ++
$status->failCount
;
137 if ( !$ignoreErrors ) {
138 // Log remaining ops as failed for recovery...
139 for ( $i = ($index +
1); $i < count( $performOps ); $i++
) {
140 $performOps[$i]->logFailure( 'attempt_aborted' );
142 return $status; // bail out
151 * Get the value of the parameter with the given name
153 * @param $name string
154 * @return mixed Returns null if the parameter is not set
156 final public function getParam( $name ) {
157 return isset( $this->params
[$name] ) ?
$this->params
[$name] : null;
161 * Check if this operation failed precheck() or attempt()
165 final public function failed() {
166 return $this->failed
;
170 * Get a new empty predicates array for precheck()
174 final public static function newPredicates() {
175 return array( 'exists' => array(), 'sha1' => array() );
179 * Check preconditions of the operation without writing anything
181 * @param $predicates Array
184 final public function precheck( array &$predicates ) {
185 if ( $this->state
!== self
::STATE_NEW
) {
186 return Status
::newFatal( 'fileop-fail-state', self
::STATE_NEW
, $this->state
);
188 $this->state
= self
::STATE_CHECKED
;
189 $status = $this->doPrecheck( $predicates );
190 if ( !$status->isOK() ) {
191 $this->failed
= true;
197 * Attempt the operation, backing up files as needed; this must be reversible
201 final public function attempt() {
202 if ( $this->state
!== self
::STATE_CHECKED
) {
203 return Status
::newFatal( 'fileop-fail-state', self
::STATE_CHECKED
, $this->state
);
204 } elseif ( $this->failed
) { // failed precheck
205 return Status
::newFatal( 'fileop-fail-attempt-precheck' );
207 $this->state
= self
::STATE_ATTEMPTED
;
208 $status = $this->doAttempt();
209 if ( !$status->isOK() ) {
210 $this->failed
= true;
211 $this->logFailure( 'attempt' );
217 * Get the file operation parameters
219 * @return Array (required params list, optional params list)
221 protected function allowedParams() {
222 return array( array(), array() );
226 * Get a list of storage paths read from for this operation
230 public function storagePathsRead() {
235 * Get a list of storage paths written to for this operation
239 public function storagePathsChanged() {
246 protected function doPrecheck( array &$predicates ) {
247 return Status
::newGood();
253 protected function doAttempt() {
254 return Status
::newGood();
258 * Check for errors with regards to the destination file already existing.
259 * This also updates the destSameAsSource and sourceSha1 member variables.
260 * A bad status will be returned if there is no chance it can be overwritten.
262 * @param $predicates Array
265 protected function precheckDestExistence( array $predicates ) {
266 $status = Status
::newGood();
267 // Get hash of source file/string and the destination file
268 $this->sourceSha1
= $this->getSourceSha1Base36(); // FS file or data string
269 if ( $this->sourceSha1
=== null ) { // file in storage?
270 $this->sourceSha1
= $this->fileSha1( $this->params
['src'], $predicates );
272 $this->destSameAsSource
= false;
273 if ( $this->fileExists( $this->params
['dst'], $predicates ) ) {
274 if ( $this->getParam( 'overwrite' ) ) {
275 return $status; // OK
276 } elseif ( $this->getParam( 'overwriteSame' ) ) {
277 $dhash = $this->fileSha1( $this->params
['dst'], $predicates );
278 // Check if hashes are valid and match each other...
279 if ( !strlen( $this->sourceSha1
) ||
!strlen( $dhash ) ) {
280 $status->fatal( 'backend-fail-hashes' );
281 } elseif ( $this->sourceSha1
!== $dhash ) {
282 // Give an error if the files are not identical
283 $status->fatal( 'backend-fail-notsame', $this->params
['dst'] );
285 $this->destSameAsSource
= true; // OK
287 return $status; // do nothing; either OK or bad status
289 $status->fatal( 'backend-fail-alreadyexists', $this->params
['dst'] );
297 * precheckDestExistence() helper function to get the source file SHA-1.
298 * Subclasses should overwride this iff the source is not in storage.
300 * @return string|bool Returns false on failure
302 protected function getSourceSha1Base36() {
307 * Check if a file will exist in storage when this operation is attempted
309 * @param $source string Storage path
310 * @param $predicates Array
313 final protected function fileExists( $source, array $predicates ) {
314 if ( isset( $predicates['exists'][$source] ) ) {
315 return $predicates['exists'][$source]; // previous op assures this
317 $params = array( 'src' => $source, 'latest' => $this->useLatest
);
318 return $this->backend
->fileExists( $params );
323 * Get the SHA-1 of a file in storage when this operation is attempted
325 * @param $source string Storage path
326 * @param $predicates Array
327 * @return string|bool False on failure
329 final protected function fileSha1( $source, array $predicates ) {
330 if ( isset( $predicates['sha1'][$source] ) ) {
331 return $predicates['sha1'][$source]; // previous op assures this
333 $params = array( 'src' => $source, 'latest' => $this->useLatest
);
334 return $this->backend
->getFileSha1Base36( $params );
339 * Log a file operation failure and preserve any temp files
341 * @param $action string
344 final protected function logFailure( $action ) {
345 $params = $this->params
;
346 $params['failedAction'] = $action;
348 wfDebugLog( 'FileOperation',
349 get_class( $this ) . ' failed: ' . FormatJson
::encode( $params ) );
350 } catch ( Exception
$e ) {
351 // bad config? debug log error?
357 * FileOp helper class to expand PHP execution time for a function.
358 * On construction, set_time_limit() is called and set to $seconds.
359 * When the object goes out of scope, the timer is restarted, with
360 * the original time limit minus the time the object existed.
362 class FileOpScopedPHPTimeout
{
363 protected $startTime; // float; seconds
364 protected $oldTimeout; // integer; seconds
366 protected static $stackDepth = 0; // integer
367 protected static $totalCalls = 0; // integer
368 protected static $totalElapsed = 0; // float; seconds
370 /* Prevent callers in infinite loops from running forever */
371 const MAX_TOTAL_CALLS
= 1000000;
372 const MAX_TOTAL_TIME
= 300; // seconds
375 * @param $seconds integer
377 public function __construct( $seconds ) {
378 if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0
379 if ( self
::$totalCalls >= self
::MAX_TOTAL_CALLS
) {
380 trigger_error( "Maximum invocations of " . __CLASS__
. " exceeded." );
381 } elseif ( self
::$totalElapsed >= self
::MAX_TOTAL_TIME
) {
382 trigger_error( "Time limit within invocations of " . __CLASS__
. " exceeded." );
383 } elseif ( self
::$stackDepth > 0 ) { // recursion guard
384 trigger_error( "Resursive invocation of " . __CLASS__
. " attempted." );
386 $this->oldTimeout
= ini_set( 'max_execution_time', $seconds );
387 $this->startTime
= microtime( true );
389 ++self
::$totalCalls; // proof against < 1us scopes
395 * Restore the original timeout.
396 * This does not account for the timer value on __construct().
398 public function __destruct() {
399 if ( $this->oldTimeout
) {
400 $elapsed = microtime( true ) - $this->startTime
;
401 // Note: a limit of 0 is treated as "forever"
402 set_time_limit( max( 1, $this->oldTimeout
- (int)$elapsed ) );
403 // If each scoped timeout is for less than one second, we end up
404 // restoring the original timeout without any decrease in value.
405 // Thus web scripts in an infinite loop can run forever unless we
406 // take some measures to prevent this. Track total time and calls.
407 self
::$totalElapsed +
= $elapsed;
414 * Store a file into the backend from a file on the file system.
415 * Parameters similar to FileBackendStore::storeInternal(), which include:
416 * src : source path on file system
417 * dst : destination storage path
418 * overwrite : do nothing and pass if an identical file exists at destination
419 * overwriteSame : override any existing file at destination
421 class StoreFileOp
extends FileOp
{
422 protected function allowedParams() {
423 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
426 protected function doPrecheck( array &$predicates ) {
427 $status = Status
::newGood();
428 // Check if the source file exists on the file system
429 if ( !is_file( $this->params
['src'] ) ) {
430 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
432 // Check if the source file is too big
433 } elseif ( filesize( $this->params
['src'] ) > $this->backend
->maxFileSizeInternal() ) {
434 $status->fatal( 'backend-fail-store', $this->params
['src'], $this->params
['dst'] );
436 // Check if a file can be placed at the destination
437 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
438 $status->fatal( 'backend-fail-store', $this->params
['src'], $this->params
['dst'] );
441 // Check if destination file exists
442 $status->merge( $this->precheckDestExistence( $predicates ) );
443 if ( $status->isOK() ) {
444 // Update file existence predicates
445 $predicates['exists'][$this->params
['dst']] = true;
446 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
448 return $status; // safe to call attempt()
451 protected function doAttempt() {
452 $status = Status
::newGood();
453 // Store the file at the destination
454 if ( !$this->destSameAsSource
) {
455 $status->merge( $this->backend
->storeInternal( $this->params
) );
460 protected function getSourceSha1Base36() {
461 wfSuppressWarnings();
462 $hash = sha1_file( $this->params
['src'] );
464 if ( $hash !== false ) {
465 $hash = wfBaseConvert( $hash, 16, 36, 31 );
470 public function storagePathsChanged() {
471 return array( $this->params
['dst'] );
476 * Create a file in the backend with the given content.
477 * Parameters similar to FileBackendStore::createInternal(), which include:
478 * content : the raw file contents
479 * dst : destination storage path
480 * overwrite : do nothing and pass if an identical file exists at destination
481 * overwriteSame : override any existing file at destination
483 class CreateFileOp
extends FileOp
{
484 protected function allowedParams() {
485 return array( array( 'content', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
488 protected function doPrecheck( array &$predicates ) {
489 $status = Status
::newGood();
490 // Check if the source data is too big
491 if ( strlen( $this->getParam( 'content' ) ) > $this->backend
->maxFileSizeInternal() ) {
492 $status->fatal( 'backend-fail-create', $this->params
['dst'] );
494 // Check if a file can be placed at the destination
495 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
496 $status->fatal( 'backend-fail-create', $this->params
['dst'] );
499 // Check if destination file exists
500 $status->merge( $this->precheckDestExistence( $predicates ) );
501 if ( $status->isOK() ) {
502 // Update file existence predicates
503 $predicates['exists'][$this->params
['dst']] = true;
504 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
506 return $status; // safe to call attempt()
509 protected function doAttempt() {
510 $status = Status
::newGood();
511 // Create the file at the destination
512 if ( !$this->destSameAsSource
) {
513 $status->merge( $this->backend
->createInternal( $this->params
) );
518 protected function getSourceSha1Base36() {
519 return wfBaseConvert( sha1( $this->params
['content'] ), 16, 36, 31 );
522 public function storagePathsChanged() {
523 return array( $this->params
['dst'] );
528 * Copy a file from one storage path to another in the backend.
529 * Parameters similar to FileBackendStore::copyInternal(), which include:
530 * src : source storage path
531 * dst : destination storage path
532 * overwrite : do nothing and pass if an identical file exists at destination
533 * overwriteSame : override any existing file at destination
535 class CopyFileOp
extends FileOp
{
536 protected function allowedParams() {
537 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
540 protected function doPrecheck( array &$predicates ) {
541 $status = Status
::newGood();
542 // Check if the source file exists
543 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
544 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
546 // Check if a file can be placed at the destination
547 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
548 $status->fatal( 'backend-fail-copy', $this->params
['src'], $this->params
['dst'] );
551 // Check if destination file exists
552 $status->merge( $this->precheckDestExistence( $predicates ) );
553 if ( $status->isOK() ) {
554 // Update file existence predicates
555 $predicates['exists'][$this->params
['dst']] = true;
556 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
558 return $status; // safe to call attempt()
561 protected function doAttempt() {
562 $status = Status
::newGood();
563 // Do nothing if the src/dst paths are the same
564 if ( $this->params
['src'] !== $this->params
['dst'] ) {
565 // Copy the file into the destination
566 if ( !$this->destSameAsSource
) {
567 $status->merge( $this->backend
->copyInternal( $this->params
) );
573 public function storagePathsRead() {
574 return array( $this->params
['src'] );
577 public function storagePathsChanged() {
578 return array( $this->params
['dst'] );
583 * Move a file from one storage path to another in the backend.
584 * Parameters similar to FileBackendStore::moveInternal(), which include:
585 * src : source storage path
586 * dst : destination storage path
587 * overwrite : do nothing and pass if an identical file exists at destination
588 * overwriteSame : override any existing file at destination
590 class MoveFileOp
extends FileOp
{
591 protected function allowedParams() {
592 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
595 protected function doPrecheck( array &$predicates ) {
596 $status = Status
::newGood();
597 // Check if the source file exists
598 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
599 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
601 // Check if a file can be placed at the destination
602 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
603 $status->fatal( 'backend-fail-move', $this->params
['src'], $this->params
['dst'] );
606 // Check if destination file exists
607 $status->merge( $this->precheckDestExistence( $predicates ) );
608 if ( $status->isOK() ) {
609 // Update file existence predicates
610 $predicates['exists'][$this->params
['src']] = false;
611 $predicates['sha1'][$this->params
['src']] = false;
612 $predicates['exists'][$this->params
['dst']] = true;
613 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
615 return $status; // safe to call attempt()
618 protected function doAttempt() {
619 $status = Status
::newGood();
620 // Do nothing if the src/dst paths are the same
621 if ( $this->params
['src'] !== $this->params
['dst'] ) {
622 if ( !$this->destSameAsSource
) {
623 // Move the file into the destination
624 $status->merge( $this->backend
->moveInternal( $this->params
) );
626 // Just delete source as the destination needs no changes
627 $params = array( 'src' => $this->params
['src'] );
628 $status->merge( $this->backend
->deleteInternal( $params ) );
634 public function storagePathsRead() {
635 return array( $this->params
['src'] );
638 public function storagePathsChanged() {
639 return array( $this->params
['dst'] );
644 * Delete a file at the given storage path from the backend.
645 * Parameters similar to FileBackendStore::deleteInternal(), which include:
646 * src : source storage path
647 * ignoreMissingSource : don't return an error if the file does not exist
649 class DeleteFileOp
extends FileOp
{
650 protected function allowedParams() {
651 return array( array( 'src' ), array( 'ignoreMissingSource' ) );
654 protected $needsDelete = true;
656 protected function doPrecheck( array &$predicates ) {
657 $status = Status
::newGood();
658 // Check if the source file exists
659 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
660 if ( !$this->getParam( 'ignoreMissingSource' ) ) {
661 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
664 $this->needsDelete
= false;
666 // Update file existence predicates
667 $predicates['exists'][$this->params
['src']] = false;
668 $predicates['sha1'][$this->params
['src']] = false;
669 return $status; // safe to call attempt()
672 protected function doAttempt() {
673 $status = Status
::newGood();
674 if ( $this->needsDelete
) {
675 // Delete the source file
676 $status->merge( $this->backend
->deleteInternal( $this->params
) );
681 public function storagePathsChanged() {
682 return array( $this->params
['src'] );
687 * Placeholder operation that has no params and does nothing
689 class NullFileOp
extends FileOp
{}