ad30f0f6d2647430476a00a41dd4834504eb5f74
9 * Helper class for representing operations with transaction support.
10 * Do not use this class from places outside FileBackend.
12 * Methods called from attemptBatch() should avoid throwing exceptions at all costs.
13 * FileOp objects should be lightweight in order to support large arrays in memory.
15 * @ingroup FileBackend
18 abstract class FileOp
{
20 protected $params = array();
21 /** @var FileBackendStore */
24 protected $state = self
::STATE_NEW
; // integer
25 protected $failed = false; // boolean
26 protected $useLatest = true; // boolean
28 protected $sourceSha1; // string
29 protected $destSameAsSource; // boolean
31 /* Object life-cycle */
33 const STATE_CHECKED
= 2;
34 const STATE_ATTEMPTED
= 3;
36 /* Timeout related parameters */
37 const MAX_BATCH_SIZE
= 1000;
38 const TIME_LIMIT_SEC
= 300; // 5 minutes
41 * Build a new file operation transaction
43 * @param $backend FileBackendStore
44 * @param $params Array
47 final public function __construct( FileBackendStore
$backend, array $params ) {
48 $this->backend
= $backend;
49 list( $required, $optional ) = $this->allowedParams();
50 foreach ( $required as $name ) {
51 if ( isset( $params[$name] ) ) {
52 $this->params
[$name] = $params[$name];
54 throw new MWException( "File operation missing parameter '$name'." );
57 foreach ( $optional as $name ) {
58 if ( isset( $params[$name] ) ) {
59 $this->params
[$name] = $params[$name];
62 $this->params
= $params;
66 * Allow stale data for file reads and existence checks
70 final protected function allowStaleReads() {
71 $this->useLatest
= false;
75 * Attempt a series of file operations.
76 * Callers are responsible for handling file locking.
78 * $opts is an array of options, including:
79 * 'force' : Errors that would normally cause a rollback do not.
80 * The remaining operations are still attempted if any fail.
81 * 'allowStale' : Don't require the latest available data.
82 * This can increase performance for non-critical writes.
83 * This has no effect unless the 'force' flag is set.
85 * @param $performOps Array List of FileOp operations
86 * @param $opts Array Batch operation options
89 final public static function attemptBatch( array $performOps, array $opts ) {
90 $status = Status
::newGood();
92 $allowStale = !empty( $opts['allowStale'] );
93 $ignoreErrors = !empty( $opts['force'] );
95 $n = count( $performOps );
96 if ( $n > self
::MAX_BATCH_SIZE
) {
97 $status->fatal( 'backend-fail-batchsize', $n, self
::MAX_BATCH_SIZE
);
101 $predicates = FileOp
::newPredicates(); // account for previous op in prechecks
102 // Do pre-checks for each operation; abort on failure...
103 foreach ( $performOps as $index => $fileOp ) {
105 $fileOp->allowStaleReads(); // allow potentially stale reads
107 $subStatus = $fileOp->precheck( $predicates );
108 $status->merge( $subStatus );
109 if ( !$subStatus->isOK() ) { // operation failed?
110 $status->success
[$index] = false;
111 ++
$status->failCount
;
112 if ( !$ignoreErrors ) {
113 return $status; // abort
118 // Restart PHP's execution timer and set the timeout to safe amount.
119 // This handles cases where the operations take a long time or where we are
120 // already running low on time left. The old timeout is restored afterwards.
121 # @TODO: re-enable this for when the number of batches is high.
122 #$scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC );
124 // Attempt each operation...
125 foreach ( $performOps as $index => $fileOp ) {
126 if ( $fileOp->failed() ) {
127 continue; // nothing to do
129 $subStatus = $fileOp->attempt();
130 $status->merge( $subStatus );
131 if ( $subStatus->isOK() ) {
132 $status->success
[$index] = true;
133 ++
$status->successCount
;
135 $status->success
[$index] = false;
136 ++
$status->failCount
;
137 // We can't continue (even with $ignoreErrors) as $predicates is wrong.
138 // Log the remaining ops as failed for recovery...
139 for ( $i = ($index +
1); $i < count( $performOps ); $i++
) {
140 $performOps[$i]->logFailure( 'attempt_aborted' );
142 return $status; // bail out
150 * Get the value of the parameter with the given name
152 * @param $name string
153 * @return mixed Returns null if the parameter is not set
155 final public function getParam( $name ) {
156 return isset( $this->params
[$name] ) ?
$this->params
[$name] : null;
160 * Check if this operation failed precheck() or attempt()
164 final public function failed() {
165 return $this->failed
;
169 * Get a new empty predicates array for precheck()
173 final public static function newPredicates() {
174 return array( 'exists' => array(), 'sha1' => array() );
178 * Check preconditions of the operation without writing anything
180 * @param $predicates Array
183 final public function precheck( array &$predicates ) {
184 if ( $this->state
!== self
::STATE_NEW
) {
185 return Status
::newFatal( 'fileop-fail-state', self
::STATE_NEW
, $this->state
);
187 $this->state
= self
::STATE_CHECKED
;
188 $status = $this->doPrecheck( $predicates );
189 if ( !$status->isOK() ) {
190 $this->failed
= true;
196 * Attempt the operation, backing up files as needed; this must be reversible
200 final public function attempt() {
201 if ( $this->state
!== self
::STATE_CHECKED
) {
202 return Status
::newFatal( 'fileop-fail-state', self
::STATE_CHECKED
, $this->state
);
203 } elseif ( $this->failed
) { // failed precheck
204 return Status
::newFatal( 'fileop-fail-attempt-precheck' );
206 $this->state
= self
::STATE_ATTEMPTED
;
207 $status = $this->doAttempt();
208 if ( !$status->isOK() ) {
209 $this->failed
= true;
210 $this->logFailure( 'attempt' );
216 * Get the file operation parameters
218 * @return Array (required params list, optional params list)
220 protected function allowedParams() {
221 return array( array(), array() );
225 * Get a list of storage paths read from for this operation
229 public function storagePathsRead() {
234 * Get a list of storage paths written to for this operation
238 public function storagePathsChanged() {
245 protected function doPrecheck( array &$predicates ) {
246 return Status
::newGood();
252 protected function doAttempt() {
253 return Status
::newGood();
257 * Check for errors with regards to the destination file already existing.
258 * This also updates the destSameAsSource and sourceSha1 member variables.
259 * A bad status will be returned if there is no chance it can be overwritten.
261 * @param $predicates Array
264 protected function precheckDestExistence( array $predicates ) {
265 $status = Status
::newGood();
266 // Get hash of source file/string and the destination file
267 $this->sourceSha1
= $this->getSourceSha1Base36(); // FS file or data string
268 if ( $this->sourceSha1
=== null ) { // file in storage?
269 $this->sourceSha1
= $this->fileSha1( $this->params
['src'], $predicates );
271 $this->destSameAsSource
= false;
272 if ( $this->fileExists( $this->params
['dst'], $predicates ) ) {
273 if ( $this->getParam( 'overwrite' ) ) {
274 return $status; // OK
275 } elseif ( $this->getParam( 'overwriteSame' ) ) {
276 $dhash = $this->fileSha1( $this->params
['dst'], $predicates );
277 // Check if hashes are valid and match each other...
278 if ( !strlen( $this->sourceSha1
) ||
!strlen( $dhash ) ) {
279 $status->fatal( 'backend-fail-hashes' );
280 } elseif ( $this->sourceSha1
!== $dhash ) {
281 // Give an error if the files are not identical
282 $status->fatal( 'backend-fail-notsame', $this->params
['dst'] );
284 $this->destSameAsSource
= true; // OK
286 return $status; // do nothing; either OK or bad status
288 $status->fatal( 'backend-fail-alreadyexists', $this->params
['dst'] );
296 * precheckDestExistence() helper function to get the source file SHA-1.
297 * Subclasses should overwride this iff the source is not in storage.
299 * @return string|bool Returns false on failure
301 protected function getSourceSha1Base36() {
306 * Check if a file will exist in storage when this operation is attempted
308 * @param $source string Storage path
309 * @param $predicates Array
312 final protected function fileExists( $source, array $predicates ) {
313 if ( isset( $predicates['exists'][$source] ) ) {
314 return $predicates['exists'][$source]; // previous op assures this
316 $params = array( 'src' => $source, 'latest' => $this->useLatest
);
317 return $this->backend
->fileExists( $params );
322 * Get the SHA-1 of a file in storage when this operation is attempted
324 * @param $source string Storage path
325 * @param $predicates Array
326 * @return string|bool False on failure
328 final protected function fileSha1( $source, array $predicates ) {
329 if ( isset( $predicates['sha1'][$source] ) ) {
330 return $predicates['sha1'][$source]; // previous op assures this
332 $params = array( 'src' => $source, 'latest' => $this->useLatest
);
333 return $this->backend
->getFileSha1Base36( $params );
338 * Log a file operation failure and preserve any temp files
340 * @param $action string
343 final protected function logFailure( $action ) {
344 $params = $this->params
;
345 $params['failedAction'] = $action;
347 wfDebugLog( 'FileOperation',
348 get_class( $this ) . ' failed: ' . FormatJson
::encode( $params ) );
349 } catch ( Exception
$e ) {
350 // bad config? debug log error?
356 * FileOp helper class to expand PHP execution time for a function.
357 * On construction, set_time_limit() is called and set to $seconds.
358 * When the object goes out of scope, the timer is restarted, with
359 * the original time limit minus the time the object existed.
361 class FileOpScopedPHPTimeout
{
362 protected $startTime; // float; seconds
363 protected $oldTimeout; // integer; seconds
365 protected static $stackDepth = 0; // integer
366 protected static $totalCalls = 0; // integer
367 protected static $totalElapsed = 0; // float; seconds
369 /* Prevent callers in infinite loops from running forever */
370 const MAX_TOTAL_CALLS
= 1000000;
371 const MAX_TOTAL_TIME
= 300; // seconds
374 * @param $seconds integer
376 public function __construct( $seconds ) {
377 if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0
378 if ( self
::$totalCalls >= self
::MAX_TOTAL_CALLS
) {
379 trigger_error( "Maximum invocations of " . __CLASS__
. " exceeded." );
380 } elseif ( self
::$totalElapsed >= self
::MAX_TOTAL_TIME
) {
381 trigger_error( "Time limit within invocations of " . __CLASS__
. " exceeded." );
382 } elseif ( self
::$stackDepth > 0 ) { // recursion guard
383 trigger_error( "Resursive invocation of " . __CLASS__
. " attempted." );
385 $this->oldTimeout
= ini_set( 'max_execution_time', $seconds );
386 $this->startTime
= microtime( true );
388 ++self
::$totalCalls; // proof against < 1us scopes
394 * Restore the original timeout.
395 * This does not account for the timer value on __construct().
397 public function __destruct() {
398 if ( $this->oldTimeout
) {
399 $elapsed = microtime( true ) - $this->startTime
;
400 // Note: a limit of 0 is treated as "forever"
401 set_time_limit( max( 1, $this->oldTimeout
- (int)$elapsed ) );
402 // If each scoped timeout is for less than one second, we end up
403 // restoring the original timeout without any decrease in value.
404 // Thus web scripts in an infinite loop can run forever unless we
405 // take some measures to prevent this. Track total time and calls.
406 self
::$totalElapsed +
= $elapsed;
413 * Store a file into the backend from a file on the file system.
414 * Parameters similar to FileBackendStore::storeInternal(), which include:
415 * src : source path on file system
416 * dst : destination storage path
417 * overwrite : do nothing and pass if an identical file exists at destination
418 * overwriteSame : override any existing file at destination
420 class StoreFileOp
extends FileOp
{
421 protected function allowedParams() {
422 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
425 protected function doPrecheck( array &$predicates ) {
426 $status = Status
::newGood();
427 // Check if the source file exists on the file system
428 if ( !is_file( $this->params
['src'] ) ) {
429 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
431 // Check if the source file is too big
432 } elseif ( filesize( $this->params
['src'] ) > $this->backend
->maxFileSizeInternal() ) {
433 $status->fatal( 'backend-fail-store', $this->params
['src'], $this->params
['dst'] );
435 // Check if a file can be placed at the destination
436 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
437 $status->fatal( 'backend-fail-store', $this->params
['src'], $this->params
['dst'] );
440 // Check if destination file exists
441 $status->merge( $this->precheckDestExistence( $predicates ) );
442 if ( $status->isOK() ) {
443 // Update file existence predicates
444 $predicates['exists'][$this->params
['dst']] = true;
445 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
447 return $status; // safe to call attempt()
450 protected function doAttempt() {
451 $status = Status
::newGood();
452 // Store the file at the destination
453 if ( !$this->destSameAsSource
) {
454 $status->merge( $this->backend
->storeInternal( $this->params
) );
459 protected function getSourceSha1Base36() {
460 wfSuppressWarnings();
461 $hash = sha1_file( $this->params
['src'] );
463 if ( $hash !== false ) {
464 $hash = wfBaseConvert( $hash, 16, 36, 31 );
469 public function storagePathsChanged() {
470 return array( $this->params
['dst'] );
475 * Create a file in the backend with the given content.
476 * Parameters similar to FileBackendStore::createInternal(), which include:
477 * content : the raw file contents
478 * dst : destination storage path
479 * overwrite : do nothing and pass if an identical file exists at destination
480 * overwriteSame : override any existing file at destination
482 class CreateFileOp
extends FileOp
{
483 protected function allowedParams() {
484 return array( array( 'content', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
487 protected function doPrecheck( array &$predicates ) {
488 $status = Status
::newGood();
489 // Check if the source data is too big
490 if ( strlen( $this->getParam( 'content' ) ) > $this->backend
->maxFileSizeInternal() ) {
491 $status->fatal( 'backend-fail-create', $this->params
['dst'] );
493 // Check if a file can be placed at the destination
494 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
495 $status->fatal( 'backend-fail-create', $this->params
['dst'] );
498 // Check if destination file exists
499 $status->merge( $this->precheckDestExistence( $predicates ) );
500 if ( $status->isOK() ) {
501 // Update file existence predicates
502 $predicates['exists'][$this->params
['dst']] = true;
503 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
505 return $status; // safe to call attempt()
508 protected function doAttempt() {
509 $status = Status
::newGood();
510 // Create the file at the destination
511 if ( !$this->destSameAsSource
) {
512 $status->merge( $this->backend
->createInternal( $this->params
) );
517 protected function getSourceSha1Base36() {
518 return wfBaseConvert( sha1( $this->params
['content'] ), 16, 36, 31 );
521 public function storagePathsChanged() {
522 return array( $this->params
['dst'] );
527 * Copy a file from one storage path to another in the backend.
528 * Parameters similar to FileBackendStore::copyInternal(), which include:
529 * src : source storage path
530 * dst : destination storage path
531 * overwrite : do nothing and pass if an identical file exists at destination
532 * overwriteSame : override any existing file at destination
534 class CopyFileOp
extends FileOp
{
535 protected function allowedParams() {
536 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
539 protected function doPrecheck( array &$predicates ) {
540 $status = Status
::newGood();
541 // Check if the source file exists
542 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
543 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
545 // Check if a file can be placed at the destination
546 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
547 $status->fatal( 'backend-fail-copy', $this->params
['src'], $this->params
['dst'] );
550 // Check if destination file exists
551 $status->merge( $this->precheckDestExistence( $predicates ) );
552 if ( $status->isOK() ) {
553 // Update file existence predicates
554 $predicates['exists'][$this->params
['dst']] = true;
555 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
557 return $status; // safe to call attempt()
560 protected function doAttempt() {
561 $status = Status
::newGood();
562 // Do nothing if the src/dst paths are the same
563 if ( $this->params
['src'] !== $this->params
['dst'] ) {
564 // Copy the file into the destination
565 if ( !$this->destSameAsSource
) {
566 $status->merge( $this->backend
->copyInternal( $this->params
) );
572 public function storagePathsRead() {
573 return array( $this->params
['src'] );
576 public function storagePathsChanged() {
577 return array( $this->params
['dst'] );
582 * Move a file from one storage path to another in the backend.
583 * Parameters similar to FileBackendStore::moveInternal(), which include:
584 * src : source storage path
585 * dst : destination storage path
586 * overwrite : do nothing and pass if an identical file exists at destination
587 * overwriteSame : override any existing file at destination
589 class MoveFileOp
extends FileOp
{
590 protected function allowedParams() {
591 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
594 protected function doPrecheck( array &$predicates ) {
595 $status = Status
::newGood();
596 // Check if the source file exists
597 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
598 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
600 // Check if a file can be placed at the destination
601 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
602 $status->fatal( 'backend-fail-move', $this->params
['src'], $this->params
['dst'] );
605 // Check if destination file exists
606 $status->merge( $this->precheckDestExistence( $predicates ) );
607 if ( $status->isOK() ) {
608 // Update file existence predicates
609 $predicates['exists'][$this->params
['src']] = false;
610 $predicates['sha1'][$this->params
['src']] = false;
611 $predicates['exists'][$this->params
['dst']] = true;
612 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
614 return $status; // safe to call attempt()
617 protected function doAttempt() {
618 $status = Status
::newGood();
619 // Do nothing if the src/dst paths are the same
620 if ( $this->params
['src'] !== $this->params
['dst'] ) {
621 if ( !$this->destSameAsSource
) {
622 // Move the file into the destination
623 $status->merge( $this->backend
->moveInternal( $this->params
) );
625 // Just delete source as the destination needs no changes
626 $params = array( 'src' => $this->params
['src'] );
627 $status->merge( $this->backend
->deleteInternal( $params ) );
633 public function storagePathsRead() {
634 return array( $this->params
['src'] );
637 public function storagePathsChanged() {
638 return array( $this->params
['dst'] );
643 * Delete a file at the given storage path from the backend.
644 * Parameters similar to FileBackendStore::deleteInternal(), which include:
645 * src : source storage path
646 * ignoreMissingSource : don't return an error if the file does not exist
648 class DeleteFileOp
extends FileOp
{
649 protected function allowedParams() {
650 return array( array( 'src' ), array( 'ignoreMissingSource' ) );
653 protected $needsDelete = true;
655 protected function doPrecheck( array &$predicates ) {
656 $status = Status
::newGood();
657 // Check if the source file exists
658 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
659 if ( !$this->getParam( 'ignoreMissingSource' ) ) {
660 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
663 $this->needsDelete
= false;
665 // Update file existence predicates
666 $predicates['exists'][$this->params
['src']] = false;
667 $predicates['sha1'][$this->params
['src']] = false;
668 return $status; // safe to call attempt()
671 protected function doAttempt() {
672 $status = Status
::newGood();
673 if ( $this->needsDelete
) {
674 // Delete the source file
675 $status->merge( $this->backend
->deleteInternal( $this->params
) );
680 public function storagePathsChanged() {
681 return array( $this->params
['src'] );
686 * Placeholder operation that has no params and does nothing
688 class NullFileOp
extends FileOp
{}