Fixed FileOp::attemptBatch() behavior when unexpected failures occur with the 'force...
[lhc/web/wiklou.git] / includes / filerepo / backend / FileOp.php
1 <?php
2 /**
3 * @file
4 * @ingroup FileBackend
5 * @author Aaron Schulz
6 */
7
8 /**
9 * Helper class for representing operations with transaction support.
10 * Do not use this class from places outside FileBackend.
11 *
12 * Methods called from attemptBatch() should avoid throwing exceptions at all costs.
13 * FileOp objects should be lightweight in order to support large arrays in memory.
14 *
15 * @ingroup FileBackend
16 * @since 1.19
17 */
18 abstract class FileOp {
19 /** @var Array */
20 protected $params = array();
21 /** @var FileBackendStore */
22 protected $backend;
23
24 protected $state = self::STATE_NEW; // integer
25 protected $failed = false; // boolean
26 protected $useLatest = true; // boolean
27
28 protected $sourceSha1; // string
29 protected $destSameAsSource; // boolean
30
31 /* Object life-cycle */
32 const STATE_NEW = 1;
33 const STATE_CHECKED = 2;
34 const STATE_ATTEMPTED = 3;
35
36 /* Timeout related parameters */
37 const MAX_BATCH_SIZE = 1000;
38 const TIME_LIMIT_SEC = 300; // 5 minutes
39
40 /**
41 * Build a new file operation transaction
42 *
43 * @param $backend FileBackendStore
44 * @param $params Array
45 * @throws MWException
46 */
47 final public function __construct( FileBackendStore $backend, array $params ) {
48 $this->backend = $backend;
49 list( $required, $optional ) = $this->allowedParams();
50 foreach ( $required as $name ) {
51 if ( isset( $params[$name] ) ) {
52 $this->params[$name] = $params[$name];
53 } else {
54 throw new MWException( "File operation missing parameter '$name'." );
55 }
56 }
57 foreach ( $optional as $name ) {
58 if ( isset( $params[$name] ) ) {
59 $this->params[$name] = $params[$name];
60 }
61 }
62 $this->params = $params;
63 }
64
65 /**
66 * Allow stale data for file reads and existence checks
67 *
68 * @return void
69 */
70 final protected function allowStaleReads() {
71 $this->useLatest = false;
72 }
73
74 /**
75 * Attempt a series of file operations.
76 * Callers are responsible for handling file locking.
77 *
78 * $opts is an array of options, including:
79 * 'force' : Errors that would normally cause a rollback do not.
80 * The remaining operations are still attempted if any fail.
81 * 'allowStale' : Don't require the latest available data.
82 * This can increase performance for non-critical writes.
83 * This has no effect unless the 'force' flag is set.
84 *
85 * @param $performOps Array List of FileOp operations
86 * @param $opts Array Batch operation options
87 * @return Status
88 */
89 final public static function attemptBatch( array $performOps, array $opts ) {
90 $status = Status::newGood();
91
92 $allowStale = !empty( $opts['allowStale'] );
93 $ignoreErrors = !empty( $opts['force'] );
94
95 $n = count( $performOps );
96 if ( $n > self::MAX_BATCH_SIZE ) {
97 $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
98 return $status;
99 }
100
101 $predicates = FileOp::newPredicates(); // account for previous op in prechecks
102 // Do pre-checks for each operation; abort on failure...
103 foreach ( $performOps as $index => $fileOp ) {
104 if ( $allowStale ) {
105 $fileOp->allowStaleReads(); // allow potentially stale reads
106 }
107 $subStatus = $fileOp->precheck( $predicates );
108 $status->merge( $subStatus );
109 if ( !$subStatus->isOK() ) { // operation failed?
110 $status->success[$index] = false;
111 ++$status->failCount;
112 if ( !$ignoreErrors ) {
113 return $status; // abort
114 }
115 }
116 }
117
118 // Restart PHP's execution timer and set the timeout to safe amount.
119 // This handles cases where the operations take a long time or where we are
120 // already running low on time left. The old timeout is restored afterwards.
121 # @TODO: re-enable this for when the number of batches is high.
122 #$scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC );
123
124 // Attempt each operation...
125 foreach ( $performOps as $index => $fileOp ) {
126 if ( $fileOp->failed() ) {
127 continue; // nothing to do
128 }
129 $subStatus = $fileOp->attempt();
130 $status->merge( $subStatus );
131 if ( $subStatus->isOK() ) {
132 $status->success[$index] = true;
133 ++$status->successCount;
134 } else {
135 $status->success[$index] = false;
136 ++$status->failCount;
137 // We can't continue (even with $ignoreErrors) as $predicates is wrong.
138 // Log the remaining ops as failed for recovery...
139 for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
140 $performOps[$i]->logFailure( 'attempt_aborted' );
141 }
142 return $status; // bail out
143 }
144 }
145
146 return $status;
147 }
148
149 /**
150 * Get the value of the parameter with the given name
151 *
152 * @param $name string
153 * @return mixed Returns null if the parameter is not set
154 */
155 final public function getParam( $name ) {
156 return isset( $this->params[$name] ) ? $this->params[$name] : null;
157 }
158
159 /**
160 * Check if this operation failed precheck() or attempt()
161 *
162 * @return bool
163 */
164 final public function failed() {
165 return $this->failed;
166 }
167
168 /**
169 * Get a new empty predicates array for precheck()
170 *
171 * @return Array
172 */
173 final public static function newPredicates() {
174 return array( 'exists' => array(), 'sha1' => array() );
175 }
176
177 /**
178 * Check preconditions of the operation without writing anything
179 *
180 * @param $predicates Array
181 * @return Status
182 */
183 final public function precheck( array &$predicates ) {
184 if ( $this->state !== self::STATE_NEW ) {
185 return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
186 }
187 $this->state = self::STATE_CHECKED;
188 $status = $this->doPrecheck( $predicates );
189 if ( !$status->isOK() ) {
190 $this->failed = true;
191 }
192 return $status;
193 }
194
195 /**
196 * Attempt the operation, backing up files as needed; this must be reversible
197 *
198 * @return Status
199 */
200 final public function attempt() {
201 if ( $this->state !== self::STATE_CHECKED ) {
202 return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
203 } elseif ( $this->failed ) { // failed precheck
204 return Status::newFatal( 'fileop-fail-attempt-precheck' );
205 }
206 $this->state = self::STATE_ATTEMPTED;
207 $status = $this->doAttempt();
208 if ( !$status->isOK() ) {
209 $this->failed = true;
210 $this->logFailure( 'attempt' );
211 }
212 return $status;
213 }
214
215 /**
216 * Get the file operation parameters
217 *
218 * @return Array (required params list, optional params list)
219 */
220 protected function allowedParams() {
221 return array( array(), array() );
222 }
223
224 /**
225 * Get a list of storage paths read from for this operation
226 *
227 * @return Array
228 */
229 public function storagePathsRead() {
230 return array();
231 }
232
233 /**
234 * Get a list of storage paths written to for this operation
235 *
236 * @return Array
237 */
238 public function storagePathsChanged() {
239 return array();
240 }
241
242 /**
243 * @return Status
244 */
245 protected function doPrecheck( array &$predicates ) {
246 return Status::newGood();
247 }
248
249 /**
250 * @return Status
251 */
252 protected function doAttempt() {
253 return Status::newGood();
254 }
255
256 /**
257 * Check for errors with regards to the destination file already existing.
258 * This also updates the destSameAsSource and sourceSha1 member variables.
259 * A bad status will be returned if there is no chance it can be overwritten.
260 *
261 * @param $predicates Array
262 * @return Status
263 */
264 protected function precheckDestExistence( array $predicates ) {
265 $status = Status::newGood();
266 // Get hash of source file/string and the destination file
267 $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string
268 if ( $this->sourceSha1 === null ) { // file in storage?
269 $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
270 }
271 $this->destSameAsSource = false;
272 if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
273 if ( $this->getParam( 'overwrite' ) ) {
274 return $status; // OK
275 } elseif ( $this->getParam( 'overwriteSame' ) ) {
276 $dhash = $this->fileSha1( $this->params['dst'], $predicates );
277 // Check if hashes are valid and match each other...
278 if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) {
279 $status->fatal( 'backend-fail-hashes' );
280 } elseif ( $this->sourceSha1 !== $dhash ) {
281 // Give an error if the files are not identical
282 $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
283 } else {
284 $this->destSameAsSource = true; // OK
285 }
286 return $status; // do nothing; either OK or bad status
287 } else {
288 $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
289 return $status;
290 }
291 }
292 return $status;
293 }
294
295 /**
296 * precheckDestExistence() helper function to get the source file SHA-1.
297 * Subclasses should overwride this iff the source is not in storage.
298 *
299 * @return string|bool Returns false on failure
300 */
301 protected function getSourceSha1Base36() {
302 return null; // N/A
303 }
304
305 /**
306 * Check if a file will exist in storage when this operation is attempted
307 *
308 * @param $source string Storage path
309 * @param $predicates Array
310 * @return bool
311 */
312 final protected function fileExists( $source, array $predicates ) {
313 if ( isset( $predicates['exists'][$source] ) ) {
314 return $predicates['exists'][$source]; // previous op assures this
315 } else {
316 $params = array( 'src' => $source, 'latest' => $this->useLatest );
317 return $this->backend->fileExists( $params );
318 }
319 }
320
321 /**
322 * Get the SHA-1 of a file in storage when this operation is attempted
323 *
324 * @param $source string Storage path
325 * @param $predicates Array
326 * @return string|bool False on failure
327 */
328 final protected function fileSha1( $source, array $predicates ) {
329 if ( isset( $predicates['sha1'][$source] ) ) {
330 return $predicates['sha1'][$source]; // previous op assures this
331 } else {
332 $params = array( 'src' => $source, 'latest' => $this->useLatest );
333 return $this->backend->getFileSha1Base36( $params );
334 }
335 }
336
337 /**
338 * Log a file operation failure and preserve any temp files
339 *
340 * @param $action string
341 * @return void
342 */
343 final protected function logFailure( $action ) {
344 $params = $this->params;
345 $params['failedAction'] = $action;
346 try {
347 wfDebugLog( 'FileOperation',
348 get_class( $this ) . ' failed: ' . FormatJson::encode( $params ) );
349 } catch ( Exception $e ) {
350 // bad config? debug log error?
351 }
352 }
353 }
354
355 /**
356 * FileOp helper class to expand PHP execution time for a function.
357 * On construction, set_time_limit() is called and set to $seconds.
358 * When the object goes out of scope, the timer is restarted, with
359 * the original time limit minus the time the object existed.
360 */
361 class FileOpScopedPHPTimeout {
362 protected $startTime; // float; seconds
363 protected $oldTimeout; // integer; seconds
364
365 protected static $stackDepth = 0; // integer
366 protected static $totalCalls = 0; // integer
367 protected static $totalElapsed = 0; // float; seconds
368
369 /* Prevent callers in infinite loops from running forever */
370 const MAX_TOTAL_CALLS = 1000000;
371 const MAX_TOTAL_TIME = 300; // seconds
372
373 /**
374 * @param $seconds integer
375 */
376 public function __construct( $seconds ) {
377 if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0
378 if ( self::$totalCalls >= self::MAX_TOTAL_CALLS ) {
379 trigger_error( "Maximum invocations of " . __CLASS__ . " exceeded." );
380 } elseif ( self::$totalElapsed >= self::MAX_TOTAL_TIME ) {
381 trigger_error( "Time limit within invocations of " . __CLASS__ . " exceeded." );
382 } elseif ( self::$stackDepth > 0 ) { // recursion guard
383 trigger_error( "Resursive invocation of " . __CLASS__ . " attempted." );
384 } else {
385 $this->oldTimeout = ini_set( 'max_execution_time', $seconds );
386 $this->startTime = microtime( true );
387 ++self::$stackDepth;
388 ++self::$totalCalls; // proof against < 1us scopes
389 }
390 }
391 }
392
393 /**
394 * Restore the original timeout.
395 * This does not account for the timer value on __construct().
396 */
397 public function __destruct() {
398 if ( $this->oldTimeout ) {
399 $elapsed = microtime( true ) - $this->startTime;
400 // Note: a limit of 0 is treated as "forever"
401 set_time_limit( max( 1, $this->oldTimeout - (int)$elapsed ) );
402 // If each scoped timeout is for less than one second, we end up
403 // restoring the original timeout without any decrease in value.
404 // Thus web scripts in an infinite loop can run forever unless we
405 // take some measures to prevent this. Track total time and calls.
406 self::$totalElapsed += $elapsed;
407 --self::$stackDepth;
408 }
409 }
410 }
411
412 /**
413 * Store a file into the backend from a file on the file system.
414 * Parameters similar to FileBackendStore::storeInternal(), which include:
415 * src : source path on file system
416 * dst : destination storage path
417 * overwrite : do nothing and pass if an identical file exists at destination
418 * overwriteSame : override any existing file at destination
419 */
420 class StoreFileOp extends FileOp {
421 protected function allowedParams() {
422 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
423 }
424
425 protected function doPrecheck( array &$predicates ) {
426 $status = Status::newGood();
427 // Check if the source file exists on the file system
428 if ( !is_file( $this->params['src'] ) ) {
429 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
430 return $status;
431 // Check if the source file is too big
432 } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
433 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
434 return $status;
435 // Check if a file can be placed at the destination
436 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
437 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
438 return $status;
439 }
440 // Check if destination file exists
441 $status->merge( $this->precheckDestExistence( $predicates ) );
442 if ( $status->isOK() ) {
443 // Update file existence predicates
444 $predicates['exists'][$this->params['dst']] = true;
445 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
446 }
447 return $status; // safe to call attempt()
448 }
449
450 protected function doAttempt() {
451 $status = Status::newGood();
452 // Store the file at the destination
453 if ( !$this->destSameAsSource ) {
454 $status->merge( $this->backend->storeInternal( $this->params ) );
455 }
456 return $status;
457 }
458
459 protected function getSourceSha1Base36() {
460 wfSuppressWarnings();
461 $hash = sha1_file( $this->params['src'] );
462 wfRestoreWarnings();
463 if ( $hash !== false ) {
464 $hash = wfBaseConvert( $hash, 16, 36, 31 );
465 }
466 return $hash;
467 }
468
469 public function storagePathsChanged() {
470 return array( $this->params['dst'] );
471 }
472 }
473
474 /**
475 * Create a file in the backend with the given content.
476 * Parameters similar to FileBackendStore::createInternal(), which include:
477 * content : the raw file contents
478 * dst : destination storage path
479 * overwrite : do nothing and pass if an identical file exists at destination
480 * overwriteSame : override any existing file at destination
481 */
482 class CreateFileOp extends FileOp {
483 protected function allowedParams() {
484 return array( array( 'content', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
485 }
486
487 protected function doPrecheck( array &$predicates ) {
488 $status = Status::newGood();
489 // Check if the source data is too big
490 if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
491 $status->fatal( 'backend-fail-create', $this->params['dst'] );
492 return $status;
493 // Check if a file can be placed at the destination
494 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
495 $status->fatal( 'backend-fail-create', $this->params['dst'] );
496 return $status;
497 }
498 // Check if destination file exists
499 $status->merge( $this->precheckDestExistence( $predicates ) );
500 if ( $status->isOK() ) {
501 // Update file existence predicates
502 $predicates['exists'][$this->params['dst']] = true;
503 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
504 }
505 return $status; // safe to call attempt()
506 }
507
508 protected function doAttempt() {
509 $status = Status::newGood();
510 // Create the file at the destination
511 if ( !$this->destSameAsSource ) {
512 $status->merge( $this->backend->createInternal( $this->params ) );
513 }
514 return $status;
515 }
516
517 protected function getSourceSha1Base36() {
518 return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
519 }
520
521 public function storagePathsChanged() {
522 return array( $this->params['dst'] );
523 }
524 }
525
526 /**
527 * Copy a file from one storage path to another in the backend.
528 * Parameters similar to FileBackendStore::copyInternal(), which include:
529 * src : source storage path
530 * dst : destination storage path
531 * overwrite : do nothing and pass if an identical file exists at destination
532 * overwriteSame : override any existing file at destination
533 */
534 class CopyFileOp extends FileOp {
535 protected function allowedParams() {
536 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
537 }
538
539 protected function doPrecheck( array &$predicates ) {
540 $status = Status::newGood();
541 // Check if the source file exists
542 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
543 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
544 return $status;
545 // Check if a file can be placed at the destination
546 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
547 $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] );
548 return $status;
549 }
550 // Check if destination file exists
551 $status->merge( $this->precheckDestExistence( $predicates ) );
552 if ( $status->isOK() ) {
553 // Update file existence predicates
554 $predicates['exists'][$this->params['dst']] = true;
555 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
556 }
557 return $status; // safe to call attempt()
558 }
559
560 protected function doAttempt() {
561 $status = Status::newGood();
562 // Do nothing if the src/dst paths are the same
563 if ( $this->params['src'] !== $this->params['dst'] ) {
564 // Copy the file into the destination
565 if ( !$this->destSameAsSource ) {
566 $status->merge( $this->backend->copyInternal( $this->params ) );
567 }
568 }
569 return $status;
570 }
571
572 public function storagePathsRead() {
573 return array( $this->params['src'] );
574 }
575
576 public function storagePathsChanged() {
577 return array( $this->params['dst'] );
578 }
579 }
580
581 /**
582 * Move a file from one storage path to another in the backend.
583 * Parameters similar to FileBackendStore::moveInternal(), which include:
584 * src : source storage path
585 * dst : destination storage path
586 * overwrite : do nothing and pass if an identical file exists at destination
587 * overwriteSame : override any existing file at destination
588 */
589 class MoveFileOp extends FileOp {
590 protected function allowedParams() {
591 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
592 }
593
594 protected function doPrecheck( array &$predicates ) {
595 $status = Status::newGood();
596 // Check if the source file exists
597 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
598 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
599 return $status;
600 // Check if a file can be placed at the destination
601 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
602 $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] );
603 return $status;
604 }
605 // Check if destination file exists
606 $status->merge( $this->precheckDestExistence( $predicates ) );
607 if ( $status->isOK() ) {
608 // Update file existence predicates
609 $predicates['exists'][$this->params['src']] = false;
610 $predicates['sha1'][$this->params['src']] = false;
611 $predicates['exists'][$this->params['dst']] = true;
612 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
613 }
614 return $status; // safe to call attempt()
615 }
616
617 protected function doAttempt() {
618 $status = Status::newGood();
619 // Do nothing if the src/dst paths are the same
620 if ( $this->params['src'] !== $this->params['dst'] ) {
621 if ( !$this->destSameAsSource ) {
622 // Move the file into the destination
623 $status->merge( $this->backend->moveInternal( $this->params ) );
624 } else {
625 // Just delete source as the destination needs no changes
626 $params = array( 'src' => $this->params['src'] );
627 $status->merge( $this->backend->deleteInternal( $params ) );
628 }
629 }
630 return $status;
631 }
632
633 public function storagePathsRead() {
634 return array( $this->params['src'] );
635 }
636
637 public function storagePathsChanged() {
638 return array( $this->params['dst'] );
639 }
640 }
641
642 /**
643 * Delete a file at the given storage path from the backend.
644 * Parameters similar to FileBackendStore::deleteInternal(), which include:
645 * src : source storage path
646 * ignoreMissingSource : don't return an error if the file does not exist
647 */
648 class DeleteFileOp extends FileOp {
649 protected function allowedParams() {
650 return array( array( 'src' ), array( 'ignoreMissingSource' ) );
651 }
652
653 protected $needsDelete = true;
654
655 protected function doPrecheck( array &$predicates ) {
656 $status = Status::newGood();
657 // Check if the source file exists
658 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
659 if ( !$this->getParam( 'ignoreMissingSource' ) ) {
660 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
661 return $status;
662 }
663 $this->needsDelete = false;
664 }
665 // Update file existence predicates
666 $predicates['exists'][$this->params['src']] = false;
667 $predicates['sha1'][$this->params['src']] = false;
668 return $status; // safe to call attempt()
669 }
670
671 protected function doAttempt() {
672 $status = Status::newGood();
673 if ( $this->needsDelete ) {
674 // Delete the source file
675 $status->merge( $this->backend->deleteInternal( $this->params ) );
676 }
677 return $status;
678 }
679
680 public function storagePathsChanged() {
681 return array( $this->params['src'] );
682 }
683 }
684
685 /**
686 * Placeholder operation that has no params and does nothing
687 */
688 class NullFileOp extends FileOp {}