[FileBackend] Made doOperations() Status handling align with documentation as well...
[lhc/web/wiklou.git] / includes / filerepo / backend / FileOp.php
1 <?php
2 /**
3 * @file
4 * @ingroup FileBackend
5 * @author Aaron Schulz
6 */
7
8 /**
9 * Helper class for representing operations with transaction support.
10 * Do not use this class from places outside FileBackend.
11 *
12 * Methods called from attemptBatch() should avoid throwing exceptions at all costs.
13 * FileOp objects should be lightweight in order to support large arrays in memory.
14 *
15 * @ingroup FileBackend
16 * @since 1.19
17 */
18 abstract class FileOp {
19 /** @var Array */
20 protected $params = array();
21 /** @var FileBackendStore */
22 protected $backend;
23
24 protected $state = self::STATE_NEW; // integer
25 protected $failed = false; // boolean
26 protected $useLatest = true; // boolean
27
28 protected $sourceSha1; // string
29 protected $destSameAsSource; // boolean
30
31 /* Object life-cycle */
32 const STATE_NEW = 1;
33 const STATE_CHECKED = 2;
34 const STATE_ATTEMPTED = 3;
35
36 /* Timeout related parameters */
37 const MAX_BATCH_SIZE = 1000;
38 const TIME_LIMIT_SEC = 300; // 5 minutes
39
40 /**
41 * Build a new file operation transaction
42 *
43 * @param $backend FileBackendStore
44 * @param $params Array
45 * @throws MWException
46 */
47 final public function __construct( FileBackendStore $backend, array $params ) {
48 $this->backend = $backend;
49 list( $required, $optional ) = $this->allowedParams();
50 foreach ( $required as $name ) {
51 if ( isset( $params[$name] ) ) {
52 $this->params[$name] = $params[$name];
53 } else {
54 throw new MWException( "File operation missing parameter '$name'." );
55 }
56 }
57 foreach ( $optional as $name ) {
58 if ( isset( $params[$name] ) ) {
59 $this->params[$name] = $params[$name];
60 }
61 }
62 $this->params = $params;
63 }
64
65 /**
66 * Allow stale data for file reads and existence checks
67 *
68 * @return void
69 */
70 final protected function allowStaleReads() {
71 $this->useLatest = false;
72 }
73
74 /**
75 * Attempt a series of file operations.
76 * Callers are responsible for handling file locking.
77 *
78 * $opts is an array of options, including:
79 * 'force' : Errors that would normally cause a rollback do not.
80 * The remaining operations are still attempted if any fail.
81 * 'allowStale' : Don't require the latest available data.
82 * This can increase performance for non-critical writes.
83 * This has no effect unless the 'force' flag is set.
84 *
85 * The resulting Status will be "OK" unless:
86 * a) unexpected operation errors occurred (network partitions, disk full...)
87 * b) significant operation errors occured and 'force' was not set
88 *
89 * @param $performOps Array List of FileOp operations
90 * @param $opts Array Batch operation options
91 * @return Status
92 */
93 final public static function attemptBatch( array $performOps, array $opts ) {
94 $status = Status::newGood();
95
96 $allowStale = !empty( $opts['allowStale'] );
97 $ignoreErrors = !empty( $opts['force'] );
98
99 $n = count( $performOps );
100 if ( $n > self::MAX_BATCH_SIZE ) {
101 $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
102 return $status;
103 }
104
105 $predicates = FileOp::newPredicates(); // account for previous op in prechecks
106 // Do pre-checks for each operation; abort on failure...
107 foreach ( $performOps as $index => $fileOp ) {
108 if ( $allowStale ) {
109 $fileOp->allowStaleReads(); // allow potentially stale reads
110 }
111 $subStatus = $fileOp->precheck( $predicates );
112 $status->merge( $subStatus );
113 if ( !$subStatus->isOK() ) { // operation failed?
114 $status->success[$index] = false;
115 ++$status->failCount;
116 if ( !$ignoreErrors ) {
117 return $status; // abort
118 }
119 }
120 }
121
122 if ( $ignoreErrors ) {
123 # Treat all precheck() fatals as merely warnings
124 $status->setResult( true, $status->value );
125 }
126
127 // Restart PHP's execution timer and set the timeout to safe amount.
128 // This handles cases where the operations take a long time or where we are
129 // already running low on time left. The old timeout is restored afterwards.
130 # @TODO: re-enable this for when the number of batches is high.
131 #$scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC );
132
133 // Attempt each operation...
134 foreach ( $performOps as $index => $fileOp ) {
135 if ( $fileOp->failed() ) {
136 continue; // nothing to do
137 }
138 $subStatus = $fileOp->attempt();
139 $status->merge( $subStatus );
140 if ( $subStatus->isOK() ) {
141 $status->success[$index] = true;
142 ++$status->successCount;
143 } else {
144 $status->success[$index] = false;
145 ++$status->failCount;
146 // We can't continue (even with $ignoreErrors) as $predicates is wrong.
147 // Log the remaining ops as failed for recovery...
148 for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
149 $performOps[$i]->logFailure( 'attempt_aborted' );
150 }
151 return $status; // bail out
152 }
153 }
154
155 return $status;
156 }
157
158 /**
159 * Get the value of the parameter with the given name
160 *
161 * @param $name string
162 * @return mixed Returns null if the parameter is not set
163 */
164 final public function getParam( $name ) {
165 return isset( $this->params[$name] ) ? $this->params[$name] : null;
166 }
167
168 /**
169 * Check if this operation failed precheck() or attempt()
170 *
171 * @return bool
172 */
173 final public function failed() {
174 return $this->failed;
175 }
176
177 /**
178 * Get a new empty predicates array for precheck()
179 *
180 * @return Array
181 */
182 final public static function newPredicates() {
183 return array( 'exists' => array(), 'sha1' => array() );
184 }
185
186 /**
187 * Check preconditions of the operation without writing anything
188 *
189 * @param $predicates Array
190 * @return Status
191 */
192 final public function precheck( array &$predicates ) {
193 if ( $this->state !== self::STATE_NEW ) {
194 return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
195 }
196 $this->state = self::STATE_CHECKED;
197 $status = $this->doPrecheck( $predicates );
198 if ( !$status->isOK() ) {
199 $this->failed = true;
200 }
201 return $status;
202 }
203
204 /**
205 * Attempt the operation, backing up files as needed; this must be reversible
206 *
207 * @return Status
208 */
209 final public function attempt() {
210 if ( $this->state !== self::STATE_CHECKED ) {
211 return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
212 } elseif ( $this->failed ) { // failed precheck
213 return Status::newFatal( 'fileop-fail-attempt-precheck' );
214 }
215 $this->state = self::STATE_ATTEMPTED;
216 $status = $this->doAttempt();
217 if ( !$status->isOK() ) {
218 $this->failed = true;
219 $this->logFailure( 'attempt' );
220 }
221 return $status;
222 }
223
224 /**
225 * Get the file operation parameters
226 *
227 * @return Array (required params list, optional params list)
228 */
229 protected function allowedParams() {
230 return array( array(), array() );
231 }
232
233 /**
234 * Get a list of storage paths read from for this operation
235 *
236 * @return Array
237 */
238 public function storagePathsRead() {
239 return array();
240 }
241
242 /**
243 * Get a list of storage paths written to for this operation
244 *
245 * @return Array
246 */
247 public function storagePathsChanged() {
248 return array();
249 }
250
251 /**
252 * @return Status
253 */
254 protected function doPrecheck( array &$predicates ) {
255 return Status::newGood();
256 }
257
258 /**
259 * @return Status
260 */
261 protected function doAttempt() {
262 return Status::newGood();
263 }
264
265 /**
266 * Check for errors with regards to the destination file already existing.
267 * This also updates the destSameAsSource and sourceSha1 member variables.
268 * A bad status will be returned if there is no chance it can be overwritten.
269 *
270 * @param $predicates Array
271 * @return Status
272 */
273 protected function precheckDestExistence( array $predicates ) {
274 $status = Status::newGood();
275 // Get hash of source file/string and the destination file
276 $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string
277 if ( $this->sourceSha1 === null ) { // file in storage?
278 $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
279 }
280 $this->destSameAsSource = false;
281 if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
282 if ( $this->getParam( 'overwrite' ) ) {
283 return $status; // OK
284 } elseif ( $this->getParam( 'overwriteSame' ) ) {
285 $dhash = $this->fileSha1( $this->params['dst'], $predicates );
286 // Check if hashes are valid and match each other...
287 if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) {
288 $status->fatal( 'backend-fail-hashes' );
289 } elseif ( $this->sourceSha1 !== $dhash ) {
290 // Give an error if the files are not identical
291 $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
292 } else {
293 $this->destSameAsSource = true; // OK
294 }
295 return $status; // do nothing; either OK or bad status
296 } else {
297 $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
298 return $status;
299 }
300 }
301 return $status;
302 }
303
304 /**
305 * precheckDestExistence() helper function to get the source file SHA-1.
306 * Subclasses should overwride this iff the source is not in storage.
307 *
308 * @return string|bool Returns false on failure
309 */
310 protected function getSourceSha1Base36() {
311 return null; // N/A
312 }
313
314 /**
315 * Check if a file will exist in storage when this operation is attempted
316 *
317 * @param $source string Storage path
318 * @param $predicates Array
319 * @return bool
320 */
321 final protected function fileExists( $source, array $predicates ) {
322 if ( isset( $predicates['exists'][$source] ) ) {
323 return $predicates['exists'][$source]; // previous op assures this
324 } else {
325 $params = array( 'src' => $source, 'latest' => $this->useLatest );
326 return $this->backend->fileExists( $params );
327 }
328 }
329
330 /**
331 * Get the SHA-1 of a file in storage when this operation is attempted
332 *
333 * @param $source string Storage path
334 * @param $predicates Array
335 * @return string|bool False on failure
336 */
337 final protected function fileSha1( $source, array $predicates ) {
338 if ( isset( $predicates['sha1'][$source] ) ) {
339 return $predicates['sha1'][$source]; // previous op assures this
340 } else {
341 $params = array( 'src' => $source, 'latest' => $this->useLatest );
342 return $this->backend->getFileSha1Base36( $params );
343 }
344 }
345
346 /**
347 * Log a file operation failure and preserve any temp files
348 *
349 * @param $action string
350 * @return void
351 */
352 final protected function logFailure( $action ) {
353 $params = $this->params;
354 $params['failedAction'] = $action;
355 try {
356 wfDebugLog( 'FileOperation',
357 get_class( $this ) . ' failed: ' . FormatJson::encode( $params ) );
358 } catch ( Exception $e ) {
359 // bad config? debug log error?
360 }
361 }
362 }
363
364 /**
365 * FileOp helper class to expand PHP execution time for a function.
366 * On construction, set_time_limit() is called and set to $seconds.
367 * When the object goes out of scope, the timer is restarted, with
368 * the original time limit minus the time the object existed.
369 */
370 class FileOpScopedPHPTimeout {
371 protected $startTime; // float; seconds
372 protected $oldTimeout; // integer; seconds
373
374 protected static $stackDepth = 0; // integer
375 protected static $totalCalls = 0; // integer
376 protected static $totalElapsed = 0; // float; seconds
377
378 /* Prevent callers in infinite loops from running forever */
379 const MAX_TOTAL_CALLS = 1000000;
380 const MAX_TOTAL_TIME = 300; // seconds
381
382 /**
383 * @param $seconds integer
384 */
385 public function __construct( $seconds ) {
386 if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0
387 if ( self::$totalCalls >= self::MAX_TOTAL_CALLS ) {
388 trigger_error( "Maximum invocations of " . __CLASS__ . " exceeded." );
389 } elseif ( self::$totalElapsed >= self::MAX_TOTAL_TIME ) {
390 trigger_error( "Time limit within invocations of " . __CLASS__ . " exceeded." );
391 } elseif ( self::$stackDepth > 0 ) { // recursion guard
392 trigger_error( "Resursive invocation of " . __CLASS__ . " attempted." );
393 } else {
394 $this->oldTimeout = ini_set( 'max_execution_time', $seconds );
395 $this->startTime = microtime( true );
396 ++self::$stackDepth;
397 ++self::$totalCalls; // proof against < 1us scopes
398 }
399 }
400 }
401
402 /**
403 * Restore the original timeout.
404 * This does not account for the timer value on __construct().
405 */
406 public function __destruct() {
407 if ( $this->oldTimeout ) {
408 $elapsed = microtime( true ) - $this->startTime;
409 // Note: a limit of 0 is treated as "forever"
410 set_time_limit( max( 1, $this->oldTimeout - (int)$elapsed ) );
411 // If each scoped timeout is for less than one second, we end up
412 // restoring the original timeout without any decrease in value.
413 // Thus web scripts in an infinite loop can run forever unless we
414 // take some measures to prevent this. Track total time and calls.
415 self::$totalElapsed += $elapsed;
416 --self::$stackDepth;
417 }
418 }
419 }
420
421 /**
422 * Store a file into the backend from a file on the file system.
423 * Parameters similar to FileBackendStore::storeInternal(), which include:
424 * src : source path on file system
425 * dst : destination storage path
426 * overwrite : do nothing and pass if an identical file exists at destination
427 * overwriteSame : override any existing file at destination
428 */
429 class StoreFileOp extends FileOp {
430 protected function allowedParams() {
431 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
432 }
433
434 protected function doPrecheck( array &$predicates ) {
435 $status = Status::newGood();
436 // Check if the source file exists on the file system
437 if ( !is_file( $this->params['src'] ) ) {
438 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
439 return $status;
440 // Check if the source file is too big
441 } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
442 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
443 return $status;
444 // Check if a file can be placed at the destination
445 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
446 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
447 return $status;
448 }
449 // Check if destination file exists
450 $status->merge( $this->precheckDestExistence( $predicates ) );
451 if ( $status->isOK() ) {
452 // Update file existence predicates
453 $predicates['exists'][$this->params['dst']] = true;
454 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
455 }
456 return $status; // safe to call attempt()
457 }
458
459 protected function doAttempt() {
460 $status = Status::newGood();
461 // Store the file at the destination
462 if ( !$this->destSameAsSource ) {
463 $status->merge( $this->backend->storeInternal( $this->params ) );
464 }
465 return $status;
466 }
467
468 protected function getSourceSha1Base36() {
469 wfSuppressWarnings();
470 $hash = sha1_file( $this->params['src'] );
471 wfRestoreWarnings();
472 if ( $hash !== false ) {
473 $hash = wfBaseConvert( $hash, 16, 36, 31 );
474 }
475 return $hash;
476 }
477
478 public function storagePathsChanged() {
479 return array( $this->params['dst'] );
480 }
481 }
482
483 /**
484 * Create a file in the backend with the given content.
485 * Parameters similar to FileBackendStore::createInternal(), which include:
486 * content : the raw file contents
487 * dst : destination storage path
488 * overwrite : do nothing and pass if an identical file exists at destination
489 * overwriteSame : override any existing file at destination
490 */
491 class CreateFileOp extends FileOp {
492 protected function allowedParams() {
493 return array( array( 'content', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
494 }
495
496 protected function doPrecheck( array &$predicates ) {
497 $status = Status::newGood();
498 // Check if the source data is too big
499 if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
500 $status->fatal( 'backend-fail-create', $this->params['dst'] );
501 return $status;
502 // Check if a file can be placed at the destination
503 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
504 $status->fatal( 'backend-fail-create', $this->params['dst'] );
505 return $status;
506 }
507 // Check if destination file exists
508 $status->merge( $this->precheckDestExistence( $predicates ) );
509 if ( $status->isOK() ) {
510 // Update file existence predicates
511 $predicates['exists'][$this->params['dst']] = true;
512 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
513 }
514 return $status; // safe to call attempt()
515 }
516
517 protected function doAttempt() {
518 $status = Status::newGood();
519 // Create the file at the destination
520 if ( !$this->destSameAsSource ) {
521 $status->merge( $this->backend->createInternal( $this->params ) );
522 }
523 return $status;
524 }
525
526 protected function getSourceSha1Base36() {
527 return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
528 }
529
530 public function storagePathsChanged() {
531 return array( $this->params['dst'] );
532 }
533 }
534
535 /**
536 * Copy a file from one storage path to another in the backend.
537 * Parameters similar to FileBackendStore::copyInternal(), which include:
538 * src : source storage path
539 * dst : destination storage path
540 * overwrite : do nothing and pass if an identical file exists at destination
541 * overwriteSame : override any existing file at destination
542 */
543 class CopyFileOp extends FileOp {
544 protected function allowedParams() {
545 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
546 }
547
548 protected function doPrecheck( array &$predicates ) {
549 $status = Status::newGood();
550 // Check if the source file exists
551 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
552 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
553 return $status;
554 // Check if a file can be placed at the destination
555 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
556 $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] );
557 return $status;
558 }
559 // Check if destination file exists
560 $status->merge( $this->precheckDestExistence( $predicates ) );
561 if ( $status->isOK() ) {
562 // Update file existence predicates
563 $predicates['exists'][$this->params['dst']] = true;
564 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
565 }
566 return $status; // safe to call attempt()
567 }
568
569 protected function doAttempt() {
570 $status = Status::newGood();
571 // Do nothing if the src/dst paths are the same
572 if ( $this->params['src'] !== $this->params['dst'] ) {
573 // Copy the file into the destination
574 if ( !$this->destSameAsSource ) {
575 $status->merge( $this->backend->copyInternal( $this->params ) );
576 }
577 }
578 return $status;
579 }
580
581 public function storagePathsRead() {
582 return array( $this->params['src'] );
583 }
584
585 public function storagePathsChanged() {
586 return array( $this->params['dst'] );
587 }
588 }
589
590 /**
591 * Move a file from one storage path to another in the backend.
592 * Parameters similar to FileBackendStore::moveInternal(), which include:
593 * src : source storage path
594 * dst : destination storage path
595 * overwrite : do nothing and pass if an identical file exists at destination
596 * overwriteSame : override any existing file at destination
597 */
598 class MoveFileOp extends FileOp {
599 protected function allowedParams() {
600 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
601 }
602
603 protected function doPrecheck( array &$predicates ) {
604 $status = Status::newGood();
605 // Check if the source file exists
606 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
607 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
608 return $status;
609 // Check if a file can be placed at the destination
610 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
611 $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] );
612 return $status;
613 }
614 // Check if destination file exists
615 $status->merge( $this->precheckDestExistence( $predicates ) );
616 if ( $status->isOK() ) {
617 // Update file existence predicates
618 $predicates['exists'][$this->params['src']] = false;
619 $predicates['sha1'][$this->params['src']] = false;
620 $predicates['exists'][$this->params['dst']] = true;
621 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
622 }
623 return $status; // safe to call attempt()
624 }
625
626 protected function doAttempt() {
627 $status = Status::newGood();
628 // Do nothing if the src/dst paths are the same
629 if ( $this->params['src'] !== $this->params['dst'] ) {
630 if ( !$this->destSameAsSource ) {
631 // Move the file into the destination
632 $status->merge( $this->backend->moveInternal( $this->params ) );
633 } else {
634 // Just delete source as the destination needs no changes
635 $params = array( 'src' => $this->params['src'] );
636 $status->merge( $this->backend->deleteInternal( $params ) );
637 }
638 }
639 return $status;
640 }
641
642 public function storagePathsRead() {
643 return array( $this->params['src'] );
644 }
645
646 public function storagePathsChanged() {
647 return array( $this->params['dst'] );
648 }
649 }
650
651 /**
652 * Delete a file at the given storage path from the backend.
653 * Parameters similar to FileBackendStore::deleteInternal(), which include:
654 * src : source storage path
655 * ignoreMissingSource : don't return an error if the file does not exist
656 */
657 class DeleteFileOp extends FileOp {
658 protected function allowedParams() {
659 return array( array( 'src' ), array( 'ignoreMissingSource' ) );
660 }
661
662 protected $needsDelete = true;
663
664 protected function doPrecheck( array &$predicates ) {
665 $status = Status::newGood();
666 // Check if the source file exists
667 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
668 if ( !$this->getParam( 'ignoreMissingSource' ) ) {
669 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
670 return $status;
671 }
672 $this->needsDelete = false;
673 }
674 // Update file existence predicates
675 $predicates['exists'][$this->params['src']] = false;
676 $predicates['sha1'][$this->params['src']] = false;
677 return $status; // safe to call attempt()
678 }
679
680 protected function doAttempt() {
681 $status = Status::newGood();
682 if ( $this->needsDelete ) {
683 // Delete the source file
684 $status->merge( $this->backend->deleteInternal( $this->params ) );
685 }
686 return $status;
687 }
688
689 public function storagePathsChanged() {
690 return array( $this->params['src'] );
691 }
692 }
693
694 /**
695 * Placeholder operation that has no params and does nothing
696 */
697 class NullFileOp extends FileOp {}