825a666b5522ef455da2d0ece6a8448dab1f1284
[lhc/web/wiklou.git] / includes / filerepo / backend / FileOp.php
1 <?php
2 /**
3 * @file
4 * @ingroup FileBackend
5 * @author Aaron Schulz
6 */
7
8 /**
9 * Helper class for representing operations with transaction support.
10 * Do not use this class from places outside FileBackend.
11 *
12 * Methods called from attemptBatch() should avoid throwing exceptions at all costs.
13 * FileOp objects should be lightweight in order to support large arrays in memory.
14 *
15 * @ingroup FileBackend
16 * @since 1.19
17 */
18 abstract class FileOp {
19 /** @var Array */
20 protected $params = array();
21 /** @var FileBackendStore */
22 protected $backend;
23
24 protected $state = self::STATE_NEW; // integer
25 protected $failed = false; // boolean
26 protected $useLatest = true; // boolean
27
28 protected $sourceSha1; // string
29 protected $destSameAsSource; // boolean
30
31 /* Object life-cycle */
32 const STATE_NEW = 1;
33 const STATE_CHECKED = 2;
34 const STATE_ATTEMPTED = 3;
35
36 /* Timeout related parameters */
37 const MAX_BATCH_SIZE = 1000;
38 const TIME_LIMIT_SEC = 300; // 5 minutes
39
40 /**
41 * Build a new file operation transaction
42 *
43 * @param $backend FileBackendStore
44 * @param $params Array
45 * @throws MWException
46 */
47 final public function __construct( FileBackendStore $backend, array $params ) {
48 $this->backend = $backend;
49 list( $required, $optional ) = $this->allowedParams();
50 foreach ( $required as $name ) {
51 if ( isset( $params[$name] ) ) {
52 $this->params[$name] = $params[$name];
53 } else {
54 throw new MWException( "File operation missing parameter '$name'." );
55 }
56 }
57 foreach ( $optional as $name ) {
58 if ( isset( $params[$name] ) ) {
59 $this->params[$name] = $params[$name];
60 }
61 }
62 $this->params = $params;
63 }
64
65 /**
66 * Whether to allow stale data for file reads and stat checks
67 *
68 * @param $allowStale bool
69 * @return void
70 */
71 final protected function allowStaleReads( $allowStale ) {
72 $this->useLatest = !$allowStale;
73 }
74
75 /**
76 * Attempt a series of file operations.
77 * Callers are responsible for handling file locking.
78 *
79 * $opts is an array of options, including:
80 * 'force' : Errors that would normally cause a rollback do not.
81 * The remaining operations are still attempted if any fail.
82 * 'allowStale' : Don't require the latest available data.
83 * This can increase performance for non-critical writes.
84 * This has no effect unless the 'force' flag is set.
85 *
86 * The resulting Status will be "OK" unless:
87 * a) unexpected operation errors occurred (network partitions, disk full...)
88 * b) significant operation errors occured and 'force' was not set
89 *
90 * @param $performOps Array List of FileOp operations
91 * @param $opts Array Batch operation options
92 * @return Status
93 */
94 final public static function attemptBatch( array $performOps, array $opts ) {
95 $status = Status::newGood();
96
97 $allowStale = !empty( $opts['allowStale'] );
98 $ignoreErrors = !empty( $opts['force'] );
99
100 $n = count( $performOps );
101 if ( $n > self::MAX_BATCH_SIZE ) {
102 $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
103 return $status;
104 }
105
106 $predicates = FileOp::newPredicates(); // account for previous op in prechecks
107 // Do pre-checks for each operation; abort on failure...
108 foreach ( $performOps as $index => $fileOp ) {
109 $fileOp->allowStaleReads( $allowStale );
110 $subStatus = $fileOp->precheck( $predicates );
111 $status->merge( $subStatus );
112 if ( !$subStatus->isOK() ) { // operation failed?
113 $status->success[$index] = false;
114 ++$status->failCount;
115 if ( !$ignoreErrors ) {
116 return $status; // abort
117 }
118 }
119 }
120
121 if ( $ignoreErrors ) {
122 # Treat all precheck() fatals as merely warnings
123 $status->setResult( true, $status->value );
124 }
125
126 // Restart PHP's execution timer and set the timeout to safe amount.
127 // This handles cases where the operations take a long time or where we are
128 // already running low on time left. The old timeout is restored afterwards.
129 # @TODO: re-enable this for when the number of batches is high.
130 #$scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC );
131
132 // Attempt each operation...
133 foreach ( $performOps as $index => $fileOp ) {
134 if ( $fileOp->failed() ) {
135 continue; // nothing to do
136 }
137 $subStatus = $fileOp->attempt();
138 $status->merge( $subStatus );
139 if ( $subStatus->isOK() ) {
140 $status->success[$index] = true;
141 ++$status->successCount;
142 } else {
143 $status->success[$index] = false;
144 ++$status->failCount;
145 // We can't continue (even with $ignoreErrors) as $predicates is wrong.
146 // Log the remaining ops as failed for recovery...
147 for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
148 $performOps[$i]->logFailure( 'attempt_aborted' );
149 }
150 return $status; // bail out
151 }
152 }
153
154 return $status;
155 }
156
157 /**
158 * Get the value of the parameter with the given name
159 *
160 * @param $name string
161 * @return mixed Returns null if the parameter is not set
162 */
163 final public function getParam( $name ) {
164 return isset( $this->params[$name] ) ? $this->params[$name] : null;
165 }
166
167 /**
168 * Check if this operation failed precheck() or attempt()
169 *
170 * @return bool
171 */
172 final public function failed() {
173 return $this->failed;
174 }
175
176 /**
177 * Get a new empty predicates array for precheck()
178 *
179 * @return Array
180 */
181 final public static function newPredicates() {
182 return array( 'exists' => array(), 'sha1' => array() );
183 }
184
185 /**
186 * Check preconditions of the operation without writing anything
187 *
188 * @param $predicates Array
189 * @return Status
190 */
191 final public function precheck( array &$predicates ) {
192 if ( $this->state !== self::STATE_NEW ) {
193 return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
194 }
195 $this->state = self::STATE_CHECKED;
196 $status = $this->doPrecheck( $predicates );
197 if ( !$status->isOK() ) {
198 $this->failed = true;
199 }
200 return $status;
201 }
202
203 /**
204 * Attempt the operation, backing up files as needed; this must be reversible
205 *
206 * @return Status
207 */
208 final public function attempt() {
209 if ( $this->state !== self::STATE_CHECKED ) {
210 return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
211 } elseif ( $this->failed ) { // failed precheck
212 return Status::newFatal( 'fileop-fail-attempt-precheck' );
213 }
214 $this->state = self::STATE_ATTEMPTED;
215 $status = $this->doAttempt();
216 if ( !$status->isOK() ) {
217 $this->failed = true;
218 $this->logFailure( 'attempt' );
219 }
220 return $status;
221 }
222
223 /**
224 * Get the file operation parameters
225 *
226 * @return Array (required params list, optional params list)
227 */
228 protected function allowedParams() {
229 return array( array(), array() );
230 }
231
232 /**
233 * Get a list of storage paths read from for this operation
234 *
235 * @return Array
236 */
237 public function storagePathsRead() {
238 return array();
239 }
240
241 /**
242 * Get a list of storage paths written to for this operation
243 *
244 * @return Array
245 */
246 public function storagePathsChanged() {
247 return array();
248 }
249
250 /**
251 * @return Status
252 */
253 protected function doPrecheck( array &$predicates ) {
254 return Status::newGood();
255 }
256
257 /**
258 * @return Status
259 */
260 protected function doAttempt() {
261 return Status::newGood();
262 }
263
264 /**
265 * Check for errors with regards to the destination file already existing.
266 * This also updates the destSameAsSource and sourceSha1 member variables.
267 * A bad status will be returned if there is no chance it can be overwritten.
268 *
269 * @param $predicates Array
270 * @return Status
271 */
272 protected function precheckDestExistence( array $predicates ) {
273 $status = Status::newGood();
274 // Get hash of source file/string and the destination file
275 $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string
276 if ( $this->sourceSha1 === null ) { // file in storage?
277 $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
278 }
279 $this->destSameAsSource = false;
280 if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
281 if ( $this->getParam( 'overwrite' ) ) {
282 return $status; // OK
283 } elseif ( $this->getParam( 'overwriteSame' ) ) {
284 $dhash = $this->fileSha1( $this->params['dst'], $predicates );
285 // Check if hashes are valid and match each other...
286 if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) {
287 $status->fatal( 'backend-fail-hashes' );
288 } elseif ( $this->sourceSha1 !== $dhash ) {
289 // Give an error if the files are not identical
290 $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
291 } else {
292 $this->destSameAsSource = true; // OK
293 }
294 return $status; // do nothing; either OK or bad status
295 } else {
296 $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
297 return $status;
298 }
299 }
300 return $status;
301 }
302
303 /**
304 * precheckDestExistence() helper function to get the source file SHA-1.
305 * Subclasses should overwride this iff the source is not in storage.
306 *
307 * @return string|bool Returns false on failure
308 */
309 protected function getSourceSha1Base36() {
310 return null; // N/A
311 }
312
313 /**
314 * Check if a file will exist in storage when this operation is attempted
315 *
316 * @param $source string Storage path
317 * @param $predicates Array
318 * @return bool
319 */
320 final protected function fileExists( $source, array $predicates ) {
321 if ( isset( $predicates['exists'][$source] ) ) {
322 return $predicates['exists'][$source]; // previous op assures this
323 } else {
324 $params = array( 'src' => $source, 'latest' => $this->useLatest );
325 return $this->backend->fileExists( $params );
326 }
327 }
328
329 /**
330 * Get the SHA-1 of a file in storage when this operation is attempted
331 *
332 * @param $source string Storage path
333 * @param $predicates Array
334 * @return string|bool False on failure
335 */
336 final protected function fileSha1( $source, array $predicates ) {
337 if ( isset( $predicates['sha1'][$source] ) ) {
338 return $predicates['sha1'][$source]; // previous op assures this
339 } else {
340 $params = array( 'src' => $source, 'latest' => $this->useLatest );
341 return $this->backend->getFileSha1Base36( $params );
342 }
343 }
344
345 /**
346 * Log a file operation failure and preserve any temp files
347 *
348 * @param $action string
349 * @return void
350 */
351 final protected function logFailure( $action ) {
352 $params = $this->params;
353 $params['failedAction'] = $action;
354 try {
355 wfDebugLog( 'FileOperation',
356 get_class( $this ) . ' failed: ' . FormatJson::encode( $params ) );
357 } catch ( Exception $e ) {
358 // bad config? debug log error?
359 }
360 }
361 }
362
363 /**
364 * FileOp helper class to expand PHP execution time for a function.
365 * On construction, set_time_limit() is called and set to $seconds.
366 * When the object goes out of scope, the timer is restarted, with
367 * the original time limit minus the time the object existed.
368 */
369 class FileOpScopedPHPTimeout {
370 protected $startTime; // float; seconds
371 protected $oldTimeout; // integer; seconds
372
373 protected static $stackDepth = 0; // integer
374 protected static $totalCalls = 0; // integer
375 protected static $totalElapsed = 0; // float; seconds
376
377 /* Prevent callers in infinite loops from running forever */
378 const MAX_TOTAL_CALLS = 1000000;
379 const MAX_TOTAL_TIME = 300; // seconds
380
381 /**
382 * @param $seconds integer
383 */
384 public function __construct( $seconds ) {
385 if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0
386 if ( self::$totalCalls >= self::MAX_TOTAL_CALLS ) {
387 trigger_error( "Maximum invocations of " . __CLASS__ . " exceeded." );
388 } elseif ( self::$totalElapsed >= self::MAX_TOTAL_TIME ) {
389 trigger_error( "Time limit within invocations of " . __CLASS__ . " exceeded." );
390 } elseif ( self::$stackDepth > 0 ) { // recursion guard
391 trigger_error( "Resursive invocation of " . __CLASS__ . " attempted." );
392 } else {
393 $this->oldTimeout = ini_set( 'max_execution_time', $seconds );
394 $this->startTime = microtime( true );
395 ++self::$stackDepth;
396 ++self::$totalCalls; // proof against < 1us scopes
397 }
398 }
399 }
400
401 /**
402 * Restore the original timeout.
403 * This does not account for the timer value on __construct().
404 */
405 public function __destruct() {
406 if ( $this->oldTimeout ) {
407 $elapsed = microtime( true ) - $this->startTime;
408 // Note: a limit of 0 is treated as "forever"
409 set_time_limit( max( 1, $this->oldTimeout - (int)$elapsed ) );
410 // If each scoped timeout is for less than one second, we end up
411 // restoring the original timeout without any decrease in value.
412 // Thus web scripts in an infinite loop can run forever unless we
413 // take some measures to prevent this. Track total time and calls.
414 self::$totalElapsed += $elapsed;
415 --self::$stackDepth;
416 }
417 }
418 }
419
420 /**
421 * Store a file into the backend from a file on the file system.
422 * Parameters similar to FileBackendStore::storeInternal(), which include:
423 * src : source path on file system
424 * dst : destination storage path
425 * overwrite : do nothing and pass if an identical file exists at destination
426 * overwriteSame : override any existing file at destination
427 */
428 class StoreFileOp extends FileOp {
429 protected function allowedParams() {
430 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
431 }
432
433 protected function doPrecheck( array &$predicates ) {
434 $status = Status::newGood();
435 // Check if the source file exists on the file system
436 if ( !is_file( $this->params['src'] ) ) {
437 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
438 return $status;
439 // Check if the source file is too big
440 } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
441 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
442 return $status;
443 // Check if a file can be placed at the destination
444 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
445 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
446 return $status;
447 }
448 // Check if destination file exists
449 $status->merge( $this->precheckDestExistence( $predicates ) );
450 if ( $status->isOK() ) {
451 // Update file existence predicates
452 $predicates['exists'][$this->params['dst']] = true;
453 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
454 }
455 return $status; // safe to call attempt()
456 }
457
458 protected function doAttempt() {
459 $status = Status::newGood();
460 // Store the file at the destination
461 if ( !$this->destSameAsSource ) {
462 $status->merge( $this->backend->storeInternal( $this->params ) );
463 }
464 return $status;
465 }
466
467 protected function getSourceSha1Base36() {
468 wfSuppressWarnings();
469 $hash = sha1_file( $this->params['src'] );
470 wfRestoreWarnings();
471 if ( $hash !== false ) {
472 $hash = wfBaseConvert( $hash, 16, 36, 31 );
473 }
474 return $hash;
475 }
476
477 public function storagePathsChanged() {
478 return array( $this->params['dst'] );
479 }
480 }
481
482 /**
483 * Create a file in the backend with the given content.
484 * Parameters similar to FileBackendStore::createInternal(), which include:
485 * content : the raw file contents
486 * dst : destination storage path
487 * overwrite : do nothing and pass if an identical file exists at destination
488 * overwriteSame : override any existing file at destination
489 */
490 class CreateFileOp extends FileOp {
491 protected function allowedParams() {
492 return array( array( 'content', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
493 }
494
495 protected function doPrecheck( array &$predicates ) {
496 $status = Status::newGood();
497 // Check if the source data is too big
498 if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
499 $status->fatal( 'backend-fail-create', $this->params['dst'] );
500 return $status;
501 // Check if a file can be placed at the destination
502 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
503 $status->fatal( 'backend-fail-create', $this->params['dst'] );
504 return $status;
505 }
506 // Check if destination file exists
507 $status->merge( $this->precheckDestExistence( $predicates ) );
508 if ( $status->isOK() ) {
509 // Update file existence predicates
510 $predicates['exists'][$this->params['dst']] = true;
511 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
512 }
513 return $status; // safe to call attempt()
514 }
515
516 protected function doAttempt() {
517 $status = Status::newGood();
518 // Create the file at the destination
519 if ( !$this->destSameAsSource ) {
520 $status->merge( $this->backend->createInternal( $this->params ) );
521 }
522 return $status;
523 }
524
525 protected function getSourceSha1Base36() {
526 return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
527 }
528
529 public function storagePathsChanged() {
530 return array( $this->params['dst'] );
531 }
532 }
533
534 /**
535 * Copy a file from one storage path to another in the backend.
536 * Parameters similar to FileBackendStore::copyInternal(), which include:
537 * src : source storage path
538 * dst : destination storage path
539 * overwrite : do nothing and pass if an identical file exists at destination
540 * overwriteSame : override any existing file at destination
541 */
542 class CopyFileOp extends FileOp {
543 protected function allowedParams() {
544 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
545 }
546
547 protected function doPrecheck( array &$predicates ) {
548 $status = Status::newGood();
549 // Check if the source file exists
550 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
551 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
552 return $status;
553 // Check if a file can be placed at the destination
554 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
555 $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] );
556 return $status;
557 }
558 // Check if destination file exists
559 $status->merge( $this->precheckDestExistence( $predicates ) );
560 if ( $status->isOK() ) {
561 // Update file existence predicates
562 $predicates['exists'][$this->params['dst']] = true;
563 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
564 }
565 return $status; // safe to call attempt()
566 }
567
568 protected function doAttempt() {
569 $status = Status::newGood();
570 // Do nothing if the src/dst paths are the same
571 if ( $this->params['src'] !== $this->params['dst'] ) {
572 // Copy the file into the destination
573 if ( !$this->destSameAsSource ) {
574 $status->merge( $this->backend->copyInternal( $this->params ) );
575 }
576 }
577 return $status;
578 }
579
580 public function storagePathsRead() {
581 return array( $this->params['src'] );
582 }
583
584 public function storagePathsChanged() {
585 return array( $this->params['dst'] );
586 }
587 }
588
589 /**
590 * Move a file from one storage path to another in the backend.
591 * Parameters similar to FileBackendStore::moveInternal(), which include:
592 * src : source storage path
593 * dst : destination storage path
594 * overwrite : do nothing and pass if an identical file exists at destination
595 * overwriteSame : override any existing file at destination
596 */
597 class MoveFileOp extends FileOp {
598 protected function allowedParams() {
599 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
600 }
601
602 protected function doPrecheck( array &$predicates ) {
603 $status = Status::newGood();
604 // Check if the source file exists
605 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
606 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
607 return $status;
608 // Check if a file can be placed at the destination
609 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
610 $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] );
611 return $status;
612 }
613 // Check if destination file exists
614 $status->merge( $this->precheckDestExistence( $predicates ) );
615 if ( $status->isOK() ) {
616 // Update file existence predicates
617 $predicates['exists'][$this->params['src']] = false;
618 $predicates['sha1'][$this->params['src']] = false;
619 $predicates['exists'][$this->params['dst']] = true;
620 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
621 }
622 return $status; // safe to call attempt()
623 }
624
625 protected function doAttempt() {
626 $status = Status::newGood();
627 // Do nothing if the src/dst paths are the same
628 if ( $this->params['src'] !== $this->params['dst'] ) {
629 if ( !$this->destSameAsSource ) {
630 // Move the file into the destination
631 $status->merge( $this->backend->moveInternal( $this->params ) );
632 } else {
633 // Just delete source as the destination needs no changes
634 $params = array( 'src' => $this->params['src'] );
635 $status->merge( $this->backend->deleteInternal( $params ) );
636 }
637 }
638 return $status;
639 }
640
641 public function storagePathsRead() {
642 return array( $this->params['src'] );
643 }
644
645 public function storagePathsChanged() {
646 return array( $this->params['dst'] );
647 }
648 }
649
650 /**
651 * Delete a file at the given storage path from the backend.
652 * Parameters similar to FileBackendStore::deleteInternal(), which include:
653 * src : source storage path
654 * ignoreMissingSource : don't return an error if the file does not exist
655 */
656 class DeleteFileOp extends FileOp {
657 protected function allowedParams() {
658 return array( array( 'src' ), array( 'ignoreMissingSource' ) );
659 }
660
661 protected $needsDelete = true;
662
663 protected function doPrecheck( array &$predicates ) {
664 $status = Status::newGood();
665 // Check if the source file exists
666 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
667 if ( !$this->getParam( 'ignoreMissingSource' ) ) {
668 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
669 return $status;
670 }
671 $this->needsDelete = false;
672 }
673 // Update file existence predicates
674 $predicates['exists'][$this->params['src']] = false;
675 $predicates['sha1'][$this->params['src']] = false;
676 return $status; // safe to call attempt()
677 }
678
679 protected function doAttempt() {
680 $status = Status::newGood();
681 if ( $this->needsDelete ) {
682 // Delete the source file
683 $status->merge( $this->backend->deleteInternal( $this->params ) );
684 }
685 return $status;
686 }
687
688 public function storagePathsChanged() {
689 return array( $this->params['src'] );
690 }
691 }
692
693 /**
694 * Placeholder operation that has no params and does nothing
695 */
696 class NullFileOp extends FileOp {}