Merge "Pass the result of the database queries in Title::getPreviousRevisionID()...
[lhc/web/wiklou.git] / includes / filerepo / backend / FileOp.php
1 <?php
2 /**
3 * Helper class for representing operations with transaction support.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
19 *
20 * @file
21 * @ingroup FileBackend
22 * @author Aaron Schulz
23 */
24
25 /**
26 * Helper class for representing operations with transaction support.
27 * Do not use this class from places outside FileBackend.
28 *
29 * Methods called from attemptBatch() should avoid throwing exceptions at all costs.
30 * FileOp objects should be lightweight in order to support large arrays in memory.
31 *
32 * @ingroup FileBackend
33 * @since 1.19
34 */
35 abstract class FileOp {
36 /** @var Array */
37 protected $params = array();
38 /** @var FileBackendStore */
39 protected $backend;
40
41 protected $state = self::STATE_NEW; // integer
42 protected $failed = false; // boolean
43 protected $useLatest = true; // boolean
44 protected $batchId; // string
45
46 protected $sourceSha1; // string
47 protected $destSameAsSource; // boolean
48
49 /* Object life-cycle */
50 const STATE_NEW = 1;
51 const STATE_CHECKED = 2;
52 const STATE_ATTEMPTED = 3;
53
54 /* Timeout related parameters */
55 const MAX_BATCH_SIZE = 1000;
56 const TIME_LIMIT_SEC = 300; // 5 minutes
57
58 /**
59 * Build a new file operation transaction
60 *
61 * @param $backend FileBackendStore
62 * @param $params Array
63 * @throws MWException
64 */
65 final public function __construct( FileBackendStore $backend, array $params ) {
66 $this->backend = $backend;
67 list( $required, $optional ) = $this->allowedParams();
68 foreach ( $required as $name ) {
69 if ( isset( $params[$name] ) ) {
70 $this->params[$name] = $params[$name];
71 } else {
72 throw new MWException( "File operation missing parameter '$name'." );
73 }
74 }
75 foreach ( $optional as $name ) {
76 if ( isset( $params[$name] ) ) {
77 $this->params[$name] = $params[$name];
78 }
79 }
80 $this->params = $params;
81 }
82
83 /**
84 * Set the batch UUID this operation belongs to
85 *
86 * @param $batchId string
87 * @return void
88 */
89 final protected function setBatchId( $batchId ) {
90 $this->batchId = $batchId;
91 }
92
93 /**
94 * Whether to allow stale data for file reads and stat checks
95 *
96 * @param $allowStale bool
97 * @return void
98 */
99 final protected function allowStaleReads( $allowStale ) {
100 $this->useLatest = !$allowStale;
101 }
102
103 /**
104 * Attempt to perform a series of file operations.
105 * Callers are responsible for handling file locking.
106 *
107 * $opts is an array of options, including:
108 * 'force' : Errors that would normally cause a rollback do not.
109 * The remaining operations are still attempted if any fail.
110 * 'allowStale' : Don't require the latest available data.
111 * This can increase performance for non-critical writes.
112 * This has no effect unless the 'force' flag is set.
113 * 'nonJournaled' : Don't log this operation batch in the file journal.
114 *
115 * The resulting Status will be "OK" unless:
116 * a) unexpected operation errors occurred (network partitions, disk full...)
117 * b) significant operation errors occured and 'force' was not set
118 *
119 * @param $performOps Array List of FileOp operations
120 * @param $opts Array Batch operation options
121 * @param $journal FileJournal Journal to log operations to
122 * @return Status
123 */
124 final public static function attemptBatch(
125 array $performOps, array $opts, FileJournal $journal
126 ) {
127 $status = Status::newGood();
128
129 $n = count( $performOps );
130 if ( $n > self::MAX_BATCH_SIZE ) {
131 $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
132 return $status;
133 }
134
135 $batchId = $journal->getTimestampedUUID();
136 $allowStale = !empty( $opts['allowStale'] );
137 $ignoreErrors = !empty( $opts['force'] );
138 $journaled = empty( $opts['nonJournaled'] );
139
140 $entries = array(); // file journal entries
141 $predicates = FileOp::newPredicates(); // account for previous op in prechecks
142 // Do pre-checks for each operation; abort on failure...
143 foreach ( $performOps as $index => $fileOp ) {
144 $fileOp->setBatchId( $batchId );
145 $fileOp->allowStaleReads( $allowStale );
146 $oldPredicates = $predicates;
147 $subStatus = $fileOp->precheck( $predicates ); // updates $predicates
148 $status->merge( $subStatus );
149 if ( $subStatus->isOK() ) {
150 if ( $journaled ) { // journal log entry
151 $entries = array_merge( $entries,
152 self::getJournalEntries( $fileOp, $oldPredicates, $predicates ) );
153 }
154 } else { // operation failed?
155 $status->success[$index] = false;
156 ++$status->failCount;
157 if ( !$ignoreErrors ) {
158 return $status; // abort
159 }
160 }
161 }
162
163 // Log the operations in file journal...
164 if ( count( $entries ) ) {
165 $subStatus = $journal->logChangeBatch( $entries, $batchId );
166 if ( !$subStatus->isOK() ) {
167 return $subStatus; // abort
168 }
169 }
170
171 if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings
172 $status->setResult( true, $status->value );
173 }
174
175 // Attempt each operation...
176 foreach ( $performOps as $index => $fileOp ) {
177 if ( $fileOp->failed() ) {
178 continue; // nothing to do
179 }
180 $subStatus = $fileOp->attempt();
181 $status->merge( $subStatus );
182 if ( $subStatus->isOK() ) {
183 $status->success[$index] = true;
184 ++$status->successCount;
185 } else {
186 $status->success[$index] = false;
187 ++$status->failCount;
188 // We can't continue (even with $ignoreErrors) as $predicates is wrong.
189 // Log the remaining ops as failed for recovery...
190 for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
191 $performOps[$i]->logFailure( 'attempt_aborted' );
192 }
193 return $status; // bail out
194 }
195 }
196
197 return $status;
198 }
199
200 /**
201 * Get the file journal entries for a single file operation
202 *
203 * @param $fileOp FileOp
204 * @param $oPredicates Array Pre-op information about files
205 * @param $nPredicates Array Post-op information about files
206 * @return Array
207 */
208 final protected static function getJournalEntries(
209 FileOp $fileOp, array $oPredicates, array $nPredicates
210 ) {
211 $nullEntries = array();
212 $updateEntries = array();
213 $deleteEntries = array();
214 $pathsUsed = array_merge( $fileOp->storagePathsRead(), $fileOp->storagePathsChanged() );
215 foreach ( $pathsUsed as $path ) {
216 $nullEntries[] = array( // assertion for recovery
217 'op' => 'null',
218 'path' => $path,
219 'newSha1' => $fileOp->fileSha1( $path, $oPredicates )
220 );
221 }
222 foreach ( $fileOp->storagePathsChanged() as $path ) {
223 if ( $nPredicates['sha1'][$path] === false ) { // deleted
224 $deleteEntries[] = array(
225 'op' => 'delete',
226 'path' => $path,
227 'newSha1' => ''
228 );
229 } else { // created/updated
230 $updateEntries[] = array(
231 'op' => $fileOp->fileExists( $path, $oPredicates ) ? 'update' : 'create',
232 'path' => $path,
233 'newSha1' => $nPredicates['sha1'][$path]
234 );
235 }
236 }
237 return array_merge( $nullEntries, $updateEntries, $deleteEntries );
238 }
239
240 /**
241 * Get the value of the parameter with the given name
242 *
243 * @param $name string
244 * @return mixed Returns null if the parameter is not set
245 */
246 final public function getParam( $name ) {
247 return isset( $this->params[$name] ) ? $this->params[$name] : null;
248 }
249
250 /**
251 * Check if this operation failed precheck() or attempt()
252 *
253 * @return bool
254 */
255 final public function failed() {
256 return $this->failed;
257 }
258
259 /**
260 * Get a new empty predicates array for precheck()
261 *
262 * @return Array
263 */
264 final public static function newPredicates() {
265 return array( 'exists' => array(), 'sha1' => array() );
266 }
267
268 /**
269 * Check preconditions of the operation without writing anything
270 *
271 * @param $predicates Array
272 * @return Status
273 */
274 final public function precheck( array &$predicates ) {
275 if ( $this->state !== self::STATE_NEW ) {
276 return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state );
277 }
278 $this->state = self::STATE_CHECKED;
279 $status = $this->doPrecheck( $predicates );
280 if ( !$status->isOK() ) {
281 $this->failed = true;
282 }
283 return $status;
284 }
285
286 /**
287 * Attempt the operation, backing up files as needed; this must be reversible
288 *
289 * @return Status
290 */
291 final public function attempt() {
292 if ( $this->state !== self::STATE_CHECKED ) {
293 return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state );
294 } elseif ( $this->failed ) { // failed precheck
295 return Status::newFatal( 'fileop-fail-attempt-precheck' );
296 }
297 $this->state = self::STATE_ATTEMPTED;
298 $status = $this->doAttempt();
299 if ( !$status->isOK() ) {
300 $this->failed = true;
301 $this->logFailure( 'attempt' );
302 }
303 return $status;
304 }
305
306 /**
307 * Get the file operation parameters
308 *
309 * @return Array (required params list, optional params list)
310 */
311 protected function allowedParams() {
312 return array( array(), array() );
313 }
314
315 /**
316 * Get a list of storage paths read from for this operation
317 *
318 * @return Array
319 */
320 public function storagePathsRead() {
321 return array();
322 }
323
324 /**
325 * Get a list of storage paths written to for this operation
326 *
327 * @return Array
328 */
329 public function storagePathsChanged() {
330 return array();
331 }
332
333 /**
334 * @return Status
335 */
336 protected function doPrecheck( array &$predicates ) {
337 return Status::newGood();
338 }
339
340 /**
341 * @return Status
342 */
343 protected function doAttempt() {
344 return Status::newGood();
345 }
346
347 /**
348 * Check for errors with regards to the destination file already existing.
349 * This also updates the destSameAsSource and sourceSha1 member variables.
350 * A bad status will be returned if there is no chance it can be overwritten.
351 *
352 * @param $predicates Array
353 * @return Status
354 */
355 protected function precheckDestExistence( array $predicates ) {
356 $status = Status::newGood();
357 // Get hash of source file/string and the destination file
358 $this->sourceSha1 = $this->getSourceSha1Base36(); // FS file or data string
359 if ( $this->sourceSha1 === null ) { // file in storage?
360 $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates );
361 }
362 $this->destSameAsSource = false;
363 if ( $this->fileExists( $this->params['dst'], $predicates ) ) {
364 if ( $this->getParam( 'overwrite' ) ) {
365 return $status; // OK
366 } elseif ( $this->getParam( 'overwriteSame' ) ) {
367 $dhash = $this->fileSha1( $this->params['dst'], $predicates );
368 // Check if hashes are valid and match each other...
369 if ( !strlen( $this->sourceSha1 ) || !strlen( $dhash ) ) {
370 $status->fatal( 'backend-fail-hashes' );
371 } elseif ( $this->sourceSha1 !== $dhash ) {
372 // Give an error if the files are not identical
373 $status->fatal( 'backend-fail-notsame', $this->params['dst'] );
374 } else {
375 $this->destSameAsSource = true; // OK
376 }
377 return $status; // do nothing; either OK or bad status
378 } else {
379 $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
380 return $status;
381 }
382 }
383 return $status;
384 }
385
386 /**
387 * precheckDestExistence() helper function to get the source file SHA-1.
388 * Subclasses should overwride this iff the source is not in storage.
389 *
390 * @return string|bool Returns false on failure
391 */
392 protected function getSourceSha1Base36() {
393 return null; // N/A
394 }
395
396 /**
397 * Check if a file will exist in storage when this operation is attempted
398 *
399 * @param $source string Storage path
400 * @param $predicates Array
401 * @return bool
402 */
403 final protected function fileExists( $source, array $predicates ) {
404 if ( isset( $predicates['exists'][$source] ) ) {
405 return $predicates['exists'][$source]; // previous op assures this
406 } else {
407 $params = array( 'src' => $source, 'latest' => $this->useLatest );
408 return $this->backend->fileExists( $params );
409 }
410 }
411
412 /**
413 * Get the SHA-1 of a file in storage when this operation is attempted
414 *
415 * @param $source string Storage path
416 * @param $predicates Array
417 * @return string|bool False on failure
418 */
419 final protected function fileSha1( $source, array $predicates ) {
420 if ( isset( $predicates['sha1'][$source] ) ) {
421 return $predicates['sha1'][$source]; // previous op assures this
422 } else {
423 $params = array( 'src' => $source, 'latest' => $this->useLatest );
424 return $this->backend->getFileSha1Base36( $params );
425 }
426 }
427
428 /**
429 * Log a file operation failure and preserve any temp files
430 *
431 * @param $action string
432 * @return void
433 */
434 final protected function logFailure( $action ) {
435 $params = $this->params;
436 $params['failedAction'] = $action;
437 try {
438 wfDebugLog( 'FileOperation', get_class( $this ) .
439 " failed (batch #{$this->batchId}): " . FormatJson::encode( $params ) );
440 } catch ( Exception $e ) {
441 // bad config? debug log error?
442 }
443 }
444 }
445
446 /**
447 * Store a file into the backend from a file on the file system.
448 * Parameters similar to FileBackendStore::storeInternal(), which include:
449 * src : source path on file system
450 * dst : destination storage path
451 * overwrite : do nothing and pass if an identical file exists at destination
452 * overwriteSame : override any existing file at destination
453 */
454 class StoreFileOp extends FileOp {
455 protected function allowedParams() {
456 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
457 }
458
459 protected function doPrecheck( array &$predicates ) {
460 $status = Status::newGood();
461 // Check if the source file exists on the file system
462 if ( !is_file( $this->params['src'] ) ) {
463 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
464 return $status;
465 // Check if the source file is too big
466 } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
467 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
468 return $status;
469 // Check if a file can be placed at the destination
470 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
471 $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
472 return $status;
473 }
474 // Check if destination file exists
475 $status->merge( $this->precheckDestExistence( $predicates ) );
476 if ( $status->isOK() ) {
477 // Update file existence predicates
478 $predicates['exists'][$this->params['dst']] = true;
479 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
480 }
481 return $status; // safe to call attempt()
482 }
483
484 protected function doAttempt() {
485 $status = Status::newGood();
486 // Store the file at the destination
487 if ( !$this->destSameAsSource ) {
488 $status->merge( $this->backend->storeInternal( $this->params ) );
489 }
490 return $status;
491 }
492
493 protected function getSourceSha1Base36() {
494 wfSuppressWarnings();
495 $hash = sha1_file( $this->params['src'] );
496 wfRestoreWarnings();
497 if ( $hash !== false ) {
498 $hash = wfBaseConvert( $hash, 16, 36, 31 );
499 }
500 return $hash;
501 }
502
503 public function storagePathsChanged() {
504 return array( $this->params['dst'] );
505 }
506 }
507
508 /**
509 * Create a file in the backend with the given content.
510 * Parameters similar to FileBackendStore::createInternal(), which include:
511 * content : the raw file contents
512 * dst : destination storage path
513 * overwrite : do nothing and pass if an identical file exists at destination
514 * overwriteSame : override any existing file at destination
515 */
516 class CreateFileOp extends FileOp {
517 protected function allowedParams() {
518 return array( array( 'content', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
519 }
520
521 protected function doPrecheck( array &$predicates ) {
522 $status = Status::newGood();
523 // Check if the source data is too big
524 if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
525 $status->fatal( 'backend-fail-create', $this->params['dst'] );
526 return $status;
527 // Check if a file can be placed at the destination
528 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
529 $status->fatal( 'backend-fail-create', $this->params['dst'] );
530 return $status;
531 }
532 // Check if destination file exists
533 $status->merge( $this->precheckDestExistence( $predicates ) );
534 if ( $status->isOK() ) {
535 // Update file existence predicates
536 $predicates['exists'][$this->params['dst']] = true;
537 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
538 }
539 return $status; // safe to call attempt()
540 }
541
542 protected function doAttempt() {
543 $status = Status::newGood();
544 // Create the file at the destination
545 if ( !$this->destSameAsSource ) {
546 $status->merge( $this->backend->createInternal( $this->params ) );
547 }
548 return $status;
549 }
550
551 protected function getSourceSha1Base36() {
552 return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 );
553 }
554
555 public function storagePathsChanged() {
556 return array( $this->params['dst'] );
557 }
558 }
559
560 /**
561 * Copy a file from one storage path to another in the backend.
562 * Parameters similar to FileBackendStore::copyInternal(), which include:
563 * src : source storage path
564 * dst : destination storage path
565 * overwrite : do nothing and pass if an identical file exists at destination
566 * overwriteSame : override any existing file at destination
567 */
568 class CopyFileOp extends FileOp {
569 protected function allowedParams() {
570 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
571 }
572
573 protected function doPrecheck( array &$predicates ) {
574 $status = Status::newGood();
575 // Check if the source file exists
576 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
577 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
578 return $status;
579 // Check if a file can be placed at the destination
580 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
581 $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] );
582 return $status;
583 }
584 // Check if destination file exists
585 $status->merge( $this->precheckDestExistence( $predicates ) );
586 if ( $status->isOK() ) {
587 // Update file existence predicates
588 $predicates['exists'][$this->params['dst']] = true;
589 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
590 }
591 return $status; // safe to call attempt()
592 }
593
594 protected function doAttempt() {
595 $status = Status::newGood();
596 // Do nothing if the src/dst paths are the same
597 if ( $this->params['src'] !== $this->params['dst'] ) {
598 // Copy the file into the destination
599 if ( !$this->destSameAsSource ) {
600 $status->merge( $this->backend->copyInternal( $this->params ) );
601 }
602 }
603 return $status;
604 }
605
606 public function storagePathsRead() {
607 return array( $this->params['src'] );
608 }
609
610 public function storagePathsChanged() {
611 return array( $this->params['dst'] );
612 }
613 }
614
615 /**
616 * Move a file from one storage path to another in the backend.
617 * Parameters similar to FileBackendStore::moveInternal(), which include:
618 * src : source storage path
619 * dst : destination storage path
620 * overwrite : do nothing and pass if an identical file exists at destination
621 * overwriteSame : override any existing file at destination
622 */
623 class MoveFileOp extends FileOp {
624 protected function allowedParams() {
625 return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
626 }
627
628 protected function doPrecheck( array &$predicates ) {
629 $status = Status::newGood();
630 // Check if the source file exists
631 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
632 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
633 return $status;
634 // Check if a file can be placed at the destination
635 } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
636 $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] );
637 return $status;
638 }
639 // Check if destination file exists
640 $status->merge( $this->precheckDestExistence( $predicates ) );
641 if ( $status->isOK() ) {
642 // Update file existence predicates
643 $predicates['exists'][$this->params['src']] = false;
644 $predicates['sha1'][$this->params['src']] = false;
645 $predicates['exists'][$this->params['dst']] = true;
646 $predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
647 }
648 return $status; // safe to call attempt()
649 }
650
651 protected function doAttempt() {
652 $status = Status::newGood();
653 // Do nothing if the src/dst paths are the same
654 if ( $this->params['src'] !== $this->params['dst'] ) {
655 if ( !$this->destSameAsSource ) {
656 // Move the file into the destination
657 $status->merge( $this->backend->moveInternal( $this->params ) );
658 } else {
659 // Just delete source as the destination needs no changes
660 $params = array( 'src' => $this->params['src'] );
661 $status->merge( $this->backend->deleteInternal( $params ) );
662 }
663 }
664 return $status;
665 }
666
667 public function storagePathsRead() {
668 return array( $this->params['src'] );
669 }
670
671 public function storagePathsChanged() {
672 return array( $this->params['src'], $this->params['dst'] );
673 }
674 }
675
676 /**
677 * Delete a file at the given storage path from the backend.
678 * Parameters similar to FileBackendStore::deleteInternal(), which include:
679 * src : source storage path
680 * ignoreMissingSource : don't return an error if the file does not exist
681 */
682 class DeleteFileOp extends FileOp {
683 protected function allowedParams() {
684 return array( array( 'src' ), array( 'ignoreMissingSource' ) );
685 }
686
687 protected $needsDelete = true;
688
689 protected function doPrecheck( array &$predicates ) {
690 $status = Status::newGood();
691 // Check if the source file exists
692 if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
693 if ( !$this->getParam( 'ignoreMissingSource' ) ) {
694 $status->fatal( 'backend-fail-notexists', $this->params['src'] );
695 return $status;
696 }
697 $this->needsDelete = false;
698 }
699 // Update file existence predicates
700 $predicates['exists'][$this->params['src']] = false;
701 $predicates['sha1'][$this->params['src']] = false;
702 return $status; // safe to call attempt()
703 }
704
705 protected function doAttempt() {
706 $status = Status::newGood();
707 if ( $this->needsDelete ) {
708 // Delete the source file
709 $status->merge( $this->backend->deleteInternal( $this->params ) );
710 }
711 return $status;
712 }
713
714 public function storagePathsChanged() {
715 return array( $this->params['src'] );
716 }
717 }
718
719 /**
720 * Placeholder operation that has no params and does nothing
721 */
722 class NullFileOp extends FileOp {}