3 * Helper class for representing operations with transaction support.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @ingroup FileBackend
22 * @author Aaron Schulz
26 * FileBackend helper class for representing operations.
27 * Do not use this class from places outside FileBackend.
29 * Methods called from FileOpBatch::attempt() should avoid throwing
30 * exceptions at all costs. FileOp objects should be lightweight in order
31 * to support large arrays in memory and serialization.
33 * @ingroup FileBackend
36 abstract class FileOp
{
38 protected $params = array();
39 /** @var FileBackendStore */
42 protected $state = self
::STATE_NEW
; // integer
43 protected $failed = false; // boolean
44 protected $async = false; // boolean
45 protected $useLatest = true; // boolean
46 protected $batchId; // string
48 protected $doOperation = true; // boolean; operation is not a no-op
49 protected $sourceSha1; // string
50 protected $destSameAsSource; // boolean
51 protected $destExists; // boolean
53 /* Object life-cycle */
55 const STATE_CHECKED
= 2;
56 const STATE_ATTEMPTED
= 3;
59 * Build a new file operation transaction
61 * @param $backend FileBackendStore
62 * @param $params Array
65 final public function __construct( FileBackendStore
$backend, array $params ) {
66 $this->backend
= $backend;
67 list( $required, $optional ) = $this->allowedParams();
68 foreach ( $required as $name ) {
69 if ( isset( $params[$name] ) ) {
70 $this->params
[$name] = self
::normalizeAnyStoragePaths( $params[$name] );
72 throw new MWException( "File operation missing parameter '$name'." );
75 foreach ( $optional as $name ) {
76 if ( isset( $params[$name] ) ) {
77 $this->params
[$name] = self
::normalizeAnyStoragePaths( $params[$name] );
80 $this->params
= $params;
84 * Normalize $item or anything in $item that is a valid storage path
86 * @param $item string|array
87 * @return string|Array
89 protected function normalizeAnyStoragePaths( $item ) {
90 if ( is_array( $item ) ) {
92 foreach ( $item as $k => $v ) {
93 $k = self
::normalizeIfValidStoragePath( $k );
94 $v = self
::normalizeIfValidStoragePath( $v );
99 return self
::normalizeIfValidStoragePath( $item );
104 * Normalize a string if it is a valid storage path
106 * @param $path string
109 protected static function normalizeIfValidStoragePath( $path ) {
110 if ( FileBackend
::isStoragePath( $path ) ) {
111 $res = FileBackend
::normalizeStoragePath( $path );
112 return ( $res !== null ) ?
$res : $path;
118 * Set the batch UUID this operation belongs to
120 * @param $batchId string
123 final public function setBatchId( $batchId ) {
124 $this->batchId
= $batchId;
128 * Whether to allow stale data for file reads and stat checks
130 * @param $allowStale bool
133 final public function allowStaleReads( $allowStale ) {
134 $this->useLatest
= !$allowStale;
138 * Get the value of the parameter with the given name
140 * @param $name string
141 * @return mixed Returns null if the parameter is not set
143 final public function getParam( $name ) {
144 return isset( $this->params
[$name] ) ?
$this->params
[$name] : null;
148 * Check if this operation failed precheck() or attempt()
152 final public function failed() {
153 return $this->failed
;
157 * Get a new empty predicates array for precheck()
161 final public static function newPredicates() {
162 return array( 'exists' => array(), 'sha1' => array() );
166 * Get a new empty dependency tracking array for paths read/written to
170 final public static function newDependencies() {
171 return array( 'read' => array(), 'write' => array() );
175 * Update a dependency tracking array to account for this operation
177 * @param $deps Array Prior path reads/writes; format of FileOp::newPredicates()
180 final public function applyDependencies( array $deps ) {
181 $deps['read'] +
= array_fill_keys( $this->storagePathsRead(), 1 );
182 $deps['write'] +
= array_fill_keys( $this->storagePathsChanged(), 1 );
187 * Check if this operation changes files listed in $paths
189 * @param $paths Array Prior path reads/writes; format of FileOp::newPredicates()
192 final public function dependsOn( array $deps ) {
193 foreach ( $this->storagePathsChanged() as $path ) {
194 if ( isset( $deps['read'][$path] ) ||
isset( $deps['write'][$path] ) ) {
195 return true; // "output" or "anti" dependency
198 foreach ( $this->storagePathsRead() as $path ) {
199 if ( isset( $deps['write'][$path] ) ) {
200 return true; // "flow" dependency
207 * Get the file journal entries for this file operation
209 * @param $oPredicates Array Pre-op info about files (format of FileOp::newPredicates)
210 * @param $nPredicates Array Post-op info about files (format of FileOp::newPredicates)
213 final public function getJournalEntries( array $oPredicates, array $nPredicates ) {
214 if ( !$this->doOperation
) {
215 return array(); // this is a no-op
217 $nullEntries = array();
218 $updateEntries = array();
219 $deleteEntries = array();
220 $pathsUsed = array_merge( $this->storagePathsRead(), $this->storagePathsChanged() );
221 foreach ( array_unique( $pathsUsed ) as $path ) {
222 $nullEntries[] = array( // assertion for recovery
225 'newSha1' => $this->fileSha1( $path, $oPredicates )
228 foreach ( $this->storagePathsChanged() as $path ) {
229 if ( $nPredicates['sha1'][$path] === false ) { // deleted
230 $deleteEntries[] = array(
235 } else { // created/updated
236 $updateEntries[] = array(
237 'op' => $this->fileExists( $path, $oPredicates ) ?
'update' : 'create',
239 'newSha1' => $nPredicates['sha1'][$path]
243 return array_merge( $nullEntries, $updateEntries, $deleteEntries );
247 * Check preconditions of the operation without writing anything.
248 * This must update $predicates for each path that the op can change
249 * except when a failing status object is returned.
251 * @param $predicates Array
254 final public function precheck( array &$predicates ) {
255 if ( $this->state
!== self
::STATE_NEW
) {
256 return Status
::newFatal( 'fileop-fail-state', self
::STATE_NEW
, $this->state
);
258 $this->state
= self
::STATE_CHECKED
;
259 $status = $this->doPrecheck( $predicates );
260 if ( !$status->isOK() ) {
261 $this->failed
= true;
269 protected function doPrecheck( array &$predicates ) {
270 return Status
::newGood();
274 * Attempt the operation
278 final public function attempt() {
279 if ( $this->state
!== self
::STATE_CHECKED
) {
280 return Status
::newFatal( 'fileop-fail-state', self
::STATE_CHECKED
, $this->state
);
281 } elseif ( $this->failed
) { // failed precheck
282 return Status
::newFatal( 'fileop-fail-attempt-precheck' );
284 $this->state
= self
::STATE_ATTEMPTED
;
285 if ( $this->doOperation
) {
286 $status = $this->doAttempt();
287 if ( !$status->isOK() ) {
288 $this->failed
= true;
289 $this->logFailure( 'attempt' );
292 $status = Status
::newGood();
300 protected function doAttempt() {
301 return Status
::newGood();
305 * Attempt the operation in the background
309 final public function attemptAsync() {
311 $result = $this->attempt();
312 $this->async
= false;
317 * Get the file operation parameters
319 * @return Array (required params list, optional params list)
321 protected function allowedParams() {
322 return array( array(), array() );
326 * Adjust params to FileBackendStore internal file calls
328 * @param $params Array
329 * @return Array (required params list, optional params list)
331 protected function setFlags( array $params ) {
332 return array( 'async' => $this->async
) +
$params;
336 * Get a list of storage paths read from for this operation
340 public function storagePathsRead() {
345 * Get a list of storage paths written to for this operation
349 public function storagePathsChanged() {
354 * Check for errors with regards to the destination file already existing.
355 * Also set the destExists, destSameAsSource and sourceSha1 member variables.
356 * A bad status will be returned if there is no chance it can be overwritten.
358 * @param $predicates Array
361 protected function precheckDestExistence( array $predicates ) {
362 $status = Status
::newGood();
363 // Get hash of source file/string and the destination file
364 $this->sourceSha1
= $this->getSourceSha1Base36(); // FS file or data string
365 if ( $this->sourceSha1
=== null ) { // file in storage?
366 $this->sourceSha1
= $this->fileSha1( $this->params
['src'], $predicates );
368 $this->destSameAsSource
= false;
369 $this->destExists
= $this->fileExists( $this->params
['dst'], $predicates );
370 if ( $this->destExists
) {
371 if ( $this->getParam( 'overwrite' ) ) {
372 return $status; // OK
373 } elseif ( $this->getParam( 'overwriteSame' ) ) {
374 $dhash = $this->fileSha1( $this->params
['dst'], $predicates );
375 // Check if hashes are valid and match each other...
376 if ( !strlen( $this->sourceSha1
) ||
!strlen( $dhash ) ) {
377 $status->fatal( 'backend-fail-hashes' );
378 } elseif ( $this->sourceSha1
!== $dhash ) {
379 // Give an error if the files are not identical
380 $status->fatal( 'backend-fail-notsame', $this->params
['dst'] );
382 $this->destSameAsSource
= true; // OK
384 return $status; // do nothing; either OK or bad status
386 $status->fatal( 'backend-fail-alreadyexists', $this->params
['dst'] );
394 * precheckDestExistence() helper function to get the source file SHA-1.
395 * Subclasses should overwride this iff the source is not in storage.
397 * @return string|bool Returns false on failure
399 protected function getSourceSha1Base36() {
404 * Check if a file will exist in storage when this operation is attempted
406 * @param $source string Storage path
407 * @param $predicates Array
410 final protected function fileExists( $source, array $predicates ) {
411 if ( isset( $predicates['exists'][$source] ) ) {
412 return $predicates['exists'][$source]; // previous op assures this
414 $params = array( 'src' => $source, 'latest' => $this->useLatest
);
415 return $this->backend
->fileExists( $params );
420 * Get the SHA-1 of a file in storage when this operation is attempted
422 * @param $source string Storage path
423 * @param $predicates Array
424 * @return string|bool False on failure
426 final protected function fileSha1( $source, array $predicates ) {
427 if ( isset( $predicates['sha1'][$source] ) ) {
428 return $predicates['sha1'][$source]; // previous op assures this
429 } elseif ( isset( $predicates['exists'][$source] ) && !$predicates['exists'][$source] ) {
430 return false; // previous op assures this
432 $params = array( 'src' => $source, 'latest' => $this->useLatest
);
433 return $this->backend
->getFileSha1Base36( $params );
438 * Get the backend this operation is for
440 * @return FileBackendStore
442 public function getBackend() {
443 return $this->backend
;
447 * Log a file operation failure and preserve any temp files
449 * @param $action string
452 final public function logFailure( $action ) {
453 $params = $this->params
;
454 $params['failedAction'] = $action;
456 wfDebugLog( 'FileOperation', get_class( $this ) .
457 " failed (batch #{$this->batchId}): " . FormatJson
::encode( $params ) );
458 } catch ( Exception
$e ) {
459 // bad config? debug log error?
465 * Create a file in the backend with the given content.
466 * Parameters for this operation are outlined in FileBackend::doOperations().
468 class CreateFileOp
extends FileOp
{
469 protected function allowedParams() {
470 return array( array( 'content', 'dst' ),
471 array( 'overwrite', 'overwriteSame', 'disposition', 'headers' ) );
474 protected function doPrecheck( array &$predicates ) {
475 $status = Status
::newGood();
476 // Check if the source data is too big
477 if ( strlen( $this->getParam( 'content' ) ) > $this->backend
->maxFileSizeInternal() ) {
478 $status->fatal( 'backend-fail-maxsize',
479 $this->params
['dst'], $this->backend
->maxFileSizeInternal() );
480 $status->fatal( 'backend-fail-create', $this->params
['dst'] );
482 // Check if a file can be placed/changed at the destination
483 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
484 $status->fatal( 'backend-fail-usable', $this->params
['dst'] );
485 $status->fatal( 'backend-fail-create', $this->params
['dst'] );
488 // Check if destination file exists
489 $status->merge( $this->precheckDestExistence( $predicates ) );
490 $this->params
['dstExists'] = $this->destExists
; // see FileBackendStore::setFileCache()
491 if ( $status->isOK() ) {
492 // Update file existence predicates
493 $predicates['exists'][$this->params
['dst']] = true;
494 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
496 return $status; // safe to call attempt()
502 protected function doAttempt() {
503 if ( !$this->destSameAsSource
) {
504 // Create the file at the destination
505 return $this->backend
->createInternal( $this->setFlags( $this->params
) );
507 return Status
::newGood();
511 * @return bool|String
513 protected function getSourceSha1Base36() {
514 return wfBaseConvert( sha1( $this->params
['content'] ), 16, 36, 31 );
520 public function storagePathsChanged() {
521 return array( $this->params
['dst'] );
526 * Store a file into the backend from a file on the file system.
527 * Parameters for this operation are outlined in FileBackend::doOperations().
529 class StoreFileOp
extends FileOp
{
533 protected function allowedParams() {
534 return array( array( 'src', 'dst' ),
535 array( 'overwrite', 'overwriteSame', 'disposition', 'headers' ) );
539 * @param $predicates array
542 protected function doPrecheck( array &$predicates ) {
543 $status = Status
::newGood();
544 // Check if the source file exists on the file system
545 if ( !is_file( $this->params
['src'] ) ) {
546 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
548 // Check if the source file is too big
549 } elseif ( filesize( $this->params
['src'] ) > $this->backend
->maxFileSizeInternal() ) {
550 $status->fatal( 'backend-fail-maxsize',
551 $this->params
['dst'], $this->backend
->maxFileSizeInternal() );
552 $status->fatal( 'backend-fail-store', $this->params
['src'], $this->params
['dst'] );
554 // Check if a file can be placed/changed at the destination
555 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
556 $status->fatal( 'backend-fail-usable', $this->params
['dst'] );
557 $status->fatal( 'backend-fail-store', $this->params
['src'], $this->params
['dst'] );
560 // Check if destination file exists
561 $status->merge( $this->precheckDestExistence( $predicates ) );
562 $this->params
['dstExists'] = $this->destExists
; // see FileBackendStore::setFileCache()
563 if ( $status->isOK() ) {
564 // Update file existence predicates
565 $predicates['exists'][$this->params
['dst']] = true;
566 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
568 return $status; // safe to call attempt()
574 protected function doAttempt() {
575 // Store the file at the destination
576 if ( !$this->destSameAsSource
) {
577 return $this->backend
->storeInternal( $this->setFlags( $this->params
) );
579 return Status
::newGood();
583 * @return bool|string
585 protected function getSourceSha1Base36() {
586 wfSuppressWarnings();
587 $hash = sha1_file( $this->params
['src'] );
589 if ( $hash !== false ) {
590 $hash = wfBaseConvert( $hash, 16, 36, 31 );
595 public function storagePathsChanged() {
596 return array( $this->params
['dst'] );
601 * Copy a file from one storage path to another in the backend.
602 * Parameters for this operation are outlined in FileBackend::doOperations().
604 class CopyFileOp
extends FileOp
{
608 protected function allowedParams() {
609 return array( array( 'src', 'dst' ),
610 array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'disposition' ) );
614 * @param $predicates array
617 protected function doPrecheck( array &$predicates ) {
618 $status = Status
::newGood();
619 // Check if the source file exists
620 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
621 if ( $this->getParam( 'ignoreMissingSource' ) ) {
622 $this->doOperation
= false; // no-op
623 // Update file existence predicates (cache 404s)
624 $predicates['exists'][$this->params
['src']] = false;
625 $predicates['sha1'][$this->params
['src']] = false;
626 return $status; // nothing to do
628 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
631 // Check if a file can be placed/changed at the destination
632 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
633 $status->fatal( 'backend-fail-usable', $this->params
['dst'] );
634 $status->fatal( 'backend-fail-copy', $this->params
['src'], $this->params
['dst'] );
637 // Check if destination file exists
638 $status->merge( $this->precheckDestExistence( $predicates ) );
639 $this->params
['dstExists'] = $this->destExists
; // see FileBackendStore::setFileCache()
640 if ( $status->isOK() ) {
641 // Update file existence predicates
642 $predicates['exists'][$this->params
['dst']] = true;
643 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
645 return $status; // safe to call attempt()
651 protected function doAttempt() {
652 // Do nothing if the src/dst paths are the same
653 if ( $this->params
['src'] !== $this->params
['dst'] ) {
654 // Copy the file into the destination
655 if ( !$this->destSameAsSource
) {
656 return $this->backend
->copyInternal( $this->setFlags( $this->params
) );
659 return Status
::newGood();
665 public function storagePathsRead() {
666 return array( $this->params
['src'] );
672 public function storagePathsChanged() {
673 return array( $this->params
['dst'] );
678 * Move a file from one storage path to another in the backend.
679 * Parameters for this operation are outlined in FileBackend::doOperations().
681 class MoveFileOp
extends FileOp
{
685 protected function allowedParams() {
686 return array( array( 'src', 'dst' ),
687 array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'disposition' ) );
691 * @param $predicates array
694 protected function doPrecheck( array &$predicates ) {
695 $status = Status
::newGood();
696 // Check if the source file exists
697 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
698 if ( $this->getParam( 'ignoreMissingSource' ) ) {
699 $this->doOperation
= false; // no-op
700 // Update file existence predicates (cache 404s)
701 $predicates['exists'][$this->params
['src']] = false;
702 $predicates['sha1'][$this->params
['src']] = false;
703 return $status; // nothing to do
705 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
708 // Check if a file can be placed/changed at the destination
709 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
710 $status->fatal( 'backend-fail-usable', $this->params
['dst'] );
711 $status->fatal( 'backend-fail-move', $this->params
['src'], $this->params
['dst'] );
714 // Check if destination file exists
715 $status->merge( $this->precheckDestExistence( $predicates ) );
716 $this->params
['dstExists'] = $this->destExists
; // see FileBackendStore::setFileCache()
717 if ( $status->isOK() ) {
718 // Update file existence predicates
719 $predicates['exists'][$this->params
['src']] = false;
720 $predicates['sha1'][$this->params
['src']] = false;
721 $predicates['exists'][$this->params
['dst']] = true;
722 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
724 return $status; // safe to call attempt()
730 protected function doAttempt() {
731 // Do nothing if the src/dst paths are the same
732 if ( $this->params
['src'] !== $this->params
['dst'] ) {
733 if ( !$this->destSameAsSource
) {
734 // Move the file into the destination
735 return $this->backend
->moveInternal( $this->setFlags( $this->params
) );
737 // Just delete source as the destination needs no changes
738 $params = array( 'src' => $this->params
['src'] );
739 return $this->backend
->deleteInternal( $this->setFlags( $params ) );
742 return Status
::newGood();
748 public function storagePathsRead() {
749 return array( $this->params
['src'] );
755 public function storagePathsChanged() {
756 return array( $this->params
['src'], $this->params
['dst'] );
761 * Delete a file at the given storage path from the backend.
762 * Parameters for this operation are outlined in FileBackend::doOperations().
764 class DeleteFileOp
extends FileOp
{
768 protected function allowedParams() {
769 return array( array( 'src' ), array( 'ignoreMissingSource' ) );
773 * @param $predicates array
776 protected function doPrecheck( array &$predicates ) {
777 $status = Status
::newGood();
778 // Check if the source file exists
779 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
780 if ( $this->getParam( 'ignoreMissingSource' ) ) {
781 $this->doOperation
= false; // no-op
782 // Update file existence predicates (cache 404s)
783 $predicates['exists'][$this->params
['src']] = false;
784 $predicates['sha1'][$this->params
['src']] = false;
785 return $status; // nothing to do
787 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
790 // Check if a file can be placed/changed at the source
791 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['src'] ) ) {
792 $status->fatal( 'backend-fail-usable', $this->params
['src'] );
793 $status->fatal( 'backend-fail-delete', $this->params
['src'] );
796 // Update file existence predicates
797 $predicates['exists'][$this->params
['src']] = false;
798 $predicates['sha1'][$this->params
['src']] = false;
799 return $status; // safe to call attempt()
805 protected function doAttempt() {
806 // Delete the source file
807 return $this->backend
->deleteInternal( $this->setFlags( $this->params
) );
813 public function storagePathsChanged() {
814 return array( $this->params
['src'] );
819 * Change metadata for a file at the given storage path in the backend.
820 * Parameters for this operation are outlined in FileBackend::doOperations().
822 class DescribeFileOp
extends FileOp
{
826 protected function allowedParams() {
827 return array( array( 'src' ), array( 'disposition', 'headers' ) );
831 * @param $predicates array
834 protected function doPrecheck( array &$predicates ) {
835 $status = Status
::newGood();
836 // Check if the source file exists
837 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
838 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
840 // Check if a file can be placed/changed at the source
841 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['src'] ) ) {
842 $status->fatal( 'backend-fail-usable', $this->params
['src'] );
843 $status->fatal( 'backend-fail-describe', $this->params
['src'] );
846 // Update file existence predicates
847 $predicates['exists'][$this->params
['src']] =
848 $this->fileExists( $this->params
['src'], $predicates );
849 $predicates['sha1'][$this->params
['src']] =
850 $this->fileSha1( $this->params
['src'], $predicates );
851 return $status; // safe to call attempt()
857 protected function doAttempt() {
858 // Update the source file's metadata
859 return $this->backend
->describeInternal( $this->setFlags( $this->params
) );
865 public function storagePathsChanged() {
866 return array( $this->params
['src'] );
871 * Placeholder operation that has no params and does nothing
873 class NullFileOp
extends FileOp
{}