3 * Helper class for representing operations with transaction support.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @ingroup FileBackend
22 * @author Aaron Schulz
26 * FileBackend helper class for representing operations.
27 * Do not use this class from places outside FileBackend.
29 * Methods called from FileOpBatch::attempt() should avoid throwing
30 * exceptions at all costs. FileOp objects should be lightweight in order
31 * to support large arrays in memory and serialization.
33 * @ingroup FileBackend
36 abstract class FileOp
{
38 protected $params = array();
39 /** @var FileBackendStore */
42 protected $state = self
::STATE_NEW
; // integer
43 protected $failed = false; // boolean
44 protected $async = false; // boolean
45 protected $useLatest = true; // boolean
46 protected $batchId; // string
48 protected $doOperation = true; // boolean; operation is not a no-op
49 protected $sourceSha1; // string
50 protected $destSameAsSource; // boolean
52 /* Object life-cycle */
54 const STATE_CHECKED
= 2;
55 const STATE_ATTEMPTED
= 3;
58 * Build a new file operation transaction
60 * @param $backend FileBackendStore
61 * @param $params Array
64 final public function __construct( FileBackendStore
$backend, array $params ) {
65 $this->backend
= $backend;
66 list( $required, $optional ) = $this->allowedParams();
67 foreach ( $required as $name ) {
68 if ( isset( $params[$name] ) ) {
69 $this->params
[$name] = self
::normalizeAnyStoragePaths( $params[$name] );
71 throw new MWException( "File operation missing parameter '$name'." );
74 foreach ( $optional as $name ) {
75 if ( isset( $params[$name] ) ) {
76 $this->params
[$name] = self
::normalizeAnyStoragePaths( $params[$name] );
79 $this->params
= $params;
83 * Normalize $item or anything in $item that is a valid storage path
85 * @param $item string|array
86 * @return string|Array
88 protected function normalizeAnyStoragePaths( $item ) {
89 if ( is_array( $item ) ) {
91 foreach ( $item as $k => $v ) {
92 $k = self
::normalizeIfValidStoragePath( $k );
93 $v = self
::normalizeIfValidStoragePath( $v );
98 return self
::normalizeIfValidStoragePath( $item );
103 * Normalize a string if it is a valid storage path
105 * @param $path string
108 protected static function normalizeIfValidStoragePath( $path ) {
109 if ( FileBackend
::isStoragePath( $path ) ) {
110 $res = FileBackend
::normalizeStoragePath( $path );
111 return ( $res !== null ) ?
$res : $path;
117 * Set the batch UUID this operation belongs to
119 * @param $batchId string
122 final public function setBatchId( $batchId ) {
123 $this->batchId
= $batchId;
127 * Whether to allow stale data for file reads and stat checks
129 * @param $allowStale bool
132 final public function allowStaleReads( $allowStale ) {
133 $this->useLatest
= !$allowStale;
137 * Get the value of the parameter with the given name
139 * @param $name string
140 * @return mixed Returns null if the parameter is not set
142 final public function getParam( $name ) {
143 return isset( $this->params
[$name] ) ?
$this->params
[$name] : null;
147 * Check if this operation failed precheck() or attempt()
151 final public function failed() {
152 return $this->failed
;
156 * Get a new empty predicates array for precheck()
160 final public static function newPredicates() {
161 return array( 'exists' => array(), 'sha1' => array() );
165 * Get a new empty dependency tracking array for paths read/written to
169 final public static function newDependencies() {
170 return array( 'read' => array(), 'write' => array() );
174 * Update a dependency tracking array to account for this operation
176 * @param $deps Array Prior path reads/writes; format of FileOp::newPredicates()
179 final public function applyDependencies( array $deps ) {
180 $deps['read'] +
= array_fill_keys( $this->storagePathsRead(), 1 );
181 $deps['write'] +
= array_fill_keys( $this->storagePathsChanged(), 1 );
186 * Check if this operation changes files listed in $paths
188 * @param $paths Array Prior path reads/writes; format of FileOp::newPredicates()
191 final public function dependsOn( array $deps ) {
192 foreach ( $this->storagePathsChanged() as $path ) {
193 if ( isset( $deps['read'][$path] ) ||
isset( $deps['write'][$path] ) ) {
194 return true; // "output" or "anti" dependency
197 foreach ( $this->storagePathsRead() as $path ) {
198 if ( isset( $deps['write'][$path] ) ) {
199 return true; // "flow" dependency
206 * Get the file journal entries for this file operation
208 * @param $oPredicates Array Pre-op info about files (format of FileOp::newPredicates)
209 * @param $nPredicates Array Post-op info about files (format of FileOp::newPredicates)
212 final public function getJournalEntries( array $oPredicates, array $nPredicates ) {
213 if ( !$this->doOperation
) {
214 return array(); // this is a no-op
216 $nullEntries = array();
217 $updateEntries = array();
218 $deleteEntries = array();
219 $pathsUsed = array_merge( $this->storagePathsRead(), $this->storagePathsChanged() );
220 foreach ( array_unique( $pathsUsed ) as $path ) {
221 $nullEntries[] = array( // assertion for recovery
224 'newSha1' => $this->fileSha1( $path, $oPredicates )
227 foreach ( $this->storagePathsChanged() as $path ) {
228 if ( $nPredicates['sha1'][$path] === false ) { // deleted
229 $deleteEntries[] = array(
234 } else { // created/updated
235 $updateEntries[] = array(
236 'op' => $this->fileExists( $path, $oPredicates ) ?
'update' : 'create',
238 'newSha1' => $nPredicates['sha1'][$path]
242 return array_merge( $nullEntries, $updateEntries, $deleteEntries );
246 * Check preconditions of the operation without writing anything.
247 * This must update $predicates for each path that the op can change
248 * except when a failing status object is returned.
250 * @param $predicates Array
253 final public function precheck( array &$predicates ) {
254 if ( $this->state
!== self
::STATE_NEW
) {
255 return Status
::newFatal( 'fileop-fail-state', self
::STATE_NEW
, $this->state
);
257 $this->state
= self
::STATE_CHECKED
;
258 $status = $this->doPrecheck( $predicates );
259 if ( !$status->isOK() ) {
260 $this->failed
= true;
268 protected function doPrecheck( array &$predicates ) {
269 return Status
::newGood();
273 * Attempt the operation
277 final public function attempt() {
278 if ( $this->state
!== self
::STATE_CHECKED
) {
279 return Status
::newFatal( 'fileop-fail-state', self
::STATE_CHECKED
, $this->state
);
280 } elseif ( $this->failed
) { // failed precheck
281 return Status
::newFatal( 'fileop-fail-attempt-precheck' );
283 $this->state
= self
::STATE_ATTEMPTED
;
284 if ( $this->doOperation
) {
285 $status = $this->doAttempt();
286 if ( !$status->isOK() ) {
287 $this->failed
= true;
288 $this->logFailure( 'attempt' );
291 $status = Status
::newGood();
299 protected function doAttempt() {
300 return Status
::newGood();
304 * Attempt the operation in the background
308 final public function attemptAsync() {
310 $result = $this->attempt();
311 $this->async
= false;
316 * Get the file operation parameters
318 * @return Array (required params list, optional params list)
320 protected function allowedParams() {
321 return array( array(), array() );
325 * Adjust params to FileBackendStore internal file calls
327 * @param $params Array
328 * @return Array (required params list, optional params list)
330 protected function setFlags( array $params ) {
331 return array( 'async' => $this->async
) +
$params;
335 * Get a list of storage paths read from for this operation
339 public function storagePathsRead() {
344 * Get a list of storage paths written to for this operation
348 public function storagePathsChanged() {
353 * Check for errors with regards to the destination file already existing.
354 * This also updates the destSameAsSource and sourceSha1 member variables.
355 * A bad status will be returned if there is no chance it can be overwritten.
357 * @param $predicates Array
360 protected function precheckDestExistence( array $predicates ) {
361 $status = Status
::newGood();
362 // Get hash of source file/string and the destination file
363 $this->sourceSha1
= $this->getSourceSha1Base36(); // FS file or data string
364 if ( $this->sourceSha1
=== null ) { // file in storage?
365 $this->sourceSha1
= $this->fileSha1( $this->params
['src'], $predicates );
367 $this->destSameAsSource
= false;
368 if ( $this->fileExists( $this->params
['dst'], $predicates ) ) {
369 if ( $this->getParam( 'overwrite' ) ) {
370 return $status; // OK
371 } elseif ( $this->getParam( 'overwriteSame' ) ) {
372 $dhash = $this->fileSha1( $this->params
['dst'], $predicates );
373 // Check if hashes are valid and match each other...
374 if ( !strlen( $this->sourceSha1
) ||
!strlen( $dhash ) ) {
375 $status->fatal( 'backend-fail-hashes' );
376 } elseif ( $this->sourceSha1
!== $dhash ) {
377 // Give an error if the files are not identical
378 $status->fatal( 'backend-fail-notsame', $this->params
['dst'] );
380 $this->destSameAsSource
= true; // OK
382 return $status; // do nothing; either OK or bad status
384 $status->fatal( 'backend-fail-alreadyexists', $this->params
['dst'] );
392 * precheckDestExistence() helper function to get the source file SHA-1.
393 * Subclasses should overwride this iff the source is not in storage.
395 * @return string|bool Returns false on failure
397 protected function getSourceSha1Base36() {
402 * Check if a file will exist in storage when this operation is attempted
404 * @param $source string Storage path
405 * @param $predicates Array
408 final protected function fileExists( $source, array $predicates ) {
409 if ( isset( $predicates['exists'][$source] ) ) {
410 return $predicates['exists'][$source]; // previous op assures this
412 $params = array( 'src' => $source, 'latest' => $this->useLatest
);
413 return $this->backend
->fileExists( $params );
418 * Get the SHA-1 of a file in storage when this operation is attempted
420 * @param $source string Storage path
421 * @param $predicates Array
422 * @return string|bool False on failure
424 final protected function fileSha1( $source, array $predicates ) {
425 if ( isset( $predicates['sha1'][$source] ) ) {
426 return $predicates['sha1'][$source]; // previous op assures this
427 } elseif ( isset( $predicates['exists'][$source] ) && !$predicates['exists'][$source] ) {
428 return false; // previous op assures this
430 $params = array( 'src' => $source, 'latest' => $this->useLatest
);
431 return $this->backend
->getFileSha1Base36( $params );
436 * Get the backend this operation is for
438 * @return FileBackendStore
440 public function getBackend() {
441 return $this->backend
;
445 * Log a file operation failure and preserve any temp files
447 * @param $action string
450 final public function logFailure( $action ) {
451 $params = $this->params
;
452 $params['failedAction'] = $action;
454 wfDebugLog( 'FileOperation', get_class( $this ) .
455 " failed (batch #{$this->batchId}): " . FormatJson
::encode( $params ) );
456 } catch ( Exception
$e ) {
457 // bad config? debug log error?
463 * Store a file into the backend from a file on the file system.
464 * Parameters for this operation are outlined in FileBackend::doOperations().
466 class StoreFileOp
extends FileOp
{
470 protected function allowedParams() {
471 return array( array( 'src', 'dst' ),
472 array( 'overwrite', 'overwriteSame', 'disposition' ) );
476 * @param $predicates array
479 protected function doPrecheck( array &$predicates ) {
480 $status = Status
::newGood();
481 // Check if the source file exists on the file system
482 if ( !is_file( $this->params
['src'] ) ) {
483 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
485 // Check if the source file is too big
486 } elseif ( filesize( $this->params
['src'] ) > $this->backend
->maxFileSizeInternal() ) {
487 $status->fatal( 'backend-fail-maxsize',
488 $this->params
['dst'], $this->backend
->maxFileSizeInternal() );
489 $status->fatal( 'backend-fail-store', $this->params
['src'], $this->params
['dst'] );
491 // Check if a file can be placed/changed at the destination
492 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
493 $status->fatal( 'backend-fail-usable', $this->params
['dst'] );
494 $status->fatal( 'backend-fail-store', $this->params
['src'], $this->params
['dst'] );
497 // Check if destination file exists
498 $status->merge( $this->precheckDestExistence( $predicates ) );
499 if ( $status->isOK() ) {
500 // Update file existence predicates
501 $predicates['exists'][$this->params
['dst']] = true;
502 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
504 return $status; // safe to call attempt()
510 protected function doAttempt() {
511 // Store the file at the destination
512 if ( !$this->destSameAsSource
) {
513 return $this->backend
->storeInternal( $this->setFlags( $this->params
) );
515 return Status
::newGood();
519 * @return bool|string
521 protected function getSourceSha1Base36() {
522 wfSuppressWarnings();
523 $hash = sha1_file( $this->params
['src'] );
525 if ( $hash !== false ) {
526 $hash = wfBaseConvert( $hash, 16, 36, 31 );
531 public function storagePathsChanged() {
532 return array( $this->params
['dst'] );
537 * Create a file in the backend with the given content.
538 * Parameters for this operation are outlined in FileBackend::doOperations().
540 class CreateFileOp
extends FileOp
{
541 protected function allowedParams() {
542 return array( array( 'content', 'dst' ),
543 array( 'overwrite', 'overwriteSame', 'disposition' ) );
546 protected function doPrecheck( array &$predicates ) {
547 $status = Status
::newGood();
548 // Check if the source data is too big
549 if ( strlen( $this->getParam( 'content' ) ) > $this->backend
->maxFileSizeInternal() ) {
550 $status->fatal( 'backend-fail-maxsize',
551 $this->params
['dst'], $this->backend
->maxFileSizeInternal() );
552 $status->fatal( 'backend-fail-create', $this->params
['dst'] );
554 // Check if a file can be placed/changed at the destination
555 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
556 $status->fatal( 'backend-fail-usable', $this->params
['dst'] );
557 $status->fatal( 'backend-fail-create', $this->params
['dst'] );
560 // Check if destination file exists
561 $status->merge( $this->precheckDestExistence( $predicates ) );
562 if ( $status->isOK() ) {
563 // Update file existence predicates
564 $predicates['exists'][$this->params
['dst']] = true;
565 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
567 return $status; // safe to call attempt()
573 protected function doAttempt() {
574 if ( !$this->destSameAsSource
) {
575 // Create the file at the destination
576 return $this->backend
->createInternal( $this->setFlags( $this->params
) );
578 return Status
::newGood();
582 * @return bool|String
584 protected function getSourceSha1Base36() {
585 return wfBaseConvert( sha1( $this->params
['content'] ), 16, 36, 31 );
591 public function storagePathsChanged() {
592 return array( $this->params
['dst'] );
597 * Copy a file from one storage path to another in the backend.
598 * Parameters for this operation are outlined in FileBackend::doOperations().
600 class CopyFileOp
extends FileOp
{
604 protected function allowedParams() {
605 return array( array( 'src', 'dst' ),
606 array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'disposition' ) );
610 * @param $predicates array
613 protected function doPrecheck( array &$predicates ) {
614 $status = Status
::newGood();
615 // Check if the source file exists
616 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
617 if ( $this->getParam( 'ignoreMissingSource' ) ) {
618 $this->doOperation
= false; // no-op
619 // Update file existence predicates (cache 404s)
620 $predicates['exists'][$this->params
['src']] = false;
621 $predicates['sha1'][$this->params
['src']] = false;
622 return $status; // nothing to do
624 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
627 // Check if a file can be placed/changed at the destination
628 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
629 $status->fatal( 'backend-fail-usable', $this->params
['dst'] );
630 $status->fatal( 'backend-fail-copy', $this->params
['src'], $this->params
['dst'] );
633 // Check if destination file exists
634 $status->merge( $this->precheckDestExistence( $predicates ) );
635 if ( $status->isOK() ) {
636 // Update file existence predicates
637 $predicates['exists'][$this->params
['dst']] = true;
638 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
640 return $status; // safe to call attempt()
646 protected function doAttempt() {
647 // Do nothing if the src/dst paths are the same
648 if ( $this->params
['src'] !== $this->params
['dst'] ) {
649 // Copy the file into the destination
650 if ( !$this->destSameAsSource
) {
651 return $this->backend
->copyInternal( $this->setFlags( $this->params
) );
654 return Status
::newGood();
660 public function storagePathsRead() {
661 return array( $this->params
['src'] );
667 public function storagePathsChanged() {
668 return array( $this->params
['dst'] );
673 * Move a file from one storage path to another in the backend.
674 * Parameters for this operation are outlined in FileBackend::doOperations().
676 class MoveFileOp
extends FileOp
{
680 protected function allowedParams() {
681 return array( array( 'src', 'dst' ),
682 array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'disposition' ) );
686 * @param $predicates array
689 protected function doPrecheck( array &$predicates ) {
690 $status = Status
::newGood();
691 // Check if the source file exists
692 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
693 if ( $this->getParam( 'ignoreMissingSource' ) ) {
694 $this->doOperation
= false; // no-op
695 // Update file existence predicates (cache 404s)
696 $predicates['exists'][$this->params
['src']] = false;
697 $predicates['sha1'][$this->params
['src']] = false;
698 return $status; // nothing to do
700 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
703 // Check if a file can be placed/changed at the destination
704 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['dst'] ) ) {
705 $status->fatal( 'backend-fail-usable', $this->params
['dst'] );
706 $status->fatal( 'backend-fail-move', $this->params
['src'], $this->params
['dst'] );
709 // Check if destination file exists
710 $status->merge( $this->precheckDestExistence( $predicates ) );
711 if ( $status->isOK() ) {
712 // Update file existence predicates
713 $predicates['exists'][$this->params
['src']] = false;
714 $predicates['sha1'][$this->params
['src']] = false;
715 $predicates['exists'][$this->params
['dst']] = true;
716 $predicates['sha1'][$this->params
['dst']] = $this->sourceSha1
;
718 return $status; // safe to call attempt()
724 protected function doAttempt() {
725 // Do nothing if the src/dst paths are the same
726 if ( $this->params
['src'] !== $this->params
['dst'] ) {
727 if ( !$this->destSameAsSource
) {
728 // Move the file into the destination
729 return $this->backend
->moveInternal( $this->setFlags( $this->params
) );
731 // Just delete source as the destination needs no changes
732 $params = array( 'src' => $this->params
['src'] );
733 return $this->backend
->deleteInternal( $this->setFlags( $params ) );
736 return Status
::newGood();
742 public function storagePathsRead() {
743 return array( $this->params
['src'] );
749 public function storagePathsChanged() {
750 return array( $this->params
['src'], $this->params
['dst'] );
755 * Delete a file at the given storage path from the backend.
756 * Parameters for this operation are outlined in FileBackend::doOperations().
758 class DeleteFileOp
extends FileOp
{
762 protected function allowedParams() {
763 return array( array( 'src' ), array( 'ignoreMissingSource' ) );
767 * @param $predicates array
770 protected function doPrecheck( array &$predicates ) {
771 $status = Status
::newGood();
772 // Check if the source file exists
773 if ( !$this->fileExists( $this->params
['src'], $predicates ) ) {
774 if ( $this->getParam( 'ignoreMissingSource' ) ) {
775 $this->doOperation
= false; // no-op
776 // Update file existence predicates (cache 404s)
777 $predicates['exists'][$this->params
['src']] = false;
778 $predicates['sha1'][$this->params
['src']] = false;
779 return $status; // nothing to do
781 $status->fatal( 'backend-fail-notexists', $this->params
['src'] );
784 // Check if a file can be placed/changed at the source
785 } elseif ( !$this->backend
->isPathUsableInternal( $this->params
['src'] ) ) {
786 $status->fatal( 'backend-fail-usable', $this->params
['src'] );
787 $status->fatal( 'backend-fail-delete', $this->params
['src'] );
790 // Update file existence predicates
791 $predicates['exists'][$this->params
['src']] = false;
792 $predicates['sha1'][$this->params
['src']] = false;
793 return $status; // safe to call attempt()
799 protected function doAttempt() {
800 // Delete the source file
801 return $this->backend
->deleteInternal( $this->setFlags( $this->params
) );
807 public function storagePathsChanged() {
808 return array( $this->params
['src'] );
813 * Placeholder operation that has no params and does nothing
815 class NullFileOp
extends FileOp
{}