/**
* Helper class for representing operations with transaction support.
- * FileBackend::doOperations() will require these classes for supported operations.
* Do not use this class from places outside FileBackend.
- *
- * Use of large fields should be avoided as we want to support
- * potentially many FileOp classes in large arrays in memory.
+ *
+ * Methods called from attemptBatch() should avoid throwing exceptions at all costs.
+ * FileOp objects should be lightweight in order to support large arrays in memory.
*
* @ingroup FileBackend
* @since 1.19
abstract class FileOp {
/** @var Array */
protected $params = array();
- /** @var FileBackendBase */
+ /** @var FileBackendStore */
protected $backend;
protected $state = self::STATE_NEW; // integer
protected $failed = false; // boolean
protected $useLatest = true; // boolean
+ protected $batchId; // string
protected $sourceSha1; // string
protected $destSameAsSource; // boolean
/**
* Build a new file operation transaction
*
- * @params $backend FileBackend
- * @params $params Array
+ * @param $backend FileBackendStore
+ * @param $params Array
+ * @throws MWException
*/
- final public function __construct( FileBackendBase $backend, array $params ) {
+ final public function __construct( FileBackendStore $backend, array $params ) {
$this->backend = $backend;
- foreach ( $this->allowedParams() as $name ) {
+ list( $required, $optional ) = $this->allowedParams();
+ foreach ( $required as $name ) {
+ if ( isset( $params[$name] ) ) {
+ $this->params[$name] = $params[$name];
+ } else {
+ throw new MWException( "File operation missing parameter '$name'." );
+ }
+ }
+ foreach ( $optional as $name ) {
if ( isset( $params[$name] ) ) {
$this->params[$name] = $params[$name];
}
}
/**
- * Allow stale data for file reads and existence checks
+ * Set the batch UUID this operation belongs to
*
+ * @param $batchId string
* @return void
*/
- final protected function allowStaleReads() {
- $this->useLatest = false;
+ final protected function setBatchId( $batchId ) {
+ $this->batchId = $batchId;
}
/**
- * Attempt a series of file operations.
+ * Whether to allow stale data for file reads and stat checks
+ *
+ * @param $allowStale bool
+ * @return void
+ */
+ final protected function allowStaleReads( $allowStale ) {
+ $this->useLatest = !$allowStale;
+ }
+
+ /**
+ * Attempt to perform a series of file operations.
* Callers are responsible for handling file locking.
*
* $opts is an array of options, including:
- * 'force' : Errors that would normally cause a rollback do not.
- * The remaining operations are still attempted if any fail.
- * 'allowStale' : Don't require the latest available data.
- * This can increase performance for non-critical writes.
- * This has no effect unless the 'force' flag is set.
+ * 'force' : Errors that would normally cause a rollback do not.
+ * The remaining operations are still attempted if any fail.
+ * 'allowStale' : Don't require the latest available data.
+ * This can increase performance for non-critical writes.
+ * This has no effect unless the 'force' flag is set.
+ * 'nonJournaled' : Don't log this operation batch in the file journal.
+ *
+ * The resulting Status will be "OK" unless:
+ * a) unexpected operation errors occurred (network partitions, disk full...)
+ * b) significant operation errors occured and 'force' was not set
*
* @param $performOps Array List of FileOp operations
* @param $opts Array Batch operation options
+ * @param $journal FileJournal Journal to log operations to
* @return Status
*/
- final public static function attemptBatch( array $performOps, array $opts ) {
+ final public static function attemptBatch(
+ array $performOps, array $opts, FileJournal $journal
+ ) {
$status = Status::newGood();
- $allowStale = !empty( $opts['allowStale'] );
- $ignoreErrors = !empty( $opts['force'] );
-
$n = count( $performOps );
if ( $n > self::MAX_BATCH_SIZE ) {
$status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
return $status;
}
+ $batchId = $journal->getTimestampedUUID();
+ $allowStale = !empty( $opts['allowStale'] );
+ $ignoreErrors = !empty( $opts['force'] );
+ $journaled = empty( $opts['nonJournaled'] );
+
+ $entries = array(); // file journal entries
$predicates = FileOp::newPredicates(); // account for previous op in prechecks
// Do pre-checks for each operation; abort on failure...
foreach ( $performOps as $index => $fileOp ) {
- if ( $allowStale ) {
- $fileOp->allowStaleReads(); // allow potentially stale reads
- }
- $subStatus = $fileOp->precheck( $predicates );
+ $fileOp->setBatchId( $batchId );
+ $fileOp->allowStaleReads( $allowStale );
+ $oldPredicates = $predicates;
+ $subStatus = $fileOp->precheck( $predicates ); // updates $predicates
$status->merge( $subStatus );
- if ( !$subStatus->isOK() ) { // operation failed?
+ if ( $subStatus->isOK() ) {
+ if ( $journaled ) { // journal log entry
+ $entries = array_merge( $entries,
+ self::getJournalEntries( $fileOp, $oldPredicates, $predicates ) );
+ }
+ } else { // operation failed?
$status->success[$index] = false;
++$status->failCount;
if ( !$ignoreErrors ) {
}
}
+ // Log the operations in file journal...
+ if ( count( $entries ) ) {
+ $subStatus = $journal->logChangeBatch( $entries, $batchId );
+ if ( !$subStatus->isOK() ) {
+ return $subStatus; // abort
+ }
+ }
+
+ if ( $ignoreErrors ) { // treat precheck() fatals as mere warnings
+ $status->setResult( true, $status->value );
+ }
+
// Restart PHP's execution timer and set the timeout to safe amount.
// This handles cases where the operations take a long time or where we are
// already running low on time left. The old timeout is restored afterwards.
- $scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC );
+ # @TODO: re-enable this for when the number of batches is high.
+ #$scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC );
// Attempt each operation...
foreach ( $performOps as $index => $fileOp ) {
} else {
$status->success[$index] = false;
++$status->failCount;
- if ( !$ignoreErrors ) {
- // Log remaining ops as failed for recovery...
- for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
- $performOps[$i]->logFailure( 'attempt_aborted' );
- }
- return $status; // bail out
+ // We can't continue (even with $ignoreErrors) as $predicates is wrong.
+ // Log the remaining ops as failed for recovery...
+ for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) {
+ $performOps[$i]->logFailure( 'attempt_aborted' );
}
+ return $status; // bail out
}
}
return $status;
}
+ /**
+ * Get the file journal entries for a single file operation
+ *
+ * @param $fileOp FileOp
+ * @param $oPredicates Array Pre-op information about files
+ * @param $nPredicates Array Post-op information about files
+ * @return Array
+ */
+ final protected static function getJournalEntries(
+ FileOp $fileOp, array $oPredicates, array $nPredicates
+ ) {
+ $nullEntries = array();
+ $updateEntries = array();
+ $deleteEntries = array();
+ $pathsUsed = array_merge( $fileOp->storagePathsRead(), $fileOp->storagePathsChanged() );
+ foreach ( $pathsUsed as $path ) {
+ $nullEntries[] = array( // assertion for recovery
+ 'op' => 'null',
+ 'path' => $path,
+ 'newSha1' => $fileOp->fileSha1( $path, $oPredicates )
+ );
+ }
+ foreach ( $fileOp->storagePathsChanged() as $path ) {
+ if ( $nPredicates['sha1'][$path] === false ) { // deleted
+ $deleteEntries[] = array(
+ 'op' => 'delete',
+ 'path' => $path,
+ 'newSha1' => ''
+ );
+ } else { // created/updated
+ $updateEntries[] = array(
+ 'op' => $fileOp->fileExists( $path, $oPredicates ) ? 'update' : 'create',
+ 'path' => $path,
+ 'newSha1' => $nPredicates['sha1'][$path]
+ );
+ }
+ }
+ return array_merge( $nullEntries, $updateEntries, $deleteEntries );
+ }
+
/**
* Get the value of the parameter with the given name
*
return $status;
}
+ /**
+ * Get the file operation parameters
+ *
+ * @return Array (required params list, optional params list)
+ */
+ protected function allowedParams() {
+ return array( array(), array() );
+ }
+
/**
* Get a list of storage paths read from for this operation
*
return array();
}
- /**
- * @return Array List of allowed parameters
- */
- protected function allowedParams() {
- return array();
- }
-
/**
* @return Status
*/
/**
* @return Status
*/
- abstract protected function doAttempt();
+ protected function doAttempt() {
+ return Status::newGood();
+ }
/**
* Check for errors with regards to the destination file already existing.
* precheckDestExistence() helper function to get the source file SHA-1.
* Subclasses should overwride this iff the source is not in storage.
*
- * @return string|false Returns false on failure
+ * @return string|bool Returns false on failure
*/
protected function getSourceSha1Base36() {
return null; // N/A
*
* @param $source string Storage path
* @param $predicates Array
- * @return string|false
+ * @return string|bool False on failure
*/
final protected function fileSha1( $source, array $predicates ) {
if ( isset( $predicates['sha1'][$source] ) ) {
$params = $this->params;
$params['failedAction'] = $action;
try {
- wfDebugLog( 'FileOperation',
- get_class( $this ) . ' failed:' . serialize( $params ) );
+ wfDebugLog( 'FileOperation', get_class( $this ) .
+ " failed (batch #{$this->batchId}): " . FormatJson::encode( $params ) );
} catch ( Exception $e ) {
// bad config? debug log error?
}
* the original time limit minus the time the object existed.
*/
class FileOpScopedPHPTimeout {
- protected $startTime; // integer seconds
- protected $oldTimeout; // integer seconds
+ protected $startTime; // float; seconds
+ protected $oldTimeout; // integer; seconds
+
+ protected static $stackDepth = 0; // integer
+ protected static $totalCalls = 0; // integer
+ protected static $totalElapsed = 0; // float; seconds
+
+ /* Prevent callers in infinite loops from running forever */
+ const MAX_TOTAL_CALLS = 1000000;
+ const MAX_TOTAL_TIME = 300; // seconds
/**
* @param $seconds integer
*/
public function __construct( $seconds ) {
if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0
- $this->oldTimeout = ini_set( 'max_execution_time', $seconds );
+ if ( self::$totalCalls >= self::MAX_TOTAL_CALLS ) {
+ trigger_error( "Maximum invocations of " . __CLASS__ . " exceeded." );
+ } elseif ( self::$totalElapsed >= self::MAX_TOTAL_TIME ) {
+ trigger_error( "Time limit within invocations of " . __CLASS__ . " exceeded." );
+ } elseif ( self::$stackDepth > 0 ) { // recursion guard
+ trigger_error( "Resursive invocation of " . __CLASS__ . " attempted." );
+ } else {
+ $this->oldTimeout = ini_set( 'max_execution_time', $seconds );
+ $this->startTime = microtime( true );
+ ++self::$stackDepth;
+ ++self::$totalCalls; // proof against < 1us scopes
+ }
}
- $this->startTime = time();
}
- /*
+ /**
* Restore the original timeout.
* This does not account for the timer value on __construct().
*/
public function __destruct() {
if ( $this->oldTimeout ) {
- $elapsed = time() - $this->startTime;
+ $elapsed = microtime( true ) - $this->startTime;
// Note: a limit of 0 is treated as "forever"
- set_time_limit( max( 1, $this->oldTimeout - $elapsed ) );
+ set_time_limit( max( 1, $this->oldTimeout - (int)$elapsed ) );
+ // If each scoped timeout is for less than one second, we end up
+ // restoring the original timeout without any decrease in value.
+ // Thus web scripts in an infinite loop can run forever unless we
+ // take some measures to prevent this. Track total time and calls.
+ self::$totalElapsed += $elapsed;
+ --self::$stackDepth;
}
}
}
/**
* Store a file into the backend from a file on the file system.
- * Parameters similar to FileBackend::storeInternal(), which include:
+ * Parameters similar to FileBackendStore::storeInternal(), which include:
* src : source path on file system
* dst : destination storage path
* overwrite : do nothing and pass if an identical file exists at destination
*/
class StoreFileOp extends FileOp {
protected function allowedParams() {
- return array( 'src', 'dst', 'overwrite', 'overwriteSame' );
+ return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
}
protected function doPrecheck( array &$predicates ) {
/**
* Create a file in the backend with the given content.
- * Parameters similar to FileBackend::createInternal(), which include:
- * content : a string of raw file contents
+ * Parameters similar to FileBackendStore::createInternal(), which include:
+ * content : the raw file contents
* dst : destination storage path
* overwrite : do nothing and pass if an identical file exists at destination
* overwriteSame : override any existing file at destination
*/
class CreateFileOp extends FileOp {
protected function allowedParams() {
- return array( 'content', 'dst', 'overwrite', 'overwriteSame' );
+ return array( array( 'content', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
}
protected function doPrecheck( array &$predicates ) {
$status = Status::newGood();
// Check if the source data is too big
- if ( strlen( $this->params['content'] ) > $this->backend->maxFileSizeInternal() ) {
+ if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) {
$status->fatal( 'backend-fail-create', $this->params['dst'] );
return $status;
// Check if a file can be placed at the destination
/**
* Copy a file from one storage path to another in the backend.
- * Parameters similar to FileBackend::copyInternal(), which include:
+ * Parameters similar to FileBackendStore::copyInternal(), which include:
* src : source storage path
* dst : destination storage path
* overwrite : do nothing and pass if an identical file exists at destination
*/
class CopyFileOp extends FileOp {
protected function allowedParams() {
- return array( 'src', 'dst', 'overwrite', 'overwriteSame' );
+ return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
}
protected function doPrecheck( array &$predicates ) {
/**
* Move a file from one storage path to another in the backend.
- * Parameters similar to FileBackend::moveInternal(), which include:
+ * Parameters similar to FileBackendStore::moveInternal(), which include:
* src : source storage path
* dst : destination storage path
* overwrite : do nothing and pass if an identical file exists at destination
*/
class MoveFileOp extends FileOp {
protected function allowedParams() {
- return array( 'src', 'dst', 'overwrite', 'overwriteSame' );
+ return array( array( 'src', 'dst' ), array( 'overwrite', 'overwriteSame' ) );
}
protected function doPrecheck( array &$predicates ) {
}
/**
- * Delete a file at the storage path.
- * Parameters similar to FileBackend::deleteInternal(), which include:
+ * Delete a file at the given storage path from the backend.
+ * Parameters similar to FileBackendStore::deleteInternal(), which include:
* src : source storage path
* ignoreMissingSource : don't return an error if the file does not exist
*/
class DeleteFileOp extends FileOp {
- protected $needsDelete = true;
-
protected function allowedParams() {
- return array( 'src', 'ignoreMissingSource' );
+ return array( array( 'src' ), array( 'ignoreMissingSource' ) );
}
+ protected $needsDelete = true;
+
protected function doPrecheck( array &$predicates ) {
$status = Status::newGood();
// Check if the source file exists
/**
* Placeholder operation that has no params and does nothing
*/
-class NullFileOp extends FileOp {
- protected function doAttempt() {
- return Status::newGood();
- }
-}
+class NullFileOp extends FileOp {}
dépôts / lhc / web / wiklou.git / blobdiff