/**
* @var string Set by subclasses
*/
- protected static $notMinimumVerisonMessage;
+ protected static $notMinimumVersionMessage;
/**
* The database connection.
public static function meetsMinimumRequirement( $serverVersion ) {
if ( version_compare( $serverVersion, static::$minimumVersion ) < 0 ) {
return Status::newFatal(
- static::$notMinimumVerisonMessage, static::$minimumVersion, $serverVersion
+ static::$notMinimumVersionMessage, static::$minimumVersion, $serverVersion
);
}
// SQL Server 2005 RTM
// @todo Are SQL Express version numbers different?)
public static $minimumVersion = '9.00.1399';
- protected static $notMinimumVerisonMessage = 'config-mssql-old';
+ protected static $notMinimumVersionMessage = 'config-mssql-old';
// These are schema-level privs
// Note: the web user will be created will full permissions if possible, this permission
public $supportedEngines = [ 'InnoDB', 'MyISAM' ];
public static $minimumVersion = '5.5.8';
- protected static $notMinimumVerisonMessage = 'config-mysql-old';
+ protected static $notMinimumVersionMessage = 'config-mysql-old';
public $webUserPrivs = [
'DELETE',
];
public static $minimumVersion = '9.0.1'; // 9iR1
- protected static $notMinimumVerisonMessage = 'config-oracle-old';
+ protected static $notMinimumVersionMessage = 'config-oracle-old';
protected $connError = null;
];
public static $minimumVersion = '9.2';
- protected static $notMinimumVerisonMessage = 'config-postgres-old';
+ protected static $notMinimumVersionMessage = 'config-postgres-old';
public $maxRoleSearchDepth = 5;
protected $pgConns = [];
class SqliteInstaller extends DatabaseInstaller {
public static $minimumVersion = '3.8.0';
- protected static $notMinimumVerisonMessage = 'config-outdated-sqlite';
+ protected static $notMinimumVersionMessage = 'config-outdated-sqlite';
/**
* @var DatabaseSqlite
) {
unset( $baseConfig[$o] ); // partition queue doesn't care about this
}
- // The class handles all aggregator calls already
- unset( $baseConfig['aggregator'] );
// Get the partition queue objects
foreach ( $partitionMap as $partition => $w ) {
if ( !isset( $params['configByPartition'][$partition] ) ) {
$mime = 'application/zip';
$opendocTypes = [
+ # In OASIS Open Document Format v1.2, Database front end document
+ # has a recommended MIME type of:
+ # application/vnd.oasis.opendocument.base
+ # Despite the type registered at the IANA being 'database' which is
+ # supposed to be normative.
+ # T35515
+ 'base',
+
'chart-template',
'chart',
'formula-template',
'text-web',
'text' ];
- // https://lists.oasis-open.org/archives/office/200505/msg00006.html
+ // The list of document types is available in OASIS Open Document
+ // Format version 1.2 under Appendix C. It is not normative though,
+ // supposedly types registered at the IANA should be.
+ // http://docs.oasis-open.org/office/v1.2/os/OpenDocument-v1.2-os-part1.html
$types = '(?:' . implode( '|', $opendocTypes ) . ')';
$opendocRegex = "/^mimetype(application\/vnd\.oasis\.opendocument\.$types)/";
/**
* Delete all objects expiring before a certain date.
- * @param string $date The reference date in MW format
- * @param callable|bool $progressCallback Optional, a function which will be called
+ * @param string|int $timestamp The reference date in MW or TS_UNIX format
+ * @param callable|null $progressCallback Optional, a function which will be called
* regularly during long-running operations with the percentage progress
* as the first parameter. [optional]
* @param int $limit Maximum number of keys to delete [default: INF]
*
* @return bool Success, false if unimplemented
*/
- public function deleteObjectsExpiringBefore( $date, $progressCallback = false, $limit = INF ) {
+ public function deleteObjectsExpiringBefore(
+ $timestamp,
+ callable $progressCallback = null,
+ $limit = INF
+ ) {
// stub
return false;
}
$this->backend->setDebug( $bool );
}
- public function deleteObjectsExpiringBefore( $date, $progressCallback = false, $limit = INF ) {
- parent::deleteObjectsExpiringBefore( $date, $progressCallback, $limit );
- return $this->backend->deleteObjectsExpiringBefore( $date, $progressCallback, $limit );
+ public function deleteObjectsExpiringBefore(
+ $timestamp,
+ callable $progressCallback = null,
+ $limit = INF
+ ) {
+ parent::deleteObjectsExpiringBefore( $timestamp, $progressCallback, $limit );
+
+ return $this->backend->deleteObjectsExpiringBefore(
+ $timestamp,
+ $progressCallback,
+ $limit
+ );
}
public function makeKeyInternal( $keyspace, $args ) {
return $this->caches[0]->unlock( $key );
}
- public function deleteObjectsExpiringBefore( $date, $progressCallback = false, $limit = INF ) {
+ public function deleteObjectsExpiringBefore(
+ $timestamp,
+ callable $progressCallback = null,
+ $limit = INF
+ ) {
$ret = false;
foreach ( $this->caches as $cache ) {
- if ( $cache->deleteObjectsExpiringBefore( $date, $progressCallback, $limit ) ) {
+ if ( $cache->deleteObjectsExpiringBefore( $timestamp, $progressCallback, $limit ) ) {
$ret = true;
}
}
return $this->writeStore->unlock( $key );
}
- public function deleteObjectsExpiringBefore( $date, $progressCallback = false, $limit = INF ) {
- return $this->writeStore->deleteObjectsExpiringBefore( $date, $progressCallback );
+ public function deleteObjectsExpiringBefore(
+ $timestamp,
+ callable $progressCallback = null,
+ $limit = INF
+ ) {
+ return $this->writeStore->deleteObjectsExpiringBefore(
+ $timestamp,
+ $progressCallback,
+ $limit
+ );
}
public function getMulti( array $keys, $flags = 0 ) {
const COOLOFF_TTL = 1;
/** Default remaining TTL at which to consider pre-emptive regeneration */
const LOW_TTL = 30;
+ /** Max TTL to store keys when a data sourced is lagged */
+ const TTL_LAGGED = 30;
/** Never consider performing "popularity" refreshes until a key reaches this age */
const AGE_NEW = 60;
/** Seconds to ramp up to the "popularity" refresh chance after a key is no longer new */
const RAMPUP_TTL = 30;
- /** Idiom for getWithSetCallback() callbacks to avoid calling set() */
+ /** Idiom for getWithSetCallback() meaning "do not store the callback result" */
const TTL_UNCACHEABLE = -1;
- /** Idiom for getWithSetCallback() callbacks to 'lockTSE' logic */
+ /** Idiom for getWithSetCallback() meaning "no regeneration mutex based on key hotness" */
const TSE_NONE = -1;
- /** Max TTL to store keys when a data sourced is lagged */
- const TTL_LAGGED = 30;
- /** Idiom for delete() for "no hold-off" */
- const HOLDOFF_NONE = 0;
- /** Idiom for set()/getWithSetCallback() for "do not augment the storage medium TTL" */
+ /** Idiom for set()/getWithSetCallback() meaning "no post-expiration persistence" */
const STALE_TTL_NONE = 0;
- /** Idiom for set()/getWithSetCallback() for "no post-expired grace period" */
+ /** Idiom for set()/getWithSetCallback() meaning "no post-expiration grace period" */
const GRACE_TTL_NONE = 0;
+ /** Idiom for delete()/touchCheckKey() meaning "no hold-off period for cache writes" */
+ const HOLDOFF_NONE = 0;
- /** Idiom for getWithSetCallback() for "no minimum required as-of timestamp" */
+ /** Idiom for getWithSetCallback() meaning "no minimum required as-of timestamp" */
const MIN_TIMESTAMP_NONE = 0.0;
/** Tiny negative float to use when CTL comes up >= 0 due to clock skew */
* @note Callable type hints are not used to avoid class-autoloading
*/
final public function getWithSetCallback( $key, $ttl, $callback, array $opts = [] ) {
+ $version = $opts['version'] ?? null;
$pcTTL = $opts['pcTTL'] ?? self::TTL_UNCACHEABLE;
// Try the process cache if enabled and the cache callback is not within a cache callback.
// Process cache use in nested callbacks is not lag-safe with regard to HOLDOFF_TTL since
// the in-memory value is further lagged than the shared one since it uses a blind TTL.
if ( $pcTTL >= 0 && $this->callbackDepth == 0 ) {
- $group = $opts['pcGroup'] ?? self::PC_PRIMARY;
- $procCache = $this->getProcessCache( $group );
- $value = $procCache->has( $key, $pcTTL ) ? $procCache->get( $key ) : false;
+ $procCache = $this->getProcessCache( $opts['pcGroup'] ?? self::PC_PRIMARY );
+ if ( $procCache->has( $key, $pcTTL ) ) {
+ return $procCache->get( $key );
+ }
} else {
- $procCache = false;
- $value = false;
+ $procCache = null;
}
- if ( $value === false ) {
- // Fetch the value over the network
- if ( isset( $opts['version'] ) ) {
- $version = $opts['version'];
- $asOf = null;
- $cur = $this->doGetWithSetCallback(
- $key,
+ if ( $version !== null ) {
+ $curAsOf = self::PASS_BY_REF;
+ $curValue = $this->doGetWithSetCallback(
+ $key,
+ $ttl,
+ // Wrap the value in an array with version metadata but hide it from $callback
+ function ( $oldValue, &$ttl, &$setOpts, $oldAsOf ) use ( $callback, $version ) {
+ if ( $this->isVersionedValue( $oldValue, $version ) ) {
+ $oldData = $oldValue[self::VFLD_DATA];
+ } else {
+ // VFLD_DATA is not set if an old, unversioned, key is present
+ $oldData = false;
+ $oldAsOf = null;
+ }
+
+ return [
+ self::VFLD_DATA => $callback( $oldData, $ttl, $setOpts, $oldAsOf ),
+ self::VFLD_VERSION => $version
+ ];
+ },
+ $opts,
+ $curAsOf
+ );
+ if ( $this->isVersionedValue( $curValue, $version ) ) {
+ // Current value has the requested version; use it
+ $value = $curValue[self::VFLD_DATA];
+ } else {
+ // Current value has a different version; use the variant key for this version.
+ // Regenerate the variant value if it is not newer than the main value at $key
+ // so that purges to they key propagate to the variant value.
+ $value = $this->doGetWithSetCallback(
+ $this->makeGlobalKey( 'WANCache-key-variant', md5( $key ), $version ),
$ttl,
- function ( $oldValue, &$ttl, &$setOpts, $oldAsOf )
- use ( $callback, $version ) {
- if ( is_array( $oldValue )
- && array_key_exists( self::VFLD_DATA, $oldValue )
- && array_key_exists( self::VFLD_VERSION, $oldValue )
- && $oldValue[self::VFLD_VERSION] === $version
- ) {
- $oldData = $oldValue[self::VFLD_DATA];
- } else {
- // VFLD_DATA is not set if an old, unversioned, key is present
- $oldData = false;
- $oldAsOf = null;
- }
-
- return [
- self::VFLD_DATA => $callback( $oldData, $ttl, $setOpts, $oldAsOf ),
- self::VFLD_VERSION => $version
- ];
- },
- $opts,
- $asOf
+ $callback,
+ [ 'version' => null, 'minAsOf' => $curAsOf ] + $opts
);
- if ( $cur[self::VFLD_VERSION] === $version ) {
- // Value created or existed before with version; use it
- $value = $cur[self::VFLD_DATA];
- } else {
- // Value existed before with a different version; use variant key.
- // Reflect purges to $key by requiring that this key value be newer.
- $value = $this->doGetWithSetCallback(
- $this->makeGlobalKey( 'WANCache-key-variant', md5( $key ), $version ),
- $ttl,
- $callback,
- // Regenerate value if not newer than $key
- [ 'version' => null, 'minAsOf' => $asOf ] + $opts
- );
- }
- } else {
- $value = $this->doGetWithSetCallback( $key, $ttl, $callback, $opts );
}
+ } else {
+ $value = $this->doGetWithSetCallback( $key, $ttl, $callback, $opts );
+ }
- // Update the process cache if enabled
- if ( $procCache && $value !== false ) {
- $procCache->set( $key, $value );
- }
+ // Update the process cache if enabled
+ if ( $procCache && $value !== false ) {
+ $procCache->set( $key, $value );
}
return $value;
$busyValue = $opts['busyValue'] ?? null;
$popWindow = $opts['hotTTR'] ?? self::HOT_TTR;
$ageNew = $opts['ageNew'] ?? self::AGE_NEW;
- $minTime = $opts['minAsOf'] ?? self::MIN_TIMESTAMP_NONE;
- $needsVersion = isset( $opts['version'] );
+ $minAsOf = $opts['minAsOf'] ?? self::MIN_TIMESTAMP_NONE;
$touchedCb = $opts['touchedCallback'] ?? null;
$initialTime = $this->getCurrentTime();
$kClass = $this->determineKeyClassForStats( $key );
- // Get the current key value
+ // Get the current key value and metadata
$curTTL = self::PASS_BY_REF;
$curInfo = self::PASS_BY_REF; /** @var array $curInfo */
$curValue = $this->get( $key, $curTTL, $checkKeys, $curInfo );
// Apply any $touchedCb invalidation timestamp to get the "last purge timestamp"
list( $curTTL, $LPT ) = $this->resolveCTL( $curValue, $curTTL, $curInfo, $touchedCb );
- // Keep track of the best candidate value and its timestamp
- $value = $curValue; // return value
- $asOf = $curInfo['asOf']; // return value timestamp
+ // Best possible return value and its corresponding "as of" timestamp
+ $value = $curValue;
+ $asOf = $curInfo['asOf'];
// Determine if a cached value regeneration is needed or desired
if (
- $this->isValid( $value, $needsVersion, $asOf, $minTime ) &&
+ $this->isValid( $value, $asOf, $minAsOf ) &&
$this->isAliveOrInGracePeriod( $curTTL, $graceTTL )
) {
$preemptiveRefresh = (
$isKeyTombstoned = ( $curInfo['tombAsOf'] !== null );
if ( $isKeyTombstoned ) {
// Get the interim key value since the key is tombstoned (write-holed)
- list( $value, $asOf ) = $this->getInterimValue( $key, $needsVersion, $minTime );
+ list( $value, $asOf ) = $this->getInterimValue( $key, $minAsOf );
// Update the "last purge time" since the $touchedCb timestamp depends on $value
$LPT = $this->resolveTouched( $value, $LPT, $touchedCb );
}
// Reduce mutex and cache set spam while keys are in the tombstone/holdoff period by
// checking if $value was genereated by a recent thread much less than a second ago.
if (
- $this->isValid( $value, $needsVersion, $asOf, $minTime, $LPT ) &&
+ $this->isValid( $value, $asOf, $minAsOf, $LPT ) &&
$this->isVolatileValueAgeNegligible( $initialTime - $asOf )
) {
$this->stats->increment( "wanobjectcache.$kClass.hit.volatile" );
$hasLock = false;
if ( $useMutex ) {
- // Acquire a datacenter-local non-blocking lock
+ // Attempt to acquire a non-blocking lock specific to the local datacenter
if ( $this->cache->add( self::MUTEX_KEY_PREFIX . $key, 1, self::LOCK_TTL ) ) {
// Lock acquired; this thread will recompute the value and update cache
$hasLock = true;
- } elseif ( $this->isValid( $value, $needsVersion, $asOf, $minTime ) ) {
- // Lock not acquired and a stale value exists; use the stale value
+ } elseif ( $this->isValid( $value, $asOf, $minAsOf ) ) {
+ // Not acquired and stale cache value exists; use the stale value
$this->stats->increment( "wanobjectcache.$kClass.hit.stale" );
return $value;
// Lock not acquired and no stale value exists
if ( $busyValue !== null ) {
// Use the busy fallback value if nothing else
- $miss = is_infinite( $minTime ) ? 'renew' : 'miss';
+ $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
$this->stats->increment( "wanobjectcache.$kClass.$miss.busy" );
return is_callable( $busyValue ) ? $busyValue() : $busyValue;
if ( $valueIsCacheable ) {
$ago = max( $this->getCurrentTime() - $initialTime, 0.0 );
- $this->stats->timing( "wanobjectcache.$kClass.regen_set_delay", 1000 * $ago );
+ $this->stats->timing( "wanobjectcache.$kClass.regen_set_delay", 1e3 * $ago );
if ( $isKeyTombstoned ) {
if ( $this->checkAndSetCooloff( $key, $kClass, $ago, $lockTSE, $hasLock ) ) {
$this->cache->changeTTL( self::MUTEX_KEY_PREFIX . $key, (int)$initialTime - 60 );
}
- $miss = is_infinite( $minTime ) ? 'renew' : 'miss';
+ $miss = is_infinite( $minAsOf ) ? 'renew' : 'miss';
$this->stats->increment( "wanobjectcache.$kClass.$miss.compute" );
return $value;
/**
* @param string $key
- * @param bool $versioned
- * @param float $minTime
+ * @param float $minAsOf Minimum acceptable "as of" timestamp
* @return array (cached value or false, cached value timestamp or null)
*/
- protected function getInterimValue( $key, $versioned, $minTime ) {
+ protected function getInterimValue( $key, $minAsOf ) {
if ( !$this->useInterimHoldOffCaching ) {
return [ false, null ]; // disabled
}
$wrapped = $this->cache->get( self::INTERIM_KEY_PREFIX . $key );
list( $value ) = $this->unwrap( $wrapped, $this->getCurrentTime() );
$valueAsOf = $wrapped[self::FLD_TIME] ?? null;
- if ( $this->isValid( $value, $versioned, $valueAsOf, $minTime ) ) {
+ if ( $this->isValid( $value, $valueAsOf, $minAsOf ) ) {
return [ $value, $valueAsOf ];
}
// Update the cache value later, such during post-send of an HTTP request
$func = $this->asyncHandler;
$func( function () use ( $key, $ttl, $callback, $opts ) {
- $asOf = null; // unused
$opts['minAsOf'] = INF; // force a refresh
- $this->doGetWithSetCallback( $key, $ttl, $callback, $opts, $asOf );
+ $this->doGetWithSetCallback( $key, $ttl, $callback, $opts );
} );
return true;
* Check if $value is not false, versioned (if needed), and not older than $minTime (if set)
*
* @param array|bool $value
- * @param bool $versioned
* @param float $asOf The time $value was generated
- * @param float $minTime The last time the main value was generated (0.0 if unknown)
+ * @param float $minAsOf Minimum acceptable "as of" timestamp
* @param float|null $purgeTime The last time the value was invalidated
* @return bool
*/
- protected function isValid( $value, $versioned, $asOf, $minTime, $purgeTime = null ) {
+ protected function isValid( $value, $asOf, $minAsOf, $purgeTime = null ) {
// Avoid reading any key not generated after the latest delete() or touch
- $safeMinTime = max( $minTime, $purgeTime + self::TINY_POSTIVE );
+ $safeMinAsOf = max( $minAsOf, $purgeTime + self::TINY_POSTIVE );
if ( $value === false ) {
return false;
- } elseif ( $versioned && !isset( $value[self::VFLD_VERSION] ) ) {
- return false;
- } elseif ( $safeMinTime > 0 && $asOf < $minTime ) {
+ } elseif ( $safeMinAsOf > 0 && $asOf < $minAsOf ) {
return false;
}
return self::PURGE_VAL_PREFIX . (float)$timestamp . ':' . (int)$holdoff;
}
+ /**
+ * @param mixed $value
+ * @param int $version
+ * @return bool
+ */
+ protected function isVersionedValue( $value, $version ) {
+ return (
+ is_array( $value ) &&
+ array_key_exists( self::VFLD_DATA, $value ) &&
+ array_key_exists( self::VFLD_VERSION, $value ) &&
+ $value[self::VFLD_VERSION] === $version
+ );
+ }
+
/**
* @param string $group
* @return MapCacheLRU
protected $serverTags;
/** @var int */
protected $numServers;
- /** @var int */
- protected $lastExpireAll = 0;
+ /** @var int UNIX timestamp */
+ protected $lastGarbageCollect = 0;
/** @var int */
protected $purgePeriod = 10;
/** @var int */
/** @var array Exceptions */
protected $connFailureErrors = [];
+ /** @var int */
+ const GARBAGE_COLLECT_DELAY_SEC = 1;
+
/**
* Constructor. Parameters are:
* - server: A server info structure in the format required by each
$keysByTable[$serverIndex][$tableName][] = $key;
}
- $this->garbageCollect(); // expire old entries if any
-
$result = true;
$exptime = (int)$expiry;
/** @noinspection PhpUnusedLocalVariableInspection */
$db = null;
try {
$db = $this->getDB( $serverIndex );
+ $this->occasionallyGarbageCollect( $db );
} catch ( DBError $e ) {
$this->handleWriteError( $e, $db, $serverIndex );
$result = false;
* @return bool
*/
protected function isExpired( $db, $exptime ) {
- return $exptime != $this->getMaxDateTime( $db ) && wfTimestamp( TS_UNIX, $exptime ) < time();
+ return (
+ $exptime != $this->getMaxDateTime( $db ) &&
+ wfTimestamp( TS_UNIX, $exptime ) < time()
+ );
}
/**
}
}
- protected function garbageCollect() {
- if ( !$this->purgePeriod || $this->replicaOnly ) {
- // Disabled
- return;
- }
- // Only purge on one in every $this->purgePeriod writes
- if ( $this->purgePeriod !== 1 && mt_rand( 0, $this->purgePeriod - 1 ) ) {
- return;
- }
- $now = time();
- // Avoid repeating the delete within a few seconds
- if ( $now > ( $this->lastExpireAll + 1 ) ) {
- $this->lastExpireAll = $now;
- $this->deleteObjectsExpiringBefore(
- wfTimestamp( TS_MW, $now ),
- false,
- $this->purgeLimit
- );
+ /**
+ * @param IDatabase $db
+ * @throws DBError
+ */
+ protected function occasionallyGarbageCollect( IDatabase $db ) {
+ if (
+ // Random purging is enabled
+ $this->purgePeriod &&
+ // This is not using a replica DB
+ !$this->replicaOnly &&
+ // Only purge on one in every $this->purgePeriod writes
+ mt_rand( 0, $this->purgePeriod - 1 ) == 0 &&
+ // Avoid repeating the delete within a few seconds
+ ( time() - $this->lastGarbageCollect ) > self::GARBAGE_COLLECT_DELAY_SEC
+ ) {
+ $garbageCollector = function () use ( $db ) {
+ $this->deleteServerObjectsExpiringBefore( $db, time(), null, $this->purgeLimit );
+ $this->lastGarbageCollect = time();
+ };
+ if ( $this->asyncHandler ) {
+ $this->lastGarbageCollect = time(); // avoid duplicate enqueues
+ ( $this->asyncHandler )( $garbageCollector );
+ } else {
+ $garbageCollector();
+ }
}
}
public function expireAll() {
- $this->deleteObjectsExpiringBefore( wfTimestampNow() );
+ $this->deleteObjectsExpiringBefore( time() );
}
public function deleteObjectsExpiringBefore(
$timestamp,
- $progressCallback = false,
+ callable $progressCallback = null,
$limit = INF
) {
/** @noinspection PhpUnusedLocalVariableInspection */
$silenceScope = $this->silenceTransactionProfiler();
- $count = 0;
- for ( $serverIndex = 0; $serverIndex < $this->numServers; $serverIndex++ ) {
+ $serverIndexes = range( 0, $this->numServers - 1 );
+ shuffle( $serverIndexes );
+
+ $ok = true;
+
+ $keysDeletedCount = 0;
+ foreach ( $serverIndexes as $numServersDone => $serverIndex ) {
$db = null;
try {
$db = $this->getDB( $serverIndex );
- $dbTimestamp = $db->timestamp( $timestamp );
- $totalSeconds = false;
- $baseConds = [ 'exptime < ' . $db->addQuotes( $dbTimestamp ) ];
- for ( $i = 0; $i < $this->shards; $i++ ) {
- $maxExpTime = false;
- while ( true ) {
- $conds = $baseConds;
- if ( $maxExpTime !== false ) {
- $conds[] = 'exptime >= ' . $db->addQuotes( $maxExpTime );
- }
- $rows = $db->select(
- $this->getTableNameByShard( $i ),
- [ 'keyname', 'exptime' ],
- $conds,
- __METHOD__,
- [ 'LIMIT' => 100, 'ORDER BY' => 'exptime' ]
- );
- if ( $rows === false || !$rows->numRows() ) {
- break;
- }
- $keys = [];
- $row = $rows->current();
- $minExpTime = $row->exptime;
- if ( $totalSeconds === false ) {
- $totalSeconds = wfTimestamp( TS_UNIX, $timestamp )
- - wfTimestamp( TS_UNIX, $minExpTime );
- }
- foreach ( $rows as $row ) {
- $keys[] = $row->keyname;
- $maxExpTime = $row->exptime;
- }
-
- $db->delete(
- $this->getTableNameByShard( $i ),
- [
- 'exptime >= ' . $db->addQuotes( $minExpTime ),
- 'exptime < ' . $db->addQuotes( $dbTimestamp ),
- 'keyname' => $keys
- ],
- __METHOD__
- );
- $count += $db->affectedRows();
- if ( $count >= $limit ) {
- return true;
- }
-
- if ( is_callable( $progressCallback ) ) {
- if ( intval( $totalSeconds ) === 0 ) {
- $percent = 0;
- } else {
- $remainingSeconds = wfTimestamp( TS_UNIX, $timestamp )
- - wfTimestamp( TS_UNIX, $maxExpTime );
- if ( $remainingSeconds > $totalSeconds ) {
- $totalSeconds = $remainingSeconds;
- }
- $processedSeconds = $totalSeconds - $remainingSeconds;
- $percent = ( $i + $processedSeconds / $totalSeconds )
- / $this->shards * 100;
- }
- $percent = ( $percent / $this->numServers )
- + ( $serverIndex / $this->numServers * 100 );
- call_user_func( $progressCallback, $percent );
- }
- }
- }
+ $this->deleteServerObjectsExpiringBefore(
+ $db,
+ $timestamp,
+ $progressCallback,
+ $limit,
+ $numServersDone,
+ $keysDeletedCount
+ );
} catch ( DBError $e ) {
$this->handleWriteError( $e, $db, $serverIndex );
- return false;
+ $ok = false;
}
}
- return true;
+ return $ok;
+ }
+
+ /**
+ * @param IDatabase $db
+ * @param string|int $timestamp
+ * @param callable|null $progressCallback
+ * @param int $limit
+ * @param int $serversDoneCount
+ * @param int &$keysDeletedCount
+ * @throws DBError
+ */
+ private function deleteServerObjectsExpiringBefore(
+ IDatabase $db,
+ $timestamp,
+ $progressCallback,
+ $limit,
+ $serversDoneCount = 0,
+ &$keysDeletedCount = 0
+ ) {
+ $cutoffUnix = wfTimestamp( TS_UNIX, $timestamp );
+ $shardIndexes = range( 0, $this->shards - 1 );
+ shuffle( $shardIndexes );
+
+ foreach ( $shardIndexes as $numShardsDone => $shardIndex ) {
+ $continue = null; // last exptime
+ $lag = null; // purge lag
+ do {
+ $res = $db->select(
+ $this->getTableNameByShard( $shardIndex ),
+ [ 'keyname', 'exptime' ],
+ array_merge(
+ [ 'exptime < ' . $db->addQuotes( $db->timestamp( $cutoffUnix ) ) ],
+ $continue ? [ 'exptime >= ' . $db->addQuotes( $continue ) ] : []
+ ),
+ __METHOD__,
+ [ 'LIMIT' => min( $limit, 100 ), 'ORDER BY' => 'exptime' ]
+ );
+
+ if ( $res->numRows() ) {
+ $row = $res->current();
+ if ( $lag === null ) {
+ $lag = max( $cutoffUnix - wfTimestamp( TS_UNIX, $row->exptime ), 1 );
+ }
+
+ $keys = [];
+ foreach ( $res as $row ) {
+ $keys[] = $row->keyname;
+ $continue = $row->exptime;
+ }
+
+ $db->delete(
+ $this->getTableNameByShard( $shardIndex ),
+ [
+ 'exptime < ' . $db->addQuotes( $db->timestamp( $cutoffUnix ) ),
+ 'keyname' => $keys
+ ],
+ __METHOD__
+ );
+ $keysDeletedCount += $db->affectedRows();
+ }
+
+ if ( is_callable( $progressCallback ) ) {
+ if ( $lag ) {
+ $remainingLag = $cutoffUnix - wfTimestamp( TS_UNIX, $continue );
+ $processedLag = max( $lag - $remainingLag, 0 );
+ $doneRatio = ( $numShardsDone + $processedLag / $lag ) / $this->shards;
+ } else {
+ $doneRatio = 1;
+ }
+
+ $overallRatio = ( $doneRatio / $this->numServers )
+ + ( $serversDoneCount / $this->numServers );
+ call_user_func( $progressCallback, $overallRatio * 100 );
+ }
+ } while ( $res->numRows() && $keysDeletedCount < $limit );
+ }
}
/**
}
list( $serverIndex ) = $this->getTableByKey( $key );
+
+ $db = null;
try {
$db = $this->getDB( $serverIndex );
$ok = $db->lock( $key, __METHOD__, $timeout );
unset( $this->locks[$key] );
list( $serverIndex ) = $this->getTableByKey( $key );
+
+ $db = null;
try {
$db = $this->getDB( $serverIndex );
$ok = $db->unlock( $key, __METHOD__ );
public $stack, $rootAccum;
/**
- * @var PPDStack
+ * @var PPDStack|false
*/
public $top;
public $out;
/**
* @throws MWException
* @param string|int $key
- * @param string|PPNode_DOM|DOMDocument $root
+ * @param string|PPNode_DOM|DOMNode|DOMNodeList $root
* @param int $flags
* @return string
*/
/**
* @throws MWException
- * @param string|PPNode_DOM|DOMDocument $root
+ * @param string|PPNode_DOM|DOMNode $root
* @param int $flags
* @return string
*/
/**
* @param string $sep
* @param int $flags
- * @param string|PPNode_DOM|DOMDocument ...$args
+ * @param string|PPNode_DOM|DOMNode ...$args
* @return string
*/
public function implodeWithFlags( $sep, $flags, ...$args ) {
* This previously called implodeWithFlags but has now been inlined to reduce stack depth
*
* @param string $sep
- * @param string|PPNode_DOM|DOMDocument ...$args
+ * @param string|PPNode_DOM|DOMNode ...$args
* @return string
*/
public function implode( $sep, ...$args ) {
* with implode()
*
* @param string $sep
- * @param string|PPNode_DOM|DOMDocument ...$args
+ * @param string|PPNode_DOM|DOMNode ...$args
* @return array
*/
public function virtualImplode( $sep, ...$args ) {
* @param string $start
* @param string $sep
* @param string $end
- * @param string|PPNode_DOM|DOMDocument ...$args
+ * @param string|PPNode_DOM|DOMNode ...$args
* @return array
*/
public function virtualBracketedImplode( $start, $sep, $end, ...$args ) {
/**
* Get an array-type node containing the children of this node.
* Returns false if this is not a tree node.
- * @return PPNode
+ * @return false|PPNode
*/
public function getChildren();
/**
* Get the first child of a tree node. False if there isn't one.
*
- * @return PPNode
+ * @return false|PPNode
*/
public function getFirstChild();
/**
* Get the next sibling of any node. False if there isn't one
- * @return PPNode
+ * @return false|PPNode
*/
public function getNextSibling();
* Get all children of this tree node which have a given name.
* Returns an array-type node, or false if this is not a tree node.
* @param string $type
- * @return bool|PPNode
+ * @return false|PPNode
*/
public function getChildrenOfType( $type );
class PPNode_DOM implements PPNode {
/**
- * @var DOMElement
+ * @var DOMElement|DOMNodeList
*/
public $node;
public $xpath;
}
/**
- * @return bool|PPNode_DOM
+ * @return false|PPNode_DOM
*/
public function getChildren() {
return $this->node->childNodes ? new self( $this->node->childNodes ) : false;
}
/**
- * @return bool|PPNode_DOM
+ * @return false|PPNode_DOM
*/
public function getFirstChild() {
return $this->node->firstChild ? new self( $this->node->firstChild ) : false;
}
/**
- * @return bool|PPNode_DOM
+ * @return false|PPNode_DOM
*/
public function getNextSibling() {
return $this->node->nextSibling ? new self( $this->node->nextSibling ) : false;
/**
* @param string $type
*
- * @return bool|PPNode_DOM
+ * @return false|PPNode_DOM
*/
public function getChildrenOfType( $type ) {
return new self( $this->getXPath()->query( $type, $this->node ) );
* return a temporary proxy object: different instances will be returned
* if this is called more than once on the same node.
*
- * @return PPNode_Hash_Tree|PPNode_Hash_Attr|PPNode_Hash_Text|bool
+ * @return PPNode_Hash_Tree|PPNode_Hash_Attr|PPNode_Hash_Text|false
*/
public function getFirstChild() {
if ( !isset( $this->rawChildren[0] ) ) {
* return a temporary proxy object: different instances will be returned
* if this is called more than once on the same node.
*
- * @return PPNode_Hash_Tree|PPNode_Hash_Attr|PPNode_Hash_Text|bool
+ * @return PPNode_Hash_Tree|PPNode_Hash_Attr|PPNode_Hash_Text|false
*/
public function getNextSibling() {
return self::factory( $this->store, $this->index + 1 );
public $mLinkID;
public $mIncludeSizes, $mPPNodeCount, $mGeneratedPPNodeCount, $mHighestExpansionDepth;
public $mDefaultSort;
- public $mTplRedirCache, $mTplDomCache, $mHeadings, $mDoubleUnderscores;
+ public $mTplRedirCache, $mHeadings, $mDoubleUnderscores;
public $mExpensiveFunctionCount; # number of expensive parser function calls
public $mShowToc, $mForceTocPosition;
+ /** @var array */
+ public $mTplDomCache;
/**
* @var User
* self::OT_HTML: all templates and extension tags
*
* @param string $text The text to transform
- * @param bool|PPFrame $frame Object describing the arguments passed to the
+ * @param false|PPFrame|array $frame Object describing the arguments passed to the
* template. Arguments may also be provided as an associative array, as
* was the usual case before MW1.12. Providing arguments this way may be
* useful for extensions wishing to perform variable replacement
* $piece['lineStart']: whether the brace was at the start of a line
* @param PPFrame $frame The current frame, contains template arguments
* @throws Exception
- * @return string The text of the template
+ * @return string|array The text of the template
*/
public function braceSubstitution( $piece, $frame ) {
// Flags
const CACHE_PREFIX = 'preprocess-xml';
+ /**
+ * @param Parser $parser
+ */
public function __construct( $parser ) {
wfDeprecated( __METHOD__, '1.34' ); // T204945
$this->parser = $parser;
const CACHE_PREFIX = 'preprocess-hash';
const CACHE_VERSION = 2;
+ /**
+ * @param Parser $parser
+ */
public function __construct( $parser ) {
$this->parser = $parser;
}
$actualType = $this->doGuessMimeType( [ $file, 'doc' ] );
$this->assertEquals( 'application/msword', $actualType );
}
+
+ /**
+ * @covers MimeAnalyzer::detectZipType
+ * @dataProvider provideOpendocumentsformatHeaders
+ */
+ function testDetectZipTypeRecognizesOpendocuments( $expected, $header ) {
+ $this->assertEquals(
+ $expected,
+ $this->mimeAnalyzer->detectZipType( $header )
+ );
+ }
+
+ /**
+ * An ODF file is a ZIP file of multiple files. The first one being
+ * 'mimetype' and is not compressed.
+ */
+ function provideOpendocumentsformatHeaders() {
+ $thirtychars = str_repeat( 0, 30 );
+ return [
+ 'Database front end document header based on ODF 1.2' => [
+ 'application/vnd.oasis.opendocument.base',
+ $thirtychars . 'mimetypeapplication/vnd.oasis.opendocument.basePK',
+ ],
+ ];
+ }
+
}
$user = User::newFromName( 'UTSysop' );
try {
- $executor->executeSpecialPage( $page, '', null, null, $user );
+ $executor->executeSpecialPage( $page, '', null, 'qqx', $user );
} catch ( \PHPUnit\Framework\Error\Deprecated $deprecated ) {
// Allow deprecation,
// this test want to check fatals or other things breaking the extension
dépôts / lhc / web / wiklou.git / commitdiff