From bebe30333df8464370073973d99ef18db35f65b2 Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Thu, 11 Jul 2019 05:35:07 -0700 Subject: [PATCH] objectcache: add MediumSpecificBagOStuff base class for non-proxy subclasses This make it much clearer what needs to be overridden Change-Id: I3073f8a0605f557c6a3a93d0d8401cddd0fb8dbe --- autoload.php | 1 + includes/libs/objectcache/APCBagOStuff.php | 2 +- includes/libs/objectcache/APCUBagOStuff.php | 2 +- includes/libs/objectcache/BagOStuff.php | 686 +------------ includes/libs/objectcache/CachedBagOStuff.php | 138 ++- includes/libs/objectcache/EmptyBagOStuff.php | 2 +- includes/libs/objectcache/HashBagOStuff.php | 2 +- .../objectcache/MediumSpecificBagOStuff.php | 932 ++++++++++++++++++ .../libs/objectcache/MemcachedBagOStuff.php | 2 +- .../objectcache/MemcachedPhpBagOStuff.php | 5 +- .../libs/objectcache/MultiWriteBagOStuff.php | 50 +- includes/libs/objectcache/RESTBagOStuff.php | 2 +- includes/libs/objectcache/RedisBagOStuff.php | 2 +- .../libs/objectcache/ReplicatedBagOStuff.php | 45 +- .../libs/objectcache/WinCacheBagOStuff.php | 2 +- includes/objectcache/SqlBagOStuff.php | 7 +- .../libs/objectcache/BagOStuffTest.php | 51 +- .../includes/session/TestBagOStuff.php | 10 +- 18 files changed, 1147 insertions(+), 794 deletions(-) create mode 100644 includes/libs/objectcache/MediumSpecificBagOStuff.php diff --git a/autoload.php b/autoload.php index 9f9f1a6b52..5410bb8a4b 100644 --- a/autoload.php +++ b/autoload.php @@ -968,6 +968,7 @@ $wgAutoloadLocalClasses = [ 'MediaWiki\\Widget\\TitlesMultiselectWidget' => __DIR__ . '/includes/widget/TitlesMultiselectWidget.php', 'MediaWiki\\Widget\\UserInputWidget' => __DIR__ . '/includes/widget/UserInputWidget.php', 'MediaWiki\\Widget\\UsersMultiselectWidget' => __DIR__ . '/includes/widget/UsersMultiselectWidget.php', + 'MediumSpecificBagOStuff' => __DIR__ . '/includes/libs/objectcache/MediumSpecificBagOStuff.php', 'MemcLockManager' => __DIR__ . '/includes/libs/lockmanager/MemcLockManager.php', 'MemcachedBagOStuff' => __DIR__ . '/includes/libs/objectcache/MemcachedBagOStuff.php', 'MemcachedClient' => __DIR__ . '/includes/libs/objectcache/MemcachedClient.php', diff --git a/includes/libs/objectcache/APCBagOStuff.php b/includes/libs/objectcache/APCBagOStuff.php index 465fe820e0..0954ac8061 100644 --- a/includes/libs/objectcache/APCBagOStuff.php +++ b/includes/libs/objectcache/APCBagOStuff.php @@ -33,7 +33,7 @@ * * @ingroup Cache */ -class APCBagOStuff extends BagOStuff { +class APCBagOStuff extends MediumSpecificBagOStuff { /** @var bool Whether to trust the APC implementation to serialization */ private $nativeSerialize; diff --git a/includes/libs/objectcache/APCUBagOStuff.php b/includes/libs/objectcache/APCUBagOStuff.php index b14ac7c4df..021cdf7b76 100644 --- a/includes/libs/objectcache/APCUBagOStuff.php +++ b/includes/libs/objectcache/APCUBagOStuff.php @@ -33,7 +33,7 @@ * * @ingroup Cache */ -class APCUBagOStuff extends BagOStuff { +class APCUBagOStuff extends MediumSpecificBagOStuff { /** @var bool Whether to trust the APC implementation to serialization */ private $nativeSerialize; diff --git a/includes/libs/objectcache/BagOStuff.php b/includes/libs/objectcache/BagOStuff.php index 4819f0ef23..906e955464 100644 --- a/includes/libs/objectcache/BagOStuff.php +++ b/includes/libs/objectcache/BagOStuff.php @@ -30,7 +30,6 @@ use Psr\Log\LoggerAwareInterface; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; use Wikimedia\ScopedCallback; -use Wikimedia\WaitConditionLoop; /** * Class representing a cache/ephemeral data store @@ -62,41 +61,20 @@ use Wikimedia\WaitConditionLoop; * @ingroup Cache */ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwareInterface { - /** @var array[] Lock tracking */ - protected $locks = []; - /** @var int ERR_* class constant */ - protected $lastError = self::ERR_NONE; - /** @var string */ - protected $keyspace = 'local'; /** @var LoggerInterface */ protected $logger; + /** @var callable|null */ protected $asyncHandler; - /** @var int Seconds */ - protected $syncTimeout; - /** @var int Bytes; chunk size of segmented cache values */ - protected $segmentationSize; - /** @var int Bytes; maximum total size of a segmented cache value */ - protected $segmentedValueMaxSize; + /** @var int[] Map of (ATTR_* class constant => QOS_* class constant) */ + protected $attrMap = []; /** @var bool */ - private $debugMode = false; - /** @var array */ - private $duplicateKeyLookups = []; - /** @var bool */ - private $reportDupes = false; - /** @var bool */ - private $dupeTrackScheduled = false; - - /** @var callable[] */ - protected $busyCallbacks = []; + protected $debugMode = false; /** @var float|null */ private $wallClockOverride; - /** @var int[] Map of (ATTR_* class constant => QOS_* class constant) */ - protected $attrMap = []; - /** Bitfield constants for get()/getMulti(); these are only advisory */ const READ_LATEST = 1; // if supported, avoid reading stale data due to replication const READ_VERIFIED = 2; // promise that the caller handles detection of staleness @@ -105,44 +83,18 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar const WRITE_CACHE_ONLY = 8; // only change state of the in-memory cache const WRITE_ALLOW_SEGMENTS = 16; // allow partitioning of the value if it is large const WRITE_PRUNE_SEGMENTS = 32; // delete all the segments if the value is partitioned - const WRITE_BACKGROUND = 64; // if supported, - - /** @var string Component to use for key construction of blob segment keys */ - const SEGMENT_COMPONENT = 'segment'; + const WRITE_BACKGROUND = 64; // if supported, do not block on completion until the next read /** - * $params include: + * Parameters include: * - logger: Psr\Log\LoggerInterface instance - * - keyspace: Default keyspace for $this->makeKey() * - asyncHandler: Callable to use for scheduling tasks after the web request ends. * In CLI mode, it should run the task immediately. - * - reportDupes: Whether to emit warning log messages for all keys that were - * requested more than once (requires an asyncHandler). - * - syncTimeout: How long to wait with WRITE_SYNC in seconds. - * - segmentationSize: The chunk size, in bytes, of segmented values. The value should - * not exceed the maximum size of values in the storage backend, as configured by - * the site administrator. - * - segmentedValueMaxSize: The maximum total size, in bytes, of segmented values. - * This should be configured to a reasonable size give the site traffic and the - * amount of I/O between application and cache servers that the network can handle. * @param array $params */ public function __construct( array $params = [] ) { $this->setLogger( $params['logger'] ?? new NullLogger() ); - - if ( isset( $params['keyspace'] ) ) { - $this->keyspace = $params['keyspace']; - } - $this->asyncHandler = $params['asyncHandler'] ?? null; - - if ( !empty( $params['reportDupes'] ) && is_callable( $this->asyncHandler ) ) { - $this->reportDupes = true; - } - - $this->syncTimeout = $params['syncTimeout'] ?? 3; - $this->segmentationSize = $params['segmentationSize'] ?? 8388608; // 8MiB - $this->segmentedValueMaxSize = $params['segmentedValueMaxSize'] ?? 67108864; // 64MiB } /** @@ -154,10 +106,10 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar } /** - * @param bool $bool + * @param bool $enabled */ - public function setDebug( $bool ) { - $this->debugMode = $bool; + public function setDebug( $enabled ) { + $this->debugMode = $enabled; } /** @@ -201,115 +153,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @param int $flags Bitfield of BagOStuff::READ_* constants [optional] * @return mixed Returns false on failure or if the item does not exist */ - public function get( $key, $flags = 0 ) { - $this->trackDuplicateKeys( $key ); - - return $this->resolveSegments( $key, $this->doGet( $key, $flags ) ); - } - - /** - * Track the number of times that a given key has been used. - * @param string $key - */ - private function trackDuplicateKeys( $key ) { - if ( !$this->reportDupes ) { - return; - } - - if ( !isset( $this->duplicateKeyLookups[$key] ) ) { - // Track that we have seen this key. This N-1 counting style allows - // easy filtering with array_filter() later. - $this->duplicateKeyLookups[$key] = 0; - } else { - $this->duplicateKeyLookups[$key] += 1; - - if ( $this->dupeTrackScheduled === false ) { - $this->dupeTrackScheduled = true; - // Schedule a callback that logs keys processed more than once by get(). - call_user_func( $this->asyncHandler, function () { - $dups = array_filter( $this->duplicateKeyLookups ); - foreach ( $dups as $key => $count ) { - $this->logger->warning( - 'Duplicate get(): "{key}" fetched {count} times', - // Count is N-1 of the actual lookup count - [ 'key' => $key, 'count' => $count + 1, ] - ); - } - } ); - } - } - } - - /** - * @param string $key - * @param int $flags Bitfield of BagOStuff::READ_* constants [optional] - * @param mixed|null &$casToken Token to use for check-and-set comparisons - * @return mixed Returns false on failure or if the item does not exist - */ - abstract protected function doGet( $key, $flags = 0, &$casToken = null ); - - /** - * Set an item - * - * @param string $key - * @param mixed $value - * @param int $exptime Either an interval in seconds or a unix timestamp for expiry - * @param int $flags Bitfield of BagOStuff::WRITE_* constants - * @return bool Success - */ - public function set( $key, $value, $exptime = 0, $flags = 0 ) { - if ( - is_int( $value ) || // avoid breaking incr()/decr() - ( $flags & self::WRITE_ALLOW_SEGMENTS ) != self::WRITE_ALLOW_SEGMENTS || - is_infinite( $this->segmentationSize ) - ) { - return $this->doSet( $key, $value, $exptime, $flags ); - } - - $serialized = $this->serialize( $value ); - $segmentSize = $this->getSegmentationSize(); - $maxTotalSize = $this->getSegmentedValueMaxSize(); - - $size = strlen( $serialized ); - if ( $size <= $segmentSize ) { - // Since the work of serializing it was already done, just use it inline - return $this->doSet( - $key, - SerializedValueContainer::newUnified( $serialized ), - $exptime, - $flags - ); - } elseif ( $size > $maxTotalSize ) { - $this->setLastError( "Key $key exceeded $maxTotalSize bytes." ); - - return false; - } - - $chunksByKey = []; - $segmentHashes = []; - $count = intdiv( $size, $segmentSize ) + ( ( $size % $segmentSize ) ? 1 : 0 ); - for ( $i = 0; $i < $count; ++$i ) { - $segment = substr( $serialized, $i * $segmentSize, $segmentSize ); - $hash = sha1( $segment ); - $chunkKey = $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $hash ); - $chunksByKey[$chunkKey] = $segment; - $segmentHashes[] = $hash; - } - - $flags &= ~self::WRITE_ALLOW_SEGMENTS; // sanity - $ok = $this->setMulti( $chunksByKey, $exptime, $flags ); - if ( $ok ) { - // Only when all segments are stored should the main key be changed - $ok = $this->doSet( - $key, - SerializedValueContainer::newSegmented( $segmentHashes ), - $exptime, - $flags - ); - } - - return $ok; - } + abstract public function get( $key, $flags = 0 ); /** * Set an item @@ -320,7 +164,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @param int $flags Bitfield of BagOStuff::WRITE_* constants * @return bool Success */ - abstract protected function doSet( $key, $value, $exptime = 0, $flags = 0 ); + abstract public function set( $key, $value, $exptime = 0, $flags = 0 ); /** * Delete an item @@ -333,38 +177,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @return bool True if the item was deleted or not found, false on failure * @param int $flags Bitfield of BagOStuff::WRITE_* constants */ - public function delete( $key, $flags = 0 ) { - if ( ( $flags & self::WRITE_PRUNE_SEGMENTS ) != self::WRITE_PRUNE_SEGMENTS ) { - return $this->doDelete( $key, $flags ); - } - - $mainValue = $this->doGet( $key, self::READ_LATEST ); - if ( !$this->doDelete( $key, $flags ) ) { - return false; - } - - if ( !SerializedValueContainer::isSegmented( $mainValue ) ) { - return true; // no segments to delete - } - - $orderedKeys = array_map( - function ( $segmentHash ) use ( $key ) { - return $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $segmentHash ); - }, - $mainValue->{SerializedValueContainer::SEGMENTED_HASHES} - ); - - return $this->deleteMulti( $orderedKeys, $flags ); - } - - /** - * Delete an item - * - * @param string $key - * @return bool True if the item was deleted or not found, false on failure - * @param int $flags Bitfield of BagOStuff::WRITE_* constants - */ - abstract protected function doDelete( $key, $flags = 0 ); + abstract public function delete( $key, $flags = 0 ); /** * Insert an item if it does not already exist @@ -394,99 +207,13 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @return bool Success * @throws InvalidArgumentException */ - public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) { - return $this->mergeViaCas( $key, $callback, $exptime, $attempts, $flags ); - } - - /** - * @see BagOStuff::merge() - * - * @param string $key - * @param callable $callback Callback method to be executed - * @param int $exptime Either an interval in seconds or a unix timestamp for expiry - * @param int $attempts The amount of times to attempt a merge in case of failure - * @param int $flags Bitfield of BagOStuff::WRITE_* constants - * @return bool Success - */ - final protected function mergeViaCas( $key, callable $callback, $exptime, $attempts, $flags ) { - do { - $casToken = null; // passed by reference - // Get the old value and CAS token from cache - $this->clearLastError(); - $currentValue = $this->resolveSegments( - $key, - $this->doGet( $key, self::READ_LATEST, $casToken ) - ); - if ( $this->getLastError() ) { - $this->logger->warning( - __METHOD__ . ' failed due to I/O error on get() for {key}.', - [ 'key' => $key ] - ); - - return false; // don't spam retries (retry only on races) - } - - // Derive the new value from the old value - $value = call_user_func( $callback, $this, $key, $currentValue, $exptime ); - $hadNoCurrentValue = ( $currentValue === false ); - unset( $currentValue ); // free RAM in case the value is large - - $this->clearLastError(); - if ( $value === false ) { - $success = true; // do nothing - } elseif ( $hadNoCurrentValue ) { - // Try to create the key, failing if it gets created in the meantime - $success = $this->add( $key, $value, $exptime, $flags ); - } else { - // Try to update the key, failing if it gets changed in the meantime - $success = $this->cas( $casToken, $key, $value, $exptime, $flags ); - } - if ( $this->getLastError() ) { - $this->logger->warning( - __METHOD__ . ' failed due to I/O error for {key}.', - [ 'key' => $key ] - ); - - return false; // IO error; don't spam retries - } - - } while ( !$success && --$attempts ); - - return $success; - } - - /** - * Check and set an item - * - * @param mixed $casToken - * @param string $key - * @param mixed $value - * @param int $exptime Either an interval in seconds or a unix timestamp for expiry - * @param int $flags Bitfield of BagOStuff::WRITE_* constants - * @return bool Success - */ - protected function cas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) { - if ( !$this->lock( $key, 0 ) ) { - return false; // non-blocking - } - - $curCasToken = null; // passed by reference - $this->doGet( $key, self::READ_LATEST, $curCasToken ); - if ( $casToken === $curCasToken ) { - $success = $this->set( $key, $value, $exptime, $flags ); - } else { - $this->logger->info( - __METHOD__ . ' failed due to race condition for {key}.', - [ 'key' => $key ] - ); - - $success = false; // mismatched or failed - } - - $this->unlock( $key ); - - return $success; - } + abstract public function merge( + $key, + callable $callback, + $exptime = 0, + $attempts = 10, + $flags = 0 + ); /** * Change the expiration on a key if it exists @@ -505,39 +232,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @return bool Success Returns false on failure or if the item does not exist * @since 1.28 */ - public function changeTTL( $key, $exptime = 0, $flags = 0 ) { - return $this->doChangeTTL( $key, $exptime, $flags ); - } - - /** - * @param string $key - * @param int $exptime - * @param int $flags - * @return bool - */ - protected function doChangeTTL( $key, $exptime, $flags ) { - $expiry = $this->convertToExpiry( $exptime ); - $delete = ( $expiry != 0 && $expiry < $this->getCurrentTime() ); - - if ( !$this->lock( $key, 0 ) ) { - return false; - } - // Use doGet() to avoid having to trigger resolveSegments() - $blob = $this->doGet( $key, self::READ_LATEST ); - if ( $blob ) { - if ( $delete ) { - $ok = $this->doDelete( $key, $flags ); - } else { - $ok = $this->doSet( $key, $blob, $exptime, $flags ); - } - } else { - $ok = false; - } - - $this->unlock( $key ); - - return $ok; - } + abstract public function changeTTL( $key, $exptime = 0, $flags = 0 ); /** * Acquire an advisory lock on a key string @@ -550,51 +245,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @param string $rclass Allow reentry if set and the current lock used this value * @return bool Success */ - public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' ) { - // Avoid deadlocks and allow lock reentry if specified - if ( isset( $this->locks[$key] ) ) { - if ( $rclass != '' && $this->locks[$key]['class'] === $rclass ) { - ++$this->locks[$key]['depth']; - return true; - } else { - return false; - } - } - - $fname = __METHOD__; - $expiry = min( $expiry ?: INF, self::TTL_DAY ); - $loop = new WaitConditionLoop( - function () use ( $key, $expiry, $fname ) { - $this->clearLastError(); - if ( $this->add( "{$key}:lock", 1, $expiry ) ) { - return WaitConditionLoop::CONDITION_REACHED; // locked! - } elseif ( $this->getLastError() ) { - $this->logger->warning( - $fname . ' failed due to I/O error for {key}.', - [ 'key' => $key ] - ); - - return WaitConditionLoop::CONDITION_ABORTED; // network partition? - } - - return WaitConditionLoop::CONDITION_CONTINUE; - }, - $timeout - ); - - $code = $loop->invoke(); - $locked = ( $code === $loop::CONDITION_REACHED ); - if ( $locked ) { - $this->locks[$key] = [ 'class' => $rclass, 'depth' => 1 ]; - } elseif ( $code === $loop::CONDITION_TIMED_OUT ) { - $this->logger->warning( - "$fname failed due to timeout for {key}.", - [ 'key' => $key, 'timeout' => $timeout ] - ); - } - - return $locked; - } + abstract public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' ); /** * Release an advisory lock on a key string @@ -602,27 +253,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @param string $key * @return bool Success */ - public function unlock( $key ) { - if ( !isset( $this->locks[$key] ) ) { - return false; - } - - if ( --$this->locks[$key]['depth'] <= 0 ) { - unset( $this->locks[$key] ); - - $ok = $this->doDelete( "{$key}:lock" ); - if ( !$ok ) { - $this->logger->warning( - __METHOD__ . ' failed to release lock for {key}.', - [ 'key' => $key ] - ); - } - - return $ok; - } - - return true; - } + abstract public function unlock( $key ); /** * Get a lightweight exclusive self-unlocking lock @@ -673,37 +304,11 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * * @return bool Success; false if unimplemented */ - public function deleteObjectsExpiringBefore( + abstract public function deleteObjectsExpiringBefore( $timestamp, callable $progress = null, $limit = INF - ) { - return false; - } - - /** - * Get an associative array containing the item for each of the keys that have items. - * @param string[] $keys List of keys; can be a map of (unused => key) for convenience - * @param int $flags Bitfield; supports READ_LATEST [optional] - * @return mixed[] Map of (key => value) for existing keys; preserves the order of $keys - */ - public function getMulti( array $keys, $flags = 0 ) { - $foundByKey = $this->doGetMulti( $keys, $flags ); - - $res = []; - foreach ( $keys as $key ) { - // Resolve one blob at a time (avoids too much I/O at once) - if ( array_key_exists( $key, $foundByKey ) ) { - // A value should not appear in the key if a segment is missing - $value = $this->resolveSegments( $key, $foundByKey[$key] ); - if ( $value !== false ) { - $res[$key] = $value; - } - } - } - - return $res; - } + ); /** * Get an associative array containing the item for each of the keys that have items. @@ -711,17 +316,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @param int $flags Bitfield; supports READ_LATEST [optional] * @return mixed[] Map of (key => value) for existing keys */ - protected function doGetMulti( array $keys, $flags = 0 ) { - $res = []; - foreach ( $keys as $key ) { - $val = $this->doGet( $key, $flags ); - if ( $val !== false ) { - $res[$key] = $val; - } - } - - return $res; - } + abstract public function getMulti( array $keys, $flags = 0 ); /** * Batch insertion/replace @@ -736,26 +331,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @return bool Success * @since 1.24 */ - public function setMulti( array $data, $exptime = 0, $flags = 0 ) { - if ( ( $flags & self::WRITE_ALLOW_SEGMENTS ) === self::WRITE_ALLOW_SEGMENTS ) { - throw new InvalidArgumentException( __METHOD__ . ' got WRITE_ALLOW_SEGMENTS' ); - } - return $this->doSetMulti( $data, $exptime, $flags ); - } - - /** - * @param mixed[] $data Map of (key => value) - * @param int $exptime Either an interval in seconds or a unix timestamp for expiry - * @param int $flags Bitfield of BagOStuff::WRITE_* constants - * @return bool Success - */ - protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) { - $res = true; - foreach ( $data as $key => $value ) { - $res = $this->doSet( $key, $value, $exptime, $flags ) && $res; - } - return $res; - } + abstract public function setMulti( array $data, $exptime = 0, $flags = 0 ); /** * Batch deletion @@ -769,25 +345,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @return bool Success * @since 1.33 */ - public function deleteMulti( array $keys, $flags = 0 ) { - if ( ( $flags & self::WRITE_ALLOW_SEGMENTS ) === self::WRITE_ALLOW_SEGMENTS ) { - throw new InvalidArgumentException( __METHOD__ . ' got WRITE_ALLOW_SEGMENTS' ); - } - return $this->doDeleteMulti( $keys, $flags ); - } - - /** - * @param string[] $keys List of keys - * @param int $flags Bitfield of BagOStuff::WRITE_* constants - * @return bool Success - */ - protected function doDeleteMulti( array $keys, $flags = 0 ) { - $res = true; - foreach ( $keys as $key ) { - $res = $this->doDelete( $key, $flags ) && $res; - } - return $res; - } + abstract public function deleteMulti( array $keys, $flags = 0 ); /** * Change the expiration of multiple keys that exist @@ -800,14 +358,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @return bool Success * @since 1.34 */ - public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) { - $res = true; - foreach ( $keys as $key ) { - $res = $this->doChangeTTL( $key, $exptime, $flags ) && $res; - } - - return $res; - } + abstract public function changeTTLMulti( array $keys, $exptime, $flags = 0 ); /** * Increase stored value of $key by $value while preserving its TTL @@ -823,9 +374,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @param int $value Value to subtract from $key (default: 1) [optional] * @return int|bool New value or false on failure */ - public function decr( $key, $value = 1 ) { - return $this->incr( $key, - $value ); - } + abstract public function decr( $key, $value = 1 ); /** * Increase stored value of $key by $value while preserving its TTL @@ -839,83 +388,20 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @return int|bool New value or false on failure * @since 1.24 */ - public function incrWithInit( $key, $ttl, $value = 1, $init = 1 ) { - $this->clearLastError(); - $newValue = $this->incr( $key, $value ); - if ( $newValue === false && !$this->getLastError() ) { - // No key set; initialize - $newValue = $this->add( $key, (int)$init, $ttl ) ? $init : false; - if ( $newValue === false && !$this->getLastError() ) { - // Raced out initializing; increment - $newValue = $this->incr( $key, $value ); - } - } - - return $newValue; - } - - /** - * Get and reassemble the chunks of blob at the given key - * - * @param string $key - * @param mixed $mainValue - * @return string|null|bool The combined string, false if missing, null on error - */ - final protected function resolveSegments( $key, $mainValue ) { - if ( SerializedValueContainer::isUnified( $mainValue ) ) { - return $this->unserialize( $mainValue->{SerializedValueContainer::UNIFIED_DATA} ); - } - - if ( SerializedValueContainer::isSegmented( $mainValue ) ) { - $orderedKeys = array_map( - function ( $segmentHash ) use ( $key ) { - return $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $segmentHash ); - }, - $mainValue->{SerializedValueContainer::SEGMENTED_HASHES} - ); - - $segmentsByKey = $this->doGetMulti( $orderedKeys ); - - $parts = []; - foreach ( $orderedKeys as $segmentKey ) { - if ( isset( $segmentsByKey[$segmentKey] ) ) { - $parts[] = $segmentsByKey[$segmentKey]; - } else { - return false; // missing segment - } - } - - return $this->unserialize( implode( '', $parts ) ); - } - - return $mainValue; - } + abstract public function incrWithInit( $key, $ttl, $value = 1, $init = 1 ); /** * Get the "last error" registered; clearLastError() should be called manually * @return int ERR_* constant for the "last error" registry * @since 1.23 */ - public function getLastError() { - return $this->lastError; - } + abstract public function getLastError(); /** * Clear the "last error" registry * @since 1.23 */ - public function clearLastError() { - $this->lastError = self::ERR_NONE; - } - - /** - * Set the "last error" registry - * @param int $err ERR_* constant - * @since 1.23 - */ - protected function setLastError( $err ) { - $this->lastError = $err; - } + abstract public function clearLastError(); /** * Let a callback be run to avoid wasting time on special blocking calls @@ -937,75 +423,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @param callable $workCallback * @since 1.28 */ - final public function addBusyCallback( callable $workCallback ) { - $this->busyCallbacks[] = $workCallback; - } - - /** - * @param string $text - */ - protected function debug( $text ) { - if ( $this->debugMode ) { - $this->logger->debug( "{class} debug: $text", [ 'class' => static::class ] ); - } - } - - /** - * @param int $exptime - * @return bool - */ - final protected function expiryIsRelative( $exptime ) { - return ( $exptime != 0 && $exptime < ( 10 * self::TTL_YEAR ) ); - } - - /** - * Convert an optionally relative timestamp to an absolute time - * - * The input value will be cast to an integer and interpreted as follows: - * - zero: no expiry; return zero (e.g. TTL_INDEFINITE) - * - negative: relative TTL; return UNIX timestamp offset by this value - * - positive (< 10 years): relative TTL; return UNIX timestamp offset by this value - * - positive (>= 10 years): absolute UNIX timestamp; return this value - * - * @param int $exptime Absolute TTL or 0 for indefinite - * @return int - */ - final protected function convertToExpiry( $exptime ) { - return $this->expiryIsRelative( $exptime ) - ? (int)$this->getCurrentTime() + $exptime - : $exptime; - } - - /** - * Convert an optionally absolute expiry time to a relative time. If an - * absolute time is specified which is in the past, use a short expiry time. - * - * @param int $exptime - * @return int - */ - final protected function convertToRelative( $exptime ) { - return $this->expiryIsRelative( $exptime ) - ? (int)$exptime - : max( $exptime - (int)$this->getCurrentTime(), 1 ); - } - - /** - * Check if a value is an integer - * - * @param mixed $value - * @return bool - */ - final protected function isInteger( $value ) { - if ( is_int( $value ) ) { - return true; - } elseif ( !is_string( $value ) ) { - return false; - } - - $integer = (int)$value; - - return ( $value === (string)$integer ); - } + abstract public function addBusyCallback( callable $workCallback ); /** * Construct a cache key. @@ -1015,13 +433,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @param array $args * @return string Colon-delimited list of $keyspace followed by escaped components of $args */ - public function makeKeyInternal( $keyspace, $args ) { - $key = $keyspace; - foreach ( $args as $arg ) { - $key .= ':' . str_replace( ':', '%3A', $arg ); - } - return strtr( $key, ' ', '_' ); - } + abstract public function makeKeyInternal( $keyspace, $args ); /** * Make a global cache key. @@ -1031,9 +443,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @param string|null $component [optional] Key component (starting with a key collection name) * @return string Colon-delimited list of $keyspace followed by escaped components of $args */ - public function makeGlobalKey( $class, $component = null ) { - return $this->makeKeyInternal( 'global', func_get_args() ); - } + abstract public function makeGlobalKey( $class, $component = null ); /** * Make a cache key, scoped to this instance's keyspace. @@ -1043,9 +453,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @param string|null $component [optional] Key component (starting with a key collection name) * @return string Colon-delimited list of $keyspace followed by escaped components of $args */ - public function makeKey( $class, $component = null ) { - return $this->makeKeyInternal( $this->keyspace, func_get_args() ); - } + abstract public function makeKey( $class, $component = null ); /** * @param int $flag ATTR_* class constant @@ -1061,7 +469,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @since 1.34 */ public function getSegmentationSize() { - return $this->segmentationSize; + return INF; } /** @@ -1069,7 +477,7 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar * @since 1.34 */ public function getSegmentedValueMaxSize() { - return $this->segmentedValueMaxSize; + return INF; } /** @@ -1110,22 +518,4 @@ abstract class BagOStuff implements IExpiringStore, IStoreKeyEncoder, LoggerAwar public function setMockTime( &$time ) { $this->wallClockOverride =& $time; } - - /** - * @param mixed $value - * @return string|int String/integer representation - * @note Special handling is usually needed for integers so incr()/decr() work - */ - protected function serialize( $value ) { - return is_int( $value ) ? $value : serialize( $value ); - } - - /** - * @param string|int $value - * @return mixed Original value or false on error - * @note Special handling is usually needed for integers so incr()/decr() work - */ - protected function unserialize( $value ) { - return $this->isInteger( $value ) ? (int)$value : unserialize( $value ); - } } diff --git a/includes/libs/objectcache/CachedBagOStuff.php b/includes/libs/objectcache/CachedBagOStuff.php index ea434e0c74..c97f56e474 100644 --- a/includes/libs/objectcache/CachedBagOStuff.php +++ b/includes/libs/objectcache/CachedBagOStuff.php @@ -44,8 +44,6 @@ class CachedBagOStuff extends BagOStuff { * @param array $params Parameters for HashBagOStuff */ public function __construct( BagOStuff $backend, $params = [] ) { - unset( $params['reportDupes'] ); // useless here - parent::__construct( $params ); $this->backend = $backend; @@ -53,17 +51,41 @@ class CachedBagOStuff extends BagOStuff { $this->attrMap = $backend->attrMap; } - protected function doGet( $key, $flags = 0, &$casToken = null ) { - $ret = $this->procCache->get( $key, $flags ); - if ( $ret === false && !$this->procCache->hasKey( $key ) ) { - $ret = $this->backend->get( $key, $flags ); - $this->set( $key, $ret, self::TTL_INDEFINITE, self::WRITE_CACHE_ONLY ); + public function setDebug( $enabled ) { + parent::setDebug( $enabled ); + $this->backend->setDebug( $enabled ); + } + + public function get( $key, $flags = 0 ) { + $value = $this->procCache->get( $key, $flags ); + if ( $value === false && !$this->procCache->hasKey( $key ) ) { + $value = $this->backend->get( $key, $flags ); + $this->set( $key, $value, self::TTL_INDEFINITE, self::WRITE_CACHE_ONLY ); + } + + return $value; + } + + public function getMulti( array $keys, $flags = 0 ) { + $valuesByKeyCached = []; + + $keysMissing = []; + foreach ( $keys as $key ) { + $value = $this->procCache->get( $key, $flags ); + if ( $value === false && !$this->procCache->hasKey( $key ) ) { + $keysMissing[] = $key; + } else { + $valuesByKeyCached[$key] = $value; + } } - return $ret; + $valuesByKeyFetched = $this->backend->getMulti( $keys, $flags ); + $this->setMulti( $valuesByKeyFetched, self::TTL_INDEFINITE, self::WRITE_CACHE_ONLY ); + + return $valuesByKeyCached + $valuesByKeyFetched; } - protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) { + public function set( $key, $value, $exptime = 0, $flags = 0 ) { $this->procCache->set( $key, $value, $exptime, $flags ); if ( ( $flags & self::WRITE_CACHE_ONLY ) != self::WRITE_CACHE_ONLY ) { $this->backend->set( $key, $value, $exptime, $flags ); @@ -72,7 +94,7 @@ class CachedBagOStuff extends BagOStuff { return true; } - protected function doDelete( $key, $flags = 0 ) { + public function delete( $key, $flags = 0 ) { $this->procCache->delete( $key, $flags ); if ( ( $flags & self::WRITE_CACHE_ONLY ) != self::WRITE_CACHE_ONLY ) { $this->backend->delete( $key, $flags ); @@ -81,19 +103,6 @@ class CachedBagOStuff extends BagOStuff { return true; } - public function deleteObjectsExpiringBefore( - $timestamp, - callable $progress = null, - $limit = INF - ) { - $this->procCache->deleteObjectsExpiringBefore( $timestamp, $progress, $limit ); - - return $this->backend->deleteObjectsExpiringBefore( $timestamp, $progress, $limit ); - } - - // These just call the backend (tested elsewhere) - // @codeCoverageIgnoreStart - public function add( $key, $value, $exptime = 0, $flags = 0 ) { if ( $this->get( $key ) === false ) { return $this->set( $key, $value, $exptime, $flags ); @@ -102,12 +111,19 @@ class CachedBagOStuff extends BagOStuff { return false; // key already set } - public function incr( $key, $value = 1 ) { - $n = $this->backend->incr( $key, $value ); + // These just call the backend (tested elsewhere) + // @codeCoverageIgnoreStart + public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) { $this->procCache->delete( $key ); - return $n; + return $this->backend->merge( $key, $callback, $exptime, $attempts, $flags ); + } + + public function changeTTL( $key, $exptime = 0, $flags = 0 ) { + $this->procCache->delete( $key ); + + return $this->backend->changeTTL( $key, $exptime, $flags ); } public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' ) { @@ -118,6 +134,16 @@ class CachedBagOStuff extends BagOStuff { return $this->backend->unlock( $key ); } + public function deleteObjectsExpiringBefore( + $timestamp, + callable $progress = null, + $limit = INF + ) { + $this->procCache->deleteObjectsExpiringBefore( $timestamp, $progress, $limit ); + + return $this->backend->deleteObjectsExpiringBefore( $timestamp, $progress, $limit ); + } + public function makeKeyInternal( $keyspace, $args ) { return $this->backend->makeKeyInternal( ...func_get_args() ); } @@ -130,11 +156,6 @@ class CachedBagOStuff extends BagOStuff { return $this->backend->makeGlobalKey( ...func_get_args() ); } - public function setDebug( $bool ) { - parent::setDebug( $bool ); - $this->backend->setDebug( $bool ); - } - public function getLastError() { return $this->backend->getLastError(); } @@ -143,5 +164,60 @@ class CachedBagOStuff extends BagOStuff { return $this->backend->clearLastError(); } + public function setMulti( array $data, $exptime = 0, $flags = 0 ) { + $this->procCache->setMulti( $data, $exptime, $flags ); + if ( ( $flags & self::WRITE_CACHE_ONLY ) != self::WRITE_CACHE_ONLY ) { + return $this->backend->setMulti( $data, $exptime, $flags ); + } + + return true; + } + + public function deleteMulti( array $keys, $flags = 0 ) { + $this->procCache->deleteMulti( $keys, $flags ); + if ( ( $flags & self::WRITE_CACHE_ONLY ) != self::WRITE_CACHE_ONLY ) { + return $this->backend->deleteMulti( $keys, $flags ); + } + + return true; + } + + public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) { + $this->procCache->changeTTLMulti( $keys, $exptime, $flags ); + if ( ( $flags & self::WRITE_CACHE_ONLY ) != self::WRITE_CACHE_ONLY ) { + return $this->backend->changeTTLMulti( $keys, $exptime, $flags ); + } + + return true; + } + + public function incr( $key, $value = 1 ) { + $this->procCache->delete( $key ); + + return $this->backend->incr( $key, $value ); + } + + public function decr( $key, $value = 1 ) { + $this->procCache->delete( $key ); + + return $this->backend->decr( $key, $value ); + } + + public function incrWithInit( $key, $ttl, $value = 1, $init = 1 ) { + $this->procCache->delete( $key ); + + return $this->backend->incrWithInit( $key, $ttl, $value, $init ); + } + + public function addBusyCallback( callable $workCallback ) { + $this->backend->addBusyCallback( $workCallback ); + } + + public function setMockTime( &$time ) { + parent::setMockTime( $time ); + $this->procCache->setMockTime( $time ); + $this->backend->setMockTime( $time ); + } + // @codeCoverageIgnoreEnd } diff --git a/includes/libs/objectcache/EmptyBagOStuff.php b/includes/libs/objectcache/EmptyBagOStuff.php index 6dc1363ff7..dab8ba1d35 100644 --- a/includes/libs/objectcache/EmptyBagOStuff.php +++ b/includes/libs/objectcache/EmptyBagOStuff.php @@ -26,7 +26,7 @@ * * @ingroup Cache */ -class EmptyBagOStuff extends BagOStuff { +class EmptyBagOStuff extends MediumSpecificBagOStuff { protected function doGet( $key, $flags = 0, &$casToken = null ) { $casToken = null; diff --git a/includes/libs/objectcache/HashBagOStuff.php b/includes/libs/objectcache/HashBagOStuff.php index c74bb6e087..83c8004c26 100644 --- a/includes/libs/objectcache/HashBagOStuff.php +++ b/includes/libs/objectcache/HashBagOStuff.php @@ -28,7 +28,7 @@ * * @ingroup Cache */ -class HashBagOStuff extends BagOStuff { +class HashBagOStuff extends MediumSpecificBagOStuff { /** @var mixed[] */ protected $bag = []; /** @var int Max entries allowed */ diff --git a/includes/libs/objectcache/MediumSpecificBagOStuff.php b/includes/libs/objectcache/MediumSpecificBagOStuff.php new file mode 100644 index 0000000000..fb088c8e98 --- /dev/null +++ b/includes/libs/objectcache/MediumSpecificBagOStuff.php @@ -0,0 +1,932 @@ +makeKey() + * - reportDupes: Whether to emit warning log messages for all keys that were + * requested more than once (requires an asyncHandler). + * - syncTimeout: How long to wait with WRITE_SYNC in seconds. + * - segmentationSize: The chunk size, in bytes, of segmented values. The value should + * not exceed the maximum size of values in the storage backend, as configured by + * the site administrator. + * - segmentedValueMaxSize: The maximum total size, in bytes, of segmented values. + * This should be configured to a reasonable size give the site traffic and the + * amount of I/O between application and cache servers that the network can handle. + * @param array $params + */ + public function __construct( array $params = [] ) { + parent::__construct( $params ); + + if ( isset( $params['keyspace'] ) ) { + $this->keyspace = $params['keyspace']; + } + + if ( !empty( $params['reportDupes'] ) && is_callable( $this->asyncHandler ) ) { + $this->reportDupes = true; + } + + $this->syncTimeout = $params['syncTimeout'] ?? 3; + $this->segmentationSize = $params['segmentationSize'] ?? 8388608; // 8MiB + $this->segmentedValueMaxSize = $params['segmentedValueMaxSize'] ?? 67108864; // 64MiB + } + + /** + * Get an item with the given key + * + * If the key includes a deterministic input hash (e.g. the key can only have + * the correct value) or complete staleness checks are handled by the caller + * (e.g. nothing relies on the TTL), then the READ_VERIFIED flag should be set. + * This lets tiered backends know they can safely upgrade a cached value to + * higher tiers using standard TTLs. + * + * @param string $key + * @param int $flags Bitfield of BagOStuff::READ_* constants [optional] + * @return mixed Returns false on failure or if the item does not exist + */ + public function get( $key, $flags = 0 ) { + $this->trackDuplicateKeys( $key ); + + return $this->resolveSegments( $key, $this->doGet( $key, $flags ) ); + } + + /** + * Track the number of times that a given key has been used. + * @param string $key + */ + private function trackDuplicateKeys( $key ) { + if ( !$this->reportDupes ) { + return; + } + + if ( !isset( $this->duplicateKeyLookups[$key] ) ) { + // Track that we have seen this key. This N-1 counting style allows + // easy filtering with array_filter() later. + $this->duplicateKeyLookups[$key] = 0; + } else { + $this->duplicateKeyLookups[$key] += 1; + + if ( $this->dupeTrackScheduled === false ) { + $this->dupeTrackScheduled = true; + // Schedule a callback that logs keys processed more than once by get(). + call_user_func( $this->asyncHandler, function () { + $dups = array_filter( $this->duplicateKeyLookups ); + foreach ( $dups as $key => $count ) { + $this->logger->warning( + 'Duplicate get(): "{key}" fetched {count} times', + // Count is N-1 of the actual lookup count + [ 'key' => $key, 'count' => $count + 1, ] + ); + } + } ); + } + } + } + + /** + * @param string $key + * @param int $flags Bitfield of BagOStuff::READ_* constants [optional] + * @param mixed|null &$casToken Token to use for check-and-set comparisons + * @return mixed Returns false on failure or if the item does not exist + */ + abstract protected function doGet( $key, $flags = 0, &$casToken = null ); + + /** + * Set an item + * + * @param string $key + * @param mixed $value + * @param int $exptime Either an interval in seconds or a unix timestamp for expiry + * @param int $flags Bitfield of BagOStuff::WRITE_* constants + * @return bool Success + */ + public function set( $key, $value, $exptime = 0, $flags = 0 ) { + if ( + is_int( $value ) || // avoid breaking incr()/decr() + ( $flags & self::WRITE_ALLOW_SEGMENTS ) != self::WRITE_ALLOW_SEGMENTS || + is_infinite( $this->segmentationSize ) + ) { + return $this->doSet( $key, $value, $exptime, $flags ); + } + + $serialized = $this->serialize( $value ); + $segmentSize = $this->getSegmentationSize(); + $maxTotalSize = $this->getSegmentedValueMaxSize(); + + $size = strlen( $serialized ); + if ( $size <= $segmentSize ) { + // Since the work of serializing it was already done, just use it inline + return $this->doSet( + $key, + SerializedValueContainer::newUnified( $serialized ), + $exptime, + $flags + ); + } elseif ( $size > $maxTotalSize ) { + $this->setLastError( "Key $key exceeded $maxTotalSize bytes." ); + + return false; + } + + $chunksByKey = []; + $segmentHashes = []; + $count = intdiv( $size, $segmentSize ) + ( ( $size % $segmentSize ) ? 1 : 0 ); + for ( $i = 0; $i < $count; ++$i ) { + $segment = substr( $serialized, $i * $segmentSize, $segmentSize ); + $hash = sha1( $segment ); + $chunkKey = $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $hash ); + $chunksByKey[$chunkKey] = $segment; + $segmentHashes[] = $hash; + } + + $flags &= ~self::WRITE_ALLOW_SEGMENTS; // sanity + $ok = $this->setMulti( $chunksByKey, $exptime, $flags ); + if ( $ok ) { + // Only when all segments are stored should the main key be changed + $ok = $this->doSet( + $key, + SerializedValueContainer::newSegmented( $segmentHashes ), + $exptime, + $flags + ); + } + + return $ok; + } + + /** + * Set an item + * + * @param string $key + * @param mixed $value + * @param int $exptime Either an interval in seconds or a unix timestamp for expiry + * @param int $flags Bitfield of BagOStuff::WRITE_* constants + * @return bool Success + */ + abstract protected function doSet( $key, $value, $exptime = 0, $flags = 0 ); + + /** + * Delete an item + * + * For large values written using WRITE_ALLOW_SEGMENTS, this only deletes the main + * segment list key unless WRITE_PRUNE_SEGMENTS is in the flags. While deleting the segment + * list key has the effect of functionally deleting the key, it leaves unused blobs in cache. + * + * @param string $key + * @param int $flags Bitfield of BagOStuff::WRITE_* constants + * @return bool True if the item was deleted or not found, false on failure + */ + public function delete( $key, $flags = 0 ) { + if ( ( $flags & self::WRITE_PRUNE_SEGMENTS ) != self::WRITE_PRUNE_SEGMENTS ) { + return $this->doDelete( $key, $flags ); + } + + $mainValue = $this->doGet( $key, self::READ_LATEST ); + if ( !$this->doDelete( $key, $flags ) ) { + return false; + } + + if ( !SerializedValueContainer::isSegmented( $mainValue ) ) { + return true; // no segments to delete + } + + $orderedKeys = array_map( + function ( $segmentHash ) use ( $key ) { + return $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $segmentHash ); + }, + $mainValue->{SerializedValueContainer::SEGMENTED_HASHES} + ); + + return $this->deleteMulti( $orderedKeys, $flags ); + } + + /** + * Delete an item + * + * @param string $key + * @param int $flags Bitfield of BagOStuff::WRITE_* constants + * @return bool True if the item was deleted or not found, false on failure + */ + abstract protected function doDelete( $key, $flags = 0 ); + + /** + * Merge changes into the existing cache value (possibly creating a new one) + * + * The callback function returns the new value given the current value + * (which will be false if not present), and takes the arguments: + * (this BagOStuff, cache key, current value, TTL). + * The TTL parameter is reference set to $exptime. It can be overriden in the callback. + * Nothing is stored nor deleted if the callback returns false. + * + * @param string $key + * @param callable $callback Callback method to be executed + * @param int $exptime Either an interval in seconds or a unix timestamp for expiry + * @param int $attempts The amount of times to attempt a merge in case of failure + * @param int $flags Bitfield of BagOStuff::WRITE_* constants + * @return bool Success + * @throws InvalidArgumentException + */ + public function merge( $key, callable $callback, $exptime = 0, $attempts = 10, $flags = 0 ) { + return $this->mergeViaCas( $key, $callback, $exptime, $attempts, $flags ); + } + + /** + * @param string $key + * @param callable $callback Callback method to be executed + * @param int $exptime Either an interval in seconds or a unix timestamp for expiry + * @param int $attempts The amount of times to attempt a merge in case of failure + * @param int $flags Bitfield of BagOStuff::WRITE_* constants + * @return bool Success + * @see BagOStuff::merge() + * + */ + final protected function mergeViaCas( $key, callable $callback, $exptime, $attempts, $flags ) { + do { + $casToken = null; // passed by reference + // Get the old value and CAS token from cache + $this->clearLastError(); + $currentValue = $this->resolveSegments( + $key, + $this->doGet( $key, self::READ_LATEST, $casToken ) + ); + if ( $this->getLastError() ) { + $this->logger->warning( + __METHOD__ . ' failed due to I/O error on get() for {key}.', + [ 'key' => $key ] + ); + + return false; // don't spam retries (retry only on races) + } + + // Derive the new value from the old value + $value = call_user_func( $callback, $this, $key, $currentValue, $exptime ); + $hadNoCurrentValue = ( $currentValue === false ); + unset( $currentValue ); // free RAM in case the value is large + + $this->clearLastError(); + if ( $value === false ) { + $success = true; // do nothing + } elseif ( $hadNoCurrentValue ) { + // Try to create the key, failing if it gets created in the meantime + $success = $this->add( $key, $value, $exptime, $flags ); + } else { + // Try to update the key, failing if it gets changed in the meantime + $success = $this->cas( $casToken, $key, $value, $exptime, $flags ); + } + if ( $this->getLastError() ) { + $this->logger->warning( + __METHOD__ . ' failed due to I/O error for {key}.', + [ 'key' => $key ] + ); + + return false; // IO error; don't spam retries + } + + } while ( !$success && --$attempts ); + + return $success; + } + + /** + * Check and set an item + * + * @param mixed $casToken + * @param string $key + * @param mixed $value + * @param int $exptime Either an interval in seconds or a unix timestamp for expiry + * @param int $flags Bitfield of BagOStuff::WRITE_* constants + * @return bool Success + */ + protected function cas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) { + if ( !$this->lock( $key, 0 ) ) { + return false; // non-blocking + } + + $curCasToken = null; // passed by reference + $this->doGet( $key, self::READ_LATEST, $curCasToken ); + if ( $casToken === $curCasToken ) { + $success = $this->set( $key, $value, $exptime, $flags ); + } else { + $this->logger->info( + __METHOD__ . ' failed due to race condition for {key}.', + [ 'key' => $key ] + ); + + $success = false; // mismatched or failed + } + + $this->unlock( $key ); + + return $success; + } + + /** + * Change the expiration on a key if it exists + * + * If an expiry in the past is given then the key will immediately be expired + * + * For large values written using WRITE_ALLOW_SEGMENTS, this only changes the TTL of the + * main segment list key. While lowering the TTL of the segment list key has the effect of + * functionally lowering the TTL of the key, it might leave unused blobs in cache for longer. + * Raising the TTL of such keys is not effective, since the expiration of a single segment + * key effectively expires the entire value. + * + * @param string $key + * @param int $exptime TTL or UNIX timestamp + * @param int $flags Bitfield of BagOStuff::WRITE_* constants (since 1.33) + * @return bool Success Returns false on failure or if the item does not exist + * @since 1.28 + */ + public function changeTTL( $key, $exptime = 0, $flags = 0 ) { + return $this->doChangeTTL( $key, $exptime, $flags ); + } + + /** + * @param string $key + * @param int $exptime + * @param int $flags + * @return bool + */ + protected function doChangeTTL( $key, $exptime, $flags ) { + $expiry = $this->convertToExpiry( $exptime ); + $delete = ( $expiry != 0 && $expiry < $this->getCurrentTime() ); + + if ( !$this->lock( $key, 0 ) ) { + return false; + } + // Use doGet() to avoid having to trigger resolveSegments() + $blob = $this->doGet( $key, self::READ_LATEST ); + if ( $blob ) { + if ( $delete ) { + $ok = $this->doDelete( $key, $flags ); + } else { + $ok = $this->doSet( $key, $blob, $exptime, $flags ); + } + } else { + $ok = false; + } + + $this->unlock( $key ); + + return $ok; + } + + /** + * Acquire an advisory lock on a key string + * + * Note that if reentry is enabled, duplicate calls ignore $expiry + * + * @param string $key + * @param int $timeout Lock wait timeout; 0 for non-blocking [optional] + * @param int $expiry Lock expiry [optional]; 1 day maximum + * @param string $rclass Allow reentry if set and the current lock used this value + * @return bool Success + */ + public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' ) { + // Avoid deadlocks and allow lock reentry if specified + if ( isset( $this->locks[$key] ) ) { + if ( $rclass != '' && $this->locks[$key]['class'] === $rclass ) { + ++$this->locks[$key]['depth']; + return true; + } else { + return false; + } + } + + $fname = __METHOD__; + $expiry = min( $expiry ?: INF, self::TTL_DAY ); + $loop = new WaitConditionLoop( + function () use ( $key, $expiry, $fname ) { + $this->clearLastError(); + if ( $this->add( "{$key}:lock", 1, $expiry ) ) { + return WaitConditionLoop::CONDITION_REACHED; // locked! + } elseif ( $this->getLastError() ) { + $this->logger->warning( + $fname . ' failed due to I/O error for {key}.', + [ 'key' => $key ] + ); + + return WaitConditionLoop::CONDITION_ABORTED; // network partition? + } + + return WaitConditionLoop::CONDITION_CONTINUE; + }, + $timeout + ); + + $code = $loop->invoke(); + $locked = ( $code === $loop::CONDITION_REACHED ); + if ( $locked ) { + $this->locks[$key] = [ 'class' => $rclass, 'depth' => 1 ]; + } elseif ( $code === $loop::CONDITION_TIMED_OUT ) { + $this->logger->warning( + "$fname failed due to timeout for {key}.", + [ 'key' => $key, 'timeout' => $timeout ] + ); + } + + return $locked; + } + + /** + * Release an advisory lock on a key string + * + * @param string $key + * @return bool Success + */ + public function unlock( $key ) { + if ( !isset( $this->locks[$key] ) ) { + return false; + } + + if ( --$this->locks[$key]['depth'] <= 0 ) { + unset( $this->locks[$key] ); + + $ok = $this->doDelete( "{$key}:lock" ); + if ( !$ok ) { + $this->logger->warning( + __METHOD__ . ' failed to release lock for {key}.', + [ 'key' => $key ] + ); + } + + return $ok; + } + + return true; + } + + /** + * Delete all objects expiring before a certain date. + * @param string|int $timestamp The reference date in MW or TS_UNIX format + * @param callable|null $progress Optional, a function which will be called + * regularly during long-running operations with the percentage progress + * as the first parameter. [optional] + * @param int $limit Maximum number of keys to delete [default: INF] + * + * @return bool Success; false if unimplemented + */ + public function deleteObjectsExpiringBefore( + $timestamp, + callable $progress = null, + $limit = INF + ) { + return false; + } + + /** + * Get an associative array containing the item for each of the keys that have items. + * @param string[] $keys List of keys; can be a map of (unused => key) for convenience + * @param int $flags Bitfield; supports READ_LATEST [optional] + * @return mixed[] Map of (key => value) for existing keys; preserves the order of $keys + */ + public function getMulti( array $keys, $flags = 0 ) { + $foundByKey = $this->doGetMulti( $keys, $flags ); + + $res = []; + foreach ( $keys as $key ) { + // Resolve one blob at a time (avoids too much I/O at once) + if ( array_key_exists( $key, $foundByKey ) ) { + // A value should not appear in the key if a segment is missing + $value = $this->resolveSegments( $key, $foundByKey[$key] ); + if ( $value !== false ) { + $res[$key] = $value; + } + } + } + + return $res; + } + + /** + * Get an associative array containing the item for each of the keys that have items. + * @param string[] $keys List of keys + * @param int $flags Bitfield; supports READ_LATEST [optional] + * @return array Map of (key => value) for existing keys + */ + protected function doGetMulti( array $keys, $flags = 0 ) { + $res = []; + foreach ( $keys as $key ) { + $val = $this->doGet( $key, $flags ); + if ( $val !== false ) { + $res[$key] = $val; + } + } + + return $res; + } + + /** + * Batch insertion/replace + * + * This does not support WRITE_ALLOW_SEGMENTS to avoid excessive read I/O + * + * @param mixed[] $data Map of (key => value) + * @param int $exptime Either an interval in seconds or a unix timestamp for expiry + * @param int $flags Bitfield of BagOStuff::WRITE_* constants (since 1.33) + * @return bool Success + * @since 1.24 + */ + public function setMulti( array $data, $exptime = 0, $flags = 0 ) { + if ( ( $flags & self::WRITE_ALLOW_SEGMENTS ) === self::WRITE_ALLOW_SEGMENTS ) { + throw new InvalidArgumentException( __METHOD__ . ' got WRITE_ALLOW_SEGMENTS' ); + } + return $this->doSetMulti( $data, $exptime, $flags ); + } + + /** + * @param mixed[] $data Map of (key => value) + * @param int $exptime Either an interval in seconds or a unix timestamp for expiry + * @param int $flags Bitfield of BagOStuff::WRITE_* constants + * @return bool Success + */ + protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) { + $res = true; + foreach ( $data as $key => $value ) { + $res = $this->doSet( $key, $value, $exptime, $flags ) && $res; + } + return $res; + } + + /** + * Batch deletion + * + * This does not support WRITE_ALLOW_SEGMENTS to avoid excessive read I/O + * + * @param string[] $keys List of keys + * @param int $flags Bitfield of BagOStuff::WRITE_* constants + * @return bool Success + * @since 1.33 + */ + public function deleteMulti( array $keys, $flags = 0 ) { + if ( ( $flags & self::WRITE_ALLOW_SEGMENTS ) === self::WRITE_ALLOW_SEGMENTS ) { + throw new InvalidArgumentException( __METHOD__ . ' got WRITE_ALLOW_SEGMENTS' ); + } + return $this->doDeleteMulti( $keys, $flags ); + } + + /** + * @param string[] $keys List of keys + * @param int $flags Bitfield of BagOStuff::WRITE_* constants + * @return bool Success + */ + protected function doDeleteMulti( array $keys, $flags = 0 ) { + $res = true; + foreach ( $keys as $key ) { + $res = $this->doDelete( $key, $flags ) && $res; + } + return $res; + } + + /** + * Change the expiration of multiple keys that exist + * + * @param string[] $keys List of keys + * @param int $exptime TTL or UNIX timestamp + * @param int $flags Bitfield of BagOStuff::WRITE_* constants (since 1.33) + * @return bool Success + * @see BagOStuff::changeTTL() + * + * @since 1.34 + */ + public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) { + $res = true; + foreach ( $keys as $key ) { + $res = $this->doChangeTTL( $key, $exptime, $flags ) && $res; + } + + return $res; + } + + /** + * Decrease stored value of $key by $value while preserving its TTL + * @param string $key + * @param int $value Value to subtract from $key (default: 1) [optional] + * @return int|bool New value or false on failure + */ + public function decr( $key, $value = 1 ) { + return $this->incr( $key, -$value ); + } + + /** + * Increase stored value of $key by $value while preserving its TTL + * + * This will create the key with value $init and TTL $ttl instead if not present + * + * @param string $key + * @param int $ttl + * @param int $value + * @param int $init + * @return int|bool New value or false on failure + * @since 1.24 + */ + public function incrWithInit( $key, $ttl, $value = 1, $init = 1 ) { + $this->clearLastError(); + $newValue = $this->incr( $key, $value ); + if ( $newValue === false && !$this->getLastError() ) { + // No key set; initialize + $newValue = $this->add( $key, (int)$init, $ttl ) ? $init : false; + if ( $newValue === false && !$this->getLastError() ) { + // Raced out initializing; increment + $newValue = $this->incr( $key, $value ); + } + } + + return $newValue; + } + + /** + * Get and reassemble the chunks of blob at the given key + * + * @param string $key + * @param mixed $mainValue + * @return string|null|bool The combined string, false if missing, null on error + */ + final protected function resolveSegments( $key, $mainValue ) { + if ( SerializedValueContainer::isUnified( $mainValue ) ) { + return $this->unserialize( $mainValue->{SerializedValueContainer::UNIFIED_DATA} ); + } + + if ( SerializedValueContainer::isSegmented( $mainValue ) ) { + $orderedKeys = array_map( + function ( $segmentHash ) use ( $key ) { + return $this->makeGlobalKey( self::SEGMENT_COMPONENT, $key, $segmentHash ); + }, + $mainValue->{SerializedValueContainer::SEGMENTED_HASHES} + ); + + $segmentsByKey = $this->doGetMulti( $orderedKeys ); + + $parts = []; + foreach ( $orderedKeys as $segmentKey ) { + if ( isset( $segmentsByKey[$segmentKey] ) ) { + $parts[] = $segmentsByKey[$segmentKey]; + } else { + return false; // missing segment + } + } + + return $this->unserialize( implode( '', $parts ) ); + } + + return $mainValue; + } + + /** + * Get the "last error" registered; clearLastError() should be called manually + * @return int ERR_* constant for the "last error" registry + * @since 1.23 + */ + public function getLastError() { + return $this->lastError; + } + + /** + * Clear the "last error" registry + * @since 1.23 + */ + public function clearLastError() { + $this->lastError = self::ERR_NONE; + } + + /** + * Set the "last error" registry + * @param int $err ERR_* constant + * @since 1.23 + */ + protected function setLastError( $err ) { + $this->lastError = $err; + } + + /** + * Let a callback be run to avoid wasting time on special blocking calls + * + * The callbacks may or may not be called ever, in any particular order. + * They are likely to be invoked when something WRITE_SYNC is used used. + * They should follow a caching pattern as shown below, so that any code + * using the work will get it's result no matter what happens. + * @code + * $result = null; + * $workCallback = function () use ( &$result ) { + * if ( !$result ) { + * $result = .... + * } + * return $result; + * } + * @endcode + * + * @param callable $workCallback + * @since 1.28 + */ + final public function addBusyCallback( callable $workCallback ) { + $this->busyCallbacks[] = $workCallback; + } + + /** + * @param int $exptime + * @return bool + */ + final protected function expiryIsRelative( $exptime ) { + return ( $exptime != 0 && $exptime < ( 10 * self::TTL_YEAR ) ); + } + + /** + * Convert an optionally relative timestamp to an absolute time + * + * The input value will be cast to an integer and interpreted as follows: + * - zero: no expiry; return zero (e.g. TTL_INDEFINITE) + * - negative: relative TTL; return UNIX timestamp offset by this value + * - positive (< 10 years): relative TTL; return UNIX timestamp offset by this value + * - positive (>= 10 years): absolute UNIX timestamp; return this value + * + * @param int $exptime Absolute TTL or 0 for indefinite + * @return int + */ + final protected function convertToExpiry( $exptime ) { + return $this->expiryIsRelative( $exptime ) + ? (int)$this->getCurrentTime() + $exptime + : $exptime; + } + + /** + * Convert an optionally absolute expiry time to a relative time. If an + * absolute time is specified which is in the past, use a short expiry time. + * + * @param int $exptime + * @return int + */ + final protected function convertToRelative( $exptime ) { + return $this->expiryIsRelative( $exptime ) + ? (int)$exptime + : max( $exptime - (int)$this->getCurrentTime(), 1 ); + } + + /** + * Check if a value is an integer + * + * @param mixed $value + * @return bool + */ + final protected function isInteger( $value ) { + if ( is_int( $value ) ) { + return true; + } elseif ( !is_string( $value ) ) { + return false; + } + + $integer = (int)$value; + + return ( $value === (string)$integer ); + } + + /** + * Construct a cache key. + * + * @param string $keyspace + * @param array $args + * @return string Colon-delimited list of $keyspace followed by escaped components of $args + * @since 1.27 + */ + public function makeKeyInternal( $keyspace, $args ) { + $key = $keyspace; + foreach ( $args as $arg ) { + $key .= ':' . str_replace( ':', '%3A', $arg ); + } + return strtr( $key, ' ', '_' ); + } + + /** + * Make a global cache key. + * + * @param string $class Key class + * @param string|null $component [optional] Key component (starting with a key collection name) + * @return string Colon-delimited list of $keyspace followed by escaped components of $args + * @since 1.27 + */ + public function makeGlobalKey( $class, $component = null ) { + return $this->makeKeyInternal( 'global', func_get_args() ); + } + + /** + * Make a cache key, scoped to this instance's keyspace. + * + * @param string $class Key class + * @param string|null $component [optional] Key component (starting with a key collection name) + * @return string Colon-delimited list of $keyspace followed by escaped components of $args + * @since 1.27 + */ + public function makeKey( $class, $component = null ) { + return $this->makeKeyInternal( $this->keyspace, func_get_args() ); + } + + /** + * @param int $flag ATTR_* class constant + * @return int QOS_* class constant + * @since 1.28 + */ + public function getQoS( $flag ) { + return $this->attrMap[$flag] ?? self::QOS_UNKNOWN; + } + + /** + * @return int|float The chunk size, in bytes, of segmented objects (INF for no limit) + * @since 1.34 + */ + public function getSegmentationSize() { + return $this->segmentationSize; + } + + /** + * @return int|float Maximum total segmented object size in bytes (INF for no limit) + * @since 1.34 + */ + public function getSegmentedValueMaxSize() { + return $this->segmentedValueMaxSize; + } + + /** + * @param mixed $value + * @return string|int String/integer representation + * @note Special handling is usually needed for integers so incr()/decr() work + */ + protected function serialize( $value ) { + return is_int( $value ) ? $value : serialize( $value ); + } + + /** + * @param string|int $value + * @return mixed Original value or false on error + * @note Special handling is usually needed for integers so incr()/decr() work + */ + protected function unserialize( $value ) { + return $this->isInteger( $value ) ? (int)$value : unserialize( $value ); + } + + /** + * @param string $text + */ + protected function debug( $text ) { + if ( $this->debugMode ) { + $this->logger->debug( "{class} debug: $text", [ 'class' => static::class ] ); + } + } +} diff --git a/includes/libs/objectcache/MemcachedBagOStuff.php b/includes/libs/objectcache/MemcachedBagOStuff.php index f75e780c80..9f1c98ab58 100644 --- a/includes/libs/objectcache/MemcachedBagOStuff.php +++ b/includes/libs/objectcache/MemcachedBagOStuff.php @@ -26,7 +26,7 @@ * * @ingroup Cache */ -abstract class MemcachedBagOStuff extends BagOStuff { +abstract class MemcachedBagOStuff extends MediumSpecificBagOStuff { function __construct( array $params ) { parent::__construct( $params ); diff --git a/includes/libs/objectcache/MemcachedPhpBagOStuff.php b/includes/libs/objectcache/MemcachedPhpBagOStuff.php index f8b91bca05..b1d5d29f16 100644 --- a/includes/libs/objectcache/MemcachedPhpBagOStuff.php +++ b/includes/libs/objectcache/MemcachedPhpBagOStuff.php @@ -53,8 +53,9 @@ class MemcachedPhpBagOStuff extends MemcachedBagOStuff { $this->client->set_servers( $params['servers'] ); } - public function setDebug( $debug ) { - $this->client->set_debug( $debug ); + public function setDebug( $enabled ) { + parent::debug( $enabled ); + $this->client->set_debug( $enabled ); } protected function doGet( $key, $flags = 0, &$casToken = null ) { diff --git a/includes/libs/objectcache/MultiWriteBagOStuff.php b/includes/libs/objectcache/MultiWriteBagOStuff.php index 8e791bae1c..6e9f387a60 100644 --- a/includes/libs/objectcache/MultiWriteBagOStuff.php +++ b/includes/libs/objectcache/MultiWriteBagOStuff.php @@ -40,7 +40,8 @@ class MultiWriteBagOStuff extends BagOStuff { /** @var int[] List of all backing cache indexes */ protected $cacheIndexes = []; - const UPGRADE_TTL = 3600; // TTL when a key is copied to a higher cache tier + /** @var int TTL when a key is copied to a higher cache tier */ + private static $UPGRADE_TTL = 3600; /** * $params include: @@ -97,9 +98,10 @@ class MultiWriteBagOStuff extends BagOStuff { $this->cacheIndexes = array_keys( $this->caches ); } - public function setDebug( $debug ) { + public function setDebug( $enabled ) { + parent::setDebug( $enabled ); foreach ( $this->caches as $cache ) { - $cache->setDebug( $debug ); + $cache->setDebug( $enabled ); } } @@ -131,7 +133,7 @@ class MultiWriteBagOStuff extends BagOStuff { $this->asyncWrites, 'set', // @TODO: consider using self::WRITE_ALLOW_SEGMENTS here? - [ $key, $value, self::UPGRADE_TTL ] + [ $key, $value, self::$UPGRADE_TTL ] ); } @@ -359,39 +361,15 @@ class MultiWriteBagOStuff extends BagOStuff { return $this->caches[0]->makeGlobalKey( ...func_get_args() ); } - protected function doGet( $key, $flags = 0, &$casToken = null ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); + public function addBusyCallback( callable $workCallback ) { + $this->caches[0]->addBusyCallback( $workCallback ); } - protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function doDelete( $key, $flags = 0 ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function doChangeTTL( $key, $exptime, $flags ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function doGetMulti( array $keys, $flags = 0 ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function doSetMulti( array $keys, $exptime = 0, $flags = 0 ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function doDeleteMulti( array $keys, $flags = 0 ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function serialize( $value ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function unserialize( $blob ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); + public function setMockTime( &$time ) { + parent::setMockTime( $time ); + foreach ( $this->caches as $cache ) { + $cache->setMockTime( $time ); + $cache->setMockTime( $time ); + } } } diff --git a/includes/libs/objectcache/RESTBagOStuff.php b/includes/libs/objectcache/RESTBagOStuff.php index 2a126891a6..aa4a9b31fc 100644 --- a/includes/libs/objectcache/RESTBagOStuff.php +++ b/includes/libs/objectcache/RESTBagOStuff.php @@ -44,7 +44,7 @@ use Psr\Log\LoggerInterface; * $wgSessionCacheType = 'sessions'; * @endcode */ -class RESTBagOStuff extends BagOStuff { +class RESTBagOStuff extends MediumSpecificBagOStuff { /** * Default connection timeout in seconds. The kernel retransmits the SYN * packet after 1 second, so 1.2 seconds allows for 1 retransmit without diff --git a/includes/libs/objectcache/RedisBagOStuff.php b/includes/libs/objectcache/RedisBagOStuff.php index 21b14f7bb1..87d26ef8fd 100644 --- a/includes/libs/objectcache/RedisBagOStuff.php +++ b/includes/libs/objectcache/RedisBagOStuff.php @@ -28,7 +28,7 @@ * @ingroup Cache * @ingroup Redis */ -class RedisBagOStuff extends BagOStuff { +class RedisBagOStuff extends MediumSpecificBagOStuff { /** @var RedisConnectionPool */ protected $redisPool; /** @var array List of server names */ diff --git a/includes/libs/objectcache/ReplicatedBagOStuff.php b/includes/libs/objectcache/ReplicatedBagOStuff.php index 295ec30c22..e49fa1086e 100644 --- a/includes/libs/objectcache/ReplicatedBagOStuff.php +++ b/includes/libs/objectcache/ReplicatedBagOStuff.php @@ -69,9 +69,10 @@ class ReplicatedBagOStuff extends BagOStuff { $this->attrMap = $this->mergeFlagMaps( [ $this->readStore, $this->writeStore ] ); } - public function setDebug( $debug ) { - $this->writeStore->setDebug( $debug ); - $this->readStore->setDebug( $debug ); + public function setDebug( $enabled ) { + parent::setDebug( $enabled ); + $this->writeStore->setDebug( $enabled ); + $this->readStore->setDebug( $enabled ); } public function get( $key, $flags = 0 ) { @@ -169,39 +170,13 @@ class ReplicatedBagOStuff extends BagOStuff { return $this->writeStore->makeGlobalKey( ...func_get_args() ); } - protected function doGet( $key, $flags = 0, &$casToken = null ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); + public function addBusyCallback( callable $workCallback ) { + $this->writeStore->addBusyCallback( $workCallback ); } - protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function doDelete( $key, $flags = 0 ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function doChangeTTL( $key, $exptime, $flags ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function doGetMulti( array $keys, $flags = 0 ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function doSetMulti( array $keys, $exptime = 0, $flags = 0 ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function doDeleteMulti( array $keys, $flags = 0 ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function serialize( $value ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); - } - - protected function unserialize( $blob ) { - throw new LogicException( __METHOD__ . ': proxy class does not need this method.' ); + public function setMockTime( &$time ) { + parent::setMockTime( $time ); + $this->writeStore->setMockTime( $time ); + $this->readStore->setMockTime( $time ); } } diff --git a/includes/libs/objectcache/WinCacheBagOStuff.php b/includes/libs/objectcache/WinCacheBagOStuff.php index 23da0bb624..0e4e3fb63d 100644 --- a/includes/libs/objectcache/WinCacheBagOStuff.php +++ b/includes/libs/objectcache/WinCacheBagOStuff.php @@ -27,7 +27,7 @@ * * @ingroup Cache */ -class WinCacheBagOStuff extends BagOStuff { +class WinCacheBagOStuff extends MediumSpecificBagOStuff { protected function doGet( $key, $flags = 0, &$casToken = null ) { $casToken = null; diff --git a/includes/objectcache/SqlBagOStuff.php b/includes/objectcache/SqlBagOStuff.php index 7e5a8a46b6..9226875e2a 100644 --- a/includes/objectcache/SqlBagOStuff.php +++ b/includes/objectcache/SqlBagOStuff.php @@ -36,7 +36,7 @@ use Wikimedia\WaitConditionLoop; * * @ingroup Cache */ -class SqlBagOStuff extends BagOStuff { +class SqlBagOStuff extends MediumSpecificBagOStuff { /** @var array[] (server index => server config) */ protected $serverInfos; /** @var string[] (server index => tag/host name) */ @@ -55,8 +55,6 @@ class SqlBagOStuff extends BagOStuff { protected $tableName = 'objectcache'; /** @var bool */ protected $replicaOnly = false; - /** @var int */ - protected $syncTimeout = 3; /** @var LoadBalancer|null */ protected $separateMainLB; @@ -159,9 +157,6 @@ class SqlBagOStuff extends BagOStuff { if ( isset( $params['shards'] ) ) { $this->shards = intval( $params['shards'] ); } - if ( isset( $params['syncTimeout'] ) ) { - $this->syncTimeout = $params['syncTimeout']; - } // Backwards-compatibility for < 1.34 $this->replicaOnly = $params['replicaOnly'] ?? ( $params['slaveOnly'] ?? false ); } diff --git a/tests/phpunit/includes/libs/objectcache/BagOStuffTest.php b/tests/phpunit/includes/libs/objectcache/BagOStuffTest.php index da532b080c..522af43662 100644 --- a/tests/phpunit/includes/libs/objectcache/BagOStuffTest.php +++ b/tests/phpunit/includes/libs/objectcache/BagOStuffTest.php @@ -6,6 +6,7 @@ use Wikimedia\TestingAccessWrapper; /** * @author Matthias Mullie * @group BagOStuff + * @covers BagOStuff */ class BagOStuffTest extends MediaWikiTestCase { /** @var BagOStuff */ @@ -31,8 +32,8 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::makeGlobalKey - * @covers BagOStuff::makeKeyInternal + * @covers MediumSpecificBagOStuff::makeGlobalKey + * @covers MediumSpecificBagOStuff::makeKeyInternal */ public function testMakeKey() { $cache = ObjectCache::newFromId( 'hash' ); @@ -65,8 +66,8 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::merge - * @covers BagOStuff::mergeViaCas + * @covers MediumSpecificBagOStuff::merge + * @covers MediumSpecificBagOStuff::mergeViaCas */ public function testMerge() { $key = $this->cache->makeKey( self::TEST_KEY ); @@ -109,7 +110,7 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::changeTTL + * @covers MediumSpecificBagOStuff::changeTTL */ public function testChangeTTL() { $key = $this->cache->makeKey( self::TEST_KEY ); @@ -130,7 +131,7 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::changeTTLMulti + * @covers MediumSpecificBagOStuff::changeTTLMulti */ public function testChangeTTLMulti() { $key1 = $this->cache->makeKey( 'test-key1' ); @@ -183,7 +184,7 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::add + * @covers MediumSpecificBagOStuff::add */ public function testAdd() { $key = $this->cache->makeKey( self::TEST_KEY ); @@ -193,7 +194,7 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::get + * @covers MediumSpecificBagOStuff::get */ public function testGet() { $value = [ 'this' => 'is', 'a' => 'test' ]; @@ -204,9 +205,9 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::get - * @covers BagOStuff::set - * @covers BagOStuff::getWithSetCallback + * @covers MediumSpecificBagOStuff::get + * @covers MediumSpecificBagOStuff::set + * @covers MediumSpecificBagOStuff::getWithSetCallback */ public function testGetWithSetCallback() { $key = $this->cache->makeKey( self::TEST_KEY ); @@ -223,7 +224,7 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::incr + * @covers MediumSpecificBagOStuff::incr */ public function testIncr() { $key = $this->cache->makeKey( self::TEST_KEY ); @@ -235,7 +236,7 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::incrWithInit + * @covers MediumSpecificBagOStuff::incrWithInit */ public function testIncrWithInit() { $key = $this->cache->makeKey( self::TEST_KEY ); @@ -247,7 +248,7 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::getMulti + * @covers MediumSpecificBagOStuff::getMulti */ public function testGetMulti() { $value1 = [ 'this' => 'is', 'a' => 'test' ]; @@ -287,8 +288,8 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::setMulti - * @covers BagOStuff::deleteMulti + * @covers MediumSpecificBagOStuff::setMulti + * @covers MediumSpecificBagOStuff::deleteMulti */ public function testSetDeleteMulti() { $map = [ @@ -319,10 +320,10 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::get - * @covers BagOStuff::getMulti - * @covers BagOStuff::merge - * @covers BagOStuff::delete + * @covers MediumSpecificBagOStuff::get + * @covers MediumSpecificBagOStuff::getMulti + * @covers MediumSpecificBagOStuff::merge + * @covers MediumSpecificBagOStuff::delete */ public function testSetSegmentable() { $key = $this->cache->makeKey( self::TEST_KEY ); @@ -369,7 +370,7 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::getScopedLock + * @covers MediumSpecificBagOStuff::getScopedLock */ public function testGetScopedLock() { $key = $this->cache->makeKey( self::TEST_KEY ); @@ -393,8 +394,8 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::__construct - * @covers BagOStuff::trackDuplicateKeys + * @covers MediumSpecificBagOStuff::__construct + * @covers MediumSpecificBagOStuff::trackDuplicateKeys */ public function testReportDupes() { $logger = $this->createMock( Psr\Log\NullLogger::class ); @@ -418,8 +419,8 @@ class BagOStuffTest extends MediaWikiTestCase { } /** - * @covers BagOStuff::lock() - * @covers BagOStuff::unlock() + * @covers MediumSpecificBagOStuff::lock() + * @covers MediumSpecificBagOStuff::unlock() */ public function testLocking() { $key = 'test'; diff --git a/tests/phpunit/includes/session/TestBagOStuff.php b/tests/phpunit/includes/session/TestBagOStuff.php index f9e30f06ab..64148b02e4 100644 --- a/tests/phpunit/includes/session/TestBagOStuff.php +++ b/tests/phpunit/includes/session/TestBagOStuff.php @@ -2,13 +2,17 @@ namespace MediaWiki\Session; +use CachedBagOStuff; +use HashBagOStuff; +use RequestContext; + /** * BagOStuff with utility functions for MediaWiki\\Session\\* testing */ -class TestBagOStuff extends \CachedBagOStuff { +class TestBagOStuff extends CachedBagOStuff { public function __construct() { - parent::__construct( new \HashBagOStuff ); + parent::__construct( new HashBagOStuff ); } /** @@ -51,7 +55,7 @@ class TestBagOStuff extends \CachedBagOStuff { * @param array|mixed $blob Session metadata and data */ public function setRawSession( $id, $blob ) { - $expiry = \RequestContext::getMain()->getConfig()->get( 'ObjectCacheSessionExpiry' ); + $expiry = RequestContext::getMain()->getConfig()->get( 'ObjectCacheSessionExpiry' ); $this->set( $this->makeKey( 'MWSession', $id ), $blob, $expiry ); } -- 2.20.1