*
* ### Deploying WANObjectCache
*
- * There are three supported ways to set up broadcasted operations:
+ * There are two supported ways to set up broadcasted operations:
*
- * - A) Configure the 'purge' EventRelayer to point to a valid PubSub endpoint
- * that has subscribed listeners on the cache servers applying the cache updates.
- * - B) Omit the 'purge' EventRelayer parameter and set up mcrouter as the underlying cache
- * backend, using a memcached BagOStuff class for the 'cache' parameter. The 'region'
- * and 'cluster' parameters must be provided and 'mcrouterAware' must be set to `true`.
+ * - A) Set up mcrouter as the underlying cache backend, using a memcached BagOStuff class
+ * for the 'cache' parameter. The 'region' and 'cluster' parameters must be provided
+ * and 'mcrouterAware' must be set to `true`.
* Configure mcrouter as follows:
* - 1) Use Route Prefixing based on region (datacenter) and cache cluster.
* See https://github.com/facebook/mcrouter/wiki/Routing-Prefix and
* configure 'set' and 'delete' operations to go to all servers in the cache
* cluster, instead of just one server determined by hashing.
* See https://github.com/facebook/mcrouter/wiki/List-of-Route-Handles.
- * - C) Omit the 'purge' EventRelayer parameter and set up dynomite as cache middleware
- * between the web servers and either memcached or redis. This will broadcast all
- * key setting operations, not just purges, which can be useful for cache warming.
- * Writes are eventually consistent via the Dynamo replication model.
- * See https://github.com/Netflix/dynomite.
+ * - B) Set up dynomite as a cache middleware between the web servers and either memcached
+ * or redis and use it as the underlying cache backend, using a memcached BagOStuff
+ * class for the 'cache' parameter. This will broadcast all key setting operations,
+ * not just purges, which can be useful for cache warming. Writes are eventually
+ * consistent via the Dynamo replication model. See https://github.com/Netflix/dynomite.
*
- * Broadcasted operations like delete() and touchCheckKey() are done asynchronously
- * in all datacenters this way, though the local one should likely be near immediate.
+ * Broadcasted operations like delete() and touchCheckKey() are intended to run
+ * immediately in the local datacenter and asynchronously in remote datacenters.
*
* This means that callers in all datacenters may see older values for however many
* milliseconds that the purge took to reach that datacenter. As with any cache, this
protected $cache;
/** @var MapCacheLRU[] Map of group PHP instance caches */
protected $processCaches = [];
- /** @var string Purge channel name */
- protected $purgeChannel;
- /** @var EventRelayer Bus that handles purge broadcasts */
- protected $purgeRelayer;
/** @bar bool Whether to use mcrouter key prefixing for routing */
protected $mcrouterAware;
/** @var string Physical region for mcrouter use */
/** @var float Unix timestamp of the oldest possible valid values */
protected $epoch;
- /** @var int ERR_* constant for the "last error" registry */
- protected $lastRelayError = self::ERR_NONE;
-
/** @var int Callback stack depth for getWithSetCallback() */
private $callbackDepth = 0;
/** @var mixed[] Temporary warm-up cache */
/** Seconds to keep lock keys around */
const LOCK_TTL = 10;
+ /** Seconds to no-op key set() calls to avoid large blob I/O stampedes */
+ const COOLOFF_TTL = 1;
/** Default remaining TTL at which to consider pre-emptive regeneration */
const LOW_TTL = 30;
/** Tiny negative float to use when CTL comes up >= 0 due to clock skew */
const TINY_NEGATIVE = -0.000001;
+ /** Tiny positive float to use when using "minTime" to assert an inequality */
+ const TINY_POSTIVE = 0.000001;
+
+ /** Seconds of delay after get() where set() storms are a consideration with 'lockTSE' */
+ const SET_DELAY_HIGH_SEC = 0.1;
+ /** Min millisecond set() backoff for keys in hold-off (far less than INTERIM_KEY_TTL) */
+ const RECENT_SET_LOW_MS = 50;
+ /** Max millisecond set() backoff for keys in hold-off (far less than INTERIM_KEY_TTL) */
+ const RECENT_SET_HIGH_MS = 100;
+
+ /** Parameter to get()/getMulti() to return extra information by reference */
+ const PASS_BY_REF = -1;
/** Cache format version number */
const VERSION = 1;
const FLD_VALUE = 1; // key to the cached value
const FLD_TTL = 2; // key to the original TTL
const FLD_TIME = 3; // key to the cache time
- const FLD_FLAGS = 4; // key to the flags bitfield
+ const FLD_FLAGS = 4; // key to the flags bitfield (reserved number)
const FLD_HOLDOFF = 5; // key to any hold-off TTL
- /** @var int Treat this value as expired-on-arrival */
- const FLG_STALE = 1;
-
- const ERR_NONE = 0; // no error
- const ERR_NO_RESPONSE = 1; // no response
- const ERR_UNREACHABLE = 2; // can't connect
- const ERR_UNEXPECTED = 3; // response gave some error
- const ERR_RELAY = 4; // relay broadcast failed
-
const VALUE_KEY_PREFIX = 'WANCache:v:';
const INTERIM_KEY_PREFIX = 'WANCache:i:';
const TIME_KEY_PREFIX = 'WANCache:t:';
const MUTEX_KEY_PREFIX = 'WANCache:m:';
+ const COOLOFF_KEY_PREFIX = 'WANCache:c:';
const PURGE_VAL_PREFIX = 'PURGED:';
const PC_PRIMARY = 'primary:1000'; // process cache name and max key count
- const DEFAULT_PURGE_CHANNEL = 'wancache-purge';
-
/**
* @param array $params
* - cache : BagOStuff object for a persistent cache
- * - channels : Map of (action => channel string). Actions include "purge".
- * - relayers : Map of (action => EventRelayer object). Actions include "purge".
* - logger : LoggerInterface object
- * - stats : LoggerInterface object
+ * - stats : StatsdDataFactoryInterface object
* - asyncHandler : A function that takes a callback and runs it later. If supplied,
* whenever a preemptive refresh would be triggered in getWithSetCallback(), the
* current cache value is still used instead. However, the async-handler function
*/
public function __construct( array $params ) {
$this->cache = $params['cache'];
- $this->purgeChannel = $params['channels']['purge'] ?? self::DEFAULT_PURGE_CHANNEL;
- $this->purgeRelayer = $params['relayers']['purge'] ?? new EventRelayerNull( [] );
$this->region = $params['region'] ?? 'main';
$this->cluster = $params['cluster'] ?? 'wan-main';
$this->mcrouterAware = !empty( $params['mcrouterAware'] );
* @return WANObjectCache
*/
public static function newEmpty() {
- return new static( [
- 'cache' => new EmptyBagOStuff()
- ] );
+ return new static( [ 'cache' => new EmptyBagOStuff() ] );
}
/**
* Consider using getWithSetCallback() instead of get() and set() cycles.
* That method has cache slam avoiding features for hot/expensive keys.
*
+ * Pass $info as WANObjectCache::PASS_BY_REF to transform it into a cache key info map.
+ * This map includes the following metadata:
+ * - asOf: UNIX timestamp of the value or null if the key is nonexistant
+ * - tombAsOf: UNIX timestamp of the tombstone or null if the key is not tombstoned
+ * - lastCKPurge: UNIX timestamp of the highest check key or null if none provided
+ *
+ * Othwerwise, $info will transform into the cached value timestamp.
+ *
* @param string $key Cache key made from makeKey() or makeGlobalKey()
* @param mixed|null &$curTTL Approximate TTL left on the key if present/tombstoned [returned]
* @param array $checkKeys List of "check" keys
- * @param float|null &$asOf UNIX timestamp of cached value; null on failure [returned]
+ * @param mixed|null &$info Key info if WANObjectCache::PASS_BY_REF [returned]
* @return mixed Value of cache key or false on failure
*/
- final public function get( $key, &$curTTL = null, array $checkKeys = [], &$asOf = null ) {
- $curTTLs = [];
- $asOfs = [];
- $values = $this->getMulti( [ $key ], $curTTLs, $checkKeys, $asOfs );
+ final public function get(
+ $key, &$curTTL = null, array $checkKeys = [], &$info = null
+ ) {
+ $curTTLs = self::PASS_BY_REF;
+ $infoByKey = self::PASS_BY_REF;
+ $values = $this->getMulti( [ $key ], $curTTLs, $checkKeys, $infoByKey );
$curTTL = $curTTLs[$key] ?? null;
- $asOf = $asOfs[$key] ?? null;
+ if ( $info === self::PASS_BY_REF ) {
+ $info = [
+ 'asOf' => $infoByKey[$key]['asOf'] ?? null,
+ 'tombAsOf' => $infoByKey[$key]['tombAsOf'] ?? null,
+ 'lastCKPurge' => $infoByKey[$key]['lastCKPurge'] ?? null
+ ];
+ } else {
+ $info = $infoByKey[$key]['asOf'] ?? null; // b/c
+ }
return $values[$key] ?? false;
}
/**
* Fetch the value of several keys from cache
*
+ * Pass $info as WANObjectCache::PASS_BY_REF to transform it into a map of cache keys
+ * to cache key info maps, each having the same style as those of WANObjectCache::get().
+ * All the cache keys listed in $keys will have an entry.
+ *
+ * Othwerwise, $info will transform into a map of (cache key => cached value timestamp).
+ * Only the cache keys listed in $keys that exists or are tombstoned will have an entry.
+ *
* @see WANObjectCache::get()
*
* @param array $keys List of cache keys made from makeKey() or makeGlobalKey()
- * @param array &$curTTLs Map of (key => approximate TTL left) for existing keys [returned]
+ * @param mixed|null &$curTTLs Map of (key => TTL left) for existing/tombstoned keys [returned]
* @param array $checkKeys List of check keys to apply to all $keys. May also apply "check"
* keys to specific cache keys only by using cache keys as keys in the $checkKeys array.
- * @param float[] &$asOfs Map of (key => UNIX timestamp of cached value; null on failure)
+ * @param mixed|null &$info Map of (key => info) if WANObjectCache::PASS_BY_REF [returned]
* @return array Map of (key => value) for keys that exist and are not tombstoned
*/
final public function getMulti(
- array $keys, &$curTTLs = [], array $checkKeys = [], array &$asOfs = []
+ array $keys,
+ &$curTTLs = [],
+ array $checkKeys = [],
+ &$info = null
) {
$result = [];
$curTTLs = [];
- $asOfs = [];
+ $infoByKey = [];
$vPrefixLen = strlen( self::VALUE_KEY_PREFIX );
$valueKeys = self::prefixCacheKeys( $keys, self::VALUE_KEY_PREFIX );
foreach ( $checkKeys as $i => $checkKeyGroup ) {
$prefixed = self::prefixCacheKeys( (array)$checkKeyGroup, self::TIME_KEY_PREFIX );
$checkKeysFlat = array_merge( $checkKeysFlat, $prefixed );
- // Is this check keys for a specific cache key, or for all keys being fetched?
+ // Are these check keys for a specific cache key, or for all keys being fetched?
if ( is_int( $i ) ) {
$checkKeysForAll = array_merge( $checkKeysForAll, $prefixed );
} else {
- $checkKeysByKey[$i] = isset( $checkKeysByKey[$i] )
- ? array_merge( $checkKeysByKey[$i], $prefixed )
- : $prefixed;
+ $checkKeysByKey[$i] = $prefixed;
}
}
// Get the main cache value for each key and validate them
foreach ( $valueKeys as $vKey ) {
- if ( !isset( $wrappedValues[$vKey] ) ) {
- continue; // not found
+ $key = substr( $vKey, $vPrefixLen ); // unprefix
+ list( $value, $curTTL, $asOf, $tombAsOf ) = isset( $wrappedValues[$vKey] )
+ ? $this->unwrap( $wrappedValues[$vKey], $now )
+ : [ false, null, null, null ]; // not found
+ // Force dependent keys to be seen as stale for a while after purging
+ // to reduce race conditions involving stale data getting cached
+ $purgeValues = $purgeValuesForAll;
+ if ( isset( $purgeValuesByKey[$key] ) ) {
+ $purgeValues = array_merge( $purgeValues, $purgeValuesByKey[$key] );
}
- $key = substr( $vKey, $vPrefixLen ); // unprefix
+ $lastCKPurge = null; // timestamp of the highest check key
+ foreach ( $purgeValues as $purge ) {
+ $lastCKPurge = max( $purge[self::FLD_TIME], $lastCKPurge );
+ $safeTimestamp = $purge[self::FLD_TIME] + $purge[self::FLD_HOLDOFF];
+ if ( $value !== false && $safeTimestamp >= $asOf ) {
+ // How long ago this value was invalidated by *this* check key
+ $ago = min( $purge[self::FLD_TIME] - $now, self::TINY_NEGATIVE );
+ // How long ago this value was invalidated by *any* known check key
+ $curTTL = min( $curTTL, $ago );
+ }
+ }
- list( $value, $curTTL ) = $this->unwrap( $wrappedValues[$vKey], $now );
if ( $value !== false ) {
$result[$key] = $value;
- // Force dependent keys to be seen as stale for a while after purging
- // to reduce race conditions involving stale data getting cached
- $purgeValues = $purgeValuesForAll;
- if ( isset( $purgeValuesByKey[$key] ) ) {
- $purgeValues = array_merge( $purgeValues, $purgeValuesByKey[$key] );
- }
- foreach ( $purgeValues as $purge ) {
- $safeTimestamp = $purge[self::FLD_TIME] + $purge[self::FLD_HOLDOFF];
- if ( $safeTimestamp >= $wrappedValues[$vKey][self::FLD_TIME] ) {
- // How long ago this value was invalidated by *this* check key
- $ago = min( $purge[self::FLD_TIME] - $now, self::TINY_NEGATIVE );
- // How long ago this value was invalidated by *any* known check key
- $curTTL = min( $curTTL, $ago );
- }
- }
}
- $curTTLs[$key] = $curTTL;
- $asOfs[$key] = ( $value !== false ) ? $wrappedValues[$vKey][self::FLD_TIME] : null;
+ if ( $curTTL !== null ) {
+ $curTTLs[$key] = $curTTL;
+ }
+
+ $infoByKey[$key] = ( $info === self::PASS_BY_REF )
+ ? [ 'asOf' => $asOf, 'tombAsOf' => $tombAsOf, 'lastCKPurge' => $lastCKPurge ]
+ : $asOf; // b/c
}
+ $info = $infoByKey;
+
return $result;
}
// Do not cache potentially uncommitted data as it might get rolled back
if ( !empty( $opts['pending'] ) ) {
- $this->logger->info( 'Rejected set() for {cachekey} due to pending writes.',
- [ 'cachekey' => $key ] );
+ $this->logger->info(
+ 'Rejected set() for {cachekey} due to pending writes.',
+ [ 'cachekey' => $key ]
+ );
return true; // no-op the write for being unsafe
}
- $wrapExtra = []; // additional wrapped value fields
+ $logicalTTL = null; // logical TTL override
// Check if there's a risk of writing stale data after the purge tombstone expired
if ( $lag === false || ( $lag + $age ) > self::MAX_READ_LAG ) {
- // Case A: read lag with "lockTSE"; save but record value as stale
- if ( $lockTSE >= 0 ) {
- $ttl = max( 1, (int)$lockTSE ); // set() expects seconds
- $wrapExtra[self::FLD_FLAGS] = self::FLG_STALE; // mark as stale
- // Case B: any long-running transaction; ignore this set()
- } elseif ( $age > self::MAX_READ_LAG ) {
- $this->logger->info( 'Rejected set() for {cachekey} due to snapshot lag.',
- [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ] );
-
- return true; // no-op the write for being unsafe
- // Case C: high replication lag; lower TTL instead of ignoring all set()s
+ // Case A: any long-running transaction
+ if ( $age > self::MAX_READ_LAG ) {
+ if ( $lockTSE >= 0 ) {
+ // Store value as *almost* stale to avoid cache and mutex stampedes
+ $logicalTTL = self::TTL_SECOND;
+ $this->logger->info(
+ 'Lowered set() TTL for {cachekey} due to snapshot lag.',
+ [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
+ );
+ } else {
+ $this->logger->info(
+ 'Rejected set() for {cachekey} due to snapshot lag.',
+ [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
+ );
+
+ return true; // no-op the write for being unsafe
+ }
+ // Case B: high replication lag; lower TTL instead of ignoring all set()s
} elseif ( $lag === false || $lag > self::MAX_READ_LAG ) {
- $ttl = $ttl ? min( $ttl, self::TTL_LAGGED ) : self::TTL_LAGGED;
- $this->logger->warning( 'Lowered set() TTL for {cachekey} due to replication lag.',
- [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ] );
- // Case D: medium length request with medium replication lag; ignore this set()
+ if ( $lockTSE >= 0 ) {
+ $logicalTTL = min( $ttl ?: INF, self::TTL_LAGGED );
+ } else {
+ $ttl = min( $ttl ?: INF, self::TTL_LAGGED );
+ }
+ $this->logger->warning(
+ 'Lowered set() TTL for {cachekey} due to replication lag.',
+ [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
+ );
+ // Case C: medium length request with medium replication lag
} else {
- $this->logger->info( 'Rejected set() for {cachekey} due to high read lag.',
- [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ] );
+ if ( $lockTSE >= 0 ) {
+ // Store value as *almost* stale to avoid cache and mutex stampedes
+ $logicalTTL = self::TTL_SECOND;
+ $this->logger->info(
+ 'Lowered set() TTL for {cachekey} due to high read lag.',
+ [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
+ );
+ } else {
+ $this->logger->info(
+ 'Rejected set() for {cachekey} due to high read lag.',
+ [ 'cachekey' => $key, 'lag' => $lag, 'age' => $age ]
+ );
- return true; // no-op the write for being unsafe
+ return true; // no-op the write for being unsafe
+ }
}
}
// Wrap that value with time/TTL/version metadata
- $wrapped = $this->wrap( $value, $ttl, $now ) + $wrapExtra;
+ $wrapped = $this->wrap( $value, $logicalTTL ?: $ttl, $now );
$func = function ( $cache, $key, $cWrapped ) use ( $wrapped ) {
return ( is_string( $cWrapped ) )
* @return bool True if the item was purged or not found, false on failure
*/
final public function delete( $key, $ttl = self::HOLDOFF_TTL ) {
- $key = self::VALUE_KEY_PREFIX . $key;
-
if ( $ttl <= 0 ) {
// Publish the purge to all datacenters
- $ok = $this->relayDelete( $key );
+ $ok = $this->relayDelete( self::VALUE_KEY_PREFIX . $key );
} else {
// Publish the purge to all datacenters
- $ok = $this->relayPurge( $key, $ttl, self::HOLDOFF_NONE );
+ $ok = $this->relayPurge( self::VALUE_KEY_PREFIX . $key, $ttl, self::HOLDOFF_NONE );
}
+ $kClass = $this->determineKeyClassForStats( $key );
+ $this->stats->increment( "wanobjectcache.$kClass.delete." . ( $ok ? 'ok' : 'error' ) );
+
return $ok;
}
*/
final public function touchCheckKey( $key, $holdoff = self::HOLDOFF_TTL ) {
// Publish the purge to all datacenters
- return $this->relayPurge( self::TIME_KEY_PREFIX . $key, self::CHECK_KEY_TTL, $holdoff );
+ $ok = $this->relayPurge( self::TIME_KEY_PREFIX . $key, self::CHECK_KEY_TTL, $holdoff );
+
+ $kClass = $this->determineKeyClassForStats( $key );
+ $this->stats->increment( "wanobjectcache.$kClass.ck_touch." . ( $ok ? 'ok' : 'error' ) );
+
+ return $ok;
}
/**
*/
final public function resetCheckKey( $key ) {
// Publish the purge to all datacenters
- return $this->relayDelete( self::TIME_KEY_PREFIX . $key );
+ $ok = $this->relayDelete( self::TIME_KEY_PREFIX . $key );
+
+ $kClass = $this->determineKeyClassForStats( $key );
+ $this->stats->increment( "wanobjectcache.$kClass.ck_reset." . ( $ok ? 'ok' : 'error' ) );
+
+ return $ok;
}
/**
* is useful if thousands or millions of keys depend on the same entity. The entity can
* simply have its "check" key updated whenever the entity is modified.
* Default: [].
- * - graceTTL: If the key is invalidated (by "checkKeys") less than this many seconds ago,
- * consider reusing the stale value. The odds of a refresh becomes more likely over time,
- * becoming certain once the grace period is reached. This can reduce traffic spikes
- * when millions of keys are compared to the same "check" key and touchCheckKey() or
- * resetCheckKey() is called on that "check" key. This option is not useful for the
- * case of the key simply expiring on account of its TTL (use "lowTTL" instead).
+ * - graceTTL: If the key is invalidated (by "checkKeys"/"touchedCallback") less than this
+ * many seconds ago, consider reusing the stale value. The odds of a refresh becomes
+ * more likely over time, becoming certain once the grace period is reached. This can
+ * reduce traffic spikes when millions of keys are compared to the same "check" key and
+ * touchCheckKey() or resetCheckKey() is called on that "check" key. This option is not
+ * useful for avoiding traffic spikes in the case of the key simply expiring on account
+ * of its TTL (use "lowTTL" instead).
* Default: WANObjectCache::GRACE_TTL_NONE.
- * - lockTSE: If the key is tombstoned or invalidated (by "checkKeys") less than this many
- * seconds ago, try to have a single thread handle cache regeneration at any given time.
- * Other threads will try to use stale values if possible. If, on miss, the time since
- * expiration is low, the assumption is that the key is hot and that a stampede is worth
- * avoiding. Setting this above WANObjectCache::HOLDOFF_TTL makes no difference. The
- * higher this is set, the higher the worst-case staleness can be. This option does not
- * by itself handle the case of the key simply expiring on account of its TTL, so make
- * sure that "lowTTL" is not disabled when using this option.
+ * - lockTSE: If the key is tombstoned or invalidated (by "checkKeys"/"touchedCallback")
+ * less than this many seconds ago, try to have a single thread handle cache regeneration
+ * at any given time. Other threads will use stale values if possible. If, on miss,
+ * the time since expiration is low, the assumption is that the key is hot and that a
+ * stampede is worth avoiding. Note that if the key falls out of cache then concurrent
+ * threads will all run the callback on cache miss until the value is saved in cache.
+ * The only stampede protection in that case is from duplicate cache sets when the
+ * callback takes longer than WANObjectCache::SET_DELAY_HIGH_SEC seconds; consider
+ * using "busyValue" if such stampedes are a problem. Note that the higher "lockTSE" is
+ * set, the higher the worst-case staleness of returned values can be. Also note that
+ * this option does not by itself handle the case of the key simply expiring on account
+ * of its TTL, so make sure that "lowTTL" is not disabled when using this option. Avoid
+ * combining this option with delete() as it can always cause a stampede due to their
+ * being no stale value available until after a thread completes the callback.
* Use WANObjectCache::TSE_NONE to disable this logic.
* Default: WANObjectCache::TSE_NONE.
* - busyValue: If no value exists and another thread is currently regenerating it, use this
$popWindow = $opts['hotTTR'] ?? self::HOT_TTR;
$ageNew = $opts['ageNew'] ?? self::AGE_NEW;
$minTime = $opts['minAsOf'] ?? self::MIN_TIMESTAMP_NONE;
- $versioned = isset( $opts['version'] );
- $touchedCallback = $opts['touchedCallback'] ?? null;
+ $needsVersion = isset( $opts['version'] );
+ $touchedCb = $opts['touchedCallback'] ?? null;
+ $initialTime = $this->getCurrentTime();
- // Get a collection name to describe this class of key
- $kClass = $this->determineKeyClass( $key );
+ $kClass = $this->determineKeyClassForStats( $key );
// Get the current key value
- $curTTL = null;
- $cValue = $this->get( $key, $curTTL, $checkKeys, $asOf ); // current value
- $value = $cValue; // return value
+ $curTTL = self::PASS_BY_REF;
+ $curInfo = self::PASS_BY_REF; /** @var array $curInfo */
+ $curValue = $this->get( $key, $curTTL, $checkKeys, $curInfo );
+ // Apply any $touchedCb invalidation timestamp to get the "last purge timestamp"
+ list( $curTTL, $LPT ) = $this->resolveCTL( $curValue, $curTTL, $curInfo, $touchedCb );
+ // Keep track of the best candidate value and its timestamp
+ $value = $curValue; // return value
+ $asOf = $curInfo['asOf']; // return value timestamp
- // Apply additional dynamic expiration logic if supplied
- $curTTL = $this->applyTouchedCallback( $value, $asOf, $curTTL, $touchedCallback );
-
- $preCallbackTime = $this->getCurrentTime();
// Determine if a cached value regeneration is needed or desired
if (
- $this->isValid( $value, $versioned, $asOf, $minTime ) &&
+ $this->isValid( $value, $needsVersion, $asOf, $minTime ) &&
$this->isAliveOrInGracePeriod( $curTTL, $graceTTL )
) {
$preemptiveRefresh = (
$this->worthRefreshExpiring( $curTTL, $lowTTL ) ||
- $this->worthRefreshPopular( $asOf, $ageNew, $popWindow, $preCallbackTime )
+ $this->worthRefreshPopular( $asOf, $ageNew, $popWindow, $initialTime )
);
if ( !$preemptiveRefresh ) {
$this->stats->increment( "wanobjectcache.$kClass.hit.good" );
return $value;
- } elseif ( $this->asyncHandler ) {
- // Update the cache value later, such during post-send of an HTTP request
- $func = $this->asyncHandler;
- $func( function () use ( $key, $ttl, $callback, $opts, $asOf ) {
- $opts['minAsOf'] = INF; // force a refresh
- $this->doGetWithSetCallback( $key, $ttl, $callback, $opts, $asOf );
- } );
+ } elseif ( $this->scheduleAsyncRefresh( $key, $ttl, $callback, $opts ) ) {
$this->stats->increment( "wanobjectcache.$kClass.hit.refresh" );
return $value;
}
}
- // Only a tombstoned key yields no value yet has a (negative) "current time left"
- $isKeyTombstoned = ( $curTTL !== null && $value === false );
+ $isKeyTombstoned = ( $curInfo['tombAsOf'] !== null );
+ if ( $isKeyTombstoned ) {
+ // Get the interim key value since the key is tombstoned (write-holed)
+ list( $value, $asOf ) = $this->getInterimValue( $key, $needsVersion, $minTime );
+ // Update the "last purge time" since the $touchedCb timestamp depends on $value
+ $LPT = $this->resolveTouched( $value, $LPT, $touchedCb );
+ }
+
+ // Reduce mutex and cache set spam while keys are in the tombstone/holdoff period by
+ // checking if $value was genereated by a recent thread much less than a second ago.
+ if (
+ $this->isValid( $value, $needsVersion, $asOf, $minTime, $LPT ) &&
+ $this->isVolatileValueAgeNegligible( $initialTime - $asOf )
+ ) {
+ $this->stats->increment( "wanobjectcache.$kClass.hit.volatile" );
+
+ return $value;
+ }
+
// Decide if only one thread should handle regeneration at a time
$useMutex =
// Note that since tombstones no-op set(), $lockTSE and $curTTL cannot be used to
- // deduce the key hotness because $curTTL will always keep increasing until the
+ // deduce the key hotness because |$curTTL| will always keep increasing until the
// tombstone expires or is overwritten by a new tombstone. Also, even if $lockTSE
// is not set, constant regeneration of a key for the tombstone lifetime might be
// very expensive. Assume tombstoned keys are possibly hot in order to reduce
// the risk of high regeneration load after the delete() method is called.
$isKeyTombstoned ||
// Assume a key is hot if requested soon ($lockTSE seconds) after invalidation.
- // This avoids stampedes when timestamps from $checkKeys/$touchedCallback bump.
+ // This avoids stampedes when timestamps from $checkKeys/$touchedCb bump.
( $curTTL !== null && $curTTL <= 0 && abs( $curTTL ) <= $lockTSE ) ||
// Assume a key is hot if there is no value and a busy fallback is given.
// This avoids stampedes on eviction or preemptive regeneration taking too long.
( $busyValue !== null && $value === false );
- $lockAcquired = false;
+ $hasLock = false;
if ( $useMutex ) {
// Acquire a datacenter-local non-blocking lock
if ( $this->cache->add( self::MUTEX_KEY_PREFIX . $key, 1, self::LOCK_TTL ) ) {
// Lock acquired; this thread will recompute the value and update cache
- $lockAcquired = true;
- } elseif ( $this->isValid( $value, $versioned, $asOf, $minTime ) ) {
+ $hasLock = true;
+ } elseif ( $this->isValid( $value, $needsVersion, $asOf, $minTime ) ) {
// Lock not acquired and a stale value exists; use the stale value
$this->stats->increment( "wanobjectcache.$kClass.hit.stale" );
return $value;
} else {
// Lock not acquired and no stale value exists
- if ( $isKeyTombstoned ) {
- // Use the INTERIM value from the last thread that regenerated it if possible
- $value = $this->getInterimValue( $key, $versioned, $minTime, $asOf );
- if ( $value !== false ) {
- $this->stats->increment( "wanobjectcache.$kClass.hit.volatile" );
-
- return $value;
- }
- }
-
if ( $busyValue !== null ) {
// Use the busy fallback value if nothing else
$miss = is_infinite( $minTime ) ? 'renew' : 'miss';
throw new InvalidArgumentException( "Invalid cache miss callback provided." );
}
+ $preCallbackTime = $this->getCurrentTime();
// Generate the new value from the callback...
$setOpts = [];
++$this->callbackDepth;
try {
- $value = call_user_func_array( $callback, [ $cValue, &$ttl, &$setOpts, $asOf ] );
+ $value = call_user_func_array( $callback, [ $curValue, &$ttl, &$setOpts, $asOf ] );
} finally {
--$this->callbackDepth;
}
$valueIsCacheable = ( $value !== false && $ttl >= 0 );
if ( $valueIsCacheable ) {
+ $ago = max( $this->getCurrentTime() - $initialTime, 0.0 );
+ $this->stats->timing( "wanobjectcache.$kClass.regen_set_delay", 1000 * $ago );
+
if ( $isKeyTombstoned ) {
- // When delete() is called, writes are write-holed by the tombstone,
- // so use a special INTERIM key to pass the new value among threads.
- $tempTTL = max( self::INTERIM_KEY_TTL, (int)$lockTSE ); // set() expects seconds
- $newAsOf = $this->getCurrentTime();
- $wrapped = $this->wrap( $value, $tempTTL, $newAsOf );
- // Avoid using set() to avoid pointless mcrouter broadcasting
- $this->setInterimValue( $key, $wrapped, $tempTTL );
- } elseif ( !$useMutex || $lockAcquired ) {
- // Save the value unless a lock-winning thread is already expected to do that
- $setOpts['lockTSE'] = $lockTSE;
- $setOpts['staleTTL'] = $staleTTL;
- // Use best known "since" timestamp if not provided
- $setOpts += [ 'since' => $preCallbackTime ];
- // Update the cache; this will fail if the key is tombstoned
- $this->set( $key, $value, $ttl, $setOpts );
+ if ( $this->checkAndSetCooloff( $key, $kClass, $ago, $lockTSE, $hasLock ) ) {
+ // Use the interim key value since the key is tombstoned (write-holed)
+ $tempTTL = max( self::INTERIM_KEY_TTL, (int)$lockTSE );
+ $this->setInterimValue( $key, $value, $tempTTL, $this->getCurrentTime() );
+ }
+ } elseif ( !$useMutex || $hasLock ) {
+ if ( $this->checkAndSetCooloff( $key, $kClass, $ago, $lockTSE, $hasLock ) ) {
+ // Save the value unless a lock-winning thread is already expected to do that
+ $setOpts['lockTSE'] = $lockTSE;
+ $setOpts['staleTTL'] = $staleTTL;
+ // Use best known "since" timestamp if not provided
+ $setOpts += [ 'since' => $preCallbackTime ];
+ // Update the cache; this will fail if the key is tombstoned
+ $this->set( $key, $value, $ttl, $setOpts );
+ }
}
}
- if ( $lockAcquired ) {
- // Avoid using delete() to avoid pointless mcrouter broadcasting
- $this->cache->changeTTL( self::MUTEX_KEY_PREFIX . $key, (int)$preCallbackTime - 60 );
+ if ( $hasLock ) {
+ $this->cache->changeTTL( self::MUTEX_KEY_PREFIX . $key, (int)$initialTime - 60 );
}
$miss = is_infinite( $minTime ) ? 'renew' : 'miss';
return $value;
}
+ /**
+ * @param float $age Age of volatile/interim key in seconds
+ * @return bool Whether the age of a volatile value is negligible
+ */
+ private function isVolatileValueAgeNegligible( $age ) {
+ return ( $age < mt_rand( self::RECENT_SET_LOW_MS, self::RECENT_SET_HIGH_MS ) / 1e3 );
+ }
+
+ /**
+ * @param string $key
+ * @param string $kClass
+ * @param float $elapsed Seconds spent regenerating the value
+ * @param float $lockTSE
+ * @param $hasLock bool
+ * @return bool Whether it is OK to proceed with a key set operation
+ */
+ private function checkAndSetCooloff( $key, $kClass, $elapsed, $lockTSE, $hasLock ) {
+ // If $lockTSE is set, the lock was bypassed because there was no stale/interim value,
+ // and $elapsed indicates that regeration is slow, then there is a risk of set()
+ // stampedes with large blobs. With a typical scale-out infrastructure, CPU and query
+ // load from $callback invocations is distributed among appservers and replica DBs,
+ // but cache operations for a given key route to a single cache server (e.g. striped
+ // consistent hashing).
+ if ( $lockTSE < 0 || $hasLock ) {
+ return true; // either not a priori hot or thread has the lock
+ } elseif ( $elapsed <= self::SET_DELAY_HIGH_SEC ) {
+ return true; // not enough time for threads to pile up
+ }
+
+ $this->cache->clearLastError();
+ if (
+ !$this->cache->add( self::COOLOFF_KEY_PREFIX . $key, 1, self::COOLOFF_TTL ) &&
+ // Don't treat failures due to I/O errors as the key being in cooloff
+ $this->cache->getLastError() === BagOStuff::ERR_NONE
+ ) {
+ $this->stats->increment( "wanobjectcache.$kClass.cooloff_bounce" );
+
+ return false;
+ }
+
+ return true;
+ }
+
/**
* @param mixed $value
- * @param float $asOf
- * @param float $curTTL
- * @param callable|null $callback
- * @return float
+ * @param float|null $curTTL
+ * @param array $curInfo
+ * @param callable|null $touchedCallback
+ * @return array (current time left or null, UNIX timestamp of last purge or null)
+ * @note Callable type hints are not used to avoid class-autoloading
*/
- protected function applyTouchedCallback( $value, $asOf, $curTTL, $callback ) {
- if ( $callback === null ) {
- return $curTTL;
+ protected function resolveCTL( $value, $curTTL, $curInfo, $touchedCallback ) {
+ if ( $touchedCallback === null || $value === false ) {
+ return [ $curTTL, max( $curInfo['tombAsOf'], $curInfo['lastCKPurge'] ) ];
}
- if ( !is_callable( $callback ) ) {
+ if ( !is_callable( $touchedCallback ) ) {
throw new InvalidArgumentException( "Invalid expiration callback provided." );
}
- if ( $value !== false ) {
- $touched = $callback( $value );
- if ( $touched !== null && $touched >= $asOf ) {
- $curTTL = min( $curTTL, self::TINY_NEGATIVE, $asOf - $touched );
- }
+ $touched = $touchedCallback( $value );
+ if ( $touched !== null && $touched >= $curInfo['asOf'] ) {
+ $curTTL = min( $curTTL, self::TINY_NEGATIVE, $curInfo['asOf'] - $touched );
+ }
+
+ return [ $curTTL, max( $curInfo['tombAsOf'], $curInfo['lastCKPurge'], $touched ) ];
+ }
+
+ /**
+ * @param mixed $value
+ * @param float|null $lastPurge
+ * @param callable|null $touchedCallback
+ * @return float|null UNIX timestamp of last purge or null
+ * @note Callable type hints are not used to avoid class-autoloading
+ */
+ protected function resolveTouched( $value, $lastPurge, $touchedCallback ) {
+ if ( $touchedCallback === null || $value === false ) {
+ return $lastPurge;
}
- return $curTTL;
+ if ( !is_callable( $touchedCallback ) ) {
+ throw new InvalidArgumentException( "Invalid expiration callback provided." );
+ }
+
+ return max( $touchedCallback( $value ), $lastPurge );
}
/**
* @param string $key
* @param bool $versioned
* @param float $minTime
- * @param mixed &$asOf
- * @return mixed
+ * @return array (cached value or false, cached value timestamp or null)
*/
- protected function getInterimValue( $key, $versioned, $minTime, &$asOf ) {
+ protected function getInterimValue( $key, $versioned, $minTime ) {
if ( !$this->useInterimHoldOffCaching ) {
- return false; // disabled
+ return [ false, null ]; // disabled
}
$wrapped = $this->cache->get( self::INTERIM_KEY_PREFIX . $key );
list( $value ) = $this->unwrap( $wrapped, $this->getCurrentTime() );
- if ( $this->isValid( $value, $versioned, $asOf, $minTime ) ) {
- $asOf = $wrapped[self::FLD_TIME];
-
- return $value;
+ $valueAsOf = $wrapped[self::FLD_TIME] ?? null;
+ if ( $this->isValid( $value, $versioned, $valueAsOf, $minTime ) ) {
+ return [ $value, $valueAsOf ];
}
- return false;
+ return [ false, null ];
}
/**
* @param string $key
- * @param array $wrapped
+ * @param mixed $value
* @param int $tempTTL
+ * @param float $newAsOf
*/
- protected function setInterimValue( $key, $wrapped, $tempTTL ) {
+ protected function setInterimValue( $key, $value, $tempTTL, $newAsOf ) {
+ $wrapped = $this->wrap( $value, $tempTTL, $newAsOf );
+
$this->cache->merge(
self::INTERIM_KEY_PREFIX . $key,
function () use ( $wrapped ) {
* @return int ERR_* class constant for the "last error" registry
*/
final public function getLastError() {
- if ( $this->lastRelayError ) {
- // If the cache and the relayer failed, focus on the latter.
- // An update not making it to the relayer means it won't show up
- // in other DCs (nor will consistent re-hashing see up-to-date values).
- // On the other hand, if just the cache update failed, then it should
- // eventually be applied by the relayer.
- return $this->lastRelayError;
- }
-
$code = $this->cache->getLastError();
switch ( $code ) {
case BagOStuff::ERR_NONE:
*/
final public function clearLastError() {
$this->cache->clearLastError();
- $this->lastRelayError = self::ERR_NONE;
}
/**
$this->makePurgeValue( $this->getCurrentTime(), self::HOLDOFF_NONE ),
$ttl
);
- } elseif ( $this->purgeRelayer instanceof EventRelayerNull ) {
+ } else {
// This handles the mcrouter and the single-DC case
$ok = $this->cache->set(
$key,
$this->makePurgeValue( $this->getCurrentTime(), self::HOLDOFF_NONE ),
$ttl
);
- } else {
- $event = $this->cache->modifySimpleRelayEvent( [
- 'cmd' => 'set',
- 'key' => $key,
- 'val' => 'PURGED:$UNIXTIME$:' . (int)$holdoff,
- 'ttl' => max( $ttl, self::TTL_SECOND ),
- 'sbt' => true, // substitute $UNIXTIME$ with actual microtime
- ] );
-
- $ok = $this->purgeRelayer->notify( $this->purgeChannel, $event );
- if ( !$ok ) {
- $this->lastRelayError = self::ERR_RELAY;
- }
}
return $ok;
// See https://github.com/facebook/mcrouter/wiki/Multi-cluster-broadcast-setup
// Wildcards select all matching routes, e.g. the WAN cluster on all DCs
$ok = $this->cache->delete( "/*/{$this->cluster}/{$key}" );
- } elseif ( $this->purgeRelayer instanceof EventRelayerNull ) {
+ } else {
// Some other proxy handles broadcasting or there is only one datacenter
$ok = $this->cache->delete( $key );
- } else {
- $event = $this->cache->modifySimpleRelayEvent( [
- 'cmd' => 'delete',
- 'key' => $key,
- ] );
-
- $ok = $this->purgeRelayer->notify( $this->purgeChannel, $event );
- if ( !$ok ) {
- $this->lastRelayError = self::ERR_RELAY;
- }
}
return $ok;
}
+ /**
+ * @param string $key
+ * @param int $ttl
+ * @param callable $callback
+ * @param array $opts
+ * @return bool Success
+ */
+ private function scheduleAsyncRefresh( $key, $ttl, $callback, $opts ) {
+ if ( !$this->asyncHandler ) {
+ return false;
+ }
+ // Update the cache value later, such during post-send of an HTTP request
+ $func = $this->asyncHandler;
+ $func( function () use ( $key, $ttl, $callback, $opts ) {
+ $asOf = null; // unused
+ $opts['minAsOf'] = INF; // force a refresh
+ $this->doGetWithSetCallback( $key, $ttl, $callback, $opts, $asOf );
+ } );
+
+ return true;
+ }
+
/**
* Check if a key is fresh or in the grace window and thus due for randomized reuse
*
* @param bool $versioned
* @param float $asOf The time $value was generated
* @param float $minTime The last time the main value was generated (0.0 if unknown)
+ * @param float|null $purgeTime The last time the value was invalidated
* @return bool
*/
- protected function isValid( $value, $versioned, $asOf, $minTime ) {
+ protected function isValid( $value, $versioned, $asOf, $minTime, $purgeTime = null ) {
+ // Avoid reading any key not generated after the latest delete() or touch
+ $safeMinTime = max( $minTime, $purgeTime + self::TINY_POSTIVE );
+
if ( $value === false ) {
return false;
} elseif ( $versioned && !isset( $value[self::VFLD_VERSION] ) ) {
return false;
- } elseif ( $minTime > 0 && $asOf < $minTime ) {
+ } elseif ( $safeMinTime > 0 && $asOf < $minTime ) {
return false;
}
/**
* Do not use this method outside WANObjectCache
*
+ * The cached value will be false if absent/tombstoned/malformed
+ *
* @param array|string|bool $wrapped
* @param float $now Unix Current timestamp (preferrably pre-query)
- * @return array (mixed; false if absent/tombstoned/malformed, current time left)
+ * @return array (cached value or false, current TTL, value timestamp, tombstone timestamp)
*/
protected function unwrap( $wrapped, $now ) {
// Check if the value is a tombstone
if ( $purge !== false ) {
// Purged values should always have a negative current $ttl
$curTTL = min( $purge[self::FLD_TIME] - $now, self::TINY_NEGATIVE );
- return [ false, $curTTL ];
+ return [ false, $curTTL, null, $purge[self::FLD_TIME] ];
}
if ( !is_array( $wrapped ) // not found
|| !isset( $wrapped[self::FLD_VERSION] ) // wrong format
|| $wrapped[self::FLD_VERSION] !== self::VERSION // wrong version
) {
- return [ false, null ];
+ return [ false, null, null, null ];
}
- $flags = $wrapped[self::FLD_FLAGS] ?? 0;
- if ( ( $flags & self::FLG_STALE ) == self::FLG_STALE ) {
- // Treat as expired, with the cache time as the expiration
- $age = $now - $wrapped[self::FLD_TIME];
- $curTTL = min( -$age, self::TINY_NEGATIVE );
- } elseif ( $wrapped[self::FLD_TTL] > 0 ) {
+ if ( $wrapped[self::FLD_TTL] > 0 ) {
// Get the approximate time left on the key
$age = $now - $wrapped[self::FLD_TIME];
$curTTL = max( $wrapped[self::FLD_TTL] - $age, 0.0 );
if ( $wrapped[self::FLD_TIME] < $this->epoch ) {
// Values this old are ignored
- return [ false, null ];
+ return [ false, null, null, null ];
}
- return [ $wrapped[self::FLD_VALUE], $curTTL ];
+ return [ $wrapped[self::FLD_VALUE], $curTTL, $wrapped[self::FLD_TIME], null ];
}
/**
/**
* @param string $key String of the format <scope>:<class>[:<class or variable>]...
- * @return string
+ * @return string A collection name to describe this class of key
*/
- protected function determineKeyClass( $key ) {
+ protected function determineKeyClassForStats( $key ) {
$parts = explode( ':', $key );
return $parts[1] ?? $parts[0]; // sanity
dépôts / lhc / web / wiklou.git / blobdiff