X-Git-Url: https://git.cyclocoop.org/%28%28?a=blobdiff_plain;f=includes%2Flibs%2Fobjectcache%2FWANObjectCache.php;h=621ad424268add3ba9e40edb0273609e1c0c3313;hb=9e56569f9f37b06b4df44af2356975f051cc7c64;hp=fb96269c5a8215c6d211f6556696d39d12bfb970;hpb=41eebe6709d3fb519d1ed3a380bb18926c67b9f8;p=lhc%2Fweb%2Fwiklou.git diff --git a/includes/libs/objectcache/WANObjectCache.php b/includes/libs/objectcache/WANObjectCache.php index fb96269c5a..621ad42426 100644 --- a/includes/libs/objectcache/WANObjectCache.php +++ b/includes/libs/objectcache/WANObjectCache.php @@ -24,7 +24,7 @@ * Multi-datacenter aware caching interface * * All operations go to the local datacenter cache, except for delete(), - * touchCheckKey(), and resetCheckKey(), which broadcast to all clusters. + * touchCheckKey(), and resetCheckKey(), which broadcast to all datacenters. * * This class is intended for caching data from primary stores. * If the get() method does not return a value, then the caller @@ -41,9 +41,9 @@ * that subscribe to the endpoint and update the caches. * * Broadcasted operations like delete() and touchCheckKey() are done - * synchronously in the local cluster, but are relayed asynchronously. + * synchronously in the local datacenter, but are relayed asynchronously. * This means that callers in other datacenters will see older values - * for a however many milliseconds the datacenters are apart. As with + * for however many milliseconds the datacenters are apart. As with * any cache, this should not be relied on for cases where reads are * used to determine writes to source (e.g. non-cache) data stores. * @@ -58,7 +58,7 @@ * @since 1.26 */ class WANObjectCache { - /** @var BagOStuff The local cluster cache */ + /** @var BagOStuff The local datacenter cache */ protected $cache; /** @var string Cache pool name */ protected $pool; @@ -68,23 +68,32 @@ class WANObjectCache { /** @var int */ protected $lastRelayError = self::ERR_NONE; + /** Max time expected to pass between delete() and DB commit finishing */ + const MAX_COMMIT_DELAY = 3; + /** Max replication lag before applying TTL_LAGGED to set() */ + const MAX_REPLICA_LAG = 5; + /** Max time since snapshot transaction start to avoid no-op of set() */ + const MAX_SNAPSHOT_LAG = 5; /** Seconds to tombstone keys on delete() */ - const HOLDOFF_TTL = 10; + const HOLDOFF_TTL = 14; // MAX_COMMIT_DELAY + MAX_REPLICA_LAG + MAX_SNAPSHOT_LAG + 1 + /** Seconds to keep dependency purge keys around */ const CHECK_KEY_TTL = 31536000; // 1 year /** Seconds to keep lock keys around */ const LOCK_TTL = 5; /** Default remaining TTL at which to consider pre-emptive regeneration */ - const LOW_TTL = 10; + const LOW_TTL = 30; /** Default time-since-expiry on a miss that makes a key "hot" */ const LOCK_TSE = 1; - /** Idiom for set()/getWithSetCallback() TTL */ + /** Idiom for set()/getWithSetCallback() TTL being "forever" */ const TTL_NONE = 0; /** Idiom for getWithSetCallback() callbacks to avoid calling set() */ const TTL_UNCACHEABLE = -1; /** Idiom for getWithSetCallback() callbacks to 'lockTSE' logic */ const TSE_NONE = -1; + /** Max TTL to store keys when a data sourced is lagged */ + const TTL_LAGGED = 30; /** Cache format version number */ const VERSION = 1; @@ -158,7 +167,9 @@ class WANObjectCache { * isolation can largely be maintained by doing the following: * - a) Calling delete() on entity change *and* creation, before DB commit * - b) Keeping transaction duration shorter than delete() hold-off TTL - * However, pre-snapshot values might still be seen due to delete() relay lag. + * + * However, pre-snapshot values might still be seen if an update was made + * in a remote datacenter but the purge from delete() didn't relay yet. * * Consider using getWithSetCallback() instead of get()/set() cycles. * That method has cache slam avoiding features for hot/expensive keys. @@ -250,13 +261,64 @@ class WANObjectCache { * the changes do not replicate to the other WAN sites. In that case, delete() * should be used instead. This method is intended for use on cache misses. * + * If the data was read from a snapshot-isolated transactions (e.g. the default + * REPEATABLE-READ in innoDB), use 'since' to avoid the following race condition: + * - a) T1 starts + * - b) T2 updates a row, calls delete(), and commits + * - c) The HOLDOFF_TTL passes, expiring the delete() tombstone + * - d) T1 reads the row and calls set() due to a cache miss + * - e) Stale value is stuck in cache + * + * Setting 'lag' helps avoids keys getting stuck in long-term stale states. + * + * Example usage: + * @code + * $dbr = wfGetDB( DB_SLAVE ); + * $setOpts = Database::getCacheSetOptions( $dbr ); + * // Fetch the row from the DB + * $row = $dbr->selectRow( ... ); + * $key = wfMemcKey( 'building', $buildingId ); + * $cache->set( $key, $row, 86400, $setOpts ); + * @endcode + * * @param string $key Cache key * @param mixed $value * @param integer $ttl Seconds to live [0=forever] + * @param array $opts Options map: + * - lag : Seconds of slave lag. Typically, this is either the slave lag + * before the data was read or, if applicable, the slave lag before + * the snapshot-isolated transaction the data was read from started. + * [Default: 0 seconds] + * - since : UNIX timestamp of the data in $value. Typically, this is either + * the current time the data was read or (if applicable) the time when + * the snapshot-isolated transaction the data was read from started. + * [Default: 0 seconds] + * - lockTSE : if excessive possible snapshot lag is detected, + * then stash the value into a temporary location + * with this TTL. This is only useful if the reads + * use getWithSetCallback() with "lockTSE" set. + * [Default: WANObjectCache::TSE_NONE] * @return bool Success */ - final public function set( $key, $value, $ttl = 0 ) { - $key = self::VALUE_KEY_PREFIX . $key; + final public function set( $key, $value, $ttl = 0, array $opts = array() ) { + $lockTSE = isset( $opts['lockTSE'] ) ? $opts['lockTSE'] : self::TSE_NONE; + $age = isset( $opts['since'] ) ? max( 0, microtime( true ) - $opts['since'] ) : 0; + $lag = isset( $opts['lag'] ) ? $opts['lag'] : 0; + + if ( $lag > self::MAX_REPLICA_LAG ) { + // Too much lag detected; lower TTL so it converges faster + $ttl = $ttl ? min( $ttl, self::TTL_LAGGED ) : self::TTL_LAGGED; + } + + if ( $age > self::MAX_SNAPSHOT_LAG ) { + if ( $lockTSE >= 0 ) { + $tempTTL = max( 1, (int)$lockTSE ); // set() expects seconds + $this->cache->set( self::STASH_KEY_PREFIX . $key, $value, $tempTTL ); + } + + return true; // no-op the write for being unsafe + } + $wrapped = $this->wrap( $value, $ttl ); $func = function ( $cache, $key, $cWrapped ) use ( $wrapped ) { @@ -265,11 +327,11 @@ class WANObjectCache { : $wrapped; }; - return $this->cache->merge( $key, $func, $ttl, 1 ); + return $this->cache->merge( self::VALUE_KEY_PREFIX . $key, $func, $ttl, 1 ); } /** - * Purge a key from all clusters + * Purge a key from all datacenters * * This should only be called when the underlying data (being cached) * changes in a significant way. This deletes the key and starts a hold-off @@ -286,6 +348,31 @@ class WANObjectCache { * - a) Replication lag is bounded to being less than HOLDOFF_TTL; or * - b) If lag is higher, the DB will have gone into read-only mode already * + * When using potentially long-running ACID transactions, a good pattern is + * to use a pre-commit hook to issue the delete. This means that immediately + * after commit, callers will see the tombstone in cache in the local datacenter + * and in the others upon relay. It also avoids the following race condition: + * - a) T1 begins, changes a row, and calls delete() + * - b) The HOLDOFF_TTL passes, expiring the delete() tombstone + * - c) T2 starts, reads the row and calls set() due to a cache miss + * - d) T1 finally commits + * - e) Stale value is stuck in cache + * + * Example usage: + * @code + * $dbw->begin(); // start of request + * ... ... + * // Update the row in the DB + * $dbw->update( ... ); + * $key = wfMemcKey( 'homes', $homeId ); + * // Purge the corresponding cache entry just before committing + * $dbw->onTransactionPreCommitOrIdle( function() use ( $cache, $key ) { + * $cache->delete( $key ); + * } ); + * ... ... + * $dbw->commit(); // end of request + * @endcode + * * If called twice on the same key, then the last hold-off TTL takes * precedence. For idempotence, the $ttl should not vary for different * delete() calls on the same key. Also note that lowering $ttl reduces @@ -299,9 +386,9 @@ class WANObjectCache { $key = self::VALUE_KEY_PREFIX . $key; // Avoid indefinite key salting for sanity $ttl = max( $ttl, 1 ); - // Update the local cluster immediately + // Update the local datacenter immediately $ok = $this->cache->set( $key, self::PURGE_VAL_PREFIX . microtime( true ), $ttl ); - // Publish the purge to all clusters + // Publish the purge to all datacenters return $this->relayPurge( $key, $ttl ) && $ok; } @@ -339,7 +426,7 @@ class WANObjectCache { } /** - * Purge a "check" key from all clusters, invalidating keys that use it + * Purge a "check" key from all datacenters, invalidating keys that use it * * This should only be called when the underlying data (being cached) * changes in a significant way, and it is impractical to call delete() @@ -367,15 +454,15 @@ class WANObjectCache { */ final public function touchCheckKey( $key ) { $key = self::TIME_KEY_PREFIX . $key; - // Update the local cluster immediately + // Update the local datacenter immediately $ok = $this->cache->set( $key, self::PURGE_VAL_PREFIX . microtime( true ), self::CHECK_KEY_TTL ); - // Publish the purge to all clusters + // Publish the purge to all datacenters return $this->relayPurge( $key, self::CHECK_KEY_TTL ) && $ok; } /** - * Delete a "check" key from all clusters, invalidating keys that use it + * Delete a "check" key from all datacenters, invalidating keys that use it * * This is similar to touchCheckKey() in that keys using it via * getWithSetCallback() will be invalidated. The differences are: @@ -384,6 +471,7 @@ class WANObjectCache { * - b) Thus, dependent keys will be known to be invalid, but not * for how long (they are treated as "just" purged), which * effects any lockTSE logic in getWithSetCallback() + * * The advantage is that this does not place high TTL keys on every cache * server, making it better for code that will cache many different keys * and either does not use lockTSE or uses a low enough TTL anyway. @@ -401,22 +489,25 @@ class WANObjectCache { */ final public function resetCheckKey( $key ) { $key = self::TIME_KEY_PREFIX . $key; - // Update the local cluster immediately + // Update the local datacenter immediately $ok = $this->cache->delete( $key ); - // Publish the purge to all clusters + // Publish the purge to all datacenters return $this->relayDelete( $key ) && $ok; } /** * Method to fetch/regenerate cache keys * - * On cache miss, the key will be set to the callback result, - * unless the callback returns false. The arguments supplied are: - * (current value or false, &$ttl) + * On cache miss, the key will be set to the callback result via set() + * unless the callback returns false. The arguments supplied to it are: + * (current value or false, &$ttl, &$setOpts) * The callback function returns the new value given the current * value (false if not present). Preemptive re-caching and $checkKeys * can result in a non-false current value. The TTL of the new value * can be set dynamically by altering $ttl in the callback (by reference). + * The $setOpts array can be altered and is given to set() when called; + * it is recommended to set the 'since' field to avoid race conditions. + * Setting 'lag' helps avoids keys getting stuck in long-term stale states. * * Usually, callbacks ignore the current value, but it can be used * to maintain "most recent X" values that come from time or sequence @@ -431,67 +522,142 @@ class WANObjectCache { * the 'lockTSE' option in $opts. If cache purges are needed, also: * - a) Pass $key into $checkKeys * - b) Use touchCheckKey( $key ) instead of delete( $key ) - * Following this pattern lets the old cache be used until a - * single thread updates it as needed. Also consider tweaking - * the 'lowTTL' parameter. * - * Example usage: + * Example usage (typical key): * @code - * $key = wfMemcKey( 'cat-recent-actions', $catId ); - * // Function that derives the new key value given the old value - * $callback = function( $cValue, &$ttl ) { ... }; - * // Get the key value from cache or from source on cache miss; - * // try to only let one cluster thread manage doing cache updates - * $opts = array( 'lockTSE' => 5, 'lowTTL' => 10 ); - * $value = $cache->getWithSetCallback( $key, $callback, 60, array(), $opts ); + * $catInfo = $cache->getWithSetCallback( + * // Key to store the cached value under + * wfMemcKey( 'cat-attributes', $catId ), + * // Time-to-live (seconds) + * 60, + * // Function that derives the new key value + * function ( $oldValue, &$ttl, array &$setOpts ) { + * $dbr = wfGetDB( DB_SLAVE ); + * // Account for any snapshot/slave lag + * $setOpts += Database::getCacheSetOptions( $dbr ); + * + * return $dbr->selectRow( ... ); + * } + * ); * @endcode * - * Example usage: + * Example usage (key that is expensive and hot): + * @code + * $catConfig = $cache->getWithSetCallback( + * // Key to store the cached value under + * wfMemcKey( 'site-cat-config' ), + * // Time-to-live (seconds) + * 86400, + * // Function that derives the new key value + * function ( $oldValue, &$ttl, array &$setOpts ) { + * $dbr = wfGetDB( DB_SLAVE ); + * // Account for any snapshot/slave lag + * $setOpts += Database::getCacheSetOptions( $dbr ); + * + * return CatConfig::newFromRow( $dbr->selectRow( ... ) ); + * }, + * array( + * // Calling touchCheckKey() on this key invalidates the cache + * 'checkKeys' => array( wfMemcKey( 'site-cat-config' ) ), + * // Try to only let one datacenter thread manage cache updates at a time + * 'lockTSE' => 30 + * ) + * ); + * @endcode + * + * Example usage (key with dynamic dependencies): + * @code + * $catState = $cache->getWithSetCallback( + * // Key to store the cached value under + * wfMemcKey( 'cat-state', $cat->getId() ), + * // Time-to-live (seconds) + * 900, + * // Function that derives the new key value + * function ( $oldValue, &$ttl, array &$setOpts ) { + * // Determine new value from the DB + * $dbr = wfGetDB( DB_SLAVE ); + * // Account for any snapshot/slave lag + * $setOpts += Database::getCacheSetOptions( $dbr ); + * + * return CatState::newFromResults( $dbr->select( ... ) ); + * }, + * array( + * // The "check" keys that represent things the value depends on; + * // Calling touchCheckKey() on any of them invalidates the cache + * 'checkKeys' => array( + * wfMemcKey( 'sustenance-bowls', $cat->getRoomId() ), + * wfMemcKey( 'people-present', $cat->getHouseId() ), + * wfMemcKey( 'cat-laws', $cat->getCityId() ), + * ) + * ) + * ); + * @endcode + * + * Example usage (hot key holding most recent 100 events): * @code - * $key = wfMemcKey( 'cat-state', $catId ); - * // The "check" keys that represent things the value depends on; - * // Calling touchCheckKey() on them invalidates "cat-state" - * $checkKeys = array( - * wfMemcKey( 'water-bowls', $houseId ), - * wfMemcKey( 'food-bowls', $houseId ), - * wfMemcKey( 'people-present', $houseId ) + * $lastCatActions = $cache->getWithSetCallback( + * // Key to store the cached value under + * wfMemcKey( 'cat-last-actions', 100 ), + * // Time-to-live (seconds) + * 10, + * // Function that derives the new key value + * function ( $oldValue, &$ttl, array &$setOpts ) { + * $dbr = wfGetDB( DB_SLAVE ); + * // Account for any snapshot/slave lag + * $setOpts += Database::getCacheSetOptions( $dbr ); + * + * // Start off with the last cached list + * $list = $oldValue ?: array(); + * // Fetch the last 100 relevant rows in descending order; + * // only fetch rows newer than $list[0] to reduce scanning + * $rows = iterator_to_array( $dbr->select( ... ) ); + * // Merge them and get the new "last 100" rows + * return array_slice( array_merge( $new, $list ), 0, 100 ); + * }, + * // Try to only let one datacenter thread manage cache updates at a time + * array( 'lockTSE' => 30 ) * ); - * // Function that derives the new key value - * $callback = function() { ... }; - * // Get the key value from cache or from source on cache miss; - * // try to only let one cluster thread manage doing cache updates - * $opts = array( 'lockTSE' => 5, 'lowTTL' => 10 ); - * $value = $cache->getWithSetCallback( $key, $callback, 60, $checkKeys, $opts ); * @endcode * * @see WANObjectCache::get() + * @see WANObjectCache::set() * * @param string $key Cache key - * @param callable $callback Value generation function * @param integer $ttl Seconds to live for key updates. Special values are: - * - WANObjectCache::TTL_NONE : cache forever - * - WANObjectCache::TTL_UNCACHEABLE : do not cache at all - * @param array $checkKeys List of "check" keys + * - WANObjectCache::TTL_NONE : Cache forever + * - WANObjectCache::TTL_UNCACHEABLE: Do not cache at all + * @param callable $callback Value generation function * @param array $opts Options map: - * - lowTTL : consider pre-emptive updates when the current TTL (sec) - * of the key is less than this. It becomes more likely - * over time, becoming a certainty once the key is expired. - * [Default: WANObjectCache::LOW_TTL seconds] - * - lockTSE : if the key is tombstoned or expired (by $checkKeys) less - * than this many seconds ago, then try to have a single - * thread handle cache regeneration at any given time. - * Other threads will try to use stale values if possible. - * If, on miss, the time since expiration is low, the assumption - * is that the key is hot and that a stampede is worth avoiding. - * Setting this above WANObjectCache::HOLDOFF_TTL makes no difference. - * The higher this is set, the higher the worst-case staleness can be. - * Use WANObjectCache::TSE_NONE to disable this logic. - * [Default: WANObjectCache::TSE_NONE] + * - checkKeys: List of "check" keys. + * - lowTTL: Consider pre-emptive updates when the current TTL (sec) of the key is less than + * this. It becomes more likely over time, becoming a certainty once the key is expired. + * Default: WANObjectCache::LOW_TTL seconds. + * - lockTSE: If the key is tombstoned or expired (by checkKeys) less than this many seconds + * ago, then try to have a single thread handle cache regeneration at any given time. + * Other threads will try to use stale values if possible. If, on miss, the time since + * expiration is low, the assumption is that the key is hot and that a stampede is worth + * avoiding. Setting this above WANObjectCache::HOLDOFF_TTL makes no difference. The + * higher this is set, the higher the worst-case staleness can be. + * Use WANObjectCache::TSE_NONE to disable this logic. Default: WANObjectCache::TSE_NONE. * @return mixed Value to use for the key */ final public function getWithSetCallback( - $key, $callback, $ttl, array $checkKeys = array(), array $opts = array() + $key, $ttl, $callback, array $opts = array(), $oldOpts = array() ) { + // Back-compat with 1.26: Swap $ttl and $callback + if ( is_int( $callback ) ) { + $temp = $ttl; + $ttl = $callback; + $callback = $temp; + } + // Back-compat with 1.26: $checkKeys as separate parameter + if ( $oldOpts || ( is_array( $opts ) && isset( $opts[0] ) ) ) { + $checkKeys = $opts; + $opts = $oldOpts; + } else { + $checkKeys = isset( $opts['checkKeys'] ) ? $opts['checkKeys'] : array(); + } + $lowTTL = isset( $opts['lowTTL'] ) ? $opts['lowTTL'] : min( self::LOW_TTL, $ttl ); $lockTSE = isset( $opts['lockTSE'] ) ? $opts['lockTSE'] : self::TSE_NONE; @@ -517,7 +683,7 @@ class WANObjectCache { $lockAcquired = false; if ( $useMutex ) { - // Acquire a cluster-local non-blocking lock + // Acquire a datacenter-local non-blocking lock if ( $this->cache->lock( $key, 0, self::LOCK_TTL ) ) { // Lock acquired; this thread should update the key $lockAcquired = true; @@ -540,7 +706,8 @@ class WANObjectCache { } // Generate the new value from the callback... - $value = call_user_func_array( $callback, array( $cValue, &$ttl ) ); + $setOpts = array(); + $value = call_user_func_array( $callback, array( $cValue, &$ttl, &$setOpts ) ); // When delete() is called, writes are write-holed by the tombstone, // so use a special stash key to pass the new value around threads. if ( $useMutex && $value !== false && $ttl >= 0 ) { @@ -554,7 +721,8 @@ class WANObjectCache { if ( $value !== false && $ttl >= 0 ) { // Update the cache; this will fail if the key is tombstoned - $this->set( $key, $value, $ttl ); + $setOpts['lockTSE'] = $lockTSE; + $this->set( $key, $value, $ttl, $setOpts ); } return $value;