From: Aaron Schulz Date: Fri, 2 Aug 2019 03:06:27 +0000 (-0400) Subject: Move WAN cache classes under a new wancache/ directory X-Git-Tag: 1.34.0-rc.0~801^2 X-Git-Url: https://git.cyclocoop.org/admin/%7B%24admin_url%7Dmembres/import.php?a=commitdiff_plain;h=6ca3de5fa078e927c74977b083dab9a61d6edef4;p=lhc%2Fweb%2Fwiklou.git Move WAN cache classes under a new wancache/ directory Change-Id: I6837761ebca7557e029e1f65beca738266e48efb --- diff --git a/autoload.php b/autoload.php index 43e26464d6..65b4e233d6 100644 --- a/autoload.php +++ b/autoload.php @@ -1594,7 +1594,7 @@ $wgAutoloadLocalClasses = [ 'VirtualRESTServiceClient' => __DIR__ . '/includes/libs/virtualrest/VirtualRESTServiceClient.php', 'WANCacheReapUpdate' => __DIR__ . '/includes/deferred/WANCacheReapUpdate.php', 'WANObjectCache' => __DIR__ . '/includes/libs/objectcache/WANObjectCache.php', - 'WANObjectCacheReaper' => __DIR__ . '/includes/libs/objectcache/WANObjectCacheReaper.php', + 'WANObjectCacheReaper' => __DIR__ . '/includes/libs/objectcache/wancache/WANObjectCacheReaper.php', 'WantedCategoriesPage' => __DIR__ . '/includes/specials/SpecialWantedcategories.php', 'WantedFilesPage' => __DIR__ . '/includes/specials/SpecialWantedfiles.php', 'WantedPagesPage' => __DIR__ . '/includes/specials/SpecialWantedpages.php', @@ -1706,7 +1706,7 @@ $wgAutoloadLocalClasses = [ 'WikitextContent' => __DIR__ . '/includes/content/WikitextContent.php', 'WikitextContentHandler' => __DIR__ . '/includes/content/WikitextContentHandler.php', 'WikitextLogFormatter' => __DIR__ . '/includes/logging/WikitextLogFormatter.php', - 'WinCacheBagOStuff' => __DIR__ . '/includes/libs/objectcache/WinCacheBagOStuff.php', + 'WinCacheBagOStuff' => __DIR__ . '/includes/libs/objectcache/wancache/WinCacheBagOStuff.php', 'WithoutInterwikiPage' => __DIR__ . '/includes/specials/SpecialWithoutinterwiki.php', 'WordLevelDiff' => __DIR__ . '/includes/diff/WordLevelDiff.php', 'WrapOldPasswords' => __DIR__ . '/maintenance/wrapOldPasswords.php', diff --git a/includes/libs/objectcache/WANObjectCacheReaper.php b/includes/libs/objectcache/WANObjectCacheReaper.php deleted file mode 100644 index fb8a75469e..0000000000 --- a/includes/libs/objectcache/WANObjectCacheReaper.php +++ /dev/null @@ -1,199 +0,0 @@ -cache = $cache; - $this->store = $store; - - $this->logChunkCallback = $logCallback; - $this->keyListCallback = $keyCallback; - if ( isset( $params['channel'] ) ) { - $this->channel = $params['channel']; - } else { - throw new UnexpectedValueException( "No channel specified." ); - } - - $this->initialStartWindow = $params['initialStartWindow'] ?? 3600; - $this->logger = $params['logger'] ?? new NullLogger(); - } - - public function setLogger( LoggerInterface $logger ) { - $this->logger = $logger; - } - - /** - * Check and reap stale keys based on a chunk of events - * - * @param int $n Number of events - * @return int Number of keys checked - */ - final public function invoke( $n = 100 ) { - $posKey = $this->store->makeGlobalKey( 'WANCache', 'reaper', $this->channel ); - $scopeLock = $this->store->getScopedLock( "$posKey:busy", 0 ); - if ( !$scopeLock ) { - return 0; - } - - $now = time(); - $status = $this->store->get( $posKey ); - if ( !$status ) { - $status = [ 'pos' => $now - $this->initialStartWindow, 'id' => null ]; - } - - // Get events for entities who's keys tombstones/hold-off should have expired by now - $events = call_user_func_array( - $this->logChunkCallback, - [ $status['pos'], $status['id'], $now - WANObjectCache::HOLDOFF_TTL - 1, $n ] - ); - - $event = null; - $keyEvents = []; - foreach ( $events as $event ) { - $keys = call_user_func_array( - $this->keyListCallback, - [ $this->cache, $event['item'] ] - ); - foreach ( $keys as $key ) { - unset( $keyEvents[$key] ); // use only the latest per key - $keyEvents[$key] = [ - 'pos' => $event['pos'], - 'id' => $event['id'] - ]; - } - } - - $purgeCount = 0; - $lastOkEvent = null; - foreach ( $keyEvents as $key => $keyEvent ) { - if ( !$this->cache->reap( $key, $keyEvent['pos'] ) ) { - break; - } - ++$purgeCount; - $lastOkEvent = $event; - } - - if ( $lastOkEvent ) { - $ok = $this->store->merge( - $posKey, - function ( $bag, $key, $curValue ) use ( $lastOkEvent ) { - if ( !$curValue ) { - // Use new position - } else { - $curCoord = [ $curValue['pos'], $curValue['id'] ]; - $newCoord = [ $lastOkEvent['pos'], $lastOkEvent['id'] ]; - if ( $newCoord < $curCoord ) { - // Keep prior position instead of rolling it back - return $curValue; - } - } - - return [ - 'pos' => $lastOkEvent['pos'], - 'id' => $lastOkEvent['id'], - 'ctime' => $curValue ? $curValue['ctime'] : date( 'c' ) - ]; - }, - IExpiringStore::TTL_INDEFINITE - ); - - $pos = $lastOkEvent['pos']; - $id = $lastOkEvent['id']; - if ( $ok ) { - $this->logger->info( "Updated cache reap position ($pos, $id)." ); - } else { - $this->logger->error( "Could not update cache reap position ($pos, $id)." ); - } - } - - ScopedCallback::consume( $scopeLock ); - - return $purgeCount; - } - - /** - * @return array|bool Returns (pos, id) map or false if not set - */ - public function getState() { - $posKey = $this->store->makeGlobalKey( 'WANCache', 'reaper', $this->channel ); - - return $this->store->get( $posKey ); - } -} diff --git a/includes/libs/objectcache/WinCacheBagOStuff.php b/includes/libs/objectcache/WinCacheBagOStuff.php deleted file mode 100644 index 0e4e3fb63d..0000000000 --- a/includes/libs/objectcache/WinCacheBagOStuff.php +++ /dev/null @@ -1,158 +0,0 @@ -unserialize( $blob ); - if ( $value !== false ) { - $casToken = (string)$blob; // don't bother hashing this - } - - return $value; - } - - protected function cas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) { - if ( !wincache_lock( $key ) ) { // optimize with FIFO lock - return false; - } - - $curCasToken = null; // passed by reference - $this->doGet( $key, self::READ_LATEST, $curCasToken ); - if ( $casToken === $curCasToken ) { - $success = $this->set( $key, $value, $exptime, $flags ); - } else { - $this->logger->info( - __METHOD__ . ' failed due to race condition for {key}.', - [ 'key' => $key ] - ); - - $success = false; // mismatched or failed - } - - wincache_unlock( $key ); - - return $success; - } - - protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) { - $result = wincache_ucache_set( $key, $this->serialize( $value ), $exptime ); - - // false positive, wincache_ucache_set returns an empty array - // in some circumstances. - // @phan-suppress-next-line PhanTypeComparisonToArray - return ( $result === [] || $result === true ); - } - - public function add( $key, $value, $exptime = 0, $flags = 0 ) { - if ( wincache_ucache_exists( $key ) ) { - return false; // avoid warnings - } - - $result = wincache_ucache_add( $key, $this->serialize( $value ), $exptime ); - - // false positive, wincache_ucache_add returns an empty array - // in some circumstances. - // @phan-suppress-next-line PhanTypeComparisonToArray - return ( $result === [] || $result === true ); - } - - protected function doDelete( $key, $flags = 0 ) { - wincache_ucache_delete( $key ); - - return true; - } - - /** - * Construct a cache key. - * - * @since 1.27 - * @param string $keyspace - * @param array $args - * @return string - */ - public function makeKeyInternal( $keyspace, $args ) { - // WinCache keys have a maximum length of 150 characters. From that, - // subtract the number of characters we need for the keyspace and for - // the separator character needed for each argument. To handle some - // custom prefixes used by thing like WANObjectCache, limit to 125. - // NOTE: Same as in memcached, except the max key length there is 255. - $charsLeft = 125 - strlen( $keyspace ) - count( $args ); - - $args = array_map( - function ( $arg ) use ( &$charsLeft ) { - // 33 = 32 characters for the MD5 + 1 for the '#' prefix. - if ( $charsLeft > 33 && strlen( $arg ) > $charsLeft ) { - $arg = '#' . md5( $arg ); - } - - $charsLeft -= strlen( $arg ); - return $arg; - }, - $args - ); - - if ( $charsLeft < 0 ) { - return $keyspace . ':BagOStuff-long-key:##' . md5( implode( ':', $args ) ); - } - - return $keyspace . ':' . implode( ':', $args ); - } - - /** - * Increase stored value of $key by $value while preserving its original TTL - * @param string $key Key to increase - * @param int $value Value to add to $key (Default 1) - * @return int|bool New value or false on failure - */ - public function incr( $key, $value = 1 ) { - if ( !wincache_lock( $key ) ) { // optimize with FIFO lock - return false; - } - - $n = $this->doGet( $key ); - if ( $this->isInteger( $n ) ) { - $n = max( $n + (int)$value, 0 ); - $oldTTL = wincache_ucache_info( false, $key )["ucache_entries"][1]["ttl_seconds"]; - $this->set( $key, $n, $oldTTL ); - } else { - $n = false; - } - - wincache_unlock( $key ); - - return $n; - } -} diff --git a/includes/libs/objectcache/wancache/WANObjectCacheReaper.php b/includes/libs/objectcache/wancache/WANObjectCacheReaper.php new file mode 100644 index 0000000000..fb8a75469e --- /dev/null +++ b/includes/libs/objectcache/wancache/WANObjectCacheReaper.php @@ -0,0 +1,199 @@ +cache = $cache; + $this->store = $store; + + $this->logChunkCallback = $logCallback; + $this->keyListCallback = $keyCallback; + if ( isset( $params['channel'] ) ) { + $this->channel = $params['channel']; + } else { + throw new UnexpectedValueException( "No channel specified." ); + } + + $this->initialStartWindow = $params['initialStartWindow'] ?? 3600; + $this->logger = $params['logger'] ?? new NullLogger(); + } + + public function setLogger( LoggerInterface $logger ) { + $this->logger = $logger; + } + + /** + * Check and reap stale keys based on a chunk of events + * + * @param int $n Number of events + * @return int Number of keys checked + */ + final public function invoke( $n = 100 ) { + $posKey = $this->store->makeGlobalKey( 'WANCache', 'reaper', $this->channel ); + $scopeLock = $this->store->getScopedLock( "$posKey:busy", 0 ); + if ( !$scopeLock ) { + return 0; + } + + $now = time(); + $status = $this->store->get( $posKey ); + if ( !$status ) { + $status = [ 'pos' => $now - $this->initialStartWindow, 'id' => null ]; + } + + // Get events for entities who's keys tombstones/hold-off should have expired by now + $events = call_user_func_array( + $this->logChunkCallback, + [ $status['pos'], $status['id'], $now - WANObjectCache::HOLDOFF_TTL - 1, $n ] + ); + + $event = null; + $keyEvents = []; + foreach ( $events as $event ) { + $keys = call_user_func_array( + $this->keyListCallback, + [ $this->cache, $event['item'] ] + ); + foreach ( $keys as $key ) { + unset( $keyEvents[$key] ); // use only the latest per key + $keyEvents[$key] = [ + 'pos' => $event['pos'], + 'id' => $event['id'] + ]; + } + } + + $purgeCount = 0; + $lastOkEvent = null; + foreach ( $keyEvents as $key => $keyEvent ) { + if ( !$this->cache->reap( $key, $keyEvent['pos'] ) ) { + break; + } + ++$purgeCount; + $lastOkEvent = $event; + } + + if ( $lastOkEvent ) { + $ok = $this->store->merge( + $posKey, + function ( $bag, $key, $curValue ) use ( $lastOkEvent ) { + if ( !$curValue ) { + // Use new position + } else { + $curCoord = [ $curValue['pos'], $curValue['id'] ]; + $newCoord = [ $lastOkEvent['pos'], $lastOkEvent['id'] ]; + if ( $newCoord < $curCoord ) { + // Keep prior position instead of rolling it back + return $curValue; + } + } + + return [ + 'pos' => $lastOkEvent['pos'], + 'id' => $lastOkEvent['id'], + 'ctime' => $curValue ? $curValue['ctime'] : date( 'c' ) + ]; + }, + IExpiringStore::TTL_INDEFINITE + ); + + $pos = $lastOkEvent['pos']; + $id = $lastOkEvent['id']; + if ( $ok ) { + $this->logger->info( "Updated cache reap position ($pos, $id)." ); + } else { + $this->logger->error( "Could not update cache reap position ($pos, $id)." ); + } + } + + ScopedCallback::consume( $scopeLock ); + + return $purgeCount; + } + + /** + * @return array|bool Returns (pos, id) map or false if not set + */ + public function getState() { + $posKey = $this->store->makeGlobalKey( 'WANCache', 'reaper', $this->channel ); + + return $this->store->get( $posKey ); + } +} diff --git a/includes/libs/objectcache/wancache/WinCacheBagOStuff.php b/includes/libs/objectcache/wancache/WinCacheBagOStuff.php new file mode 100644 index 0000000000..0e4e3fb63d --- /dev/null +++ b/includes/libs/objectcache/wancache/WinCacheBagOStuff.php @@ -0,0 +1,158 @@ +unserialize( $blob ); + if ( $value !== false ) { + $casToken = (string)$blob; // don't bother hashing this + } + + return $value; + } + + protected function cas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) { + if ( !wincache_lock( $key ) ) { // optimize with FIFO lock + return false; + } + + $curCasToken = null; // passed by reference + $this->doGet( $key, self::READ_LATEST, $curCasToken ); + if ( $casToken === $curCasToken ) { + $success = $this->set( $key, $value, $exptime, $flags ); + } else { + $this->logger->info( + __METHOD__ . ' failed due to race condition for {key}.', + [ 'key' => $key ] + ); + + $success = false; // mismatched or failed + } + + wincache_unlock( $key ); + + return $success; + } + + protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) { + $result = wincache_ucache_set( $key, $this->serialize( $value ), $exptime ); + + // false positive, wincache_ucache_set returns an empty array + // in some circumstances. + // @phan-suppress-next-line PhanTypeComparisonToArray + return ( $result === [] || $result === true ); + } + + public function add( $key, $value, $exptime = 0, $flags = 0 ) { + if ( wincache_ucache_exists( $key ) ) { + return false; // avoid warnings + } + + $result = wincache_ucache_add( $key, $this->serialize( $value ), $exptime ); + + // false positive, wincache_ucache_add returns an empty array + // in some circumstances. + // @phan-suppress-next-line PhanTypeComparisonToArray + return ( $result === [] || $result === true ); + } + + protected function doDelete( $key, $flags = 0 ) { + wincache_ucache_delete( $key ); + + return true; + } + + /** + * Construct a cache key. + * + * @since 1.27 + * @param string $keyspace + * @param array $args + * @return string + */ + public function makeKeyInternal( $keyspace, $args ) { + // WinCache keys have a maximum length of 150 characters. From that, + // subtract the number of characters we need for the keyspace and for + // the separator character needed for each argument. To handle some + // custom prefixes used by thing like WANObjectCache, limit to 125. + // NOTE: Same as in memcached, except the max key length there is 255. + $charsLeft = 125 - strlen( $keyspace ) - count( $args ); + + $args = array_map( + function ( $arg ) use ( &$charsLeft ) { + // 33 = 32 characters for the MD5 + 1 for the '#' prefix. + if ( $charsLeft > 33 && strlen( $arg ) > $charsLeft ) { + $arg = '#' . md5( $arg ); + } + + $charsLeft -= strlen( $arg ); + return $arg; + }, + $args + ); + + if ( $charsLeft < 0 ) { + return $keyspace . ':BagOStuff-long-key:##' . md5( implode( ':', $args ) ); + } + + return $keyspace . ':' . implode( ':', $args ); + } + + /** + * Increase stored value of $key by $value while preserving its original TTL + * @param string $key Key to increase + * @param int $value Value to add to $key (Default 1) + * @return int|bool New value or false on failure + */ + public function incr( $key, $value = 1 ) { + if ( !wincache_lock( $key ) ) { // optimize with FIFO lock + return false; + } + + $n = $this->doGet( $key ); + if ( $this->isInteger( $n ) ) { + $n = max( $n + (int)$value, 0 ); + $oldTTL = wincache_ucache_info( false, $key )["ucache_entries"][1]["ttl_seconds"]; + $this->set( $key, $n, $oldTTL ); + } else { + $n = false; + } + + wincache_unlock( $key ); + + return $n; + } +}