From: Aaron Schulz Date: Fri, 23 Sep 2016 07:20:19 +0000 (-0700) Subject: Move RedisConnectionPool to /libs/redis X-Git-Tag: 1.31.0-rc.0~5344^2 X-Git-Url: http://git.cyclocoop.org/data/Fool?a=commitdiff_plain;h=10593ffaabe7f4bb959aeef8eb8d640398ee7349;p=lhc%2Fweb%2Fwiklou.git Move RedisConnectionPool to /libs/redis Change-Id: Ied4a85d7172ab76b90f6d9ce4d47a83c3fd7d111 --- diff --git a/autoload.php b/autoload.php index cb3e39120a..2b70387862 100644 --- a/autoload.php +++ b/autoload.php @@ -1138,8 +1138,8 @@ $wgAutoloadLocalClasses = [ 'RedirectSpecialArticle' => __DIR__ . '/includes/specialpage/RedirectSpecialPage.php', 'RedirectSpecialPage' => __DIR__ . '/includes/specialpage/RedirectSpecialPage.php', 'RedisBagOStuff' => __DIR__ . '/includes/objectcache/RedisBagOStuff.php', - 'RedisConnRef' => __DIR__ . '/includes/clientpool/RedisConnectionPool.php', - 'RedisConnectionPool' => __DIR__ . '/includes/clientpool/RedisConnectionPool.php', + 'RedisConnRef' => __DIR__ . '/includes/libs/redis/RedisConnRef.php', + 'RedisConnectionPool' => __DIR__ . '/includes/libs/redis/RedisConnectionPool.php', 'RedisLockManager' => __DIR__ . '/includes/filebackend/lockmanager/RedisLockManager.php', 'RedisPubSubFeedEngine' => __DIR__ . '/includes/rcfeed/RedisPubSubFeedEngine.php', 'RefreshFileHeaders' => __DIR__ . '/maintenance/refreshFileHeaders.php', diff --git a/includes/clientpool/RedisConnectionPool.php b/includes/clientpool/RedisConnectionPool.php deleted file mode 100644 index a9bc59373a..0000000000 --- a/includes/clientpool/RedisConnectionPool.php +++ /dev/null @@ -1,581 +0,0 @@ - ((connection info array),...) */ - protected $connections = []; - /** @var array (server name => UNIX timestamp) */ - protected $downServers = []; - - /** @var array (pool ID => RedisConnectionPool) */ - protected static $instances = []; - - /** integer; seconds to cache servers as "down". */ - const SERVER_DOWN_TTL = 30; - - /** - * @var LoggerInterface - */ - protected $logger; - - /** - * @param array $options - * @throws Exception - */ - protected function __construct( array $options ) { - if ( !class_exists( 'Redis' ) ) { - throw new Exception( __CLASS__ . ' requires a Redis client library. ' . - 'See https://www.mediawiki.org/wiki/Redis#Setup' ); - } - if ( isset( $options['logger'] ) ) { - $this->setLogger( $options['logger'] ); - } else { - $this->setLogger( LoggerFactory::getInstance( 'redis' ) ); - } - $this->connectTimeout = $options['connectTimeout']; - $this->readTimeout = $options['readTimeout']; - $this->persistent = $options['persistent']; - $this->password = $options['password']; - if ( !isset( $options['serializer'] ) || $options['serializer'] === 'php' ) { - $this->serializer = Redis::SERIALIZER_PHP; - } elseif ( $options['serializer'] === 'igbinary' ) { - $this->serializer = Redis::SERIALIZER_IGBINARY; - } elseif ( $options['serializer'] === 'none' ) { - $this->serializer = Redis::SERIALIZER_NONE; - } else { - throw new InvalidArgumentException( "Invalid serializer specified." ); - } - } - - /** - * @param LoggerInterface $logger - * @return null - */ - public function setLogger( LoggerInterface $logger ) { - $this->logger = $logger; - } - - /** - * @param array $options - * @return array - */ - protected static function applyDefaultConfig( array $options ) { - if ( !isset( $options['connectTimeout'] ) ) { - $options['connectTimeout'] = 1; - } - if ( !isset( $options['readTimeout'] ) ) { - $options['readTimeout'] = 1; - } - if ( !isset( $options['persistent'] ) ) { - $options['persistent'] = false; - } - if ( !isset( $options['password'] ) ) { - $options['password'] = null; - } - - return $options; - } - - /** - * @param array $options - * $options include: - * - connectTimeout : The timeout for new connections, in seconds. - * Optional, default is 1 second. - * - readTimeout : The timeout for operation reads, in seconds. - * Commands like BLPOP can fail if told to wait longer than this. - * Optional, default is 1 second. - * - persistent : Set this to true to allow connections to persist across - * multiple web requests. False by default. - * - password : The authentication password, will be sent to Redis in clear text. - * Optional, if it is unspecified, no AUTH command will be sent. - * - serializer : Set to "php", "igbinary", or "none". Default is "php". - * @return RedisConnectionPool - */ - public static function singleton( array $options ) { - $options = self::applyDefaultConfig( $options ); - // Map the options to a unique hash... - ksort( $options ); // normalize to avoid pool fragmentation - $id = sha1( serialize( $options ) ); - // Initialize the object at the hash as needed... - if ( !isset( self::$instances[$id] ) ) { - self::$instances[$id] = new self( $options ); - LoggerFactory::getInstance( 'redis' )->debug( - "Creating a new " . __CLASS__ . " instance with id $id." - ); - } - - return self::$instances[$id]; - } - - /** - * Destroy all singleton() instances - * @since 1.27 - */ - public static function destroySingletons() { - self::$instances = []; - } - - /** - * Get a connection to a redis server. Based on code in RedisBagOStuff.php. - * - * @param string $server A hostname/port combination or the absolute path of a UNIX socket. - * If a hostname is specified but no port, port 6379 will be used. - * @return RedisConnRef|bool Returns false on failure - * @throws MWException - */ - public function getConnection( $server ) { - // Check the listing "dead" servers which have had a connection errors. - // Servers are marked dead for a limited period of time, to - // avoid excessive overhead from repeated connection timeouts. - if ( isset( $this->downServers[$server] ) ) { - $now = time(); - if ( $now > $this->downServers[$server] ) { - // Dead time expired - unset( $this->downServers[$server] ); - } else { - // Server is dead - $this->logger->debug( - 'Server "{redis_server}" is marked down for another ' . - ( $this->downServers[$server] - $now ) . 'seconds', - [ 'redis_server' => $server ] - ); - - return false; - } - } - - // Check if a connection is already free for use - if ( isset( $this->connections[$server] ) ) { - foreach ( $this->connections[$server] as &$connection ) { - if ( $connection['free'] ) { - $connection['free'] = false; - --$this->idlePoolSize; - - return new RedisConnRef( - $this, $server, $connection['conn'], $this->logger - ); - } - } - } - - if ( substr( $server, 0, 1 ) === '/' ) { - // UNIX domain socket - // These are required by the redis extension to start with a slash, but - // we still need to set the port to a special value to make it work. - $host = $server; - $port = 0; - } else { - // TCP connection - $hostPort = IP::splitHostAndPort( $server ); - if ( !$server || !$hostPort ) { - throw new InvalidArgumentException( - __CLASS__ . ": invalid configured server \"$server\"" - ); - } - list( $host, $port ) = $hostPort; - if ( $port === false ) { - $port = 6379; - } - } - - $conn = new Redis(); - try { - if ( $this->persistent ) { - $result = $conn->pconnect( $host, $port, $this->connectTimeout ); - } else { - $result = $conn->connect( $host, $port, $this->connectTimeout ); - } - if ( !$result ) { - $this->logger->error( - 'Could not connect to server "{redis_server}"', - [ 'redis_server' => $server ] - ); - // Mark server down for some time to avoid further timeouts - $this->downServers[$server] = time() + self::SERVER_DOWN_TTL; - - return false; - } - if ( $this->password !== null ) { - if ( !$conn->auth( $this->password ) ) { - $this->logger->error( - 'Authentication error connecting to "{redis_server}"', - [ 'redis_server' => $server ] - ); - } - } - } catch ( RedisException $e ) { - $this->downServers[$server] = time() + self::SERVER_DOWN_TTL; - $this->logger->error( - 'Redis exception connecting to "{redis_server}"', - [ - 'redis_server' => $server, - 'exception' => $e, - ] - ); - - return false; - } - - if ( $conn ) { - $conn->setOption( Redis::OPT_READ_TIMEOUT, $this->readTimeout ); - $conn->setOption( Redis::OPT_SERIALIZER, $this->serializer ); - $this->connections[$server][] = [ 'conn' => $conn, 'free' => false ]; - - return new RedisConnRef( $this, $server, $conn, $this->logger ); - } else { - return false; - } - } - - /** - * Mark a connection to a server as free to return to the pool - * - * @param string $server - * @param Redis $conn - * @return bool - */ - public function freeConnection( $server, Redis $conn ) { - $found = false; - - foreach ( $this->connections[$server] as &$connection ) { - if ( $connection['conn'] === $conn && !$connection['free'] ) { - $connection['free'] = true; - ++$this->idlePoolSize; - break; - } - } - - $this->closeExcessIdleConections(); - - return $found; - } - - /** - * Close any extra idle connections if there are more than the limit - */ - protected function closeExcessIdleConections() { - if ( $this->idlePoolSize <= count( $this->connections ) ) { - return; // nothing to do (no more connections than servers) - } - - foreach ( $this->connections as &$serverConnections ) { - foreach ( $serverConnections as $key => &$connection ) { - if ( $connection['free'] ) { - unset( $serverConnections[$key] ); - if ( --$this->idlePoolSize <= count( $this->connections ) ) { - return; // done (no more connections than servers) - } - } - } - } - } - - /** - * The redis extension throws an exception in response to various read, write - * and protocol errors. Sometimes it also closes the connection, sometimes - * not. The safest response for us is to explicitly destroy the connection - * object and let it be reopened during the next request. - * - * @param string $server - * @param RedisConnRef $cref - * @param RedisException $e - * @deprecated since 1.23 - */ - public function handleException( $server, RedisConnRef $cref, RedisException $e ) { - $this->handleError( $cref, $e ); - } - - /** - * The redis extension throws an exception in response to various read, write - * and protocol errors. Sometimes it also closes the connection, sometimes - * not. The safest response for us is to explicitly destroy the connection - * object and let it be reopened during the next request. - * - * @param RedisConnRef $cref - * @param RedisException $e - */ - public function handleError( RedisConnRef $cref, RedisException $e ) { - $server = $cref->getServer(); - $this->logger->error( - 'Redis exception on server "{redis_server}"', - [ - 'redis_server' => $server, - 'exception' => $e, - ] - ); - foreach ( $this->connections[$server] as $key => $connection ) { - if ( $cref->isConnIdentical( $connection['conn'] ) ) { - $this->idlePoolSize -= $connection['free'] ? 1 : 0; - unset( $this->connections[$server][$key] ); - break; - } - } - } - - /** - * Re-send an AUTH request to the redis server (useful after disconnects). - * - * This works around an upstream bug in phpredis. phpredis hides disconnects by transparently - * reconnecting, but it neglects to re-authenticate the new connection. To the user of the - * phpredis client API this manifests as a seemingly random tendency of connections to lose - * their authentication status. - * - * This method is for internal use only. - * - * @see https://github.com/nicolasff/phpredis/issues/403 - * - * @param string $server - * @param Redis $conn - * @return bool Success - */ - public function reauthenticateConnection( $server, Redis $conn ) { - if ( $this->password !== null ) { - if ( !$conn->auth( $this->password ) ) { - $this->logger->error( - 'Authentication error connecting to "{redis_server}"', - [ 'redis_server' => $server ] - ); - - return false; - } - } - - return true; - } - - /** - * Adjust or reset the connection handle read timeout value - * - * @param Redis $conn - * @param int $timeout Optional - */ - public function resetTimeout( Redis $conn, $timeout = null ) { - $conn->setOption( Redis::OPT_READ_TIMEOUT, $timeout ?: $this->readTimeout ); - } - - /** - * Make sure connections are closed for sanity - */ - function __destruct() { - foreach ( $this->connections as $server => &$serverConnections ) { - foreach ( $serverConnections as $key => &$connection ) { - $connection['conn']->close(); - } - } - } -} - -/** - * Helper class to handle automatically marking connectons as reusable (via RAII pattern) - * - * This class simply wraps the Redis class and can be used the same way - * - * @ingroup Redis - * @since 1.21 - */ -class RedisConnRef { - /** @var RedisConnectionPool */ - protected $pool; - /** @var Redis */ - protected $conn; - - protected $server; // string - protected $lastError; // string - - /** - * @var LoggerInterface - */ - protected $logger; - - /** - * @param RedisConnectionPool $pool - * @param string $server - * @param Redis $conn - * @param LoggerInterface $logger - */ - public function __construct( - RedisConnectionPool $pool, $server, Redis $conn, LoggerInterface $logger - ) { - $this->pool = $pool; - $this->server = $server; - $this->conn = $conn; - $this->logger = $logger; - } - - /** - * @return string - * @since 1.23 - */ - public function getServer() { - return $this->server; - } - - public function getLastError() { - return $this->lastError; - } - - public function clearLastError() { - $this->lastError = null; - } - - public function __call( $name, $arguments ) { - $conn = $this->conn; // convenience - - // Work around https://github.com/nicolasff/phpredis/issues/70 - $lname = strtolower( $name ); - if ( ( $lname === 'blpop' || $lname == 'brpop' ) - && is_array( $arguments[0] ) && isset( $arguments[1] ) - ) { - $this->pool->resetTimeout( $conn, $arguments[1] + 1 ); - } elseif ( $lname === 'brpoplpush' && isset( $arguments[2] ) ) { - $this->pool->resetTimeout( $conn, $arguments[2] + 1 ); - } - - $conn->clearLastError(); - try { - $res = call_user_func_array( [ $conn, $name ], $arguments ); - if ( preg_match( '/^ERR operation not permitted\b/', $conn->getLastError() ) ) { - $this->pool->reauthenticateConnection( $this->server, $conn ); - $conn->clearLastError(); - $res = call_user_func_array( [ $conn, $name ], $arguments ); - $this->logger->info( - "Used automatic re-authentication for method '$name'.", - [ 'redis_server' => $this->server ] - ); - } - } catch ( RedisException $e ) { - $this->pool->resetTimeout( $conn ); // restore - throw $e; - } - - $this->lastError = $conn->getLastError() ?: $this->lastError; - - $this->pool->resetTimeout( $conn ); // restore - - return $res; - } - - /** - * @param string $script - * @param array $params - * @param int $numKeys - * @return mixed - * @throws RedisException - */ - public function luaEval( $script, array $params, $numKeys ) { - $sha1 = sha1( $script ); // 40 char hex - $conn = $this->conn; // convenience - $server = $this->server; // convenience - - // Try to run the server-side cached copy of the script - $conn->clearLastError(); - $res = $conn->evalSha( $sha1, $params, $numKeys ); - // If we got a permission error reply that means that (a) we are not in - // multi()/pipeline() and (b) some connection problem likely occurred. If - // the password the client gave was just wrong, an exception should have - // been thrown back in getConnection() previously. - if ( preg_match( '/^ERR operation not permitted\b/', $conn->getLastError() ) ) { - $this->pool->reauthenticateConnection( $server, $conn ); - $conn->clearLastError(); - $res = $conn->eval( $script, $params, $numKeys ); - $this->logger->info( - "Used automatic re-authentication for Lua script '$sha1'.", - [ 'redis_server' => $server ] - ); - } - // If the script is not in cache, use eval() to retry and cache it - if ( preg_match( '/^NOSCRIPT/', $conn->getLastError() ) ) { - $conn->clearLastError(); - $res = $conn->eval( $script, $params, $numKeys ); - $this->logger->info( - "Used eval() for Lua script '$sha1'.", - [ 'redis_server' => $server ] - ); - } - - if ( $conn->getLastError() ) { // script bug? - $this->logger->error( - 'Lua script error on server "{redis_server}": {lua_error}', - [ - 'redis_server' => $server, - 'lua_error' => $conn->getLastError() - ] - ); - } - - $this->lastError = $conn->getLastError() ?: $this->lastError; - - return $res; - } - - /** - * @param Redis $conn - * @return bool - */ - public function isConnIdentical( Redis $conn ) { - return $this->conn === $conn; - } - - function __destruct() { - $this->pool->freeConnection( $this->server, $this->conn ); - } -} diff --git a/includes/filebackend/lockmanager/LockManagerGroup.php b/includes/filebackend/lockmanager/LockManagerGroup.php index 9ad2faf251..1e66e6e011 100644 --- a/includes/filebackend/lockmanager/LockManagerGroup.php +++ b/includes/filebackend/lockmanager/LockManagerGroup.php @@ -21,6 +21,7 @@ * @ingroup LockManager */ use MediaWiki\MediaWikiServices; +use MediaWiki\Logger\LoggerFactory; /** * Class to handle file lock manager registration @@ -124,6 +125,8 @@ class LockManagerGroup { $config['dbServers']['localDBMaster'] = $dbw; $config['srvCache'] = ObjectCache::getLocalServerInstance( 'hash' ); } + $config['logger'] = LoggerFactory::getInstance( 'LockManager' ); + $this->managers[$name]['instance'] = new $class( $config ); } diff --git a/includes/filebackend/lockmanager/RedisLockManager.php b/includes/filebackend/lockmanager/RedisLockManager.php index 6fd819d637..267aecead1 100644 --- a/includes/filebackend/lockmanager/RedisLockManager.php +++ b/includes/filebackend/lockmanager/RedisLockManager.php @@ -20,6 +20,7 @@ * @file * @ingroup LockManager */ +use Psr\Log\LoggerInterface; /** * Manage locks using redis servers. @@ -51,6 +52,8 @@ class RedisLockManager extends QuorumLockManager { /** @var array Map server names to hostname/IP and port numbers */ protected $lockServers = []; + /** @var LoggerInterface */ + protected $logger; /** @var string Random UUID */ protected $session = ''; @@ -76,6 +79,7 @@ class RedisLockManager extends QuorumLockManager { $this->redisPool = RedisConnectionPool::singleton( $config['redisConfig'] ); $this->session = wfRandomString( 32 ); + $this->logger = \MediaWiki\Logger\LoggerFactory::getInstance( 'redis' ); } protected function getLocksOnServer( $lockSrv, array $pathsByType ) { @@ -84,7 +88,7 @@ class RedisLockManager extends QuorumLockManager { $pathList = call_user_func_array( 'array_merge', array_values( $pathsByType ) ); $server = $this->lockServers[$lockSrv]; - $conn = $this->redisPool->getConnection( $server ); + $conn = $this->redisPool->getConnection( $server, $this->logger ); if ( !$conn ) { foreach ( $pathList as $path ) { $status->fatal( 'lockmanager-fail-acquirelock', $path ); @@ -177,7 +181,7 @@ LUA; $pathList = call_user_func_array( 'array_merge', array_values( $pathsByType ) ); $server = $this->lockServers[$lockSrv]; - $conn = $this->redisPool->getConnection( $server ); + $conn = $this->redisPool->getConnection( $server, $this->logger ); if ( !$conn ) { foreach ( $pathList as $path ) { $status->fatal( 'lockmanager-fail-releaselock', $path ); @@ -246,7 +250,9 @@ LUA; } protected function isServerUp( $lockSrv ) { - return (bool)$this->redisPool->getConnection( $this->lockServers[$lockSrv] ); + $conn = $this->redisPool->getConnection( $this->lockServers[$lockSrv], $this->logger ); + + return (bool)$conn; } /** diff --git a/includes/jobqueue/JobQueueRedis.php b/includes/jobqueue/JobQueueRedis.php index a356e84c65..cbde5e45a7 100644 --- a/includes/jobqueue/JobQueueRedis.php +++ b/includes/jobqueue/JobQueueRedis.php @@ -20,6 +20,7 @@ * @file * @author Aaron Schulz */ +use Psr\Log\LoggerInterface; /** * Class to handle job queues stored in Redis @@ -66,6 +67,8 @@ class JobQueueRedis extends JobQueue { /** @var RedisConnectionPool */ protected $redisPool; + /** @var LoggerInterface */ + protected $logger; /** @var string Server address */ protected $server; @@ -96,6 +99,7 @@ class JobQueueRedis extends JobQueue { "Non-daemonized mode is no longer supported. Please install the " . "mediawiki/services/jobrunner service and update \$wgJobTypeConf as needed." ); } + $this->logger = \MediaWiki\Logger\LoggerFactory::getInstance( 'redis' ); } protected function supportedOrders() { @@ -745,7 +749,7 @@ LUA; * @throws JobQueueConnectionError */ protected function getConnection() { - $conn = $this->redisPool->getConnection( $this->server ); + $conn = $this->redisPool->getConnection( $this->server, $this->logger ); if ( !$conn ) { throw new JobQueueConnectionError( "Unable to connect to redis server {$this->server}." ); diff --git a/includes/jobqueue/aggregator/JobQueueAggregatorRedis.php b/includes/jobqueue/aggregator/JobQueueAggregatorRedis.php index 906a48e379..6ae883712e 100644 --- a/includes/jobqueue/aggregator/JobQueueAggregatorRedis.php +++ b/includes/jobqueue/aggregator/JobQueueAggregatorRedis.php @@ -20,6 +20,7 @@ * @file * @author Aaron Schulz */ +use Psr\Log\LoggerInterface; /** * Class to handle tracking information about all queues using PhpRedis @@ -33,6 +34,8 @@ class JobQueueAggregatorRedis extends JobQueueAggregator { /** @var RedisConnectionPool */ protected $redisPool; + /** @var LoggerInterface */ + protected $logger; /** @var array List of Redis server addresses */ protected $servers; @@ -52,6 +55,7 @@ class JobQueueAggregatorRedis extends JobQueueAggregator { : [ $params['redisServer'] ]; // b/c $params['redisConfig']['serializer'] = 'none'; $this->redisPool = RedisConnectionPool::singleton( $params['redisConfig'] ); + $this->logger = \MediaWiki\Logger\LoggerFactory::getInstance( 'redis' ); } protected function doNotifyQueueEmpty( $wiki, $type ) { @@ -104,7 +108,7 @@ class JobQueueAggregatorRedis extends JobQueueAggregator { protected function getConnection() { $conn = false; foreach ( $this->servers as $server ) { - $conn = $this->redisPool->getConnection( $server ); + $conn = $this->redisPool->getConnection( $server, $this->logger ); if ( $conn ) { break; } diff --git a/includes/libs/redis/RedisConnRef.php b/includes/libs/redis/RedisConnRef.php new file mode 100644 index 0000000000..f2bb8554c6 --- /dev/null +++ b/includes/libs/redis/RedisConnRef.php @@ -0,0 +1,182 @@ +pool = $pool; + $this->server = $server; + $this->conn = $conn; + $this->logger = $logger; + } + + public function setLogger( LoggerInterface $logger ) { + $this->logger = $logger; + } + + /** + * @return string + * @since 1.23 + */ + public function getServer() { + return $this->server; + } + + public function getLastError() { + return $this->lastError; + } + + public function clearLastError() { + $this->lastError = null; + } + + public function __call( $name, $arguments ) { + $conn = $this->conn; // convenience + + // Work around https://github.com/nicolasff/phpredis/issues/70 + $lname = strtolower( $name ); + if ( ( $lname === 'blpop' || $lname == 'brpop' ) + && is_array( $arguments[0] ) && isset( $arguments[1] ) + ) { + $this->pool->resetTimeout( $conn, $arguments[1] + 1 ); + } elseif ( $lname === 'brpoplpush' && isset( $arguments[2] ) ) { + $this->pool->resetTimeout( $conn, $arguments[2] + 1 ); + } + + $conn->clearLastError(); + try { + $res = call_user_func_array( [ $conn, $name ], $arguments ); + if ( preg_match( '/^ERR operation not permitted\b/', $conn->getLastError() ) ) { + $this->pool->reauthenticateConnection( $this->server, $conn ); + $conn->clearLastError(); + $res = call_user_func_array( [ $conn, $name ], $arguments ); + $this->logger->info( + "Used automatic re-authentication for method '$name'.", + [ 'redis_server' => $this->server ] + ); + } + } catch ( RedisException $e ) { + $this->pool->resetTimeout( $conn ); // restore + throw $e; + } + + $this->lastError = $conn->getLastError() ?: $this->lastError; + + $this->pool->resetTimeout( $conn ); // restore + + return $res; + } + + /** + * @param string $script + * @param array $params + * @param int $numKeys + * @return mixed + * @throws RedisException + */ + public function luaEval( $script, array $params, $numKeys ) { + $sha1 = sha1( $script ); // 40 char hex + $conn = $this->conn; // convenience + $server = $this->server; // convenience + + // Try to run the server-side cached copy of the script + $conn->clearLastError(); + $res = $conn->evalSha( $sha1, $params, $numKeys ); + // If we got a permission error reply that means that (a) we are not in + // multi()/pipeline() and (b) some connection problem likely occurred. If + // the password the client gave was just wrong, an exception should have + // been thrown back in getConnection() previously. + if ( preg_match( '/^ERR operation not permitted\b/', $conn->getLastError() ) ) { + $this->pool->reauthenticateConnection( $server, $conn ); + $conn->clearLastError(); + $res = $conn->eval( $script, $params, $numKeys ); + $this->logger->info( + "Used automatic re-authentication for Lua script '$sha1'.", + [ 'redis_server' => $server ] + ); + } + // If the script is not in cache, use eval() to retry and cache it + if ( preg_match( '/^NOSCRIPT/', $conn->getLastError() ) ) { + $conn->clearLastError(); + $res = $conn->eval( $script, $params, $numKeys ); + $this->logger->info( + "Used eval() for Lua script '$sha1'.", + [ 'redis_server' => $server ] + ); + } + + if ( $conn->getLastError() ) { // script bug? + $this->logger->error( + 'Lua script error on server "{redis_server}": {lua_error}', + [ + 'redis_server' => $server, + 'lua_error' => $conn->getLastError() + ] + ); + } + + $this->lastError = $conn->getLastError() ?: $this->lastError; + + return $res; + } + + /** + * @param Redis $conn + * @return bool + */ + public function isConnIdentical( Redis $conn ) { + return $this->conn === $conn; + } + + function __destruct() { + $this->pool->freeConnection( $this->server, $this->conn ); + } +} diff --git a/includes/libs/redis/RedisConnectionPool.php b/includes/libs/redis/RedisConnectionPool.php new file mode 100644 index 0000000000..49d09a9f6a --- /dev/null +++ b/includes/libs/redis/RedisConnectionPool.php @@ -0,0 +1,417 @@ + ((connection info array),...) */ + protected $connections = []; + /** @var array (server name => UNIX timestamp) */ + protected $downServers = []; + + /** @var array (pool ID => RedisConnectionPool) */ + protected static $instances = []; + + /** integer; seconds to cache servers as "down". */ + const SERVER_DOWN_TTL = 30; + + /** + * @var LoggerInterface + */ + protected $logger; + + /** + * @param array $options + * @throws Exception + */ + protected function __construct( array $options ) { + if ( !class_exists( 'Redis' ) ) { + throw new RuntimeException( + __CLASS__ . ' requires a Redis client library. ' . + 'See https://www.mediawiki.org/wiki/Redis#Setup' ); + } + $this->logger = isset( $options['logger'] ) + ? $options['logger'] + : new \Psr\Log\NullLogger(); + $this->connectTimeout = $options['connectTimeout']; + $this->readTimeout = $options['readTimeout']; + $this->persistent = $options['persistent']; + $this->password = $options['password']; + if ( !isset( $options['serializer'] ) || $options['serializer'] === 'php' ) { + $this->serializer = Redis::SERIALIZER_PHP; + } elseif ( $options['serializer'] === 'igbinary' ) { + $this->serializer = Redis::SERIALIZER_IGBINARY; + } elseif ( $options['serializer'] === 'none' ) { + $this->serializer = Redis::SERIALIZER_NONE; + } else { + throw new InvalidArgumentException( "Invalid serializer specified." ); + } + } + + /** + * @param LoggerInterface $logger + * @return null + */ + public function setLogger( LoggerInterface $logger ) { + $this->logger = $logger; + } + + /** + * @param array $options + * @return array + */ + protected static function applyDefaultConfig( array $options ) { + if ( !isset( $options['connectTimeout'] ) ) { + $options['connectTimeout'] = 1; + } + if ( !isset( $options['readTimeout'] ) ) { + $options['readTimeout'] = 1; + } + if ( !isset( $options['persistent'] ) ) { + $options['persistent'] = false; + } + if ( !isset( $options['password'] ) ) { + $options['password'] = null; + } + + return $options; + } + + /** + * @param array $options + * $options include: + * - connectTimeout : The timeout for new connections, in seconds. + * Optional, default is 1 second. + * - readTimeout : The timeout for operation reads, in seconds. + * Commands like BLPOP can fail if told to wait longer than this. + * Optional, default is 1 second. + * - persistent : Set this to true to allow connections to persist across + * multiple web requests. False by default. + * - password : The authentication password, will be sent to Redis in clear text. + * Optional, if it is unspecified, no AUTH command will be sent. + * - serializer : Set to "php", "igbinary", or "none". Default is "php". + * @return RedisConnectionPool + */ + public static function singleton( array $options ) { + $options = self::applyDefaultConfig( $options ); + // Map the options to a unique hash... + ksort( $options ); // normalize to avoid pool fragmentation + $id = sha1( serialize( $options ) ); + // Initialize the object at the hash as needed... + if ( !isset( self::$instances[$id] ) ) { + self::$instances[$id] = new self( $options ); + } + + return self::$instances[$id]; + } + + /** + * Destroy all singleton() instances + * @since 1.27 + */ + public static function destroySingletons() { + self::$instances = []; + } + + /** + * Get a connection to a redis server. Based on code in RedisBagOStuff.php. + * + * @param string $server A hostname/port combination or the absolute path of a UNIX socket. + * If a hostname is specified but no port, port 6379 will be used. + * @param LoggerInterface $logger PSR-3 logger intance. [optional] + * @return RedisConnRef|bool Returns false on failure + * @throws MWException + */ + public function getConnection( $server, LoggerInterface $logger = null ) { + $logger = $logger ?: $this->logger; + // Check the listing "dead" servers which have had a connection errors. + // Servers are marked dead for a limited period of time, to + // avoid excessive overhead from repeated connection timeouts. + if ( isset( $this->downServers[$server] ) ) { + $now = time(); + if ( $now > $this->downServers[$server] ) { + // Dead time expired + unset( $this->downServers[$server] ); + } else { + // Server is dead + $logger->debug( + 'Server "{redis_server}" is marked down for another ' . + ( $this->downServers[$server] - $now ) . 'seconds', + [ 'redis_server' => $server ] + ); + + return false; + } + } + + // Check if a connection is already free for use + if ( isset( $this->connections[$server] ) ) { + foreach ( $this->connections[$server] as &$connection ) { + if ( $connection['free'] ) { + $connection['free'] = false; + --$this->idlePoolSize; + + return new RedisConnRef( + $this, $server, $connection['conn'], $logger + ); + } + } + } + + if ( !$server ) { + throw new InvalidArgumentException( + __CLASS__ . ": invalid configured server \"$server\"" ); + } elseif ( substr( $server, 0, 1 ) === '/' ) { + // UNIX domain socket + // These are required by the redis extension to start with a slash, but + // we still need to set the port to a special value to make it work. + $host = $server; + $port = 0; + } else { + // TCP connection + if ( preg_match( '/^\[(.+)\]:(\d+)$/', $server, $m ) ) { + list( $host, $port ) = [ $m[1], (int)$m[2] ]; // (ip, port) + } elseif ( preg_match( '/^([^:]+):(\d+)$/', $server, $m ) ) { + list( $host, $port ) = [ $m[1], (int)$m[2] ]; // (ip or path, port) + } else { + list( $host, $port ) = [ $server, 6379 ]; // (ip or path, port) + } + } + + $conn = new Redis(); + try { + if ( $this->persistent ) { + $result = $conn->pconnect( $host, $port, $this->connectTimeout ); + } else { + $result = $conn->connect( $host, $port, $this->connectTimeout ); + } + if ( !$result ) { + $logger->error( + 'Could not connect to server "{redis_server}"', + [ 'redis_server' => $server ] + ); + // Mark server down for some time to avoid further timeouts + $this->downServers[$server] = time() + self::SERVER_DOWN_TTL; + + return false; + } + if ( $this->password !== null ) { + if ( !$conn->auth( $this->password ) ) { + $logger->error( + 'Authentication error connecting to "{redis_server}"', + [ 'redis_server' => $server ] + ); + } + } + } catch ( RedisException $e ) { + $this->downServers[$server] = time() + self::SERVER_DOWN_TTL; + $logger->error( + 'Redis exception connecting to "{redis_server}"', + [ + 'redis_server' => $server, + 'exception' => $e, + ] + ); + + return false; + } + + if ( $conn ) { + $conn->setOption( Redis::OPT_READ_TIMEOUT, $this->readTimeout ); + $conn->setOption( Redis::OPT_SERIALIZER, $this->serializer ); + $this->connections[$server][] = [ 'conn' => $conn, 'free' => false ]; + + return new RedisConnRef( $this, $server, $conn, $logger ); + } else { + return false; + } + } + + /** + * Mark a connection to a server as free to return to the pool + * + * @param string $server + * @param Redis $conn + * @return bool + */ + public function freeConnection( $server, Redis $conn ) { + $found = false; + + foreach ( $this->connections[$server] as &$connection ) { + if ( $connection['conn'] === $conn && !$connection['free'] ) { + $connection['free'] = true; + ++$this->idlePoolSize; + break; + } + } + + $this->closeExcessIdleConections(); + + return $found; + } + + /** + * Close any extra idle connections if there are more than the limit + */ + protected function closeExcessIdleConections() { + if ( $this->idlePoolSize <= count( $this->connections ) ) { + return; // nothing to do (no more connections than servers) + } + + foreach ( $this->connections as &$serverConnections ) { + foreach ( $serverConnections as $key => &$connection ) { + if ( $connection['free'] ) { + unset( $serverConnections[$key] ); + if ( --$this->idlePoolSize <= count( $this->connections ) ) { + return; // done (no more connections than servers) + } + } + } + } + } + + /** + * The redis extension throws an exception in response to various read, write + * and protocol errors. Sometimes it also closes the connection, sometimes + * not. The safest response for us is to explicitly destroy the connection + * object and let it be reopened during the next request. + * + * @param string $server + * @param RedisConnRef $cref + * @param RedisException $e + * @deprecated since 1.23 + */ + public function handleException( $server, RedisConnRef $cref, RedisException $e ) { + $this->handleError( $cref, $e ); + } + + /** + * The redis extension throws an exception in response to various read, write + * and protocol errors. Sometimes it also closes the connection, sometimes + * not. The safest response for us is to explicitly destroy the connection + * object and let it be reopened during the next request. + * + * @param RedisConnRef $cref + * @param RedisException $e + */ + public function handleError( RedisConnRef $cref, RedisException $e ) { + $server = $cref->getServer(); + $this->logger->error( + 'Redis exception on server "{redis_server}"', + [ + 'redis_server' => $server, + 'exception' => $e, + ] + ); + foreach ( $this->connections[$server] as $key => $connection ) { + if ( $cref->isConnIdentical( $connection['conn'] ) ) { + $this->idlePoolSize -= $connection['free'] ? 1 : 0; + unset( $this->connections[$server][$key] ); + break; + } + } + } + + /** + * Re-send an AUTH request to the redis server (useful after disconnects). + * + * This works around an upstream bug in phpredis. phpredis hides disconnects by transparently + * reconnecting, but it neglects to re-authenticate the new connection. To the user of the + * phpredis client API this manifests as a seemingly random tendency of connections to lose + * their authentication status. + * + * This method is for internal use only. + * + * @see https://github.com/nicolasff/phpredis/issues/403 + * + * @param string $server + * @param Redis $conn + * @return bool Success + */ + public function reauthenticateConnection( $server, Redis $conn ) { + if ( $this->password !== null ) { + if ( !$conn->auth( $this->password ) ) { + $this->logger->error( + 'Authentication error connecting to "{redis_server}"', + [ 'redis_server' => $server ] + ); + + return false; + } + } + + return true; + } + + /** + * Adjust or reset the connection handle read timeout value + * + * @param Redis $conn + * @param int $timeout Optional + */ + public function resetTimeout( Redis $conn, $timeout = null ) { + $conn->setOption( Redis::OPT_READ_TIMEOUT, $timeout ?: $this->readTimeout ); + } + + /** + * Make sure connections are closed for sanity + */ + function __destruct() { + foreach ( $this->connections as $server => &$serverConnections ) { + foreach ( $serverConnections as $key => &$connection ) { + /** @var Redis $conn */ + $conn = $connection['conn']; + $conn->close(); + } + } + } +} diff --git a/includes/objectcache/RedisBagOStuff.php b/includes/objectcache/RedisBagOStuff.php index 64cd6864ba..d852f82ea5 100644 --- a/includes/objectcache/RedisBagOStuff.php +++ b/includes/objectcache/RedisBagOStuff.php @@ -351,7 +351,7 @@ class RedisBagOStuff extends BagOStuff { while ( ( $tag = array_shift( $candidates ) ) !== null ) { $server = $this->serverTagMap[$tag]; - $conn = $this->redisPool->getConnection( $server ); + $conn = $this->redisPool->getConnection( $server, $this->logger ); if ( !$conn ) { continue; } diff --git a/includes/poolcounter/PoolCounterRedis.php b/includes/poolcounter/PoolCounterRedis.php index 5e8db070aa..99556ed4f8 100644 --- a/includes/poolcounter/PoolCounterRedis.php +++ b/includes/poolcounter/PoolCounterRedis.php @@ -18,6 +18,7 @@ * @file * @author Aaron Schulz */ +use Psr\Log\LoggerInterface; /** * Version of PoolCounter that uses Redis @@ -55,6 +56,8 @@ class PoolCounterRedis extends PoolCounter { protected $ring; /** @var RedisConnectionPool */ protected $pool; + /** @var LoggerInterface */ + protected $logger; /** @var array (server label => host) map */ protected $serversByLabel; /** @var string SHA-1 of the key */ @@ -87,6 +90,7 @@ class PoolCounterRedis extends PoolCounter { $conf['redisConfig']['serializer'] = 'none'; // for use with Lua $this->pool = RedisConnectionPool::singleton( $conf['redisConfig'] ); + $this->logger = \MediaWiki\Logger\LoggerFactory::getInstance( 'redis' ); $this->keySha1 = sha1( $this->key ); $met = ini_get( 'max_execution_time' ); // usually 0 in CLI mode @@ -107,7 +111,7 @@ class PoolCounterRedis extends PoolCounter { $servers = $this->ring->getLocations( $this->key, 3 ); ArrayUtils::consistentHashSort( $servers, $this->key ); foreach ( $servers as $server ) { - $conn = $this->pool->getConnection( $this->serversByLabel[$server] ); + $conn = $this->pool->getConnection( $this->serversByLabel[$server], $this->logger ); if ( $conn ) { break; }