From: Aaron Schulz Date: Fri, 23 Sep 2016 07:30:31 +0000 (-0700) Subject: Move RedisLockManager to /libs X-Git-Tag: 1.31.0-rc.0~5343^2 X-Git-Url: http://git.cyclocoop.org/%7B%24admin_url%7Dmembres/cotisations/gestion/rappel_supprimer.php?a=commitdiff_plain;h=49e0691b718a918ed55c9b6b570b1265221d9818;p=lhc%2Fweb%2Fwiklou.git Move RedisLockManager to /libs Change-Id: Ia476f117243cf0f6984e373d460b24db18584704 --- diff --git a/autoload.php b/autoload.php index 2b70387862..b3747fe3d9 100644 --- a/autoload.php +++ b/autoload.php @@ -1140,7 +1140,7 @@ $wgAutoloadLocalClasses = [ 'RedisBagOStuff' => __DIR__ . '/includes/objectcache/RedisBagOStuff.php', 'RedisConnRef' => __DIR__ . '/includes/libs/redis/RedisConnRef.php', 'RedisConnectionPool' => __DIR__ . '/includes/libs/redis/RedisConnectionPool.php', - 'RedisLockManager' => __DIR__ . '/includes/filebackend/lockmanager/RedisLockManager.php', + 'RedisLockManager' => __DIR__ . '/includes/libs/lockmanager/RedisLockManager.php', 'RedisPubSubFeedEngine' => __DIR__ . '/includes/rcfeed/RedisPubSubFeedEngine.php', 'RefreshFileHeaders' => __DIR__ . '/maintenance/refreshFileHeaders.php', 'RefreshImageMetadata' => __DIR__ . '/maintenance/refreshImageMetadata.php', diff --git a/includes/filebackend/lockmanager/MySqlLockManager.php b/includes/filebackend/lockmanager/MySqlLockManager.php index 441ffea2d5..125b5e2de0 100644 --- a/includes/filebackend/lockmanager/MySqlLockManager.php +++ b/includes/filebackend/lockmanager/MySqlLockManager.php @@ -15,6 +15,12 @@ class MySqlLockManager extends DBLockManager { self::LOCK_EX => self::LOCK_EX ]; + public function __construct( array $config ) { + parent::__construct( $config ); + + $this->session = substr( $this->session, 0, 31 ); // fit to field + } + protected function initConnection( $lockDb, IDatabase $db ) { # Let this transaction see lock rows from other transactions $db->query( "SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;" ); diff --git a/includes/filebackend/lockmanager/RedisLockManager.php b/includes/filebackend/lockmanager/RedisLockManager.php deleted file mode 100644 index 267aecead1..0000000000 --- a/includes/filebackend/lockmanager/RedisLockManager.php +++ /dev/null @@ -1,282 +0,0 @@ - self::LOCK_SH, - self::LOCK_UW => self::LOCK_SH, - self::LOCK_EX => self::LOCK_EX - ]; - - /** @var RedisConnectionPool */ - protected $redisPool; - - /** @var array Map server names to hostname/IP and port numbers */ - protected $lockServers = []; - - /** @var LoggerInterface */ - protected $logger; - /** @var string Random UUID */ - protected $session = ''; - - /** - * Construct a new instance from configuration. - * - * @param array $config Parameters include: - * - lockServers : Associative array of server names to ":" strings. - * - srvsByBucket : Array of 1-16 consecutive integer keys, starting from 0, - * each having an odd-numbered list of server names (peers) as values. - * - redisConfig : Configuration for RedisConnectionPool::__construct(). - * @throws Exception - */ - public function __construct( array $config ) { - parent::__construct( $config ); - - $this->lockServers = $config['lockServers']; - // Sanitize srvsByBucket config to prevent PHP errors - $this->srvsByBucket = array_filter( $config['srvsByBucket'], 'is_array' ); - $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive - - $config['redisConfig']['serializer'] = 'none'; - $this->redisPool = RedisConnectionPool::singleton( $config['redisConfig'] ); - - $this->session = wfRandomString( 32 ); - $this->logger = \MediaWiki\Logger\LoggerFactory::getInstance( 'redis' ); - } - - protected function getLocksOnServer( $lockSrv, array $pathsByType ) { - $status = StatusValue::newGood(); - - $pathList = call_user_func_array( 'array_merge', array_values( $pathsByType ) ); - - $server = $this->lockServers[$lockSrv]; - $conn = $this->redisPool->getConnection( $server, $this->logger ); - if ( !$conn ) { - foreach ( $pathList as $path ) { - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } - - return $status; - } - - $pathsByKey = []; // (type:hash => path) map - foreach ( $pathsByType as $type => $paths ) { - $typeString = ( $type == LockManager::LOCK_SH ) ? 'SH' : 'EX'; - foreach ( $paths as $path ) { - $pathsByKey[$this->recordKeyForPath( $path, $typeString )] = $path; - } - } - - try { - static $script = -<<luaEval( $script, - array_merge( - array_keys( $pathsByKey ), // KEYS[0], KEYS[1],...,KEYS[N] - [ - $this->session, // ARGV[1] - $this->lockTTL, // ARGV[2] - time() // ARGV[3] - ] - ), - count( $pathsByKey ) # number of first argument(s) that are keys - ); - } catch ( RedisException $e ) { - $res = false; - $this->redisPool->handleError( $conn, $e ); - } - - if ( $res === false ) { - foreach ( $pathList as $path ) { - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } - } else { - foreach ( $res as $key ) { - $status->fatal( 'lockmanager-fail-acquirelock', $pathsByKey[$key] ); - } - } - - return $status; - } - - protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { - $status = StatusValue::newGood(); - - $pathList = call_user_func_array( 'array_merge', array_values( $pathsByType ) ); - - $server = $this->lockServers[$lockSrv]; - $conn = $this->redisPool->getConnection( $server, $this->logger ); - if ( !$conn ) { - foreach ( $pathList as $path ) { - $status->fatal( 'lockmanager-fail-releaselock', $path ); - } - - return $status; - } - - $pathsByKey = []; // (type:hash => path) map - foreach ( $pathsByType as $type => $paths ) { - $typeString = ( $type == LockManager::LOCK_SH ) ? 'SH' : 'EX'; - foreach ( $paths as $path ) { - $pathsByKey[$this->recordKeyForPath( $path, $typeString )] = $path; - } - } - - try { - static $script = -<< 0 then - -- Remove the whole structure if it is now empty - if redis.call('hLen',resourceKey) == 0 then - redis.call('del',resourceKey) - end - else - failed[#failed+1] = requestKey - end - end - return failed -LUA; - $res = $conn->luaEval( $script, - array_merge( - array_keys( $pathsByKey ), // KEYS[0], KEYS[1],...,KEYS[N] - [ - $this->session, // ARGV[1] - ] - ), - count( $pathsByKey ) # number of first argument(s) that are keys - ); - } catch ( RedisException $e ) { - $res = false; - $this->redisPool->handleError( $conn, $e ); - } - - if ( $res === false ) { - foreach ( $pathList as $path ) { - $status->fatal( 'lockmanager-fail-releaselock', $path ); - } - } else { - foreach ( $res as $key ) { - $status->fatal( 'lockmanager-fail-releaselock', $pathsByKey[$key] ); - } - } - - return $status; - } - - protected function releaseAllLocks() { - return StatusValue::newGood(); // not supported - } - - protected function isServerUp( $lockSrv ) { - $conn = $this->redisPool->getConnection( $this->lockServers[$lockSrv], $this->logger ); - - return (bool)$conn; - } - - /** - * @param string $path - * @param string $type One of (EX,SH) - * @return string - */ - protected function recordKeyForPath( $path, $type ) { - return implode( ':', - [ __CLASS__, 'locks', "$type:" . $this->sha1Base36Absolute( $path ) ] ); - } - - /** - * Make sure remaining locks get cleared for sanity - */ - function __destruct() { - while ( count( $this->locksHeld ) ) { - $pathsByType = []; - foreach ( $this->locksHeld as $path => $locks ) { - foreach ( $locks as $type => $count ) { - $pathsByType[$type][] = $path; - } - } - $this->unlockByType( $pathsByType ); - } - } -} diff --git a/includes/libs/lockmanager/DBLockManager.php b/includes/libs/lockmanager/DBLockManager.php index c36ff483f8..b0581468f1 100644 --- a/includes/libs/lockmanager/DBLockManager.php +++ b/includes/libs/lockmanager/DBLockManager.php @@ -43,8 +43,6 @@ abstract class DBLockManager extends QuorumLockManager { protected $lockExpiry; // integer number of seconds protected $safeDelay; // integer number of seconds - - protected $session = 0; // random integer /** @var IDatabase[] Map Database connections (DB name => Database) */ protected $conns = []; @@ -91,12 +89,6 @@ abstract class DBLockManager extends QuorumLockManager { $this->statusCache = isset( $config['srvCache'] ) ? $config['srvCache'] : new HashBagOStuff(); - - $random = []; - for ( $i = 1; $i <= 5; ++$i ) { - $random[] = mt_rand( 0, 0xFFFFFFF ); - } - $this->session = substr( md5( implode( '-', $random ) ), 0, 31 ); } /** diff --git a/includes/libs/lockmanager/LockManager.php b/includes/libs/lockmanager/LockManager.php index 80add5b8b7..42391a0986 100644 --- a/includes/libs/lockmanager/LockManager.php +++ b/includes/libs/lockmanager/LockManager.php @@ -3,6 +3,7 @@ * @defgroup LockManager Lock management * @ingroup FileBackend */ +use Psr\Log\LoggerInterface; /** * Resource locking handling. @@ -43,6 +44,9 @@ * @since 1.19 */ abstract class LockManager { + /** @var LoggerInterface */ + protected $logger; + /** @var array Mapping of lock types to the type actually used */ protected $lockTypeMap = [ self::LOCK_SH => self::LOCK_SH, @@ -56,6 +60,9 @@ abstract class LockManager { protected $domain; // string; domain (usually wiki ID) protected $lockTTL; // integer; maximum time locks can be held + /** @var string Random 32-char hex number */ + protected $session; + /** Lock types; stronger locks have higher values */ const LOCK_SH = 1; // shared lock (for reads) const LOCK_UW = 2; // shared lock (for reads used to write elsewhere) @@ -79,6 +86,14 @@ abstract class LockManager { $met = ini_get( 'max_execution_time' ); // this is 0 in CLI mode $this->lockTTL = max( 5 * 60, 2 * (int)$met ); } + + $random = []; + for ( $i = 1; $i <= 5; ++$i ) { + $random[] = mt_rand( 0, 0xFFFFFFF ); + } + $this->session = md5( implode( '-', $random ) ); + + $this->logger = isset( $config['logger'] ) ? $config['logger'] : new \Psr\Log\NullLogger(); } /** diff --git a/includes/libs/lockmanager/RedisLockManager.php b/includes/libs/lockmanager/RedisLockManager.php new file mode 100644 index 0000000000..a9f5ca364c --- /dev/null +++ b/includes/libs/lockmanager/RedisLockManager.php @@ -0,0 +1,273 @@ + self::LOCK_SH, + self::LOCK_UW => self::LOCK_SH, + self::LOCK_EX => self::LOCK_EX + ]; + + /** @var RedisConnectionPool */ + protected $redisPool; + + /** @var array Map server names to hostname/IP and port numbers */ + protected $lockServers = []; + + /** + * Construct a new instance from configuration. + * + * @param array $config Parameters include: + * - lockServers : Associative array of server names to ":" strings. + * - srvsByBucket : Array of 1-16 consecutive integer keys, starting from 0, + * each having an odd-numbered list of server names (peers) as values. + * - redisConfig : Configuration for RedisConnectionPool::__construct(). + * @throws Exception + */ + public function __construct( array $config ) { + parent::__construct( $config ); + + $this->lockServers = $config['lockServers']; + // Sanitize srvsByBucket config to prevent PHP errors + $this->srvsByBucket = array_filter( $config['srvsByBucket'], 'is_array' ); + $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive + + $config['redisConfig']['serializer'] = 'none'; + $this->redisPool = RedisConnectionPool::singleton( $config['redisConfig'] ); + } + + protected function getLocksOnServer( $lockSrv, array $pathsByType ) { + $status = StatusValue::newGood(); + + $pathList = call_user_func_array( 'array_merge', array_values( $pathsByType ) ); + + $server = $this->lockServers[$lockSrv]; + $conn = $this->redisPool->getConnection( $server, $this->logger ); + if ( !$conn ) { + foreach ( $pathList as $path ) { + $status->fatal( 'lockmanager-fail-acquirelock', $path ); + } + + return $status; + } + + $pathsByKey = []; // (type:hash => path) map + foreach ( $pathsByType as $type => $paths ) { + $typeString = ( $type == LockManager::LOCK_SH ) ? 'SH' : 'EX'; + foreach ( $paths as $path ) { + $pathsByKey[$this->recordKeyForPath( $path, $typeString )] = $path; + } + } + + try { + static $script = +<<luaEval( $script, + array_merge( + array_keys( $pathsByKey ), // KEYS[0], KEYS[1],...,KEYS[N] + [ + $this->session, // ARGV[1] + $this->lockTTL, // ARGV[2] + time() // ARGV[3] + ] + ), + count( $pathsByKey ) # number of first argument(s) that are keys + ); + } catch ( RedisException $e ) { + $res = false; + $this->redisPool->handleError( $conn, $e ); + } + + if ( $res === false ) { + foreach ( $pathList as $path ) { + $status->fatal( 'lockmanager-fail-acquirelock', $path ); + } + } else { + foreach ( $res as $key ) { + $status->fatal( 'lockmanager-fail-acquirelock', $pathsByKey[$key] ); + } + } + + return $status; + } + + protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { + $status = StatusValue::newGood(); + + $pathList = call_user_func_array( 'array_merge', array_values( $pathsByType ) ); + + $server = $this->lockServers[$lockSrv]; + $conn = $this->redisPool->getConnection( $server, $this->logger ); + if ( !$conn ) { + foreach ( $pathList as $path ) { + $status->fatal( 'lockmanager-fail-releaselock', $path ); + } + + return $status; + } + + $pathsByKey = []; // (type:hash => path) map + foreach ( $pathsByType as $type => $paths ) { + $typeString = ( $type == LockManager::LOCK_SH ) ? 'SH' : 'EX'; + foreach ( $paths as $path ) { + $pathsByKey[$this->recordKeyForPath( $path, $typeString )] = $path; + } + } + + try { + static $script = +<< 0 then + -- Remove the whole structure if it is now empty + if redis.call('hLen',resourceKey) == 0 then + redis.call('del',resourceKey) + end + else + failed[#failed+1] = requestKey + end + end + return failed +LUA; + $res = $conn->luaEval( $script, + array_merge( + array_keys( $pathsByKey ), // KEYS[0], KEYS[1],...,KEYS[N] + [ + $this->session, // ARGV[1] + ] + ), + count( $pathsByKey ) # number of first argument(s) that are keys + ); + } catch ( RedisException $e ) { + $res = false; + $this->redisPool->handleError( $conn, $e ); + } + + if ( $res === false ) { + foreach ( $pathList as $path ) { + $status->fatal( 'lockmanager-fail-releaselock', $path ); + } + } else { + foreach ( $res as $key ) { + $status->fatal( 'lockmanager-fail-releaselock', $pathsByKey[$key] ); + } + } + + return $status; + } + + protected function releaseAllLocks() { + return StatusValue::newGood(); // not supported + } + + protected function isServerUp( $lockSrv ) { + $conn = $this->redisPool->getConnection( $this->lockServers[$lockSrv], $this->logger ); + + return (bool)$conn; + } + + /** + * @param string $path + * @param string $type One of (EX,SH) + * @return string + */ + protected function recordKeyForPath( $path, $type ) { + return implode( ':', + [ __CLASS__, 'locks', "$type:" . $this->sha1Base36Absolute( $path ) ] ); + } + + /** + * Make sure remaining locks get cleared for sanity + */ + function __destruct() { + while ( count( $this->locksHeld ) ) { + $pathsByType = []; + foreach ( $this->locksHeld as $path => $locks ) { + foreach ( $locks as $type => $count ) { + $pathsByType[$type][] = $path; + } + } + $this->unlockByType( $pathsByType ); + } + } +}