From: Aaron Schulz Date: Mon, 19 Sep 2016 08:06:45 +0000 (-0700) Subject: Move more LockManager classes to /libs X-Git-Tag: 1.31.0-rc.0~5365 X-Git-Url: http://git.cyclocoop.org/%27.parametre_url%28%20%20%20generer_action_auteur%28%27charger_plugin%27%2C%20%27update_flux%27%29%2C%27update_flux%27%2C%20%27oui%27%29.%27?a=commitdiff_plain;h=2073a2569596e395c63c970cc537dd986b790ea5;p=lhc%2Fweb%2Fwiklou.git Move more LockManager classes to /libs Change-Id: I4bfa79f430827b8717a57bb61cb7c36bc7fcb489 --- diff --git a/autoload.php b/autoload.php index a352884aa0..cb3e39120a 100644 --- a/autoload.php +++ b/autoload.php @@ -306,7 +306,7 @@ $wgAutoloadLocalClasses = [ 'DBError' => __DIR__ . '/includes/libs/rdbms/exception/DBError.php', 'DBExpectedError' => __DIR__ . '/includes/libs/rdbms/exception/DBExpectedError.php', 'DBFileJournal' => __DIR__ . '/includes/filebackend/filejournal/DBFileJournal.php', - 'DBLockManager' => __DIR__ . '/includes/filebackend/lockmanager/DBLockManager.php', + 'DBLockManager' => __DIR__ . '/includes/libs/lockmanager/DBLockManager.php', 'DBMasterPos' => __DIR__ . '/includes/libs/rdbms/database/position/DBMasterPos.php', 'DBQueryError' => __DIR__ . '/includes/libs/rdbms/exception/DBQueryError.php', 'DBReadOnlyError' => __DIR__ . '/includes/libs/rdbms/exception/DBReadOnlyError.php', @@ -1069,7 +1069,7 @@ $wgAutoloadLocalClasses = [ 'PopulateRecentChangesSource' => __DIR__ . '/maintenance/populateRecentChangesSource.php', 'PopulateRevisionLength' => __DIR__ . '/maintenance/populateRevisionLength.php', 'PopulateRevisionSha1' => __DIR__ . '/maintenance/populateRevisionSha1.php', - 'PostgreSqlLockManager' => __DIR__ . '/includes/filebackend/lockmanager/PostgreSqlLockManager.php', + 'PostgreSqlLockManager' => __DIR__ . '/includes/libs/lockmanager/PostgreSqlLockManager.php', 'PostgresBlob' => __DIR__ . '/includes/libs/rdbms/encasing/PostgresBlob.php', 'PostgresField' => __DIR__ . '/includes/libs/rdbms/field/PostgresField.php', 'PostgresInstaller' => __DIR__ . '/includes/installer/PostgresInstaller.php', diff --git a/includes/filebackend/lockmanager/DBLockManager.php b/includes/filebackend/lockmanager/DBLockManager.php deleted file mode 100644 index c36ff483f8..0000000000 --- a/includes/filebackend/lockmanager/DBLockManager.php +++ /dev/null @@ -1,235 +0,0 @@ - server config or IDatabase) */ - protected $dbServers; // (DB name => server config array) - /** @var BagOStuff */ - protected $statusCache; - - protected $lockExpiry; // integer number of seconds - protected $safeDelay; // integer number of seconds - - protected $session = 0; // random integer - /** @var IDatabase[] Map Database connections (DB name => Database) */ - protected $conns = []; - - /** - * Construct a new instance from configuration. - * - * @param array $config Parameters include: - * - dbServers : Associative array of DB names to server configuration. - * Configuration is an associative array that includes: - * - host : DB server name - * - dbname : DB name - * - type : DB type (mysql,postgres,...) - * - user : DB user - * - password : DB user password - * - tablePrefix : DB table prefix - * - flags : DB flags (see DatabaseBase) - * - dbsByBucket : Array of 1-16 consecutive integer keys, starting from 0, - * each having an odd-numbered list of DB names (peers) as values. - * - lockExpiry : Lock timeout (seconds) for dropped connections. [optional] - * This tells the DB server how long to wait before assuming - * connection failure and releasing all the locks for a session. - * - srvCache : A BagOStuff instance using APC or the like. - */ - public function __construct( array $config ) { - parent::__construct( $config ); - - $this->dbServers = $config['dbServers']; - // Sanitize srvsByBucket config to prevent PHP errors - $this->srvsByBucket = array_filter( $config['dbsByBucket'], 'is_array' ); - $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive - - if ( isset( $config['lockExpiry'] ) ) { - $this->lockExpiry = $config['lockExpiry']; - } else { - $met = ini_get( 'max_execution_time' ); - $this->lockExpiry = $met ? $met : 60; // use some sane amount if 0 - } - $this->safeDelay = ( $this->lockExpiry <= 0 ) - ? 60 // pick a safe-ish number to match DB timeout default - : $this->lockExpiry; // cover worst case - - // Tracks peers that couldn't be queried recently to avoid lengthy - // connection timeouts. This is useless if each bucket has one peer. - $this->statusCache = isset( $config['srvCache'] ) - ? $config['srvCache'] - : new HashBagOStuff(); - - $random = []; - for ( $i = 1; $i <= 5; ++$i ) { - $random[] = mt_rand( 0, 0xFFFFFFF ); - } - $this->session = substr( md5( implode( '-', $random ) ), 0, 31 ); - } - - /** - * @TODO change this code to work in one batch - * @param string $lockSrv - * @param array $pathsByType - * @return StatusValue - */ - protected function getLocksOnServer( $lockSrv, array $pathsByType ) { - $status = StatusValue::newGood(); - foreach ( $pathsByType as $type => $paths ) { - $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) ); - } - - return $status; - } - - abstract protected function doGetLocksOnServer( $lockSrv, array $paths, $type ); - - protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { - return StatusValue::newGood(); - } - - /** - * @see QuorumLockManager::isServerUp() - * @param string $lockSrv - * @return bool - */ - protected function isServerUp( $lockSrv ) { - if ( !$this->cacheCheckFailures( $lockSrv ) ) { - return false; // recent failure to connect - } - try { - $this->getConnection( $lockSrv ); - } catch ( DBError $e ) { - $this->cacheRecordFailure( $lockSrv ); - - return false; // failed to connect - } - - return true; - } - - /** - * Get (or reuse) a connection to a lock DB - * - * @param string $lockDb - * @return IDatabase - * @throws DBError - * @throws UnexpectedValueException - */ - protected function getConnection( $lockDb ) { - if ( !isset( $this->conns[$lockDb] ) ) { - if ( $this->dbServers[$lockDb] instanceof IDatabase ) { - // Direct injected connection hande for $lockDB - $db = $this->dbServers[$lockDb]; - } elseif ( is_array( $this->dbServers[$lockDb] ) ) { - // Parameters to construct a new database connection - $config = $this->dbServers[$lockDb]; - $db = DatabaseBase::factory( $config['type'], $config ); - } else { - throw new UnexpectedValueException( "No server called '$lockDb'." ); - } - - $db->clearFlag( DBO_TRX ); - # If the connection drops, try to avoid letting the DB rollback - # and release the locks before the file operations are finished. - # This won't handle the case of DB server restarts however. - $options = []; - if ( $this->lockExpiry > 0 ) { - $options['connTimeout'] = $this->lockExpiry; - } - $db->setSessionOptions( $options ); - $this->initConnection( $lockDb, $db ); - - $this->conns[$lockDb] = $db; - } - - return $this->conns[$lockDb]; - } - - /** - * Do additional initialization for new lock DB connection - * - * @param string $lockDb - * @param IDatabase $db - * @throws DBError - */ - protected function initConnection( $lockDb, IDatabase $db ) { - } - - /** - * Checks if the DB has not recently had connection/query errors. - * This just avoids wasting time on doomed connection attempts. - * - * @param string $lockDb - * @return bool - */ - protected function cacheCheckFailures( $lockDb ) { - return ( $this->safeDelay > 0 ) - ? !$this->statusCache->get( $this->getMissKey( $lockDb ) ) - : true; - } - - /** - * Log a lock request failure to the cache - * - * @param string $lockDb - * @return bool Success - */ - protected function cacheRecordFailure( $lockDb ) { - return ( $this->safeDelay > 0 ) - ? $this->statusCache->set( $this->getMissKey( $lockDb ), 1, $this->safeDelay ) - : true; - } - - /** - * Get a cache key for recent query misses for a DB - * - * @param string $lockDb - * @return string - */ - protected function getMissKey( $lockDb ) { - return 'dblockmanager:downservers:' . str_replace( ' ', '_', $lockDb ); - } - - /** - * Make sure remaining locks get cleared for sanity - */ - function __destruct() { - $this->releaseAllLocks(); - foreach ( $this->conns as $db ) { - $db->close(); - } - } -} diff --git a/includes/filebackend/lockmanager/PostgreSqlLockManager.php b/includes/filebackend/lockmanager/PostgreSqlLockManager.php deleted file mode 100644 index d6b1ce822d..0000000000 --- a/includes/filebackend/lockmanager/PostgreSqlLockManager.php +++ /dev/null @@ -1,79 +0,0 @@ - self::LOCK_SH, - self::LOCK_UW => self::LOCK_SH, - self::LOCK_EX => self::LOCK_EX - ]; - - protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) { - $status = StatusValue::newGood(); - if ( !count( $paths ) ) { - return $status; // nothing to lock - } - - $db = $this->getConnection( $lockSrv ); // checked in isServerUp() - $bigints = array_unique( array_map( - function ( $key ) { - return Wikimedia\base_convert( substr( $key, 0, 15 ), 16, 10 ); - }, - array_map( [ $this, 'sha1Base16Absolute' ], $paths ) - ) ); - - // Try to acquire all the locks... - $fields = []; - foreach ( $bigints as $bigint ) { - $fields[] = ( $type == self::LOCK_SH ) - ? "pg_try_advisory_lock_shared({$db->addQuotes( $bigint )}) AS K$bigint" - : "pg_try_advisory_lock({$db->addQuotes( $bigint )}) AS K$bigint"; - } - $res = $db->query( 'SELECT ' . implode( ', ', $fields ), __METHOD__ ); - $row = $res->fetchRow(); - - if ( in_array( 'f', $row ) ) { - // Release any acquired locks if some could not be acquired... - $fields = []; - foreach ( $row as $kbigint => $ok ) { - if ( $ok === 't' ) { // locked - $bigint = substr( $kbigint, 1 ); // strip off the "K" - $fields[] = ( $type == self::LOCK_SH ) - ? "pg_advisory_unlock_shared({$db->addQuotes( $bigint )})" - : "pg_advisory_unlock({$db->addQuotes( $bigint )})"; - } - } - if ( count( $fields ) ) { - $db->query( 'SELECT ' . implode( ', ', $fields ), __METHOD__ ); - } - foreach ( $paths as $path ) { - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } - } - - return $status; - } - - /** - * @see QuorumLockManager::releaseAllLocks() - * @return StatusValue - */ - protected function releaseAllLocks() { - $status = StatusValue::newGood(); - - foreach ( $this->conns as $lockDb => $db ) { - try { - $db->query( "SELECT pg_advisory_unlock_all()", __METHOD__ ); - } catch ( DBError $e ) { - $status->fatal( 'lockmanager-fail-db-release', $lockDb ); - } - } - - return $status; - } -} diff --git a/includes/libs/lockmanager/DBLockManager.php b/includes/libs/lockmanager/DBLockManager.php new file mode 100644 index 0000000000..c36ff483f8 --- /dev/null +++ b/includes/libs/lockmanager/DBLockManager.php @@ -0,0 +1,235 @@ + server config or IDatabase) */ + protected $dbServers; // (DB name => server config array) + /** @var BagOStuff */ + protected $statusCache; + + protected $lockExpiry; // integer number of seconds + protected $safeDelay; // integer number of seconds + + protected $session = 0; // random integer + /** @var IDatabase[] Map Database connections (DB name => Database) */ + protected $conns = []; + + /** + * Construct a new instance from configuration. + * + * @param array $config Parameters include: + * - dbServers : Associative array of DB names to server configuration. + * Configuration is an associative array that includes: + * - host : DB server name + * - dbname : DB name + * - type : DB type (mysql,postgres,...) + * - user : DB user + * - password : DB user password + * - tablePrefix : DB table prefix + * - flags : DB flags (see DatabaseBase) + * - dbsByBucket : Array of 1-16 consecutive integer keys, starting from 0, + * each having an odd-numbered list of DB names (peers) as values. + * - lockExpiry : Lock timeout (seconds) for dropped connections. [optional] + * This tells the DB server how long to wait before assuming + * connection failure and releasing all the locks for a session. + * - srvCache : A BagOStuff instance using APC or the like. + */ + public function __construct( array $config ) { + parent::__construct( $config ); + + $this->dbServers = $config['dbServers']; + // Sanitize srvsByBucket config to prevent PHP errors + $this->srvsByBucket = array_filter( $config['dbsByBucket'], 'is_array' ); + $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive + + if ( isset( $config['lockExpiry'] ) ) { + $this->lockExpiry = $config['lockExpiry']; + } else { + $met = ini_get( 'max_execution_time' ); + $this->lockExpiry = $met ? $met : 60; // use some sane amount if 0 + } + $this->safeDelay = ( $this->lockExpiry <= 0 ) + ? 60 // pick a safe-ish number to match DB timeout default + : $this->lockExpiry; // cover worst case + + // Tracks peers that couldn't be queried recently to avoid lengthy + // connection timeouts. This is useless if each bucket has one peer. + $this->statusCache = isset( $config['srvCache'] ) + ? $config['srvCache'] + : new HashBagOStuff(); + + $random = []; + for ( $i = 1; $i <= 5; ++$i ) { + $random[] = mt_rand( 0, 0xFFFFFFF ); + } + $this->session = substr( md5( implode( '-', $random ) ), 0, 31 ); + } + + /** + * @TODO change this code to work in one batch + * @param string $lockSrv + * @param array $pathsByType + * @return StatusValue + */ + protected function getLocksOnServer( $lockSrv, array $pathsByType ) { + $status = StatusValue::newGood(); + foreach ( $pathsByType as $type => $paths ) { + $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) ); + } + + return $status; + } + + abstract protected function doGetLocksOnServer( $lockSrv, array $paths, $type ); + + protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { + return StatusValue::newGood(); + } + + /** + * @see QuorumLockManager::isServerUp() + * @param string $lockSrv + * @return bool + */ + protected function isServerUp( $lockSrv ) { + if ( !$this->cacheCheckFailures( $lockSrv ) ) { + return false; // recent failure to connect + } + try { + $this->getConnection( $lockSrv ); + } catch ( DBError $e ) { + $this->cacheRecordFailure( $lockSrv ); + + return false; // failed to connect + } + + return true; + } + + /** + * Get (or reuse) a connection to a lock DB + * + * @param string $lockDb + * @return IDatabase + * @throws DBError + * @throws UnexpectedValueException + */ + protected function getConnection( $lockDb ) { + if ( !isset( $this->conns[$lockDb] ) ) { + if ( $this->dbServers[$lockDb] instanceof IDatabase ) { + // Direct injected connection hande for $lockDB + $db = $this->dbServers[$lockDb]; + } elseif ( is_array( $this->dbServers[$lockDb] ) ) { + // Parameters to construct a new database connection + $config = $this->dbServers[$lockDb]; + $db = DatabaseBase::factory( $config['type'], $config ); + } else { + throw new UnexpectedValueException( "No server called '$lockDb'." ); + } + + $db->clearFlag( DBO_TRX ); + # If the connection drops, try to avoid letting the DB rollback + # and release the locks before the file operations are finished. + # This won't handle the case of DB server restarts however. + $options = []; + if ( $this->lockExpiry > 0 ) { + $options['connTimeout'] = $this->lockExpiry; + } + $db->setSessionOptions( $options ); + $this->initConnection( $lockDb, $db ); + + $this->conns[$lockDb] = $db; + } + + return $this->conns[$lockDb]; + } + + /** + * Do additional initialization for new lock DB connection + * + * @param string $lockDb + * @param IDatabase $db + * @throws DBError + */ + protected function initConnection( $lockDb, IDatabase $db ) { + } + + /** + * Checks if the DB has not recently had connection/query errors. + * This just avoids wasting time on doomed connection attempts. + * + * @param string $lockDb + * @return bool + */ + protected function cacheCheckFailures( $lockDb ) { + return ( $this->safeDelay > 0 ) + ? !$this->statusCache->get( $this->getMissKey( $lockDb ) ) + : true; + } + + /** + * Log a lock request failure to the cache + * + * @param string $lockDb + * @return bool Success + */ + protected function cacheRecordFailure( $lockDb ) { + return ( $this->safeDelay > 0 ) + ? $this->statusCache->set( $this->getMissKey( $lockDb ), 1, $this->safeDelay ) + : true; + } + + /** + * Get a cache key for recent query misses for a DB + * + * @param string $lockDb + * @return string + */ + protected function getMissKey( $lockDb ) { + return 'dblockmanager:downservers:' . str_replace( ' ', '_', $lockDb ); + } + + /** + * Make sure remaining locks get cleared for sanity + */ + function __destruct() { + $this->releaseAllLocks(); + foreach ( $this->conns as $db ) { + $db->close(); + } + } +} diff --git a/includes/libs/lockmanager/PostgreSqlLockManager.php b/includes/libs/lockmanager/PostgreSqlLockManager.php new file mode 100644 index 0000000000..d6b1ce822d --- /dev/null +++ b/includes/libs/lockmanager/PostgreSqlLockManager.php @@ -0,0 +1,79 @@ + self::LOCK_SH, + self::LOCK_UW => self::LOCK_SH, + self::LOCK_EX => self::LOCK_EX + ]; + + protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) { + $status = StatusValue::newGood(); + if ( !count( $paths ) ) { + return $status; // nothing to lock + } + + $db = $this->getConnection( $lockSrv ); // checked in isServerUp() + $bigints = array_unique( array_map( + function ( $key ) { + return Wikimedia\base_convert( substr( $key, 0, 15 ), 16, 10 ); + }, + array_map( [ $this, 'sha1Base16Absolute' ], $paths ) + ) ); + + // Try to acquire all the locks... + $fields = []; + foreach ( $bigints as $bigint ) { + $fields[] = ( $type == self::LOCK_SH ) + ? "pg_try_advisory_lock_shared({$db->addQuotes( $bigint )}) AS K$bigint" + : "pg_try_advisory_lock({$db->addQuotes( $bigint )}) AS K$bigint"; + } + $res = $db->query( 'SELECT ' . implode( ', ', $fields ), __METHOD__ ); + $row = $res->fetchRow(); + + if ( in_array( 'f', $row ) ) { + // Release any acquired locks if some could not be acquired... + $fields = []; + foreach ( $row as $kbigint => $ok ) { + if ( $ok === 't' ) { // locked + $bigint = substr( $kbigint, 1 ); // strip off the "K" + $fields[] = ( $type == self::LOCK_SH ) + ? "pg_advisory_unlock_shared({$db->addQuotes( $bigint )})" + : "pg_advisory_unlock({$db->addQuotes( $bigint )})"; + } + } + if ( count( $fields ) ) { + $db->query( 'SELECT ' . implode( ', ', $fields ), __METHOD__ ); + } + foreach ( $paths as $path ) { + $status->fatal( 'lockmanager-fail-acquirelock', $path ); + } + } + + return $status; + } + + /** + * @see QuorumLockManager::releaseAllLocks() + * @return StatusValue + */ + protected function releaseAllLocks() { + $status = StatusValue::newGood(); + + foreach ( $this->conns as $lockDb => $db ) { + try { + $db->query( "SELECT pg_advisory_unlock_all()", __METHOD__ ); + } catch ( DBError $e ) { + $status->fatal( 'lockmanager-fail-db-release', $lockDb ); + } + } + + return $status; + } +}