From: Tim Starling Date: Mon, 4 Jul 2011 12:01:10 +0000 (+0000) Subject: Allow SqlBagOStuff data to be split over many tables, to avoid lock contention perfor... X-Git-Tag: 1.31.0-rc.0~29088 X-Git-Url: http://git.cyclocoop.org/%22%20.%20generer_url_aide%28?a=commitdiff_plain;h=9f4064540ebe4188e0f2c9d3ac25791c8c9ff828;p=lhc%2Fweb%2Fwiklou.git Allow SqlBagOStuff data to be split over many tables, to avoid lock contention performance issues on servers with a high write load. See bugs.mysql.com/bug.php?id=61735 and http://bugs.mysql.com/bug.php?id=61736 . --- diff --git a/includes/objectcache/SqlBagOStuff.php b/includes/objectcache/SqlBagOStuff.php index 7be345b3d5..4174913b22 100644 --- a/includes/objectcache/SqlBagOStuff.php +++ b/includes/objectcache/SqlBagOStuff.php @@ -19,6 +19,8 @@ class SqlBagOStuff extends BagOStuff { var $serverInfo; var $lastExpireAll = 0; var $purgePeriod = 100; + var $shards = 1; + var $tableName = 'objectcache'; /** * Constructor. Parameters are: @@ -32,6 +34,16 @@ class SqlBagOStuff extends BagOStuff { * request. If this is set to zero, purging will never be * done. * + * - tableName: The table name to use, default is "objectcache". + * + * - shards: The number of tables to use for data storage. If this is + * more than 1, table names will be formed in the style + * objectcacheNNN where NNN is the shard index, between 0 and + * shards-1. The number of digits will be the minimum number + * required to hold the largest shard index. Data will be + * distributed across all tables by key hash. This is for + * MySQL bugs 61735 and 61736. + * * @param $params array */ public function __construct( $params ) { @@ -42,6 +54,12 @@ class SqlBagOStuff extends BagOStuff { if ( isset( $params['purgePeriod'] ) ) { $this->purgePeriod = intval( $params['purgePeriod'] ); } + if ( isset( $params['tableName'] ) ) { + $this->tableName = $params['tableName']; + } + if ( isset( $params['shards'] ) ) { + $this->shards = intval( $params['shards'] ); + } } /** @@ -72,11 +90,37 @@ class SqlBagOStuff extends BagOStuff { return $this->db; } + /** + * Get the table name for a given key + */ + protected function getTableByKey( $key ) { + if ( $this->shards > 1 ) { + $hash = hexdec( substr( md5( $key ), 0, 8 ) ) & 0x7fffffff; + return $this->getTableByShard( $hash % $this->shards ); + } else { + return $this->tableName; + } + } + + /** + * Get the table name for a given shard index + */ + protected function getTableByShard( $index ) { + if ( $this->shards > 1 ) { + $decimals = strlen( $this->shards - 1 ); + return $this->tableName . + sprintf( "%0{$decimals}d", $index ); + } else { + return $this->tableName; + } + } + public function get( $key ) { # expire old entries if any $this->garbageCollect(); $db = $this->getDB(); - $row = $db->selectRow( 'objectcache', array( 'value', 'exptime' ), + $tableName = $this->getTableByKey( $key ); + $row = $db->selectRow( $tableName, array( 'value', 'exptime' ), array( 'keyname' => $key ), __METHOD__ ); if ( !$row ) { @@ -92,7 +136,7 @@ class SqlBagOStuff extends BagOStuff { $db->begin(); # Put the expiry time in the WHERE condition to avoid deleting a # newly-inserted value - $db->delete( 'objectcache', + $db->delete( $tableName, array( 'keyname' => $key, 'exptime' => $row->exptime @@ -129,7 +173,9 @@ class SqlBagOStuff extends BagOStuff { $db->begin(); // (bug 24425) use a replace if the db supports it instead of // delete/insert to avoid clashes with conflicting keynames - $db->replace( 'objectcache', array( 'keyname' ), + $db->replace( + $this->getTableByKey( $key ), + array( 'keyname' ), array( 'keyname' => $key, 'value' => $db->encodeBlob( $this->serialize( $value ) ), @@ -150,7 +196,10 @@ class SqlBagOStuff extends BagOStuff { try { $db->begin(); - $db->delete( 'objectcache', array( 'keyname' => $key ), __METHOD__ ); + $db->delete( + $this->getTableByKey( $key ), + array( 'keyname' => $key ), + __METHOD__ ); $db->commit(); } catch ( DBQueryError $e ) { $this->handleWriteError( $e ); @@ -163,19 +212,24 @@ class SqlBagOStuff extends BagOStuff { public function incr( $key, $step = 1 ) { $db = $this->getDB(); + $tableName = $this->getTableByKey( $key ); $step = intval( $step ); try { $db->begin(); - $row = $db->selectRow( 'objectcache', array( 'value', 'exptime' ), - array( 'keyname' => $key ), __METHOD__, array( 'FOR UPDATE' ) ); + $row = $db->selectRow( + $tableName, + array( 'value', 'exptime' ), + array( 'keyname' => $key ), + __METHOD__, + array( 'FOR UPDATE' ) ); if ( $row === false ) { // Missing $db->commit(); return null; } - $db->delete( 'objectcache', array( 'keyname' => $key ), __METHOD__ ); + $db->delete( $tableName, array( 'keyname' => $key ), __METHOD__ ); if ( $this->isExpired( $row->exptime ) ) { // Expired, do not reinsert $db->commit(); @@ -185,7 +239,7 @@ class SqlBagOStuff extends BagOStuff { $oldValue = intval( $this->unserialize( $db->decodeBlob( $row->value ) ) ); $newValue = $oldValue + $step; - $db->insert( 'objectcache', + $db->insert( $tableName, array( 'keyname' => $key, 'value' => $db->encodeBlob( $this->serialize( $newValue ) ), @@ -208,11 +262,14 @@ class SqlBagOStuff extends BagOStuff { public function keys() { $db = $this->getDB(); - $res = $db->select( 'objectcache', array( 'keyname' ), false, __METHOD__ ); $result = array(); - foreach ( $res as $row ) { - $result[] = $row->keyname; + for ( $i = 0; $i < $this->shards; $i++ ) { + $res = $db->select( $this->getTableByShard( $i ), + array( 'keyname' ), false, __METHOD__ ); + foreach ( $res as $row ) { + $result[] = $row->keyname; + } } return $result; @@ -252,9 +309,14 @@ class SqlBagOStuff extends BagOStuff { $now = $db->timestamp(); try { - $db->begin(); - $db->delete( 'objectcache', array( 'exptime < ' . $db->addQuotes( $now ) ), __METHOD__ ); - $db->commit(); + for ( $i = 0; $i < $this->shards; $i++ ) { + $db->begin(); + $db->delete( + $this->getTableByShard( $i ), + array( 'exptime < ' . $db->addQuotes( $now ) ), + __METHOD__ ); + $db->commit(); + } } catch ( DBQueryError $e ) { $this->handleWriteError( $e ); } @@ -264,9 +326,11 @@ class SqlBagOStuff extends BagOStuff { $db = $this->getDB(); try { - $db->begin(); - $db->delete( 'objectcache', '*', __METHOD__ ); - $db->commit(); + for ( $i = 0; $i < $this->shards; $i++ ) { + $db->begin(); + $db->delete( $this->getTableByShard( $i ), '*', __METHOD__ ); + $db->commit(); + } } catch ( DBQueryError $e ) { $this->handleWriteError( $e ); } @@ -328,6 +392,27 @@ class SqlBagOStuff extends BagOStuff { wfDebug( __METHOD__ . ": ignoring query error\n" ); $db->ignoreErrors( false ); } + + /** + * Create shard tables. For use from eval.php. + */ + public function createTables() { + $db = $this->getDB(); + if ( $db->getType() !== 'mysql' + || version_compare( $db->getServerVersion(), '4.1.0', '<' ) ) + { + throw new MWException( __METHOD__ . ' is not supported on this DB server' ); + } + + for ( $i = 0; $i < $this->shards; $i++ ) { + $db->begin(); + $db->query( + 'CREATE TABLE ' . $db->tableName( $this->getTableByShard( $i ) ) . + ' LIKE ' . $db->tableName( 'objectcache' ), + __METHOD__ ); + $db->commit(); + } + } } /**