From 987d2e4c7728193c2ea3d7a0a10d765c9480ae3c Mon Sep 17 00:00:00 2001 From: Kunal Mehta Date: Wed, 12 Nov 2014 10:34:11 -0800 Subject: [PATCH] Use cdb library and provide a back-compat layer The new cdb library is pulled in via composer. Since the library uses namespaces, a backwards-compatability layer is provided for the old class names: * CdbReader * CdbWriter * CdbException The PHP/DBA-specific classes should never have been used directly. Depends on I98302bdf1 in mediawiki/vendor Change-Id: I39549ac8540b262cf91f7d1830d36327afb3033d --- autoload.php | 13 +- composer.json | 3 +- .../cdb/CdbException.php => CdbCompat.php} | 24 +- includes/cache/LocalisationCache.php | 3 + includes/interwiki/Interwiki.php | 2 + includes/libs/cdb/CdbFunctions.php | 97 -------- includes/libs/cdb/CdbReader.php | 82 ------ includes/libs/cdb/CdbReaderDBA.php | 44 ---- includes/libs/cdb/CdbReaderPHP.php | 214 ---------------- includes/libs/cdb/CdbWriter.php | 96 ------- includes/libs/cdb/CdbWriterDBA.php | 52 ---- includes/libs/cdb/CdbWriterPHP.php | 234 ------------------ maintenance/cdb.php | 2 + tests/phpunit/includes/libs/cdb/CdbTest.php | 90 ------- 14 files changed, 33 insertions(+), 923 deletions(-) rename includes/{libs/cdb/CdbException.php => CdbCompat.php} (64%) delete mode 100644 includes/libs/cdb/CdbFunctions.php delete mode 100644 includes/libs/cdb/CdbReader.php delete mode 100644 includes/libs/cdb/CdbReaderDBA.php delete mode 100644 includes/libs/cdb/CdbReaderPHP.php delete mode 100644 includes/libs/cdb/CdbWriter.php delete mode 100644 includes/libs/cdb/CdbWriterDBA.php delete mode 100644 includes/libs/cdb/CdbWriterPHP.php delete mode 100644 tests/phpunit/includes/libs/cdb/CdbTest.php diff --git a/autoload.php b/autoload.php index bc039e9bfe..5239edcacc 100644 --- a/autoload.php +++ b/autoload.php @@ -189,14 +189,9 @@ $wgAutoloadLocalClasses = array( 'CategoryPage' => __DIR__ . '/includes/page/CategoryPage.php', 'CategoryPager' => __DIR__ . '/includes/specials/SpecialCategories.php', 'CategoryViewer' => __DIR__ . '/includes/CategoryViewer.php', - 'CdbException' => __DIR__ . '/includes/libs/cdb/CdbException.php', - 'CdbFunctions' => __DIR__ . '/includes/libs/cdb/CdbFunctions.php', - 'CdbReader' => __DIR__ . '/includes/libs/cdb/CdbReader.php', - 'CdbReaderDBA' => __DIR__ . '/includes/libs/cdb/CdbReaderDBA.php', - 'CdbReaderPHP' => __DIR__ . '/includes/libs/cdb/CdbReaderPHP.php', - 'CdbWriter' => __DIR__ . '/includes/libs/cdb/CdbWriter.php', - 'CdbWriterDBA' => __DIR__ . '/includes/libs/cdb/CdbWriterDBA.php', - 'CdbWriterPHP' => __DIR__ . '/includes/libs/cdb/CdbWriterPHP.php', + 'CdbException' => __DIR__ . '/includes/CdbCompat.php', + 'CdbReader' => __DIR__ . '/includes/CdbCompat.php', + 'CdbWriter' => __DIR__ . '/includes/CdbCompat.php', 'CgzCopyTransaction' => __DIR__ . '/maintenance/storage/recompressTracked.php', 'ChangePassword' => __DIR__ . '/maintenance/changePassword.php', 'ChangeTags' => __DIR__ . '/includes/ChangeTags.php', @@ -1316,4 +1311,4 @@ $wgAutoloadLocalClasses = array( 'lessc_formatter_lessjs' => __DIR__ . '/includes/libs/lessc.inc.php', 'lessc_parser' => __DIR__ . '/includes/libs/lessc.inc.php', 'profile_point' => __DIR__ . '/profileinfo.php', -); \ No newline at end of file +); diff --git a/composer.json b/composer.json index 61f30cee08..834e14fd35 100644 --- a/composer.json +++ b/composer.json @@ -18,7 +18,8 @@ "require": { "php": ">=5.3.3", "psr/log": "1.0.0", - "cssjanus/cssjanus": "1.1.0" + "cssjanus/cssjanus": "1.1.0", + "cdb/cdb": "1.0.0" }, "require-dev": { "phpunit/phpunit": "*" diff --git a/includes/libs/cdb/CdbException.php b/includes/CdbCompat.php similarity index 64% rename from includes/libs/cdb/CdbException.php rename to includes/CdbCompat.php index 6cda529d28..0c00b39e0e 100644 --- a/includes/libs/cdb/CdbException.php +++ b/includes/CdbCompat.php @@ -18,9 +18,25 @@ * @file */ +/*** + * This file contains a set of backwards-compatability class names + * after the cdb functions were moved out into a separate library + * and put under a proper namespace + * + * @since 1.25 + */ + +/** + * @deprecated since 1.25 + */ +abstract class CdbReader extends \Cdb\Reader {} + +/** + * @deprecated since 1.25 + */ +abstract class CdbWriter extends \Cdb\Writer {} + /** - * Exception for Cdb errors. - * This explicitly doesn't subclass MWException to encourage reuse. + * @deprecated since 1.25 */ -class CdbException extends Exception { -} +class CdbException extends \Cdb\Exception {} diff --git a/includes/cache/LocalisationCache.php b/includes/cache/LocalisationCache.php index ae27fba3a4..2a3cd38fd7 100644 --- a/includes/cache/LocalisationCache.php +++ b/includes/cache/LocalisationCache.php @@ -20,6 +20,9 @@ * @file */ +use Cdb\Exception as CdbException; +use Cdb\Reader as CdbReader; +use Cdb\Writer as CdbWriter; /** * Class for caching the contents of localisation files, Messages*.php * and *.i18n.php. diff --git a/includes/interwiki/Interwiki.php b/includes/interwiki/Interwiki.php index 55b2506961..37a9fcfc0a 100644 --- a/includes/interwiki/Interwiki.php +++ b/includes/interwiki/Interwiki.php @@ -19,6 +19,8 @@ * * @file */ +use \Cdb\Exception as CdbException; +use \Cdb\Reader as CdbReader; /** * The interwiki class diff --git a/includes/libs/cdb/CdbFunctions.php b/includes/libs/cdb/CdbFunctions.php deleted file mode 100644 index e74924cb7e..0000000000 --- a/includes/libs/cdb/CdbFunctions.php +++ /dev/null @@ -1,97 +0,0 @@ -> $b ) | ( 0x40000000 >> ( $b - 1 ) ); - } else { - return $a >> $b; - } - } - - /** - * The CDB hash function. - * - * @param string $s - * - * @return int - */ - public static function hash( $s ) { - $h = 5381; - $len = strlen( $s ); - for ( $i = 0; $i < $len; $i++ ) { - $h5 = ( $h << 5 ) & 0xffffffff; - // Do a 32-bit sum - // Inlined here for speed - $sum = ( $h & 0x3fffffff ) + ( $h5 & 0x3fffffff ); - $h = - ( - ( $sum & 0x40000000 ? 1 : 0 ) - + ( $h & 0x80000000 ? 2 : 0 ) - + ( $h & 0x40000000 ? 1 : 0 ) - + ( $h5 & 0x80000000 ? 2 : 0 ) - + ( $h5 & 0x40000000 ? 1 : 0 ) - ) << 30 - | ( $sum & 0x3fffffff ); - $h ^= ord( $s[$i] ); - $h &= 0xffffffff; - } - - return $h; - } -} diff --git a/includes/libs/cdb/CdbReader.php b/includes/libs/cdb/CdbReader.php deleted file mode 100644 index 0ca9b9dc29..0000000000 --- a/includes/libs/cdb/CdbReader.php +++ /dev/null @@ -1,82 +0,0 @@ -handle = dba_open( $fileName, 'r-', 'cdb' ); - if ( !$this->handle ) { - throw new CdbException( 'Unable to open CDB file "' . $fileName . '"' ); - } - } - - public function close() { - if ( isset( $this->handle ) ) { - dba_close( $this->handle ); - } - unset( $this->handle ); - } - - public function get( $key ) { - return dba_fetch( $key, $this->handle ); - } -} diff --git a/includes/libs/cdb/CdbReaderPHP.php b/includes/libs/cdb/CdbReaderPHP.php deleted file mode 100644 index e448414f5e..0000000000 --- a/includes/libs/cdb/CdbReaderPHP.php +++ /dev/null @@ -1,214 +0,0 @@ -fileName = $fileName; - $this->handle = fopen( $fileName, 'rb' ); - if ( !$this->handle ) { - throw new CdbException( 'Unable to open CDB file "' . $this->fileName . '".' ); - } - $this->findStart(); - } - - public function close() { - if ( isset( $this->handle ) ) { - fclose( $this->handle ); - } - unset( $this->handle ); - } - - /** - * @param mixed $key - * @return bool|string - */ - public function get( $key ) { - // strval is required - if ( $this->find( strval( $key ) ) ) { - return $this->read( $this->dlen, $this->dpos ); - } else { - return false; - } - } - - /** - * @param string $key - * @param int $pos - * @return bool - */ - protected function match( $key, $pos ) { - $buf = $this->read( strlen( $key ), $pos ); - - return $buf === $key; - } - - protected function findStart() { - $this->loop = 0; - } - - /** - * @throws CdbException - * @param int $length - * @param int $pos - * @return string - */ - protected function read( $length, $pos ) { - if ( fseek( $this->handle, $pos ) == -1 ) { - // This can easily happen if the internal pointers are incorrect - throw new CdbException( - 'Seek failed, file "' . $this->fileName . '" may be corrupted.' ); - } - - if ( $length == 0 ) { - return ''; - } - - $buf = fread( $this->handle, $length ); - if ( $buf === false || strlen( $buf ) !== $length ) { - throw new CdbException( - 'Read from CDB file failed, file "' . $this->fileName . '" may be corrupted.' ); - } - - return $buf; - } - - /** - * Unpack an unsigned integer and throw an exception if it needs more than 31 bits - * @param string $s - * @throws CdbException - * @return mixed - */ - protected function unpack31( $s ) { - $data = unpack( 'V', $s ); - if ( $data[1] > 0x7fffffff ) { - throw new CdbException( - 'Error in CDB file "' . $this->fileName . '", integer too big.' ); - } - - return $data[1]; - } - - /** - * Unpack a 32-bit signed integer - * @param string $s - * @return int - */ - protected function unpackSigned( $s ) { - $data = unpack( 'va/vb', $s ); - - return $data['a'] | ( $data['b'] << 16 ); - } - - /** - * @param string $key - * @return bool - */ - protected function findNext( $key ) { - if ( !$this->loop ) { - $u = CdbFunctions::hash( $key ); - $buf = $this->read( 8, ( $u << 3 ) & 2047 ); - $this->hslots = $this->unpack31( substr( $buf, 4 ) ); - if ( !$this->hslots ) { - return false; - } - $this->hpos = $this->unpack31( substr( $buf, 0, 4 ) ); - $this->khash = $u; - $u = CdbFunctions::unsignedShiftRight( $u, 8 ); - $u = CdbFunctions::unsignedMod( $u, $this->hslots ); - $u <<= 3; - $this->kpos = $this->hpos + $u; - } - - while ( $this->loop < $this->hslots ) { - $buf = $this->read( 8, $this->kpos ); - $pos = $this->unpack31( substr( $buf, 4 ) ); - if ( !$pos ) { - return false; - } - $this->loop += 1; - $this->kpos += 8; - if ( $this->kpos == $this->hpos + ( $this->hslots << 3 ) ) { - $this->kpos = $this->hpos; - } - $u = $this->unpackSigned( substr( $buf, 0, 4 ) ); - if ( $u === $this->khash ) { - $buf = $this->read( 8, $pos ); - $keyLen = $this->unpack31( substr( $buf, 0, 4 ) ); - if ( $keyLen == strlen( $key ) && $this->match( $key, $pos + 8 ) ) { - // Found - $this->dlen = $this->unpack31( substr( $buf, 4 ) ); - $this->dpos = $pos + 8 + $keyLen; - - return true; - } - } - } - - return false; - } - - /** - * @param mixed $key - * @return bool - */ - protected function find( $key ) { - $this->findStart(); - - return $this->findNext( $key ); - } -} - diff --git a/includes/libs/cdb/CdbWriter.php b/includes/libs/cdb/CdbWriter.php deleted file mode 100644 index b0a90c31d6..0000000000 --- a/includes/libs/cdb/CdbWriter.php +++ /dev/null @@ -1,96 +0,0 @@ -handle ) ) { - $this->close(); - } - } - - /** - * Are we running on Windows? - * @return bool - */ - protected function isWindows() { - return substr( php_uname(), 0, 7 ) == 'Windows'; - } -} diff --git a/includes/libs/cdb/CdbWriterDBA.php b/includes/libs/cdb/CdbWriterDBA.php deleted file mode 100644 index 1de371d425..0000000000 --- a/includes/libs/cdb/CdbWriterDBA.php +++ /dev/null @@ -1,52 +0,0 @@ -realFileName = $fileName; - $this->tmpFileName = $fileName . '.tmp.' . mt_rand( 0, 0x7fffffff ); - $this->handle = dba_open( $this->tmpFileName, 'n', 'cdb_make' ); - if ( !$this->handle ) { - throw new CdbException( 'Unable to open CDB file for write "' . $fileName . '"' ); - } - } - - public function set( $key, $value ) { - return dba_insert( $key, $value, $this->handle ); - } - - public function close() { - if ( isset( $this->handle ) ) { - dba_close( $this->handle ); - } - if ( $this->isWindows() ) { - unlink( $this->realFileName ); - } - if ( !rename( $this->tmpFileName, $this->realFileName ) ) { - throw new CdbException( 'Unable to move the new CDB file into place.' ); - } - unset( $this->handle ); - } -} diff --git a/includes/libs/cdb/CdbWriterPHP.php b/includes/libs/cdb/CdbWriterPHP.php deleted file mode 100644 index bfc0d8780f..0000000000 --- a/includes/libs/cdb/CdbWriterPHP.php +++ /dev/null @@ -1,234 +0,0 @@ -realFileName = $fileName; - $this->tmpFileName = $fileName . '.tmp.' . mt_rand( 0, 0x7fffffff ); - $this->handle = fopen( $this->tmpFileName, 'wb' ); - if ( !$this->handle ) { - $this->throwException( - 'Unable to open CDB file "' . $this->tmpFileName . '" for write.' ); - } - $this->hplist = array(); - $this->numentries = 0; - $this->pos = 2048; // leaving space for the pointer array, 256 * 8 - if ( fseek( $this->handle, $this->pos ) == -1 ) { - $this->throwException( 'fseek failed in file "' . $this->tmpFileName . '".' ); - } - } - - /** - * @param string $key - * @param string $value - */ - public function set( $key, $value ) { - if ( strval( $key ) === '' ) { - // DBA cross-check hack - return; - } - $this->addbegin( strlen( $key ), strlen( $value ) ); - $this->write( $key ); - $this->write( $value ); - $this->addend( strlen( $key ), strlen( $value ), CdbFunctions::hash( $key ) ); - } - - /** - * @throws CdbException - */ - public function close() { - $this->finish(); - if ( isset( $this->handle ) ) { - fclose( $this->handle ); - } - if ( $this->isWindows() && file_exists( $this->realFileName ) ) { - unlink( $this->realFileName ); - } - if ( !rename( $this->tmpFileName, $this->realFileName ) ) { - $this->throwException( 'Unable to move the new CDB file into place.' ); - } - unset( $this->handle ); - } - - /** - * @throws CdbException - * @param string $buf - */ - protected function write( $buf ) { - $len = fwrite( $this->handle, $buf ); - if ( $len !== strlen( $buf ) ) { - $this->throwException( 'Error writing to CDB file "' . $this->tmpFileName . '".' ); - } - } - - /** - * @throws CdbException - * @param int $len - */ - protected function posplus( $len ) { - $newpos = $this->pos + $len; - if ( $newpos > 0x7fffffff ) { - $this->throwException( - 'A value in the CDB file "' . $this->tmpFileName . '" is too large.' ); - } - $this->pos = $newpos; - } - - /** - * @param int $keylen - * @param int $datalen - * @param int $h - */ - protected function addend( $keylen, $datalen, $h ) { - $this->hplist[] = array( - 'h' => $h, - 'p' => $this->pos - ); - - $this->numentries++; - $this->posplus( 8 ); - $this->posplus( $keylen ); - $this->posplus( $datalen ); - } - - /** - * @throws CdbException - * @param int $keylen - * @param int $datalen - */ - protected function addbegin( $keylen, $datalen ) { - if ( $keylen > 0x7fffffff ) { - $this->throwException( 'Key length too long in file "' . $this->tmpFileName . '".' ); - } - if ( $datalen > 0x7fffffff ) { - $this->throwException( 'Data length too long in file "' . $this->tmpFileName . '".' ); - } - $buf = pack( 'VV', $keylen, $datalen ); - $this->write( $buf ); - } - - /** - * @throws CdbException - */ - protected function finish() { - // Hack for DBA cross-check - $this->hplist = array_reverse( $this->hplist ); - - // Calculate the number of items that will be in each hashtable - $counts = array_fill( 0, 256, 0 ); - foreach ( $this->hplist as $item ) { - ++$counts[255 & $item['h']]; - } - - // Fill in $starts with the *end* indexes - $starts = array(); - $pos = 0; - for ( $i = 0; $i < 256; ++$i ) { - $pos += $counts[$i]; - $starts[$i] = $pos; - } - - // Excessively clever and indulgent code to simultaneously fill $packedTables - // with the packed hashtables, and adjust the elements of $starts - // to actually point to the starts instead of the ends. - $packedTables = array_fill( 0, $this->numentries, false ); - foreach ( $this->hplist as $item ) { - $packedTables[--$starts[255 & $item['h']]] = $item; - } - - $final = ''; - for ( $i = 0; $i < 256; ++$i ) { - $count = $counts[$i]; - - // The size of the hashtable will be double the item count. - // The rest of the slots will be empty. - $len = $count + $count; - $final .= pack( 'VV', $this->pos, $len ); - - $hashtable = array(); - for ( $u = 0; $u < $len; ++$u ) { - $hashtable[$u] = array( 'h' => 0, 'p' => 0 ); - } - - // Fill the hashtable, using the next empty slot if the hashed slot - // is taken. - for ( $u = 0; $u < $count; ++$u ) { - $hp = $packedTables[$starts[$i] + $u]; - $where = CdbFunctions::unsignedMod( - CdbFunctions::unsignedShiftRight( $hp['h'], 8 ), $len ); - while ( $hashtable[$where]['p'] ) { - if ( ++$where == $len ) { - $where = 0; - } - } - $hashtable[$where] = $hp; - } - - // Write the hashtable - for ( $u = 0; $u < $len; ++$u ) { - $buf = pack( 'vvV', - $hashtable[$u]['h'] & 0xffff, - CdbFunctions::unsignedShiftRight( $hashtable[$u]['h'], 16 ), - $hashtable[$u]['p'] ); - $this->write( $buf ); - $this->posplus( 8 ); - } - } - - // Write the pointer array at the start of the file - rewind( $this->handle ); - if ( ftell( $this->handle ) != 0 ) { - $this->throwException( 'Error rewinding to start of file "' . $this->tmpFileName . '".' ); - } - $this->write( $final ); - } - - /** - * Clean up the temp file and throw an exception - * - * @param string $msg - * @throws CdbException - */ - protected function throwException( $msg ) { - if ( $this->handle ) { - fclose( $this->handle ); - unlink( $this->tmpFileName ); - } - throw new CdbException( $msg ); - } -} diff --git a/maintenance/cdb.php b/maintenance/cdb.php index 86c686b412..2e252adb03 100644 --- a/maintenance/cdb.php +++ b/maintenance/cdb.php @@ -21,6 +21,8 @@ * @todo document * @ingroup Maintenance */ +use \Cdb\Exception as CdbException; +use \Cdb\Reader as CdbReader; /** */ require_once __DIR__ . '/commandLine.inc'; diff --git a/tests/phpunit/includes/libs/cdb/CdbTest.php b/tests/phpunit/includes/libs/cdb/CdbTest.php deleted file mode 100644 index 487ee1fc82..0000000000 --- a/tests/phpunit/includes/libs/cdb/CdbTest.php +++ /dev/null @@ -1,90 +0,0 @@ -markTestSkipped( 'Native CDB support is not available' ); - } - } - - /** - * @group medium - */ - public function testCdb() { - $dir = wfTempDir(); - if ( !is_writable( $dir ) ) { - $this->markTestSkipped( "Temp dir isn't writable" ); - } - - $phpcdbfile = $this->getNewTempFile(); - $dbacdbfile = $this->getNewTempFile(); - - $w1 = new CdbWriterPHP( $phpcdbfile ); - $w2 = new CdbWriterDBA( $dbacdbfile ); - - $data = array(); - for ( $i = 0; $i < 1000; $i++ ) { - $key = $this->randomString(); - $value = $this->randomString(); - $w1->set( $key, $value ); - $w2->set( $key, $value ); - - if ( !isset( $data[$key] ) ) { - $data[$key] = $value; - } - } - - $w1->close(); - $w2->close(); - - $this->assertEquals( - md5_file( $phpcdbfile ), - md5_file( $dbacdbfile ), - 'same hash' - ); - - $r1 = new CdbReaderPHP( $phpcdbfile ); - $r2 = new CdbReaderDBA( $dbacdbfile ); - - foreach ( $data as $key => $value ) { - if ( $key === '' ) { - // Known bug - continue; - } - $v1 = $r1->get( $key ); - $v2 = $r2->get( $key ); - - $v1 = $v1 === false ? '(not found)' : $v1; - $v2 = $v2 === false ? '(not found)' : $v2; - - # cdbAssert( 'Mismatch', $key, $v1, $v2 ); - $this->cdbAssert( "PHP error", $key, $v1, $value ); - $this->cdbAssert( "DBA error", $key, $v2, $value ); - } - } - - private function randomString() { - $len = mt_rand( 0, 10 ); - $s = ''; - for ( $j = 0; $j < $len; $j++ ) { - $s .= chr( mt_rand( 0, 255 ) ); - } - - return $s; - } - - private function cdbAssert( $msg, $key, $v1, $v2 ) { - $this->assertEquals( - $v2, - $v1, - $msg . ', k=' . bin2hex( $key ) - ); - } -} -- 2.20.1