From: Aaron Schulz Date: Tue, 24 Dec 2013 23:22:58 +0000 (-0800) Subject: Added per-node sequential ID method and made SquidUpdate use it X-Git-Tag: 1.31.0-rc.0~17410^2 X-Git-Url: http://git.cyclocoop.org/%28?a=commitdiff_plain;h=f8f452554e6e2f8319ab1c420d4d76c3a5022d88;p=lhc%2Fweb%2Fwiklou.git Added per-node sequential ID method and made SquidUpdate use it * This replaces a live wmf hack Change-Id: Ie01fd27386d5d7197968b6a8157b7220ba685f7b --- diff --git a/includes/deferred/SquidUpdate.php b/includes/deferred/SquidUpdate.php index bac9f10d65..85592e87bd 100644 --- a/includes/deferred/SquidUpdate.php +++ b/includes/deferred/SquidUpdate.php @@ -216,6 +216,11 @@ class SquidUpdate { // Remove duplicate URLs from collection $urlArr = array_unique( $urlArr ); + // Get sequential trx IDs for packet loss counting + $ids = UIDGenerator::newSequentialPerNodeIDs( + 'squidhtcppurge', 32, count( $urlArr ), UIDGenerator::QUICK_VOLATILE + ); + foreach ( $urlArr as $url ) { if ( !is_string( $url ) ) { wfProfileOut( __METHOD__ ); @@ -243,7 +248,8 @@ class SquidUpdate { // Construct a minimal HTCP request diagram // as per RFC 2756 // Opcode 'CLR', no response desired, no auth - $htcpTransID = rand(); + $htcpTransID = current( $ids ); + next( $ids ); $htcpSpecifier = pack( 'na4na*na8n', 4, 'HEAD', strlen( $url ), $url, diff --git a/includes/utils/UIDGenerator.php b/includes/utils/UIDGenerator.php index 10ff957bca..47cef8b6f4 100644 --- a/includes/utils/UIDGenerator.php +++ b/includes/utils/UIDGenerator.php @@ -40,6 +40,7 @@ class UIDGenerator { protected $fileHandles = array(); // cache file handles const QUICK_RAND = 1; // get randomness from fast and insecure sources + const QUICK_VOLATILE = 2; // use an APC like in-memory counter if available protected function __construct() { $idFile = wfTempDir() . '/mw-' . __CLASS__ . '-UID-nodeid'; @@ -214,6 +215,95 @@ class UIDGenerator { return str_replace( '-', '', self::newUUIDv4( $flags ) ); } + /** + * Return an ID that is sequential *only* for this node and bucket + * + * These IDs are suitable for per-host sequence numbers, e.g. for some packet protocols. + * If UIDGenerator::QUICK_VOLATILE is used the counter might reset on server restart. + * + * @param string $bucket Arbitrary bucket name (should be ASCII) + * @param integer $bits Bit size (<=48) of resulting numbers before wrap-around + * @param integer $flags (supports UIDGenerator::QUICK_VOLATILE) + * @return float Integer value as float + * @since 1.23 + */ + public static function newSequentialPerNodeID( $bucket, $bits = 48, $flags = 0 ) { + return current( self::newSequentialPerNodeIDs( $bucket, $bits, 1, $flags ) ); + } + + /** + * Return IDs that are sequential *only* for this node and bucket + * + * @see UIDGenerator::newSequentialPerNodeID() + * @param string $bucket Arbitrary bucket name (should be ASCII) + * @param integer $bits Bit size (16 to 48) of resulting numbers before wrap-around + * @param integer $count Number of IDs to return (1 to 10000) + * @param integer $flags (supports UIDGenerator::QUICK_VOLATILE) + * @return array Ordered list of float integer values + * @since 1.23 + */ + public static function newSequentialPerNodeIDs( $bucket, $bits, $count, $flags = 0 ) { + if ( $count <= 0 ) { + return array(); // nothing to do + } elseif ( $count > 10000 ) { + throw new MWException( "Number of requested IDs ($count) is too high." ); + } elseif ( $bits < 16 || $bits > 48 ) { + throw new MWException( "Requested bit size ($bits) is out of range." ); + } + + $counter = null; // post-increment persistent counter value + + // Use APC/eAccelerator/xcache if requested, available, and not in CLI mode; + // Counter values would not survive accross script instances in CLI mode. + $cache = null; + if ( ( $flags & self::QUICK_VOLATILE ) && PHP_SAPI !== 'cli' ) { + try { + $cache = ObjectCache::newAccelerator( array() ); + } catch ( MWException $e ) {} // not supported + } + if ( $cache ) { + $counter = $cache->incr( $bucket, $count ); + if ( $counter === false ) { + if ( !$cache->add( $bucket, $count ) ) { + throw new MWException( 'Unable to set value to ' . get_class( $cache ) ); + } + $counter = $count; + } + } + + // Note: use of fmod() avoids "division by zero" on 32 bit machines + if ( $counter === null ) { + $path = wfTempDir() . '/mw-' . __CLASS__ . '-' . rawurlencode( $bucket ) . '-48'; + $handle = fopen( $path, 'cb+' ); + // Acquire the UID lock file + if ( $handle === false ) { + throw new MWException( "Could not open '{$path}'." ); + } elseif ( !flock( $handle, LOCK_EX ) ) { + fclose( $handle ); + throw new MWException( "Could not acquire '{$path}'." ); + } + // Fetch the counter value and increment it... + rewind( $handle ); + $counter = floor( trim( fgets( $handle ) ) ) + $count; // fetch as float + // Write back the new counter value + ftruncate( $handle, 0 ); + rewind( $handle ); + fwrite( $handle, fmod( $counter, pow( 2, 48 ) ) ); // warp-around as needed + fflush( $handle ); + // Release the UID lock file + flock( $handle, LOCK_UN ); + fclose( $handle ); + } + $ids = array(); + $divisor = pow( 2, $bits ); + $currentId = floor( $counter - $count ); // pre-increment counter value + for ( $i = 0; $i < $count; ++$i ) { + $ids[] = fmod( ++$currentId, $divisor ); + } + + return $ids; + } + /** * Get a (time,counter,clock sequence) where (time,counter) is higher * than any previous (time,counter) value for the given clock sequence. @@ -237,6 +327,7 @@ class UIDGenerator { if ( $handle === false ) { throw new MWException( "Could not open '{$this->$lockFile}'." ); } elseif ( !flock( $handle, LOCK_EX ) ) { + fclose( $handle ); throw new MWException( "Could not acquire '{$this->$lockFile}'." ); } // Get the current timestamp, clock sequence number, last time, and counter diff --git a/tests/phpunit/includes/utils/UIDGeneratorTest.php b/tests/phpunit/includes/utils/UIDGeneratorTest.php index 8f78ae5126..1a1bbaf286 100644 --- a/tests/phpunit/includes/utils/UIDGeneratorTest.php +++ b/tests/phpunit/includes/utils/UIDGeneratorTest.php @@ -95,4 +95,32 @@ class UIDGeneratorTest extends MediaWikiTestCase { } } + /** + * @covers UIDGenerator::newSequentialPerNodeID + */ + public function testNewSequentialID() { + $id1 = UIDGenerator::newSequentialPerNodeID( 'test', 32 ); + $id2 = UIDGenerator::newSequentialPerNodeID( 'test', 32 ); + + $this->assertType( 'float', $id1, "ID returned as float" ); + $this->assertType( 'float', $id2, "ID returned as float" ); + $this->assertGreaterThan( 0, $id1, "ID greater than 1" ); + $this->assertGreaterThan( $id1, $id2, "IDs increasing in value" ); + } + + /** + * @covers UIDGenerator::newSequentialPerNodeIDs + */ + public function testNewSequentialIDs() { + $ids = UIDGenerator::newSequentialPerNodeIDs( 'test', 32, 5 ); + $lastId = null; + foreach ( $ids as $id ) { + $this->assertType( 'float', $id, "ID returned as float" ); + $this->assertGreaterThan( 0, $id, "ID greater than 1" ); + if ( $lastId ) { + $this->assertGreaterThan( $lastId, $id, "IDs increasing in value" ); + } + $lastId = $id; + } + } }