From: Aaron Schulz Date: Wed, 6 Feb 2013 22:45:33 +0000 (-0800) Subject: [JobQueue] Added unit tests for job queue code. X-Git-Tag: 1.31.0-rc.0~20774^2 X-Git-Url: http://git.cyclocoop.org/url?a=commitdiff_plain;h=5e81bffce0f2769ad91f7722b0f49d58fe4db1c1;p=lhc%2Fweb%2Fwiklou.git [JobQueue] Added unit tests for job queue code. Change-Id: I13f2dd555ea807b5d13ea9fbee6626d6a8f5aab3 --- diff --git a/includes/job/JobQueue.php b/includes/job/JobQueue.php index 4e0acd24dd..7ce654b3be 100644 --- a/includes/job/JobQueue.php +++ b/includes/job/JobQueue.php @@ -352,4 +352,21 @@ abstract class JobQueue { protected function doGetPeriodicTasks() { return array(); } + + /** + * Clear any process and persistent caches + * + * @return void + */ + final public function flushCaches() { + wfProfileIn( __METHOD__ ); + $this->doFlushCaches(); + wfProfileOut( __METHOD__ ); + } + + /** + * @see JobQueue::flushCaches() + * @return void + */ + protected function doFlushCaches() {} } diff --git a/includes/job/JobQueueDB.php b/includes/job/JobQueueDB.php index 4df5c0755a..6e42305b18 100644 --- a/includes/job/JobQueueDB.php +++ b/includes/job/JobQueueDB.php @@ -106,6 +106,10 @@ class JobQueueDB extends JobQueue { protected function doGetAcquiredCount() { global $wgMemc; + if ( $this->claimTTL <= 0 ) { + return 0; // no acknowledgements + } + $key = $this->getCacheKey( 'acquiredcount' ); $count = $wgMemc->get( $key ); @@ -566,6 +570,17 @@ class JobQueueDB extends JobQueue { ); } + /** + * @return void + */ + protected function doFlushCaches() { + global $wgMemc; + + foreach ( array( 'empty', 'size', 'acquiredcount' ) as $type ) { + $wgMemc->delete( $this->getCacheKey( $type ) ); + } + } + /** * @return Array (DatabaseBase, ScopedCallback) */ diff --git a/tests/phpunit/includes/jobqueue/JobQueueTest.php b/tests/phpunit/includes/jobqueue/JobQueueTest.php new file mode 100644 index 0000000000..46ba9743f6 --- /dev/null +++ b/tests/phpunit/includes/jobqueue/JobQueueTest.php @@ -0,0 +1,270 @@ +tablesUsed[] = 'job'; + } + + protected function setUp() { + global $wgMemc; + parent::setUp(); + $this->old['wgMemc'] = $wgMemc; + $wgMemc = new HashBagOStuff(); + $this->queueRand = JobQueue::factory( array( 'class' => 'JobQueueDB', + 'wiki' => wfWikiID(), 'type' => 'null', 'order' => 'random' ) ); + $this->queueRandTTL = JobQueue::factory( array( 'class' => 'JobQueueDB', + 'wiki' => wfWikiID(), 'type' => 'null', 'order' => 'random', 'claimTTL' => 10 ) ); + $this->queueFifo = JobQueue::factory( array( 'class' => 'JobQueueDB', + 'wiki' => wfWikiID(), 'type' => 'null', 'order' => 'fifo' ) ); + $this->queueFifoTTL = JobQueue::factory( array( 'class' => 'JobQueueDB', + 'wiki' => wfWikiID(), 'type' => 'null', 'order' => 'fifo', 'claimTTL' => 10 ) ); + } + + protected function tearDown() { + global $wgMemc; + parent::tearDown(); + foreach ( array( 'queueRand', 'queueRandTTL', 'queueFifo', 'queueFifoTTL' ) as $q ) { + do { + $job = $this->$q->pop(); + if ( $job ) $this->$q->ack( $job ); + } while ( $job ); + } + $this->queueRand = null; + $this->queueRandTTL = null; + $this->queueFifo = null; + $this->queueFifoTTL = null; + $wgMemc = $this->old['wgMemc']; + } + + /** + * @dataProvider provider_queueLists + */ + function testProperties( $queue, $order, $recycles, $desc ) { + $queue = $this->$queue; + + $this->assertEquals( wfWikiID(), $queue->getWiki(), "Proper wiki ID ($desc)" ); + $this->assertEquals( 'null', $queue->getType(), "Proper job type ($desc)" ); + } + + /** + * @dataProvider provider_queueLists + */ + function testBasicOperations( $queue, $order, $recycles, $desc ) { + $queue = $this->$queue; + $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" ); + + $queue->flushCaches(); + $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" ); + $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" ); + + $this->assertTrue( $queue->push( $this->newJob() ), "Push worked ($desc)" ); + $this->assertTrue( $queue->batchPush( array( $this->newJob() ) ), "Push worked ($desc)" ); + + $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" ); + + $queue->flushCaches(); + $this->assertEquals( 2, $queue->getSize(), "Queue size is correct ($desc)" ); + $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" ); + + $job1 = $queue->pop(); + $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" ); + + $queue->flushCaches(); + $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" ); + + $queue->flushCaches(); + if ( $recycles ) { + $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" ); + } else { + $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" ); + } + + $job2 = $queue->pop(); + $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" ); + $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" ); + + $queue->flushCaches(); + if ( $recycles ) { + $this->assertEquals( 2, $queue->getAcquiredCount(), "Active job count ($desc)" ); + } else { + $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" ); + } + + $queue->ack( $job1 ); + + $queue->flushCaches(); + if ( $recycles ) { + $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" ); + } else { + $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" ); + } + + $queue->ack( $job2 ); + + $queue->flushCaches(); + $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" ); + } + + /** + * @dataProvider provider_queueLists + */ + function testBasicDeduplication( $queue, $order, $recycles, $desc ) { + $queue = $this->$queue; + + $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" ); + + $queue->flushCaches(); + $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" ); + $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" ); + + $this->assertTrue( $queue->batchPush( + array( $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() ) ), + "Push worked ($desc)" ); + + $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" ); + + $queue->flushCaches(); + $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" ); + $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" ); + + $this->assertTrue( $queue->batchPush( + array( $this->newDedupedJob(), $this->newDedupedJob(), $this->newDedupedJob() ) ), + "Push worked ($desc)" ); + + $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" ); + + $queue->flushCaches(); + $this->assertEquals( 1, $queue->getSize(), "Queue size is correct ($desc)" ); + $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" ); + + $job1 = $queue->pop(); + $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" ); + + $queue->flushCaches(); + $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" ); + if ( $recycles ) { + $this->assertEquals( 1, $queue->getAcquiredCount(), "Active job count ($desc)" ); + } else { + $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" ); + } + + $queue->ack( $job1 ); + + $queue->flushCaches(); + $this->assertEquals( 0, $queue->getAcquiredCount(), "Active job count ($desc)" ); + } + + /** + * @dataProvider provider_queueLists + */ + function testRootDeduplication( $queue, $order, $recycles, $desc ) { + $queue = $this->$queue; + + $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" ); + + $queue->flushCaches(); + $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" ); + $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" ); + + $id = wfRandomString( 32 ); + $root1 = Job::newRootJobParams( "nulljobspam:$id" ); // task ID/timestamp + for ( $i=0; $i<5; ++$i ) { + $this->assertTrue( $queue->push( $this->newJob( 0, $root1 ) ), "Push worked ($desc)" ); + } + $queue->deduplicateRootJob( $this->newJob( 0, $root1 ) ); + sleep( 1 ); // roo job timestamp will increase + $root2 = Job::newRootJobParams( "nulljobspam:$id" ); // task ID/timestamp + $this->assertNotEquals( $root1['rootJobTimestamp'], $root2['rootJobTimestamp'], + "Root job signatures have different timestamps." ); + for ( $i=0; $i<5; ++$i ) { + $this->assertTrue( $queue->push( $this->newJob( 0, $root2 ) ), "Push worked ($desc)" ); + } + $queue->deduplicateRootJob( $this->newJob( 0, $root2 ) ); + + $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" ); + + $queue->flushCaches(); + $this->assertEquals( 10, $queue->getSize(), "Queue size is correct ($desc)" ); + $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" ); + + $dupcount = 0; + $jobs = array(); + do { + $job = $queue->pop(); + if ( $job ) { + $jobs[] = $job; + $queue->ack( $job ); + } + if ( $job instanceof DuplicateJob ) ++$dupcount; + } while ( $job ); + + $this->assertEquals( 10, count( $jobs ), "Correct number of jobs popped ($desc)" ); + $this->assertEquals( 5, $dupcount, "Correct number of duplicate jobs popped ($desc)" ); + } + + /** + * @dataProvider provider_fifoQueueLists + */ + function testJobOrder( $queue, $recycles, $desc ) { + $queue = $this->$queue; + + $this->assertTrue( $queue->isEmpty(), "Queue is empty ($desc)" ); + + $queue->flushCaches(); + $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" ); + $this->assertEquals( 0, $queue->getAcquiredCount(), "Queue is empty ($desc)" ); + + for ( $i=0; $i<10; ++$i ) { + $this->assertTrue( $queue->push( $this->newJob( $i ) ), "Push worked ($desc)" ); + } + + for ( $i=0; $i<10; ++$i ) { + $job = $queue->pop(); + $this->assertTrue( $job instanceof Job, "Jobs popped from queue ($desc)" ); + $params = $job->getParams(); + $this->assertEquals( $i, $params['i'], "Job popped from queue is FIFO ($desc)" ); + $queue->ack( $job ); + } + + $this->assertFalse( $queue->isEmpty(), "Queue is not empty ($desc)" ); + + $queue->flushCaches(); + $this->assertEquals( 0, $queue->getSize(), "Queue is empty ($desc)" ); + $this->assertEquals( 0, $queue->getAcquiredCount(), "No jobs active ($desc)" ); + } + + function provider_queueLists() { + return array( + array( 'queueRand', 'rand', false, 'Random queue without ack()' ), + array( 'queueRandTTL', 'rand', true, 'Random queue with ack()' ), + array( 'queueFifo', 'fifo', false, 'Ordered queue without ack()' ), + array( 'queueFifoTTL', 'fifo', true, 'Ordered queue with ack()' ) + ); + } + + function provider_fifoQueueLists() { + return array( + array( 'queueFifo', false, 'Ordered queue without ack()' ), + array( 'queueFifoTTL', true, 'Ordered queue with ack()' ) + ); + } + + function newJob( $i = 0, $rootJob = array() ) { + return new NullJob( Title::newMainPage(), + array( 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 0, 'i' => $i ) + $rootJob ); + } + + function newDedupedJob( $i = 0, $rootJob = array() ) { + return new NullJob( Title::newMainPage(), + array( 'lives' => 0, 'usleep' => 0, 'removeDuplicates' => 1, 'i' => $i ) + $rootJob ); + } +}