* The implementation of buildStringCast() in Wikimedia\Rdbms\Database has
changed to explicitly cast. Subclasses relying on the base-class
implementation should check whether they need to override it now.
+* BagOStuff::add is now abstract and must explicitly be defined in subclasses.
== Compatibility ==
MediaWiki 1.33 requires PHP 7.0.13 or later. Although HHVM 3.18.5 or later is
$mainConfig->get( 'DBmwschema' ),
$mainConfig->get( 'DBprefix' )
),
- 'profiler' => Profiler::instance(),
+ 'profiler' => function ( $section ) {
+ return Profiler::instance()->scopedProfileIn( $section );
+ },
'trxProfiler' => Profiler::instance()->getTransactionProfiler(),
'replLogger' => LoggerFactory::getInstance( 'DBReplication' ),
'queryLogger' => LoggerFactory::getInstance( 'DBQuery' ),
'wanCache' => MediaWikiServices::getInstance()->getMainWANObjectCache(),
'srvCache' => ObjectCache::getLocalServerInstance( 'hash' ),
'logger' => LoggerFactory::getInstance( 'FileOperation' ),
- 'profiler' => Profiler::instance()
+ 'profiler' => function ( $section ) {
+ return Profiler::instance()->scopedProfileIn( $section );
+ }
];
$config['lockManager'] =
LockManagerGroup::singleton( $config['wikiId'] )->get( $config['lockManager'] );
protected $fileJournal;
/** @var LoggerInterface */
protected $logger;
- /** @var object|string Class name or object With profileIn/profileOut methods */
+ /** @var callable|null */
protected $profiler;
/** @var callable */
* - obResetFunc : alternative callback to clear the output buffer
* - streamMimeFunc : alternative method to determine the content type from the path
* - logger : Optional PSR logger object.
- * - profiler : Optional class name or object With profileIn/profileOut methods.
+ * - profiler : Optional callback that takes a section name argument and returns
+ * a ScopedCallback instance that ends the profile section in its destructor.
* @throws InvalidArgumentException
*/
public function __construct( array $config ) {
$this->statusWrapper = $config['statusWrapper'] ?? null;
$this->profiler = $config['profiler'] ?? null;
+ if ( !is_callable( $this->profiler ) ) {
+ $this->profiler = null;
+ }
$this->logger = $config['logger'] ?? new \Psr\Log\NullLogger();
$this->statusWrapper = $config['statusWrapper'] ?? null;
$this->tmpDirectory = $config['tmpDirectory'] ?? null;
* @return ScopedCallback|null
*/
protected function scopedProfileSection( $section ) {
- if ( $this->profiler ) {
- call_user_func( [ $this->profiler, 'profileIn' ], $section );
- return new ScopedCallback( [ $this->profiler, 'profileOut' ], [ $section ] );
- }
-
- return null;
+ return $this->profiler ? ( $this->profiler )( $section ) : null;
}
protected function resetOutputBuffer() {
return true;
}
+ public function add( $key, $value, $exptime = 0, $flags = 0 ) {
+ return apc_add(
+ $key . self::KEY_SUFFIX,
+ $this->setSerialize( $value ),
+ $exptime
+ );
+ }
+
protected function setSerialize( $value ) {
if ( !$this->nativeSerialize && !$this->isInteger( $value ) ) {
$value = serialize( $value );
return true;
}
+ public function add( $key, $value, $exptime = 0, $flags = 0 ) {
+ return apcu_add(
+ $key . self::KEY_SUFFIX,
+ $this->setSerialize( $value ),
+ $exptime
+ );
+ }
+
public function delete( $key, $flags = 0 ) {
apcu_delete( $key . self::KEY_SUFFIX );
function () use ( $key, $expiry, $fname ) {
$this->clearLastError();
if ( $this->add( "{$key}:lock", 1, $expiry ) ) {
- return true; // locked!
+ return WaitConditionLoop::CONDITION_REACHED; // locked!
} elseif ( $this->getLastError() ) {
$this->logger->warning(
$fname . ' failed due to I/O error for {key}.',
return $res;
}
+ /**
+ * Batch deletion
+ * @param string[] $keys List of keys
+ * @param int $flags Bitfield of BagOStuff::WRITE_* constants
+ * @return bool Success
+ * @since 1.33
+ */
+ public function deleteMulti( array $keys, $flags = 0 ) {
+ $res = true;
+ foreach ( $keys as $key ) {
+ $res = $this->delete( $key, $flags ) && $res;
+ }
+
+ return $res;
+ }
+
/**
* Insertion
* @param string $key
* @param int $flags Bitfield of BagOStuff::WRITE_* constants (since 1.33)
* @return bool Success
*/
- public function add( $key, $value, $exptime = 0, $flags = 0 ) {
- // @note: avoid lock() here since that method uses *this* method by default
- if ( $this->get( $key ) === false ) {
- return $this->set( $key, $value, $exptime, $flags );
- }
-
- return false; // key already set
- }
+ abstract public function add( $key, $value, $exptime = 0, $flags = 0 );
/**
* Increase stored value of $key by $value while preserving its TTL
// These just call the backend (tested elsewhere)
// @codeCoverageIgnoreStart
+ public function add( $key, $value, $exptime = 0, $flags = 0 ) {
+ if ( $this->get( $key ) === false ) {
+ return $this->set( $key, $value, $exptime, $flags );
+ }
+
+ return false; // key already set
+ }
+
public function lock( $key, $timeout = 6, $expiry = 6, $rclass = '' ) {
return $this->backend->lock( $key, $timeout, $expiry, $rclass );
}
return true;
}
+ public function add( $key, $value, $exptime = 0, $flags = 0 ) {
+ if ( $this->get( $key ) === false ) {
+ return $this->set( $key, $value, $exptime, $flags );
+ }
+
+ return false; // key already set
+ }
+
public function delete( $key, $flags = 0 ) {
unset( $this->bag[$key] );
return $this->handleError( "Failed to store $key", $rcode, $rerr );
}
+ public function add( $key, $value, $exptime = 0, $flags = 0 ) {
+ if ( $this->get( $key ) === false ) {
+ return $this->set( $key, $value, $exptime, $flags );
+ }
+
+ return false; // key already set
+ }
+
public function delete( $key, $flags = 0 ) {
// @TODO: respect WRITE_SYNC (e.g. EACH_QUORUM)
$req = [
return $result;
}
+ public function deleteMulti( array $keys, $flags = 0 ) {
+ $batches = [];
+ $conns = [];
+ foreach ( $keys as $key ) {
+ list( $server, $conn ) = $this->getConnection( $key );
+ if ( !$conn ) {
+ continue;
+ }
+ $conns[$server] = $conn;
+ $batches[$server][] = $key;
+ }
+
+ $result = true;
+ foreach ( $batches as $server => $batchKeys ) {
+ $conn = $conns[$server];
+ try {
+ $conn->multi( Redis::PIPELINE );
+ foreach ( $batchKeys as $key ) {
+ $conn->delete( $key );
+ }
+ $batchResult = $conn->exec();
+ if ( $batchResult === false ) {
+ $this->debug( "deleteMulti request to $server failed" );
+ continue;
+ }
+ foreach ( $batchResult as $value ) {
+ if ( $value === false ) {
+ $result = false;
+ }
+ }
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ $result = false;
+ }
+ }
+
+ return $result;
+ }
+
public function add( $key, $value, $expiry = 0, $flags = 0 ) {
list( $server, $conn ) = $this->getConnection( $key );
if ( !$conn ) {
return $this->writeStore->set( $key, $value, $exptime, $flags );
}
+ public function setMulti( array $keys, $exptime = 0, $flags = 0 ) {
+ return $this->writeStore->setMulti( $keys, $exptime, $flags );
+ }
+
public function delete( $key, $flags = 0 ) {
return $this->writeStore->delete( $key, $flags );
}
+ public function deleteMulti( array $keys, $flags = 0 ) {
+ return $this->writeStore->deleteMulti( $keys, $flags );
+ }
+
public function add( $key, $value, $exptime = 0, $flags = 0 ) {
- return $this->writeStore->add( $key, $value, $exptime );
+ return $this->writeStore->add( $key, $value, $exptime, $flags );
}
public function incr( $key, $value = 1 ) {
public function set( $key, $value, $expire = 0, $flags = 0 ) {
$result = wincache_ucache_set( $key, serialize( $value ), $expire );
- /* wincache_ucache_set returns an empty array on success if $value
- * was an array, bool otherwise */
- return ( is_array( $result ) && $result === [] ) || $result;
+ return ( $result === [] || $result === true );
+ }
+
+ public function add( $key, $value, $exptime = 0, $flags = 0 ) {
+ $result = wincache_ucache_add( $key, serialize( $value ), $exptime );
+
+ return ( $result === [] || $result === true );
}
public function delete( $key, $flags = 0 ) {
/** @var int[] Prior flags member variable values */
private $priorFlags = [];
- /** @var mixed Class name or object With profileIn/profileOut methods */
+ /** @var callable|null */
protected $profiler;
/** @var TransactionProfiler */
protected $trxProfiler;
$this->srvCache = $params['srvCache'] ?? new HashBagOStuff();
- $this->profiler = $params['profiler'];
+ $this->profiler = is_callable( $params['profiler'] ) ? $params['profiler'] : null;
$this->trxProfiler = $params['trxProfiler'];
$this->connLogger = $params['connLogger'];
$this->queryLogger = $params['queryLogger'];
* used to adjust lock timeouts or encoding modes and the like.
* - connLogger: Optional PSR-3 logger interface instance.
* - queryLogger: Optional PSR-3 logger interface instance.
- * - profiler: Optional class name or object with profileIn()/profileOut() methods.
- * These will be called in query(), using a simplified version of the SQL that also
- * includes the agent as a SQL comment.
+ * - profiler : Optional callback that takes a section name argument and returns
+ * a ScopedCallback instance that ends the profile section in its destructor.
+ * These will be called in query(), using a simplified version of the SQL that
+ * also includes the agent as a SQL comment.
* - trxProfiler: Optional TransactionProfiler instance.
* - errorLogger: Optional callback that takes an Exception and logs it.
* - deprecationLogger: Optional callback that takes a string and logs it.
$queryProf .= $this->trxShortId ? " [TRX#{$this->trxShortId}]" : "";
$startTime = microtime( true );
- if ( $this->profiler ) {
- $this->profiler->profileIn( $queryProf );
- }
+ $ps = $this->profiler ? ( $this->profiler )( $queryProf ) : null;
$this->affectedRowCount = null;
$ret = $this->doQuery( $commentedSql );
$this->affectedRowCount = $this->affectedRows();
- if ( $this->profiler ) {
- $this->profiler->profileOut( $queryProf );
- }
+ unset( $ps ); // profile out (if set)
$queryRuntime = max( microtime( true ) - $startTime, 0.0 );
- unset( $queryProfSection ); // profile out (if set)
-
if ( $ret !== false ) {
$this->lastPing = $startTime;
if ( $isEffectiveWrite && $this->trxLevel ) {
}
public function setMulti( array $data, $expiry = 0, $flags = 0 ) {
+ return $this->insertMulti( $data, $expiry, $flags, true );
+ }
+
+ private function insertMulti( array $data, $expiry, $flags, $replace ) {
$keysByTable = [];
foreach ( $data as $key => $value ) {
list( $serverIndex, $tableName ) = $this->getTableByKey( $key );
}
try {
- $db->replace(
- $tableName,
- [ 'keyname' ],
- $rows,
- __METHOD__
- );
+ if ( $replace ) {
+ $db->replace( $tableName, [ 'keyname' ], $rows, __METHOD__ );
+ } else {
+ $db->insert( $tableName, $rows, __METHOD__, [ 'IGNORE' ] );
+ $result = ( $db->affectedRows() > 0 && $result );
+ }
} catch ( DBError $e ) {
$this->handleWriteError( $e, $db, $serverIndex );
$result = false;
}
}
+ }
+ if ( ( $flags & self::WRITE_SYNC ) == self::WRITE_SYNC ) {
+ $result = $this->waitForReplication() && $result;
}
return $result;
public function set( $key, $value, $exptime = 0, $flags = 0 ) {
$ok = $this->setMulti( [ $key => $value ], $exptime );
- if ( ( $flags & self::WRITE_SYNC ) == self::WRITE_SYNC ) {
- $ok = $this->waitForReplication() && $ok;
- }
return $ok;
}
+ public function add( $key, $value, $exptime = 0, $flags = 0 ) {
+ $added = $this->insertMulti( [ $key => $value ], $exptime, $flags, false );
+
+ return $added;
+ }
+
protected function cas( $casToken, $key, $value, $exptime = 0, $flags = 0 ) {
list( $serverIndex, $tableName ) = $this->getTableByKey( $key );
$db = null;
return (bool)$db->affectedRows();
}
- public function delete( $key, $flags = 0 ) {
- $ok = true;
+ public function deleteMulti( array $keys, $flags = 0 ) {
+ $keysByTable = [];
+ foreach ( $keys as $key ) {
+ list( $serverIndex, $tableName ) = $this->getTableByKey( $key );
+ $keysByTable[$serverIndex][$tableName][] = $key;
+ }
- list( $serverIndex, $tableName ) = $this->getTableByKey( $key );
- $db = null;
+ $result = true;
$silenceScope = $this->silenceTransactionProfiler();
- try {
- $db = $this->getDB( $serverIndex );
- $db->delete(
- $tableName,
- [ 'keyname' => $key ],
- __METHOD__ );
- } catch ( DBError $e ) {
- $this->handleWriteError( $e, $db, $serverIndex );
- $ok = false;
+ foreach ( $keysByTable as $serverIndex => $serverKeys ) {
+ $db = null;
+ try {
+ $db = $this->getDB( $serverIndex );
+ } catch ( DBError $e ) {
+ $this->handleWriteError( $e, $db, $serverIndex );
+ $result = false;
+ continue;
+ }
+
+ foreach ( $serverKeys as $tableName => $tableKeys ) {
+ try {
+ $db->delete( $tableName, [ 'keyname' => $tableKeys ], __METHOD__ );
+ } catch ( DBError $e ) {
+ $this->handleWriteError( $e, $db, $serverIndex );
+ $result = false;
+ }
+
+ }
}
+
if ( ( $flags & self::WRITE_SYNC ) == self::WRITE_SYNC ) {
- $ok = $this->waitForReplication() && $ok;
+ $result = $this->waitForReplication() && $result;
}
+ return $result;
+ }
+
+ public function delete( $key, $flags = 0 ) {
+ $ok = $this->deleteMulti( [ $key ], $flags );
+
return $ok;
}
[ 'value', 'exptime' ],
[ 'keyname' => $key ],
__METHOD__,
- [ 'FOR UPDATE' ] );
+ [ 'FOR UPDATE' ]
+ );
if ( $row === false ) {
// Missing
-
- return null;
+ return false;
}
$db->delete( $tableName, [ 'keyname' => $key ], __METHOD__ );
if ( $this->isExpired( $db, $row->exptime ) ) {
// Expired, do not reinsert
-
- return null;
+ return false;
}
$oldValue = intval( $this->unserialize( $db->decodeBlob( $row->value ) ) );
$newValue = $oldValue + $step;
- $db->insert( $tableName,
+ $db->insert(
+ $tableName,
[
'keyname' => $key,
'value' => $db->encodeBlob( $this->serialize( $newValue ) ),
'exptime' => $row->exptime
- ], __METHOD__, 'IGNORE' );
+ ],
+ __METHOD__,
+ 'IGNORE'
+ );
if ( $db->affectedRows() == 0 ) {
// Race condition. See T30611
- $newValue = null;
+ $newValue = false;
}
} catch ( DBError $e ) {
$this->handleWriteError( $e, $db, $serverIndex );
}
}
- // Kept BC for now, remove when possible
public function profileIn( $functionname ) {
+ wfDeprecated( __METHOD__, '1.33' );
}
public function profileOut( $functionname ) {
+ wfDeprecated( __METHOD__, '1.33' );
}
/**
Hooks::run( 'UserSaveSettings', [ $this ] );
$this->clearSharedCache();
- $this->getUserPage()->invalidateCache();
+ $this->getUserPage()->purgeSquid();
}
/**
public function __construct( $testName, array $opts = [] ) {
$this->testName = $testName;
- $this->profiler = new ProfilerStub( [] );
+ $this->profiler = null;
$this->trxProfiler = new TransactionProfiler();
$this->cliMode = $opts['cliMode'] ?? true;
$this->connLogger = new \Psr\Log\NullLogger();
}
$this->cache->delete( $this->cache->makeKey( self::TEST_KEY ) );
+ $this->cache->delete( $this->cache->makeKey( self::TEST_KEY ) . ':lock' );
}
/**
* @covers BagOStuff::mergeViaCas
*/
public function testMerge() {
- $calls = 0;
$key = $this->cache->makeKey( self::TEST_KEY );
- $callback = function ( BagOStuff $cache, $key, $oldVal ) use ( &$calls ) {
+ $locks = false;
+ $checkLockingCallback = function ( BagOStuff $cache, $key, $oldVal ) use ( &$locks ) {
+ $locks = $cache->get( "$key:lock" );
+
+ return false;
+ };
+
+ $this->cache->merge( $key, $checkLockingCallback, 5 );
+ $this->assertFalse( $this->cache->get( $key ) );
+
+ $calls = 0;
+ $casRace = false; // emulate a race
+ $callback = function ( BagOStuff $cache, $key, $oldVal ) use ( &$calls, &$casRace ) {
++$calls;
+ if ( $casRace ) {
+ // Uses CAS instead?
+ $cache->set( $key, 'conflict', 5 );
+ }
return ( $oldVal === false ) ? 'merged' : $oldVal . 'merged';
};
$this->assertEquals( 'mergedmerged', $this->cache->get( $key ) );
$calls = 0;
- $this->cache->lock( $key );
- $this->assertFalse( $this->cache->merge( $key, $callback, 1 ), 'Non-blocking merge' );
- $this->cache->unlock( $key );
- $this->assertEquals( 0, $calls );
+ if ( $locks ) {
+ // merge were something else already was merging (e.g. had the lock)
+ $this->cache->lock( $key );
+ $this->assertFalse(
+ $this->cache->merge( $key, $callback, 5, 1 ),
+ 'Non-blocking merge (locking)'
+ );
+ $this->cache->unlock( $key );
+ $this->assertEquals( 0, $calls );
+ } else {
+ $casRace = true;
+ $this->assertFalse(
+ $this->cache->merge( $key, $callback, 5, 1 ),
+ 'Non-blocking merge (CAS)'
+ );
+ $this->assertEquals( 1, $calls );
+ }
}
/**
* @covers BagOStuff::merge
* @covers BagOStuff::mergeViaLock
+ * @dataProvider provideTestMerge_fork
*/
- public function testMerge_fork() {
+ public function testMerge_fork( $exists, $winsLocking, $resLocking, $resCAS ) {
$key = $this->cache->makeKey( self::TEST_KEY );
- $callback = function ( BagOStuff $cache, $key, $oldVal ) {
- return ( $oldVal === false ) ? 'merged' : $oldVal . 'merged';
+ $pCallback = function ( BagOStuff $cache, $key, $oldVal ) {
+ return ( $oldVal === false ) ? 'init-parent' : $oldVal . '-merged-parent';
+ };
+ $cCallback = function ( BagOStuff $cache, $key, $oldVal ) {
+ return ( $oldVal === false ) ? 'init-child' : $oldVal . '-merged-child';
};
+
+ if ( $exists ) {
+ $this->cache->set( $key, 'x', 5 );
+ }
+
/*
* Test concurrent merges by forking this process, if:
* - not manually called with --use-bagostuff
$fork &= !$this->cache instanceof MultiWriteBagOStuff;
if ( $fork ) {
$pid = null;
+ $locked = false;
// Function to start merge(), run another merge() midway through, then finish
- $outerFunc = function ( BagOStuff $cache, $key, $oldVal ) use ( $callback, &$pid ) {
+ $func = function ( BagOStuff $cache, $key, $cur )
+ use ( $pCallback, $cCallback, &$pid, &$locked )
+ {
$pid = pcntl_fork();
if ( $pid == -1 ) {
return false;
} elseif ( $pid ) {
+ $locked = $cache->get( "$key:lock" ); // parent has lock?
pcntl_wait( $status );
- return $callback( $cache, $key, $oldVal );
+ return $pCallback( $cache, $key, $cur );
} else {
- $this->cache->merge( $key, $callback, 0, 1 );
+ $this->cache->merge( $key, $cCallback, 0, 1 );
// Bail out of the outer merge() in the child process since it does not
// need to attempt to write anything. Success is checked by the parent.
parent::tearDown(); // avoid phpunit notices
};
// attempt a merge - this should fail
- $merged = $this->cache->merge( $key, $outerFunc, 0, 1 );
+ $merged = $this->cache->merge( $key, $func, 0, 1 );
if ( $pid == -1 ) {
return; // can't fork, ignore this test...
}
- // merge has failed because child process was merging (and we only attempted once)
- $this->assertFalse( $merged );
-
- // make sure the child's merge is completed and verify
- $this->assertEquals( $this->cache->get( $key ), 'mergedmerged' );
+ if ( $locked ) {
+ // merge succeed since child was locked out
+ $this->assertEquals( $winsLocking, $merged );
+ $this->assertEquals( $this->cache->get( $key ), $resLocking );
+ } else {
+ // merge has failed because child process was merging (and we only attempted once)
+ $this->assertEquals( !$winsLocking, $merged );
+ $this->assertEquals( $this->cache->get( $key ), $resCAS );
+ }
} else {
$this->markTestSkipped( 'No pcntl methods available' );
}
}
+ function provideTestMerge_fork() {
+ return [
+ // (already exists, parent wins if locking, result if locking, result if CAS)
+ [ false, true, 'init-parent', 'init-child' ],
+ [ true, true, 'x-merged-parent', 'x-merged-child' ]
+ ];
+ }
+
/**
* @covers BagOStuff::changeTTL
*/
$this->cache->delete( $key4 );
}
+ /**
+ * @covers BagOStuff::setMulti
+ * @covers BagOStuff::deleteMulti
+ */
+ public function testSetDeleteMulti() {
+ $map = [
+ $this->cache->makeKey( 'test-1' ) => 'Siberian',
+ $this->cache->makeKey( 'test-2' ) => [ 'Huskies' ],
+ $this->cache->makeKey( 'test-3' ) => [ 'are' => 'the' ],
+ $this->cache->makeKey( 'test-4' ) => (object)[ 'greatest' => 'animal' ],
+ $this->cache->makeKey( 'test-5' ) => 4,
+ $this->cache->makeKey( 'test-6' ) => 'ever'
+ ];
+
+ $this->cache->setMulti( $map, 5 );
+ $this->assertEquals(
+ $map,
+ $this->cache->getMulti( array_keys( $map ) )
+ );
+
+ $this->assertTrue( $this->cache->deleteMulti( array_keys( $map ), 5 ) );
+
+ $this->assertEquals(
+ [],
+ $this->cache->getMulti( array_keys( $map ) )
+ );
+ }
+
/**
* @covers BagOStuff::getScopedLock
*/