3 * Generator of database load balancing objects.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
24 namespace Wikimedia\Rdbms
;
26 use Psr\Log\LoggerAwareInterface
;
27 use Psr\Log\LoggerInterface
;
28 use Psr\Log\NullLogger
;
29 use Wikimedia\WaitConditionLoop
;
33 * Helper class for mitigating DB replication lag in order to provide "session consistency"
35 * This helps to ensure a consistent ordering of events as seen by an client
37 * Kind of like Hawking's [[Chronology Protection Agency]].
39 class ChronologyProtector
implements LoggerAwareInterface
{
42 /** @var LoggerInterface */
45 /** @var string Storage key name */
47 /** @var string Hash of client parameters */
49 /** @var string[] Map of client information fields for logging */
50 protected $clientLogInfo;
51 /** @var int|null Expected minimum index of the last write to the position store */
52 protected $waitForPosIndex;
53 /** @var int Max seconds to wait on positions to appear */
54 protected $waitForPosStoreTimeout = self
::POS_STORE_WAIT_TIMEOUT
;
55 /** @var bool Whether to no-op all method calls */
56 protected $enabled = true;
57 /** @var bool Whether to check and wait on positions */
58 protected $wait = true;
60 /** @var bool Whether the client data was loaded */
61 protected $initialized = false;
62 /** @var DBMasterPos[] Map of (DB master name => position) */
63 protected $startupPositions = [];
64 /** @var DBMasterPos[] Map of (DB master name => position) */
65 protected $shutdownPositions = [];
66 /** @var float[] Map of (DB master name => 1) */
67 protected $shutdownTouchDBs = [];
69 /** @var int Seconds to store positions */
70 const POSITION_TTL
= 60;
71 /** @var int Seconds to store position write index cookies (safely less than POSITION_TTL) */
72 const POSITION_COOKIE_TTL
= 10;
73 /** @var int Max time to wait for positions to appear */
74 const POS_STORE_WAIT_TIMEOUT
= 5;
77 * @param BagOStuff $store
78 * @param array $client Map of (ip: <IP>, agent: <user-agent> [, clientId: <hash>] )
79 * @param int|null $posIndex Write counter index
80 * @param string $secret Secret string for HMAC hashing [optional]
83 public function __construct( BagOStuff
$store, array $client, $posIndex, $secret = '' ) {
84 $this->store
= $store;
85 if ( isset( $client['clientId'] ) ) {
86 $this->clientId
= $client['clientId'];
88 $this->clientId
= ( $secret != '' )
89 ?
hash_hmac( 'md5', $client['ip'] . "\n" . $client['agent'], $secret )
90 : md5( $client['ip'] . "\n" . $client['agent'] );
92 $this->key
= $store->makeGlobalKey( __CLASS__
, $this->clientId
, 'v2' );
93 $this->waitForPosIndex
= $posIndex;
95 $this->clientLogInfo
= [
96 'clientIP' => $client['ip'],
97 'clientAgent' => $client['agent'],
98 'clientId' => $client['clientId'] ??
null
101 $this->logger
= new NullLogger();
104 public function setLogger( LoggerInterface
$logger ) {
105 $this->logger
= $logger;
109 * @return string Client ID hash
112 public function getClientId() {
113 return $this->clientId
;
117 * @param bool $enabled Whether to no-op all method calls
120 public function setEnabled( $enabled ) {
121 $this->enabled
= $enabled;
125 * @param bool $enabled Whether to check and wait on positions
128 public function setWaitEnabled( $enabled ) {
129 $this->wait
= $enabled;
133 * Apply the "session consistency" DB replication position to a new ILoadBalancer
135 * If the stash has a previous master position recorded, this will try to make
136 * sure that the next query to a replica DB of that master will see changes up
137 * to that position by delaying execution. The delay may timeout and allow stale
138 * data if no non-lagged replica DBs are available.
140 * This method should only be called from LBFactory.
142 * @param ILoadBalancer $lb
145 public function applySessionReplicationPosition( ILoadBalancer
$lb ) {
146 if ( !$this->enabled
) {
150 $masterName = $lb->getServerName( $lb->getWriterIndex() );
151 $startupPositions = $this->getStartupMasterPositions();
153 $pos = $startupPositions[$masterName] ??
null;
154 if ( $pos instanceof DBMasterPos
) {
155 $this->logger
->debug( __METHOD__
. ": pos for DB '$masterName' set to '$pos'\n" );
156 $lb->waitFor( $pos );
161 * Save the "session consistency" DB replication position for an end-of-life ILoadBalancer
163 * This saves the replication position of the master DB if this request made writes to it.
165 * This method should only be called from LBFactory.
167 * @param ILoadBalancer $lb
170 public function storeSessionReplicationPosition( ILoadBalancer
$lb ) {
171 if ( !$this->enabled
) {
173 } elseif ( !$lb->hasOrMadeRecentMasterChanges( INF
) ) {
174 // Only save the position if writes have been done on the connection
178 $masterName = $lb->getServerName( $lb->getWriterIndex() );
179 if ( $lb->hasStreamingReplicaServers() ) {
180 $pos = $lb->getMasterPos();
182 $this->logger
->debug( __METHOD__
. ": LB for '$masterName' has pos $pos\n" );
183 $this->shutdownPositions
[$masterName] = $pos;
186 $this->logger
->debug( __METHOD__
. ": DB '$masterName' touched\n" );
188 $this->shutdownTouchDBs
[$masterName] = 1;
192 * Notify the ChronologyProtector that the LBFactory is done calling shutdownLB() for now.
193 * May commit chronology data to persistent storage.
195 * @param callable|null $workCallback Work to do instead of waiting on syncing positions
196 * @param string $mode One of (sync, async); whether to wait on remote datacenters
197 * @param int|null &$cpIndex DB position key write counter; incremented on update
198 * @return DBMasterPos[] Empty on success; returns the (db name => position) map on failure
200 public function shutdown( callable
$workCallback = null, $mode = 'sync', &$cpIndex = null ) {
201 if ( !$this->enabled
) {
205 $store = $this->store
;
206 // Some callers might want to know if a user recently touched a DB.
207 // These writes do not need to block on all datacenters receiving them.
208 foreach ( $this->shutdownTouchDBs
as $dbName => $unused ) {
210 $this->getTouchedKey( $this->store
, $dbName ),
216 if ( $this->shutdownPositions
=== [] ) {
217 $this->logger
->debug( __METHOD__
. ": no master positions to save\n" );
219 return []; // nothing to save
222 $this->logger
->debug(
223 __METHOD__
. ": saving master pos for " .
224 implode( ', ', array_keys( $this->shutdownPositions
) ) . "\n"
227 // CP-protected writes should overwhelmingly go to the master datacenter, so use a
228 // DC-local lock to merge the values. Use a DC-local get() and a synchronous all-DC
229 // set(). This makes it possible for the BagOStuff class to write in parallel to all
230 // DCs with one RTT. The use of WRITE_SYNC avoids needing READ_LATEST for the get().
231 if ( $store->lock( $this->key
, 3 ) ) {
232 if ( $workCallback ) {
233 // Let the store run the work before blocking on a replication sync barrier.
234 // If replication caught up while the work finished, the barrier will be fast.
235 $store->addBusyCallback( $workCallback );
239 $this->mergePositions(
240 $store->get( $this->key
),
241 $this->shutdownPositions
,
245 ( $mode === 'sync' ) ?
$store::WRITE_SYNC
: 0
247 $store->unlock( $this->key
);
253 $cpIndex = null; // nothing saved
254 $bouncedPositions = $this->shutdownPositions
;
255 // Raced out too many times or stash is down
256 $this->logger
->warning( __METHOD__
. ": failed to save master pos for " .
257 implode( ', ', array_keys( $this->shutdownPositions
) ) . "\n"
259 } elseif ( $mode === 'sync' &&
260 $store->getQoS( $store::ATTR_SYNCWRITES
) < $store::QOS_SYNCWRITES_BE
262 // Positions may not be in all datacenters, force LBFactory to play it safe
263 $this->logger
->info( __METHOD__
. ": store may not support synchronous writes." );
264 $bouncedPositions = $this->shutdownPositions
;
266 $bouncedPositions = [];
269 return $bouncedPositions;
273 * @param string $dbName DB master name (e.g. "db1052")
274 * @return float|bool UNIX timestamp when client last touched the DB; false if not on record
277 public function getTouched( $dbName ) {
278 return $this->store
->get( $this->getTouchedKey( $this->store
, $dbName ) );
282 * @param BagOStuff $store
283 * @param string $dbName
286 private function getTouchedKey( BagOStuff
$store, $dbName ) {
287 return $store->makeGlobalKey( __CLASS__
, 'mtime', $this->clientId
, $dbName );
291 * Load in previous master positions for the client
293 protected function getStartupMasterPositions() {
294 if ( $this->initialized
) {
295 return $this->startupPositions
;
298 $this->initialized
= true;
299 $this->logger
->debug( __METHOD__
. ": client ID is {$this->clientId} (read)\n" );
302 // If there is an expectation to see master positions from a certain write
303 // index or higher, then block until it appears, or until a timeout is reached.
304 // Since the write index restarts each time the key is created, it is possible that
305 // a lagged store has a matching key write index. However, in that case, it should
306 // already be expired and thus treated as non-existing, maintaining correctness.
307 if ( $this->waitForPosIndex
> 0 ) {
309 $indexReached = null; // highest index reached in the position store
310 $loop = new WaitConditionLoop(
311 function () use ( &$data, &$indexReached ) {
312 $data = $this->store
->get( $this->key
);
313 if ( !is_array( $data ) ) {
314 return WaitConditionLoop
::CONDITION_CONTINUE
; // not found yet
315 } elseif ( !isset( $data['writeIndex'] ) ) {
316 return WaitConditionLoop
::CONDITION_REACHED
; // b/c
318 $indexReached = max( $data['writeIndex'], $indexReached );
320 return ( $data['writeIndex'] >= $this->waitForPosIndex
)
321 ? WaitConditionLoop
::CONDITION_REACHED
322 : WaitConditionLoop
::CONDITION_CONTINUE
;
324 $this->waitForPosStoreTimeout
326 $result = $loop->invoke();
327 $waitedMs = $loop->getLastWaitTime() * 1e3
;
329 if ( $result == $loop::CONDITION_REACHED
) {
330 $this->logger
->debug(
331 __METHOD__
. ": expected and found position index.",
333 'cpPosIndex' => $this->waitForPosIndex
,
334 'waitTimeMs' => $waitedMs
335 ] +
$this->clientLogInfo
338 $this->logger
->warning(
339 __METHOD__
. ": expected but failed to find position index.",
341 'cpPosIndex' => $this->waitForPosIndex
,
342 'indexReached' => $indexReached,
343 'waitTimeMs' => $waitedMs
344 ] +
$this->clientLogInfo
348 $data = $this->store
->get( $this->key
);
351 $this->startupPositions
= $data ?
$data['positions'] : [];
352 $this->logger
->debug( __METHOD__
. ": key is {$this->key} (read)\n" );
354 $this->startupPositions
= [];
355 $this->logger
->debug( __METHOD__
. ": key is {$this->key} (unread)\n" );
358 return $this->startupPositions
;
362 * @param array|bool $curValue
363 * @param DBMasterPos[] $shutdownPositions
364 * @param int|null &$cpIndex
367 protected function mergePositions( $curValue, array $shutdownPositions, &$cpIndex = null ) {
368 /** @var DBMasterPos[] $curPositions */
369 $curPositions = $curValue['positions'] ??
[];
370 // Use the newest positions for each DB master
371 foreach ( $shutdownPositions as $db => $pos ) {
373 !isset( $curPositions[$db] ) ||
374 !( $curPositions[$db] instanceof DBMasterPos
) ||
375 $pos->asOfTime() > $curPositions[$db]->asOfTime()
377 $curPositions[$db] = $pos;
381 $cpIndex = $curValue['writeIndex'] ??
0;
384 'positions' => $curPositions,
385 'writeIndex' => ++
$cpIndex