3 * Generator of database load balancing objects.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
24 namespace Wikimedia\Rdbms
;
26 use Psr\Log\LoggerAwareInterface
;
27 use Psr\Log\LoggerInterface
;
28 use Psr\Log\NullLogger
;
29 use Wikimedia\WaitConditionLoop
;
33 * Class for ensuring a consistent ordering of events as seen by the user, despite replication.
34 * Kind of like Hawking's [[Chronology Protection Agency]].
36 class ChronologyProtector
implements LoggerAwareInterface
{
39 /** @var LoggerInterface */
42 /** @var string Storage key name */
44 /** @var string Hash of client parameters */
46 /** @var int|null Expected minimum index of the last write to the position store */
47 protected $waitForPosIndex;
48 /** @var int Max seconds to wait on positions to appear */
49 protected $waitForPosStoreTimeout = self
::POS_STORE_WAIT_TIMEOUT
;
50 /** @var bool Whether to no-op all method calls */
51 protected $enabled = true;
52 /** @var bool Whether to check and wait on positions */
53 protected $wait = true;
55 /** @var bool Whether the client data was loaded */
56 protected $initialized = false;
57 /** @var DBMasterPos[] Map of (DB master name => position) */
58 protected $startupPositions = [];
59 /** @var DBMasterPos[] Map of (DB master name => position) */
60 protected $shutdownPositions = [];
61 /** @var float[] Map of (DB master name => 1) */
62 protected $shutdownTouchDBs = [];
64 /** @var int Seconds to store positions */
65 const POSITION_TTL
= 60;
66 /** @var int Max time to wait for positions to appear */
67 const POS_STORE_WAIT_TIMEOUT
= 5;
70 * @param BagOStuff $store
71 * @param array[] $client Map of (ip: <IP>, agent: <user-agent>)
72 * @param int|null $posIndex Write counter index [optional]
75 public function __construct( BagOStuff
$store, array $client, $posIndex = null ) {
76 $this->store
= $store;
77 $this->clientId
= md5( $client['ip'] . "\n" . $client['agent'] );
78 $this->key
= $store->makeGlobalKey( __CLASS__
, $this->clientId
, 'v2' );
79 $this->waitForPosIndex
= $posIndex;
80 $this->logger
= new NullLogger();
83 public function setLogger( LoggerInterface
$logger ) {
84 $this->logger
= $logger;
88 * @param bool $enabled Whether to no-op all method calls
91 public function setEnabled( $enabled ) {
92 $this->enabled
= $enabled;
96 * @param bool $enabled Whether to check and wait on positions
99 public function setWaitEnabled( $enabled ) {
100 $this->wait
= $enabled;
104 * Initialise a ILoadBalancer to give it appropriate chronology protection.
106 * If the stash has a previous master position recorded, this will try to
107 * make sure that the next query to a replica DB of that master will see changes up
108 * to that position by delaying execution. The delay may timeout and allow stale
109 * data if no non-lagged replica DBs are available.
111 * @param ILoadBalancer $lb
114 public function initLB( ILoadBalancer
$lb ) {
115 if ( !$this->enabled ||
$lb->getServerCount() <= 1 ) {
116 return; // non-replicated setup or disabled
119 $this->initPositions();
121 $masterName = $lb->getServerName( $lb->getWriterIndex() );
123 isset( $this->startupPositions
[$masterName] ) &&
124 $this->startupPositions
[$masterName] instanceof DBMasterPos
126 $pos = $this->startupPositions
[$masterName];
127 $this->logger
->debug( __METHOD__
. ": LB for '$masterName' set to pos $pos\n" );
128 $lb->waitFor( $pos );
133 * Notify the ChronologyProtector that the ILoadBalancer is about to shut
134 * down. Saves replication positions.
136 * @param ILoadBalancer $lb
139 public function shutdownLB( ILoadBalancer
$lb ) {
140 if ( !$this->enabled
) {
141 return; // not enabled
142 } elseif ( !$lb->hasOrMadeRecentMasterChanges( INF
) ) {
143 // Only save the position if writes have been done on the connection
147 $masterName = $lb->getServerName( $lb->getWriterIndex() );
148 if ( $lb->getServerCount() > 1 ) {
149 $pos = $lb->getMasterPos();
151 $this->logger
->debug( __METHOD__
. ": LB for '$masterName' has pos $pos\n" );
152 $this->shutdownPositions
[$masterName] = $pos;
155 $this->logger
->debug( __METHOD__
. ": DB '$masterName' touched\n" );
157 $this->shutdownTouchDBs
[$masterName] = 1;
161 * Notify the ChronologyProtector that the LBFactory is done calling shutdownLB() for now.
162 * May commit chronology data to persistent storage.
164 * @param callable|null $workCallback Work to do instead of waiting on syncing positions
165 * @param string $mode One of (sync, async); whether to wait on remote datacenters
166 * @param int|null &$cpIndex DB position key write counter; incremented on update
167 * @return DBMasterPos[] Empty on success; returns the (db name => position) map on failure
169 public function shutdown( callable
$workCallback = null, $mode = 'sync', &$cpIndex = null ) {
170 if ( !$this->enabled
) {
174 $store = $this->store
;
175 // Some callers might want to know if a user recently touched a DB.
176 // These writes do not need to block on all datacenters receiving them.
177 foreach ( $this->shutdownTouchDBs
as $dbName => $unused ) {
179 $this->getTouchedKey( $this->store
, $dbName ),
185 if ( !count( $this->shutdownPositions
) ) {
186 return []; // nothing to save
189 $this->logger
->debug( __METHOD__
. ": saving master pos for " .
190 implode( ', ', array_keys( $this->shutdownPositions
) ) . "\n"
193 // CP-protected writes should overwhemingly go to the master datacenter, so get DC-local
194 // lock to merge the values. Use a DC-local get() and a synchronous all-DC set(). This
195 // makes it possible for the BagOStuff class to write in parallel to all DCs with one RTT.
196 if ( $store->lock( $this->key
, 3 ) ) {
197 if ( $workCallback ) {
198 // Let the store run the work before blocking on a replication sync barrier. By the
199 // time it's done with the work, the barrier should be fast if replication caught up.
200 $store->addBusyCallback( $workCallback );
204 $this->mergePositions(
205 $store->get( $this->key
),
206 $this->shutdownPositions
,
210 ( $mode === 'sync' ) ?
$store::WRITE_SYNC
: 0
212 $store->unlock( $this->key
);
218 $cpIndex = null; // nothing saved
219 $bouncedPositions = $this->shutdownPositions
;
220 // Raced out too many times or stash is down
221 $this->logger
->warning( __METHOD__
. ": failed to save master pos for " .
222 implode( ', ', array_keys( $this->shutdownPositions
) ) . "\n"
224 } elseif ( $mode === 'sync' &&
225 $store->getQoS( $store::ATTR_SYNCWRITES
) < $store::QOS_SYNCWRITES_BE
227 // Positions may not be in all datacenters, force LBFactory to play it safe
228 $this->logger
->info( __METHOD__
. ": store may not support synchronous writes." );
229 $bouncedPositions = $this->shutdownPositions
;
231 $bouncedPositions = [];
234 return $bouncedPositions;
238 * @param string $dbName DB master name (e.g. "db1052")
239 * @return float|bool UNIX timestamp when client last touched the DB; false if not on record
242 public function getTouched( $dbName ) {
243 return $this->store
->get( $this->getTouchedKey( $this->store
, $dbName ) );
247 * @param BagOStuff $store
248 * @param string $dbName
251 private function getTouchedKey( BagOStuff
$store, $dbName ) {
252 return $store->makeGlobalKey( __CLASS__
, 'mtime', $this->clientId
, $dbName );
256 * Load in previous master positions for the client
258 protected function initPositions() {
259 if ( $this->initialized
) {
263 $this->initialized
= true;
265 // If there is an expectation to see master positions from a certain write
266 // index or higher, then block until it appears, or until a timeout is reached.
267 // Since the write index restarts each time the key is created, it is possible that
268 // a lagged store has a matching key write index. However, in that case, it should
269 // already be expired and thus treated as non-existing, maintaining correctness.
270 if ( $this->waitForPosIndex
> 0 ) {
272 $indexReached = null; // highest index reached in the position store
273 $loop = new WaitConditionLoop(
274 function () use ( &$data, &$indexReached ) {
275 $data = $this->store
->get( $this->key
);
276 if ( !is_array( $data ) ) {
277 return WaitConditionLoop
::CONDITION_CONTINUE
; // not found yet
278 } elseif ( !isset( $data['writeIndex'] ) ) {
279 return WaitConditionLoop
::CONDITION_REACHED
; // b/c
281 $indexReached = max( $data['writeIndex'], $indexReached );
283 return ( $data['writeIndex'] >= $this->waitForPosIndex
)
284 ? WaitConditionLoop
::CONDITION_REACHED
285 : WaitConditionLoop
::CONDITION_CONTINUE
;
287 $this->waitForPosStoreTimeout
289 $result = $loop->invoke();
290 $waitedMs = $loop->getLastWaitTime() * 1e3
;
292 if ( $result == $loop::CONDITION_REACHED
) {
293 $this->logger
->debug(
294 __METHOD__
. ": expected and found position index.",
296 'cpPosIndex' => $this->waitForPosIndex
,
297 'waitTimeMs' => $waitedMs
301 $this->logger
->warning(
302 __METHOD__
. ": expected but failed to find position index.",
304 'cpPosIndex' => $this->waitForPosIndex
,
305 'indexReached' => $indexReached,
306 'waitTimeMs' => $waitedMs
311 $data = $this->store
->get( $this->key
);
314 $this->startupPositions
= $data ?
$data['positions'] : [];
315 $this->logger
->debug( __METHOD__
. ": key is {$this->key} (read)\n" );
317 $this->startupPositions
= [];
318 $this->logger
->debug( __METHOD__
. ": key is {$this->key} (unread)\n" );
323 * @param array|bool $curValue
324 * @param DBMasterPos[] $shutdownPositions
325 * @param int|null &$cpIndex
328 protected function mergePositions( $curValue, array $shutdownPositions, &$cpIndex = null ) {
329 /** @var DBMasterPos[] $curPositions */
330 $curPositions = isset( $curValue['positions'] ) ?
$curValue['positions'] : [];
331 // Use the newest positions for each DB master
332 foreach ( $shutdownPositions as $db => $pos ) {
334 !isset( $curPositions[$db] ) ||
335 !( $curPositions[$db] instanceof DBMasterPos
) ||
336 $pos->asOfTime() > $curPositions[$db]->asOfTime()
338 $curPositions[$db] = $pos;
342 $cpIndex = isset( $curValue['writeIndex'] ) ?
$curValue['writeIndex'] : 0;
345 'positions' => $curPositions,
346 'writeIndex' => ++
$cpIndex