3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation; either version 2 of the License, or
6 * (at your option) any later version.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 * http://www.gnu.org/copyleft/gpl.html
22 namespace Wikimedia\Rdbms
;
24 use Psr\Log\LoggerInterface
;
25 use Psr\Log\NullLogger
;
26 use Wikimedia\ScopedCallback
;
31 * Basic DB load monitor with no external dependencies
32 * Uses memcached to cache the replication lag for a short time
36 class LoadMonitor
implements ILoadMonitor
{
37 /** @var ILoadBalancer */
41 /** @var WANObjectCache */
43 /** @var LoggerInterface */
44 protected $replLogger;
46 /** @var float Moving average ratio (e.g. 0.1 for 10% weight to new weight) */
47 private $movingAveRatio;
48 /** @var int Amount of replication lag in seconds before warnings are logged */
49 private $lagWarnThreshold;
51 /** @var int cache key version */
53 /** @var int Default 'max lag' in seconds when unspecified */
54 const LAG_WARN_THRESHOLD
= 10;
57 * @param ILoadBalancer $lb
58 * @param BagOStuff $srvCache
59 * @param WANObjectCache $wCache
60 * @param array $options
61 * - movingAveRatio: moving average constant for server weight updates based on lag
62 * - lagWarnThreshold: how many seconds of lag trigger warnings
64 public function __construct(
65 ILoadBalancer
$lb, BagOStuff
$srvCache, WANObjectCache
$wCache, array $options = []
68 $this->srvCache
= $srvCache;
69 $this->wanCache
= $wCache;
70 $this->replLogger
= new NullLogger();
72 $this->movingAveRatio
= $options['movingAveRatio'] ??
0.1;
73 $this->lagWarnThreshold
= $options['lagWarnThreshold'] ?? self
::LAG_WARN_THRESHOLD
;
76 public function setLogger( LoggerInterface
$logger ) {
77 $this->replLogger
= $logger;
80 final public function scaleLoads( array &$weightByServer, $domain ) {
81 $serverIndexes = array_keys( $weightByServer );
82 $states = $this->getServerStates( $serverIndexes, $domain );
83 $newScalesByServer = $states['weightScales'];
84 foreach ( $weightByServer as $i => $weight ) {
85 if ( isset( $newScalesByServer[$i] ) ) {
86 $weightByServer[$i] = $weight * $newScalesByServer[$i];
87 } else { // server recently added to config?
88 $host = $this->parent
->getServerName( $i );
89 $this->replLogger
->error( __METHOD__
. ": host $host not in cache" );
94 final public function getLagTimes( array $serverIndexes, $domain ) {
95 return $this->getServerStates( $serverIndexes, $domain )['lagTimes'];
98 protected function getServerStates( array $serverIndexes, $domain ) {
99 $writerIndex = $this->parent
->getWriterIndex();
100 if ( count( $serverIndexes ) == 1 && reset( $serverIndexes ) == $writerIndex ) {
101 # Single server only, just return zero without caching
103 'lagTimes' => [ $writerIndex => 0 ],
104 'weightScales' => [ $writerIndex => 1.0 ]
108 $key = $this->getCacheKey( $serverIndexes );
109 # Randomize TTLs to reduce stampedes (4.0 - 5.0 sec)
110 $ttl = mt_rand( 4e6
, 5e6
) / 1e6
;
111 # Keep keys around longer as fallbacks
114 # (a) Check the local APC cache
115 $value = $this->srvCache
->get( $key );
116 if ( $value && $value['timestamp'] > ( microtime( true ) - $ttl ) ) {
117 $this->replLogger
->debug( __METHOD__
. ": got lag times ($key) from local cache" );
118 return $value; // cache hit
120 $staleValue = $value ?
: false;
122 # (b) Check the shared cache and backfill APC
123 $value = $this->wanCache
->get( $key );
124 if ( $value && $value['timestamp'] > ( microtime( true ) - $ttl ) ) {
125 $this->srvCache
->set( $key, $value, $staleTTL );
126 $this->replLogger
->debug( __METHOD__
. ": got lag times ($key) from main cache" );
128 return $value; // cache hit
130 $staleValue = $value ?
: $staleValue;
132 # (c) Cache key missing or expired; regenerate and backfill
133 if ( $this->srvCache
->lock( $key, 0, 10 ) ) {
134 # Let only this process update the cache value on this server
135 $sCache = $this->srvCache
;
136 /** @noinspection PhpUnusedLocalVariableInspection */
137 $unlocker = new ScopedCallback( function () use ( $sCache, $key ) {
138 $sCache->unlock( $key );
140 } elseif ( $staleValue ) {
141 # Could not acquire lock but an old cache exists, so use it
147 $movAveRatio = $this->movingAveRatio
;
148 foreach ( $serverIndexes as $i ) {
149 if ( $i == $this->parent
->getWriterIndex() ) {
150 $lagTimes[$i] = 0; // master always has no lag
151 $weightScales[$i] = 1.0; // nominal weight
155 # Handles with open transactions are avoided since they might be subject
156 # to REPEATABLE-READ snapshots, which could affect the lag estimate query.
157 $flags = ILoadBalancer
::CONN_TRX_AUTOCOMMIT | ILoadBalancer
::CONN_SILENCE_ERRORS
;
158 $conn = $this->parent
->getAnyOpenConnection( $i, $flags );
160 $close = false; // already open
162 $conn = $this->parent
->getConnection( $i, [], ILoadBalancer
::DOMAIN_ANY
, $flags );
163 $close = true; // new connection
166 $lastWeight = $staleValue['weightScales'][$i] ??
1.0;
167 $coefficient = $this->getWeightScale( $i, $conn ?
: null );
168 $newWeight = $movAveRatio * $coefficient +
( 1 - $movAveRatio ) * $lastWeight;
170 // Scale from 10% to 100% of nominal weight
171 $weightScales[$i] = max( $newWeight, 0.10 );
173 $host = $this->parent
->getServerName( $i );
176 $lagTimes[$i] = false;
177 $this->replLogger
->error(
178 __METHOD__
. ": host {db_server} is unreachable",
179 [ 'db_server' => $host ]
184 $lagTimes[$i] = $conn->getLag();
185 if ( $lagTimes[$i] === false ) {
186 $this->replLogger
->error(
187 __METHOD__
. ": host {db_server} is not replicating?",
188 [ 'db_server' => $host ]
190 } elseif ( $lagTimes[$i] > $this->lagWarnThreshold
) {
191 $this->replLogger
->warning(
192 "Server {host} has {lag} seconds of lag (>= {maxlag})",
195 'lag' => $lagTimes[$i],
196 'maxlag' => $this->lagWarnThreshold
202 # Close the connection to avoid sleeper connections piling up.
203 # Note that the caller will pick one of these DBs and reconnect,
204 # which is slightly inefficient, but this only matters for the lag
205 # time cache miss cache, which is far less common that cache hits.
206 $this->parent
->closeConnection( $conn );
210 # Add a timestamp key so we know when it was cached
212 'lagTimes' => $lagTimes,
213 'weightScales' => $weightScales,
214 'timestamp' => microtime( true )
216 $this->wanCache
->set( $key, $value, $staleTTL );
217 $this->srvCache
->set( $key, $value, $staleTTL );
218 $this->replLogger
->info( __METHOD__
. ": re-calculated lag times ($key)" );
224 * @param int $index Server index
225 * @param IDatabase|null $conn Connection handle or null on connection failure
228 protected function getWeightScale( $index, IDatabase
$conn = null ) {
229 return $conn ?
1.0 : 0.0;
232 private function getCacheKey( array $serverIndexes ) {
233 sort( $serverIndexes );
234 // Lag is per-server, not per-DB, so key on the master DB name
235 return $this->srvCache
->makeGlobalKey(
238 $this->parent
->getServerName( $this->parent
->getWriterIndex() ),
239 implode( '-', $serverIndexes )