3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation; either version 2 of the License, or
6 * (at your option) any later version.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 * http://www.gnu.org/copyleft/gpl.html
22 namespace Wikimedia\Rdbms
;
24 use Psr\Log\LoggerInterface
;
25 use Psr\Log\NullLogger
;
26 use Wikimedia\ScopedCallback
;
31 * Basic DB load monitor with no external dependencies
32 * Uses memcached to cache the replication lag for a short time
36 class LoadMonitor
implements ILoadMonitor
{
37 /** @var ILoadBalancer */
41 /** @var WANObjectCache */
43 /** @var LoggerInterface */
44 protected $replLogger;
46 /** @var float Moving average ratio (e.g. 0.1 for 10% weight to new weight) */
47 private $movingAveRatio;
48 /** @var int Amount of replication lag in seconds before warnings are logged */
49 private $lagWarnThreshold;
51 /** @var int cache key version */
53 /** @var int Default 'max lag' in seconds when unspecified */
54 const LAG_WARN_THRESHOLD
= 10;
57 * @param ILoadBalancer $lb
58 * @param BagOStuff $srvCache
59 * @param WANObjectCache $wCache
60 * @param array $options
61 * - movingAveRatio: moving average constant for server weight updates based on lag
62 * - lagWarnThreshold: how many seconds of lag trigger warnings
64 public function __construct(
65 ILoadBalancer
$lb, BagOStuff
$srvCache, WANObjectCache
$wCache, array $options = []
68 $this->srvCache
= $srvCache;
69 $this->wanCache
= $wCache;
70 $this->replLogger
= new NullLogger();
72 $this->movingAveRatio
= isset( $options['movingAveRatio'] )
73 ?
$options['movingAveRatio']
75 $this->lagWarnThreshold
= isset( $options['lagWarnThreshold'] )
76 ?
$options['lagWarnThreshold']
77 : self
::LAG_WARN_THRESHOLD
;
80 public function setLogger( LoggerInterface
$logger ) {
81 $this->replLogger
= $logger;
84 final public function scaleLoads( array &$weightByServer, $domain ) {
85 $serverIndexes = array_keys( $weightByServer );
86 $states = $this->getServerStates( $serverIndexes, $domain );
87 $newScalesByServer = $states['weightScales'];
88 foreach ( $weightByServer as $i => $weight ) {
89 if ( isset( $newScalesByServer[$i] ) ) {
90 $weightByServer[$i] = $weight * $newScalesByServer[$i];
91 } else { // server recently added to config?
92 $host = $this->parent
->getServerName( $i );
93 $this->replLogger
->error( __METHOD__
. ": host $host not in cache" );
98 final public function getLagTimes( array $serverIndexes, $domain ) {
99 return $this->getServerStates( $serverIndexes, $domain )['lagTimes'];
102 protected function getServerStates( array $serverIndexes, $domain ) {
103 $writerIndex = $this->parent
->getWriterIndex();
104 if ( count( $serverIndexes ) == 1 && reset( $serverIndexes ) == $writerIndex ) {
105 # Single server only, just return zero without caching
107 'lagTimes' => [ $writerIndex => 0 ],
108 'weightScales' => [ $writerIndex => 1.0 ]
112 $key = $this->getCacheKey( $serverIndexes );
113 # Randomize TTLs to reduce stampedes (4.0 - 5.0 sec)
114 $ttl = mt_rand( 4e6
, 5e6
) / 1e6
;
115 # Keep keys around longer as fallbacks
118 # (a) Check the local APC cache
119 $value = $this->srvCache
->get( $key );
120 if ( $value && $value['timestamp'] > ( microtime( true ) - $ttl ) ) {
121 $this->replLogger
->debug( __METHOD__
. ": got lag times ($key) from local cache" );
122 return $value; // cache hit
124 $staleValue = $value ?
: false;
126 # (b) Check the shared cache and backfill APC
127 $value = $this->wanCache
->get( $key );
128 if ( $value && $value['timestamp'] > ( microtime( true ) - $ttl ) ) {
129 $this->srvCache
->set( $key, $value, $staleTTL );
130 $this->replLogger
->debug( __METHOD__
. ": got lag times ($key) from main cache" );
132 return $value; // cache hit
134 $staleValue = $value ?
: $staleValue;
136 # (c) Cache key missing or expired; regenerate and backfill
137 if ( $this->srvCache
->lock( $key, 0, 10 ) ) {
138 # Let only this process update the cache value on this server
139 $sCache = $this->srvCache
;
140 /** @noinspection PhpUnusedLocalVariableInspection */
141 $unlocker = new ScopedCallback( function () use ( $sCache, $key ) {
142 $sCache->unlock( $key );
144 } elseif ( $staleValue ) {
145 # Could not acquire lock but an old cache exists, so use it
151 $movAveRatio = $this->movingAveRatio
;
152 foreach ( $serverIndexes as $i ) {
153 if ( $i == $this->parent
->getWriterIndex() ) {
154 $lagTimes[$i] = 0; // master always has no lag
155 $weightScales[$i] = 1.0; // nominal weight
159 # Handles with open transactions are avoided since they might be subject
160 # to REPEATABLE-READ snapshots, which could affect the lag estimate query.
161 $flags = ILoadBalancer
::CONN_TRX_AUTOCOMMIT
;
162 $conn = $this->parent
->getAnyOpenConnection( $i, $flags );
164 $close = false; // already open
166 $conn = $this->parent
->openConnection( $i, ILoadBalancer
::DOMAIN_ANY
, $flags );
167 $close = true; // new connection
170 $lastWeight = isset( $staleValue['weightScales'][$i] )
171 ?
$staleValue['weightScales'][$i]
173 $coefficient = $this->getWeightScale( $i, $conn ?
: null );
174 $newWeight = $movAveRatio * $coefficient +
( 1 - $movAveRatio ) * $lastWeight;
176 // Scale from 10% to 100% of nominal weight
177 $weightScales[$i] = max( $newWeight, 0.10 );
179 $host = $this->parent
->getServerName( $i );
182 $lagTimes[$i] = false;
183 $this->replLogger
->error(
184 __METHOD__
. ": host {db_server} is unreachable",
185 [ 'db_server' => $host ]
190 if ( $conn->getLBInfo( 'is static' ) ) {
193 $lagTimes[$i] = $conn->getLag();
194 if ( $lagTimes[$i] === false ) {
195 $this->replLogger
->error(
196 __METHOD__
. ": host {db_server} is not replicating?",
197 [ 'db_server' => $host ]
199 } elseif ( $lagTimes[$i] > $this->lagWarnThreshold
) {
200 $this->replLogger
->error(
201 "Server {host} has {lag} seconds of lag (>= {maxlag})",
204 'lag' => $lagTimes[$i],
205 'maxlag' => $this->lagWarnThreshold
212 # Close the connection to avoid sleeper connections piling up.
213 # Note that the caller will pick one of these DBs and reconnect,
214 # which is slightly inefficient, but this only matters for the lag
215 # time cache miss cache, which is far less common that cache hits.
216 $this->parent
->closeConnection( $conn );
220 # Add a timestamp key so we know when it was cached
222 'lagTimes' => $lagTimes,
223 'weightScales' => $weightScales,
224 'timestamp' => microtime( true )
226 $this->wanCache
->set( $key, $value, $staleTTL );
227 $this->srvCache
->set( $key, $value, $staleTTL );
228 $this->replLogger
->info( __METHOD__
. ": re-calculated lag times ($key)" );
234 * @param int $index Server index
235 * @param IDatabase|null $conn Connection handle or null on connection failure
238 protected function getWeightScale( $index, IDatabase
$conn = null ) {
239 return $conn ?
1.0 : 0.0;
242 private function getCacheKey( array $serverIndexes ) {
243 sort( $serverIndexes );
244 // Lag is per-server, not per-DB, so key on the master DB name
245 return $this->srvCache
->makeGlobalKey(
248 $this->parent
->getServerName( $this->parent
->getWriterIndex() ),
249 implode( '-', $serverIndexes )