3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation; either version 2 of the License, or
6 * (at your option) any later version.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 * http://www.gnu.org/copyleft/gpl.html
22 use Psr\Log\LoggerInterface
;
25 * Basic DB load monitor with no external dependencies
26 * Uses memcached to cache the replication lag for a short time
30 class LoadMonitor
implements ILoadMonitor
{
31 /** @var ILoadBalancer */
37 /** @var LoggerInterface */
38 protected $replLogger;
40 /** @var float Moving average ratio (e.g. 0.1 for 10% weight to new weight) */
41 private $movingAveRatio;
43 const VERSION
= 1; // cache key version
45 public function __construct(
46 ILoadBalancer
$lb, BagOStuff
$srvCache, BagOStuff
$cache, array $options = []
49 $this->srvCache
= $srvCache;
50 $this->mainCache
= $cache;
51 $this->replLogger
= new \Psr\Log\
NullLogger();
53 $this->movingAveRatio
= isset( $options['movingAveRatio'] )
54 ?
$options['movingAveRatio']
58 public function setLogger( LoggerInterface
$logger ) {
59 $this->replLogger
= $logger;
62 public function scaleLoads( array &$weightByServer, $domain ) {
63 $serverIndexes = array_keys( $weightByServer );
64 $states = $this->getServerStates( $serverIndexes, $domain );
65 $coefficientsByServer = $states['weightScales'];
66 foreach ( $weightByServer as $i => $weight ) {
67 if ( isset( $coefficientsByServer[$i] ) ) {
68 $weightByServer[$i] = $weight * $coefficientsByServer[$i];
69 } else { // server recently added to config?
70 $host = $this->parent
->getServerName( $i );
71 $this->replLogger
->error( __METHOD__
. ": host $host not in cache" );
76 public function getLagTimes( array $serverIndexes, $domain ) {
77 $states = $this->getServerStates( $serverIndexes, $domain );
79 return $states['lagTimes'];
82 protected function getServerStates( array $serverIndexes, $domain ) {
83 $writerIndex = $this->parent
->getWriterIndex();
84 if ( count( $serverIndexes ) == 1 && reset( $serverIndexes ) == $writerIndex ) {
85 # Single server only, just return zero without caching
87 'lagTimes' => [ $writerIndex => 0 ],
88 'weightScales' => [ $writerIndex => 1.0 ]
92 $key = $this->getCacheKey( $serverIndexes );
93 # Randomize TTLs to reduce stampedes (4.0 - 5.0 sec)
94 $ttl = mt_rand( 4e6
, 5e6
) / 1e6
;
95 # Keep keys around longer as fallbacks
98 # (a) Check the local APC cache
99 $value = $this->srvCache
->get( $key );
100 if ( $value && $value['timestamp'] > ( microtime( true ) - $ttl ) ) {
101 $this->replLogger
->debug( __METHOD__
. ": got lag times ($key) from local cache" );
102 return $value; // cache hit
104 $staleValue = $value ?
: false;
106 # (b) Check the shared cache and backfill APC
107 $value = $this->mainCache
->get( $key );
108 if ( $value && $value['timestamp'] > ( microtime( true ) - $ttl ) ) {
109 $this->srvCache
->set( $key, $value, $staleTTL );
110 $this->replLogger
->debug( __METHOD__
. ": got lag times ($key) from main cache" );
112 return $value; // cache hit
114 $staleValue = $value ?
: $staleValue;
116 # (c) Cache key missing or expired; regenerate and backfill
117 if ( $this->mainCache
->lock( $key, 0, 10 ) ) {
118 # Let this process alone update the cache value
119 $cache = $this->mainCache
;
120 /** @noinspection PhpUnusedLocalVariableInspection */
121 $unlocker = new ScopedCallback( function () use ( $cache, $key ) {
122 $cache->unlock( $key );
124 } elseif ( $staleValue ) {
125 # Could not acquire lock but an old cache exists, so use it
131 $movAveRatio = $this->movingAveRatio
;
132 foreach ( $serverIndexes as $i ) {
133 if ( $i == $this->parent
->getWriterIndex() ) {
134 $lagTimes[$i] = 0; // master always has no lag
135 $weightScales[$i] = 1.0; // nominal weight
139 $conn = $this->parent
->getAnyOpenConnection( $i );
141 $close = false; // already open
143 $conn = $this->parent
->openConnection( $i, $domain );
144 $close = true; // new connection
147 $lastWeight = isset( $staleValue['weightScales'][$i] )
148 ?
$staleValue['weightScales'][$i]
150 $coefficient = $this->getWeightScale( $i, $conn ?
: null );
151 $newWeight = $movAveRatio * $coefficient +
( 1 - $movAveRatio ) * $lastWeight;
153 // Scale from 10% to 100% of nominal weight
154 $weightScales[$i] = max( $newWeight, .10 );
157 $lagTimes[$i] = false;
158 $host = $this->parent
->getServerName( $i );
159 $this->replLogger
->error( __METHOD__
. ": host $host is unreachable" );
163 if ( $conn->getLBInfo( 'is static' ) ) {
166 $lagTimes[$i] = $conn->getLag();
167 if ( $lagTimes[$i] === false ) {
168 $host = $this->parent
->getServerName( $i );
169 $this->replLogger
->error( __METHOD__
. ": host $host is not replicating?" );
174 # Close the connection to avoid sleeper connections piling up.
175 # Note that the caller will pick one of these DBs and reconnect,
176 # which is slightly inefficient, but this only matters for the lag
177 # time cache miss cache, which is far less common that cache hits.
178 $this->parent
->closeConnection( $conn );
182 # Add a timestamp key so we know when it was cached
184 'lagTimes' => $lagTimes,
185 'weightScales' => $weightScales,
186 'timestamp' => microtime( true )
188 $this->mainCache
->set( $key, $value, $staleTTL );
189 $this->srvCache
->set( $key, $value, $staleTTL );
190 $this->replLogger
->info( __METHOD__
. ": re-calculated lag times ($key)" );
196 * @param integer $index Server index
197 * @param IDatabase|null $conn Connection handle or null on connection failure
200 protected function getWeightScale( $index, IDatabase
$conn = null ) {
201 return $conn ?
1.0 : 0.0;
204 private function getCacheKey( array $serverIndexes ) {
205 sort( $serverIndexes );
206 // Lag is per-server, not per-DB, so key on the master DB name
207 return $this->srvCache
->makeGlobalKey(
210 $this->parent
->getServerName( $this->parent
->getWriterIndex() ),
211 implode( '-', $serverIndexes )