includes/libs/rdbms/loadbalancer/LoadBalancer.php

   1 <?php
   2 /**
   3  * Database load balancing manager
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  */
  22 namespace Wikimedia\Rdbms;
  23
  24 use Psr\Log\LoggerInterface;
  25 use Psr\Log\NullLogger;
  26 use Wikimedia\ScopedCallback;
  27 use BagOStuff;
  28 use EmptyBagOStuff;
  29 use WANObjectCache;
  30 use ArrayUtils;
  31 use UnexpectedValueException;
  32 use InvalidArgumentException;
  33 use RuntimeException;
  34 use Exception;
  35
  36 /**
  37  * Database connection, tracking, load balancing, and transaction manager for a cluster
  38  *
  39  * @ingroup Database
  40  */
  41 class LoadBalancer implements ILoadBalancer {
  42         /** @var array[] Map of (server index => server config array) */
  43         private $servers;
  44         /** @var Database[][][] Map of (connection category => server index => IDatabase[]) */
  45         private $conns;
  46         /** @var float[] Map of (server index => weight) */
  47         private $loads;
  48         /** @var array[] Map of (group => server index => weight) */
  49         private $groupLoads;
  50         /** @var bool Whether to disregard replica DB lag as a factor in replica DB selection */
  51         private $allowLagged;
  52         /** @var int Seconds to spend waiting on replica DB lag to resolve */
  53         private $waitTimeout;
  54         /** @var array The LoadMonitor configuration */
  55         private $loadMonitorConfig;
  56         /** @var array[] $aliases Map of (table => (dbname, schema, prefix) map) */
  57         private $tableAliases = [];
  58         /** @var string[] Map of (index alias => index) */
  59         private $indexAliases = [];
  60
  61         /** @var ILoadMonitor */
  62         private $loadMonitor;
  63         /** @var callable|null Callback to run before the first connection attempt */
  64         private $chronologyCallback;
  65         /** @var BagOStuff */
  66         private $srvCache;
  67         /** @var WANObjectCache */
  68         private $wanCache;
  69         /** @var object|string Class name or object With profileIn/profileOut methods */
  70         protected $profiler;
  71         /** @var TransactionProfiler */
  72         protected $trxProfiler;
  73         /** @var LoggerInterface */
  74         protected $replLogger;
  75         /** @var LoggerInterface */
  76         protected $connLogger;
  77         /** @var LoggerInterface */
  78         protected $queryLogger;
  79         /** @var LoggerInterface */
  80         protected $perfLogger;
  81
  82         /** @var Database DB connection object that caused a problem */
  83         private $errorConnection;
  84         /** @var int The generic (not query grouped) replica DB index (of $mServers) */
  85         private $readIndex;
  86         /** @var bool|DBMasterPos False if not set */
  87         private $waitForPos;
  88         /** @var bool Whether the generic reader fell back to a lagged replica DB */
  89         private $laggedReplicaMode = false;
  90         /** @var bool Whether the generic reader fell back to a lagged replica DB */
  91         private $allReplicasDownMode = false;
  92         /** @var string The last DB selection or connection error */
  93         private $lastError = 'Unknown error';
  94         /** @var string|bool Reason the LB is read-only or false if not */
  95         private $readOnlyReason = false;
  96         /** @var int Total connections opened */
  97         private $connsOpened = 0;
  98         /** @var string|bool String if a requested DBO_TRX transaction round is active */
  99         private $trxRoundId = false;
 100         /** @var array[] Map of (name => callable) */
 101         private $trxRecurringCallbacks = [];
 102         /** @var DatabaseDomain Local Domain ID and default for selectDB() calls */
 103         private $localDomain;
 104         /** @var string Alternate ID string for the domain instead of DatabaseDomain::getId() */
 105         private $localDomainIdAlias;
 106         /** @var string Current server name */
 107         private $host;
 108         /** @var bool Whether this PHP instance is for a CLI script */
 109         protected $cliMode;
 110         /** @var string Agent name for query profiling */
 111         protected $agent;
 112
 113         /** @var callable Exception logger */
 114         private $errorLogger;
 115         /** @var callable Deprecation logger */
 116         private $deprecationLogger;
 117
 118         /** @var bool */
 119         private $disabled = false;
 120         /** @var bool Whether any connection has been attempted yet */
 121         private $connectionAttempted = false;
 122         /** @var int */
 123         private $maxLag = self::MAX_LAG_DEFAULT;
 124
 125         /** @var int Warn when this many connection are held */
 126         const CONN_HELD_WARN_THRESHOLD = 10;
 127
 128         /** @var int Default 'maxLag' when unspecified */
 129         const MAX_LAG_DEFAULT = 10;
 130         /** @var int Default 'waitTimeout' when unspecified */
 131         const MAX_WAIT_DEFAULT = 10;
 132         /** @var int Seconds to cache master server read-only status */
 133         const TTL_CACHE_READONLY = 5;
 134
 135         const KEY_LOCAL = 'local';
 136         const KEY_FOREIGN_FREE = 'foreignFree';
 137         const KEY_FOREIGN_INUSE = 'foreignInUse';
 138
 139         const KEY_LOCAL_NOROUND = 'localAutoCommit';
 140         const KEY_FOREIGN_FREE_NOROUND = 'foreignFreeAutoCommit';
 141         const KEY_FOREIGN_INUSE_NOROUND = 'foreignInUseAutoCommit';
 142
 143         public function __construct( array $params ) {
 144                 if ( !isset( $params['servers'] ) ) {
 145                         throw new InvalidArgumentException( __CLASS__ . ': missing servers parameter' );
 146                 }
 147                 $this->servers = $params['servers'];
 148                 foreach ( $this->servers as $i => $server ) {
 149                         if ( $i == 0 ) {
 150                                 $this->servers[$i]['master'] = true;
 151                         } else {
 152                                 $this->servers[$i]['replica'] = true;
 153                         }
 154                 }
 155
 156                 $localDomain = isset( $params['localDomain'] )
 157                         ? DatabaseDomain::newFromId( $params['localDomain'] )
 158                         : DatabaseDomain::newUnspecified();
 159                 $this->setLocalDomain( $localDomain );
 160
 161                 $this->waitTimeout = isset( $params['waitTimeout'] )
 162                         ? $params['waitTimeout']
 163                         : self::MAX_WAIT_DEFAULT;
 164
 165                 $this->readIndex = -1;
 166                 $this->conns = [
 167                         // Connection were transaction rounds may be applied
 168                         self::KEY_LOCAL => [],
 169                         self::KEY_FOREIGN_INUSE => [],
 170                         self::KEY_FOREIGN_FREE => [],
 171                         // Auto-committing counterpart connections that ignore transaction rounds
 172                         self::KEY_LOCAL_NOROUND => [],
 173                         self::KEY_FOREIGN_INUSE_NOROUND => [],
 174                         self::KEY_FOREIGN_FREE_NOROUND => []
 175                 ];
 176                 $this->loads = [];
 177                 $this->waitForPos = false;
 178                 $this->allowLagged = false;
 179
 180                 if ( isset( $params['readOnlyReason'] ) && is_string( $params['readOnlyReason'] ) ) {
 181                         $this->readOnlyReason = $params['readOnlyReason'];
 182                 }
 183
 184                 if ( isset( $params['maxLag'] ) ) {
 185                         $this->maxLag = $params['maxLag'];
 186                 }
 187
 188                 if ( isset( $params['loadMonitor'] ) ) {
 189                         $this->loadMonitorConfig = $params['loadMonitor'];
 190                 } else {
 191                         $this->loadMonitorConfig = [ 'class' => 'LoadMonitorNull' ];
 192                 }
 193                 $this->loadMonitorConfig += [ 'lagWarnThreshold' => $this->maxLag ];
 194
 195                 foreach ( $params['servers'] as $i => $server ) {
 196                         $this->loads[$i] = $server['load'];
 197                         if ( isset( $server['groupLoads'] ) ) {
 198                                 foreach ( $server['groupLoads'] as $group => $ratio ) {
 199                                         if ( !isset( $this->groupLoads[$group] ) ) {
 200                                                 $this->groupLoads[$group] = [];
 201                                         }
 202                                         $this->groupLoads[$group][$i] = $ratio;
 203                                 }
 204                         }
 205                 }
 206
 207                 if ( isset( $params['srvCache'] ) ) {
 208                         $this->srvCache = $params['srvCache'];
 209                 } else {
 210                         $this->srvCache = new EmptyBagOStuff();
 211                 }
 212                 if ( isset( $params['wanCache'] ) ) {
 213                         $this->wanCache = $params['wanCache'];
 214                 } else {
 215                         $this->wanCache = WANObjectCache::newEmpty();
 216                 }
 217                 $this->profiler = isset( $params['profiler'] ) ? $params['profiler'] : null;
 218                 if ( isset( $params['trxProfiler'] ) ) {
 219                         $this->trxProfiler = $params['trxProfiler'];
 220                 } else {
 221                         $this->trxProfiler = new TransactionProfiler();
 222                 }
 223
 224                 $this->errorLogger = isset( $params['errorLogger'] )
 225                         ? $params['errorLogger']
 226                         : function ( Exception $e ) {
 227                                 trigger_error( get_class( $e ) . ': ' . $e->getMessage(), E_USER_WARNING );
 228                         };
 229                 $this->deprecationLogger = isset( $params['deprecationLogger'] )
 230                         ? $params['deprecationLogger']
 231                         : function ( $msg ) {
 232                                 trigger_error( $msg, E_USER_DEPRECATED );
 233                         };
 234
 235                 foreach ( [ 'replLogger', 'connLogger', 'queryLogger', 'perfLogger' ] as $key ) {
 236                         $this->$key = isset( $params[$key] ) ? $params[$key] : new NullLogger();
 237                 }
 238
 239                 $this->host = isset( $params['hostname'] )
 240                         ? $params['hostname']
 241                         : ( gethostname() ?: 'unknown' );
 242                 $this->cliMode = isset( $params['cliMode'] )
 243                         ? $params['cliMode']
 244                         : ( PHP_SAPI === 'cli' || PHP_SAPI === 'phpdbg' );
 245                 $this->agent = isset( $params['agent'] ) ? $params['agent'] : '';
 246
 247                 if ( isset( $params['chronologyCallback'] ) ) {
 248                         $this->chronologyCallback = $params['chronologyCallback'];
 249                 }
 250         }
 251
 252         /**
 253          * Get the local (and default) database domain ID of connection handles
 254          *
 255          * @see DatabaseDomain
 256          * @return string Database domain ID; this specifies DB name, schema, and table prefix
 257          * @since 1.31
 258          */
 259         public function getLocalDomainID() {
 260                 return $this->localDomain->getId();
 261         }
 262
 263         /**
 264          * Get a LoadMonitor instance
 265          *
 266          * @return ILoadMonitor
 267          */
 268         private function getLoadMonitor() {
 269                 if ( !isset( $this->loadMonitor ) ) {
 270                         $compat = [
 271                                 'LoadMonitor' => LoadMonitor::class,
 272                                 'LoadMonitorNull' => LoadMonitorNull::class,
 273                                 'LoadMonitorMySQL' => LoadMonitorMySQL::class,
 274                         ];
 275
 276                         $class = $this->loadMonitorConfig['class'];
 277                         if ( isset( $compat[$class] ) ) {
 278                                 $class = $compat[$class];
 279                         }
 280
 281                         $this->loadMonitor = new $class(
 282                                 $this, $this->srvCache, $this->wanCache, $this->loadMonitorConfig );
 283                         $this->loadMonitor->setLogger( $this->replLogger );
 284                 }
 285
 286                 return $this->loadMonitor;
 287         }
 288
 289         /**
 290          * @param array $loads
 291          * @param bool|string $domain Domain to get non-lagged for
 292          * @param int $maxLag Restrict the maximum allowed lag to this many seconds
 293          * @return bool|int|string
 294          */
 295         private function getRandomNonLagged( array $loads, $domain = false, $maxLag = INF ) {
 296                 $lags = $this->getLagTimes( $domain );
 297
 298                 # Unset excessively lagged servers
 299                 foreach ( $lags as $i => $lag ) {
 300                         if ( $i != 0 ) {
 301                                 # How much lag this server nominally is allowed to have
 302                                 $maxServerLag = isset( $this->servers[$i]['max lag'] )
 303                                         ? $this->servers[$i]['max lag']
 304                                         : $this->maxLag; // default
 305                                 # Constrain that futher by $maxLag argument
 306                                 $maxServerLag = min( $maxServerLag, $maxLag );
 307
 308                                 $host = $this->getServerName( $i );
 309                                 if ( $lag === false && !is_infinite( $maxServerLag ) ) {
 310                                         $this->replLogger->error(
 311                                                 __METHOD__ .
 312                                                 ": server {host} is not replicating?", [ 'host' => $host ] );
 313                                         unset( $loads[$i] );
 314                                 } elseif ( $lag > $maxServerLag ) {
 315                                         $this->replLogger->debug(
 316                                                 __METHOD__ .
 317                                                 ": server {host} has {lag} seconds of lag (>= {maxlag})",
 318                                                 [ 'host' => $host, 'lag' => $lag, 'maxlag' => $maxServerLag ]
 319                                         );
 320                                         unset( $loads[$i] );
 321                                 }
 322                         }
 323                 }
 324
 325                 # Find out if all the replica DBs with non-zero load are lagged
 326                 $sum = 0;
 327                 foreach ( $loads as $load ) {
 328                         $sum += $load;
 329                 }
 330                 if ( $sum == 0 ) {
 331                         # No appropriate DB servers except maybe the master and some replica DBs with zero load
 332                         # Do NOT use the master
 333                         # Instead, this function will return false, triggering read-only mode,
 334                         # and a lagged replica DB will be used instead.
 335                         return false;
 336                 }
 337
 338                 if ( count( $loads ) == 0 ) {
 339                         return false;
 340                 }
 341
 342                 # Return a random representative of the remainder
 343                 return ArrayUtils::pickRandom( $loads );
 344         }
 345
 346         public function getReaderIndex( $group = false, $domain = false ) {
 347                 if ( count( $this->servers ) == 1 ) {
 348                         // Skip the load balancing if there's only one server
 349                         return $this->getWriterIndex();
 350                 } elseif ( $group === false && $this->readIndex >= 0 ) {
 351                         // Shortcut if the generic reader index was already cached
 352                         return $this->readIndex;
 353                 }
 354
 355                 if ( $group !== false ) {
 356                         // Use the server weight array for this load group
 357                         if ( isset( $this->groupLoads[$group] ) ) {
 358                                 $loads = $this->groupLoads[$group];
 359                         } else {
 360                                 // No loads for this group, return false and the caller can use some other group
 361                                 $this->connLogger->info( __METHOD__ . ": no loads for group $group" );
 362
 363                                 return false;
 364                         }
 365                 } else {
 366                         // Use the generic load group
 367                         $loads = $this->loads;
 368                 }
 369
 370                 // Scale the configured load ratios according to each server's load and state
 371                 $this->getLoadMonitor()->scaleLoads( $loads, $domain );
 372
 373                 // Pick a server to use, accounting for weights, load, lag, and "waitForPos"
 374                 list( $i, $laggedReplicaMode ) = $this->pickReaderIndex( $loads, $domain );
 375                 if ( $i === false ) {
 376                         // Replica DB connection unsuccessful
 377                         return false;
 378                 }
 379
 380                 if ( $this->waitForPos && $i != $this->getWriterIndex() ) {
 381                         // Before any data queries are run, wait for the server to catch up to the
 382                         // specified position. This is used to improve session consistency. Note that
 383                         // when LoadBalancer::waitFor() sets "waitForPos", the waiting triggers here,
 384                         // so update laggedReplicaMode as needed for consistency.
 385                         if ( !$this->doWait( $i ) ) {
 386                                 $laggedReplicaMode = true;
 387                         }
 388                 }
 389
 390                 if ( $this->readIndex <= 0 && $this->loads[$i] > 0 && $group === false ) {
 391                         // Cache the generic reader index for future ungrouped DB_REPLICA handles
 392                         $this->readIndex = $i;
 393                         // Record if the generic reader index is in "lagged replica DB" mode
 394                         if ( $laggedReplicaMode ) {
 395                                 $this->laggedReplicaMode = true;
 396                         }
 397                 }
 398
 399                 $serverName = $this->getServerName( $i );
 400                 $this->connLogger->debug( __METHOD__ . ": using server $serverName for group '$group'" );
 401
 402                 return $i;
 403         }
 404
 405         /**
 406          * @param array $loads List of server weights
 407          * @param string|bool $domain
 408          * @return array (reader index, lagged replica mode) or false on failure
 409          */
 410         private function pickReaderIndex( array $loads, $domain = false ) {
 411                 if ( !count( $loads ) ) {
 412                         throw new InvalidArgumentException( "Empty server array given to LoadBalancer" );
 413                 }
 414
 415                 /** @var int|bool $i Index of selected server */
 416                 $i = false;
 417                 /** @var bool $laggedReplicaMode Whether server is considered lagged */
 418                 $laggedReplicaMode = false;
 419
 420                 // Quickly look through the available servers for a server that meets criteria...
 421                 $currentLoads = $loads;
 422                 while ( count( $currentLoads ) ) {
 423                         if ( $this->allowLagged || $laggedReplicaMode ) {
 424                                 $i = ArrayUtils::pickRandom( $currentLoads );
 425                         } else {
 426                                 $i = false;
 427                                 if ( $this->waitForPos && $this->waitForPos->asOfTime() ) {
 428                                         // "chronologyCallback" sets "waitForPos" for session consistency.
 429                                         // This triggers doWait() after connect, so it's especially good to
 430                                         // avoid lagged servers so as to avoid excessive delay in that method.
 431                                         $ago = microtime( true ) - $this->waitForPos->asOfTime();
 432                                         // Aim for <= 1 second of waiting (being too picky can backfire)
 433                                         $i = $this->getRandomNonLagged( $currentLoads, $domain, $ago + 1 );
 434                                 }
 435                                 if ( $i === false ) {
 436                                         // Any server with less lag than it's 'max lag' param is preferable
 437                                         $i = $this->getRandomNonLagged( $currentLoads, $domain );
 438                                 }
 439                                 if ( $i === false && count( $currentLoads ) != 0 ) {
 440                                         // All replica DBs lagged. Switch to read-only mode
 441                                         $this->replLogger->error(
 442                                                 __METHOD__ . ": all replica DBs lagged. Switch to read-only mode" );
 443                                         $i = ArrayUtils::pickRandom( $currentLoads );
 444                                         $laggedReplicaMode = true;
 445                                 }
 446                         }
 447
 448                         if ( $i === false ) {
 449                                 // pickRandom() returned false.
 450                                 // This is permanent and means the configuration or the load monitor
 451                                 // wants us to return false.
 452                                 $this->connLogger->debug( __METHOD__ . ": pickRandom() returned false" );
 453
 454                                 return [ false, false ];
 455                         }
 456
 457                         $serverName = $this->getServerName( $i );
 458                         $this->connLogger->debug( __METHOD__ . ": Using reader #$i: $serverName..." );
 459
 460                         $conn = $this->openConnection( $i, $domain );
 461                         if ( !$conn ) {
 462                                 $this->connLogger->warning( __METHOD__ . ": Failed connecting to $i/$domain" );
 463                                 unset( $currentLoads[$i] ); // avoid this server next iteration
 464                                 $i = false;
 465                                 continue;
 466                         }
 467
 468                         // Decrement reference counter, we are finished with this connection.
 469                         // It will be incremented for the caller later.
 470                         if ( $domain !== false ) {
 471                                 $this->reuseConnection( $conn );
 472                         }
 473
 474                         // Return this server
 475                         break;
 476                 }
 477
 478                 // If all servers were down, quit now
 479                 if ( !count( $currentLoads ) ) {
 480                         $this->connLogger->error( __METHOD__ . ": all servers down" );
 481                 }
 482
 483                 return [ $i, $laggedReplicaMode ];
 484         }
 485
 486         public function waitFor( $pos ) {
 487                 $oldPos = $this->waitForPos;
 488                 try {
 489                         $this->waitForPos = $pos;
 490                         // If a generic reader connection was already established, then wait now
 491                         $i = $this->readIndex;
 492                         if ( $i > 0 ) {
 493                                 if ( !$this->doWait( $i ) ) {
 494                                         $this->laggedReplicaMode = true;
 495                                 }
 496                         }
 497                 } finally {
 498                         // Restore the older position if it was higher since this is used for lag-protection
 499                         $this->setWaitForPositionIfHigher( $oldPos );
 500                 }
 501         }
 502
 503         public function waitForOne( $pos, $timeout = null ) {
 504                 $oldPos = $this->waitForPos;
 505                 try {
 506                         $this->waitForPos = $pos;
 507
 508                         $i = $this->readIndex;
 509                         if ( $i <= 0 ) {
 510                                 // Pick a generic replica DB if there isn't one yet
 511                                 $readLoads = $this->loads;
 512                                 unset( $readLoads[$this->getWriterIndex()] ); // replica DBs only
 513                                 $readLoads = array_filter( $readLoads ); // with non-zero load
 514                                 $i = ArrayUtils::pickRandom( $readLoads );
 515                         }
 516
 517                         if ( $i > 0 ) {
 518                                 $ok = $this->doWait( $i, true, $timeout );
 519                         } else {
 520                                 $ok = true; // no applicable loads
 521                         }
 522                 } finally {
 523                         # Restore the old position, as this is not used for lag-protection but for throttling
 524                         $this->waitForPos = $oldPos;
 525                 }
 526
 527                 return $ok;
 528         }
 529
 530         public function waitForAll( $pos, $timeout = null ) {
 531                 $timeout = $timeout ?: $this->waitTimeout;
 532
 533                 $oldPos = $this->waitForPos;
 534                 try {
 535                         $this->waitForPos = $pos;
 536                         $serverCount = count( $this->servers );
 537
 538                         $ok = true;
 539                         for ( $i = 1; $i < $serverCount; $i++ ) {
 540                                 if ( $this->loads[$i] > 0 ) {
 541                                         $start = microtime( true );
 542                                         $ok = $this->doWait( $i, true, $timeout ) && $ok;
 543                                         $timeout -= intval( microtime( true ) - $start );
 544                                         if ( $timeout <= 0 ) {
 545                                                 break; // timeout reached
 546                                         }
 547                                 }
 548                         }
 549                 } finally {
 550                         # Restore the old position, as this is not used for lag-protection but for throttling
 551                         $this->waitForPos = $oldPos;
 552                 }
 553
 554                 return $ok;
 555         }
 556
 557         /**
 558          * @param DBMasterPos|bool $pos
 559          */
 560         private function setWaitForPositionIfHigher( $pos ) {
 561                 if ( !$pos ) {
 562                         return;
 563                 }
 564
 565                 if ( !$this->waitForPos || $pos->hasReached( $this->waitForPos ) ) {
 566                         $this->waitForPos = $pos;
 567                 }
 568         }
 569
 570         public function getAnyOpenConnection( $i, $flags = 0 ) {
 571                 $autocommit = ( ( $flags & self::CONN_TRX_AUTOCOMMIT ) == self::CONN_TRX_AUTOCOMMIT );
 572                 foreach ( $this->conns as $connsByServer ) {
 573                         if ( !isset( $connsByServer[$i] ) ) {
 574                                 continue;
 575                         }
 576
 577                         foreach ( $connsByServer[$i] as $conn ) {
 578                                 if ( !$autocommit || $conn->getLBInfo( 'autoCommitOnly' ) ) {
 579                                         return $conn;
 580                                 }
 581                         }
 582                 }
 583
 584                 return false;
 585         }
 586
 587         /**
 588          * Wait for a given replica DB to catch up to the master pos stored in $this
 589          * @param int $index Server index
 590          * @param bool $open Check the server even if a new connection has to be made
 591          * @param int $timeout Max seconds to wait; default is "waitTimeout" given to __construct()
 592          * @return bool
 593          */
 594         protected function doWait( $index, $open = false, $timeout = null ) {
 595                 $timeout = max( 1, intval( $timeout ?: $this->waitTimeout ) );
 596
 597                 // Check if we already know that the DB has reached this point
 598                 $server = $this->getServerName( $index );
 599                 $key = $this->srvCache->makeGlobalKey( __CLASS__, 'last-known-pos', $server, 'v1' );
 600                 /** @var DBMasterPos $knownReachedPos */
 601                 $knownReachedPos = $this->srvCache->get( $key );
 602                 if (
 603                         $knownReachedPos instanceof DBMasterPos &&
 604                         $knownReachedPos->hasReached( $this->waitForPos )
 605                 ) {
 606                         $this->replLogger->debug(
 607                                 __METHOD__ .
 608                                 ': replica DB {dbserver} known to be caught up (pos >= $knownReachedPos).',
 609                                 [ 'dbserver' => $server ]
 610                         );
 611                         return true;
 612                 }
 613
 614                 // Find a connection to wait on, creating one if needed and allowed
 615                 $close = false; // close the connection afterwards
 616                 $conn = $this->getAnyOpenConnection( $index );
 617                 if ( !$conn ) {
 618                         if ( !$open ) {
 619                                 $this->replLogger->debug(
 620                                         __METHOD__ . ': no connection open for {dbserver}',
 621                                         [ 'dbserver' => $server ]
 622                                 );
 623
 624                                 return false;
 625                         } else {
 626                                 $conn = $this->openConnection( $index, self::DOMAIN_ANY );
 627                                 if ( !$conn ) {
 628                                         $this->replLogger->warning(
 629                                                 __METHOD__ . ': failed to connect to {dbserver}',
 630                                                 [ 'dbserver' => $server ]
 631                                         );
 632
 633                                         return false;
 634                                 }
 635                                 // Avoid connection spam in waitForAll() when connections
 636                                 // are made just for the sake of doing this lag check.
 637                                 $close = true;
 638                         }
 639                 }
 640
 641                 $this->replLogger->info(
 642                         __METHOD__ .
 643                         ': waiting for replica DB {dbserver} to catch up...',
 644                         [ 'dbserver' => $server ]
 645                 );
 646
 647                 $result = $conn->masterPosWait( $this->waitForPos, $timeout );
 648
 649                 if ( $result === null ) {
 650                         $this->replLogger->warning(
 651                                 __METHOD__ . ': Errored out waiting on {host} pos {pos}',
 652                                 [
 653                                         'host' => $server,
 654                                         'pos' => $this->waitForPos,
 655                                         'trace' => ( new RuntimeException() )->getTraceAsString()
 656                                 ]
 657                         );
 658                         $ok = false;
 659                 } elseif ( $result == -1 ) {
 660                         $this->replLogger->warning(
 661                                 __METHOD__ . ': Timed out waiting on {host} pos {pos}',
 662                                 [
 663                                         'host' => $server,
 664                                         'pos' => $this->waitForPos,
 665                                         'trace' => ( new RuntimeException() )->getTraceAsString()
 666                                 ]
 667                         );
 668                         $ok = false;
 669                 } else {
 670                         $this->replLogger->debug( __METHOD__ . ": done waiting" );
 671                         $ok = true;
 672                         // Remember that the DB reached this point
 673                         $this->srvCache->set( $key, $this->waitForPos, BagOStuff::TTL_DAY );
 674                 }
 675
 676                 if ( $close ) {
 677                         $this->closeConnection( $conn );
 678                 }
 679
 680                 return $ok;
 681         }
 682
 683         public function getConnection( $i, $groups = [], $domain = false, $flags = 0 ) {
 684                 if ( $i === null || $i === false ) {
 685                         throw new InvalidArgumentException( 'Attempt to call ' . __METHOD__ .
 686                                 ' with invalid server index' );
 687                 }
 688
 689                 if ( $this->localDomain->equals( $domain ) || $domain === $this->localDomainIdAlias ) {
 690                         $domain = false; // local connection requested
 691                 }
 692
 693                 if ( ( $flags & self::CONN_TRX_AUTOCOMMIT ) === self::CONN_TRX_AUTOCOMMIT ) {
 694                         // Assuming all servers are of the same type (or similar), which is overwhelmingly
 695                         // the case, use the master server information to get the attributes. The information
 696                         // for $i cannot be used since it might be DB_REPLICA, which might require connection
 697                         // attempts in order to be resolved into a real server index.
 698                         $attributes = $this->getServerAttributes( $this->getWriterIndex() );
 699                         if ( $attributes[Database::ATTR_DB_LEVEL_LOCKING] ) {
 700                                 // Callers sometimes want to (a) escape REPEATABLE-READ stateness without locking
 701                                 // rows (e.g. FOR UPDATE) or (b) make small commits during a larger transactions
 702                                 // to reduce lock contention. None of these apply for sqlite and using separate
 703                                 // connections just causes self-deadlocks.
 704                                 $flags &= ~self::CONN_TRX_AUTOCOMMIT;
 705                                 $this->connLogger->info( __METHOD__ .
 706                                         ': ignoring CONN_TRX_AUTOCOMMIT to avoid deadlocks.' );
 707                         }
 708                 }
 709
 710                 $groups = ( $groups === false || $groups === [] )
 711                         ? [ false ] // check one "group": the generic pool
 712                         : (array)$groups;
 713
 714                 $masterOnly = ( $i == self::DB_MASTER || $i == $this->getWriterIndex() );
 715                 $oldConnsOpened = $this->connsOpened; // connections open now
 716
 717                 if ( $i == self::DB_MASTER ) {
 718                         $i = $this->getWriterIndex();
 719                 } elseif ( $i == self::DB_REPLICA ) {
 720                         # Try to find an available server in any the query groups (in order)
 721                         foreach ( $groups as $group ) {
 722                                 $groupIndex = $this->getReaderIndex( $group, $domain );
 723                                 if ( $groupIndex !== false ) {
 724                                         $i = $groupIndex;
 725                                         break;
 726                                 }
 727                         }
 728                 }
 729
 730                 # Operation-based index
 731                 if ( $i == self::DB_REPLICA ) {
 732                         $this->lastError = 'Unknown error'; // reset error string
 733                         # Try the general server pool if $groups are unavailable.
 734                         $i = ( $groups === [ false ] )
 735                                 ? false // don't bother with this if that is what was tried above
 736                                 : $this->getReaderIndex( false, $domain );
 737                         # Couldn't find a working server in getReaderIndex()?
 738                         if ( $i === false ) {
 739                                 $this->lastError = 'No working replica DB server: ' . $this->lastError;
 740                                 // Throw an exception
 741                                 $this->reportConnectionError();
 742                                 return null; // not reached
 743                         }
 744                 }
 745
 746                 # Now we have an explicit index into the servers array
 747                 $conn = $this->openConnection( $i, $domain, $flags );
 748                 if ( !$conn ) {
 749                         // Throw an exception
 750                         $this->reportConnectionError();
 751                         return null; // not reached
 752                 }
 753
 754                 # Profile any new connections that happen
 755                 if ( $this->connsOpened > $oldConnsOpened ) {
 756                         $host = $conn->getServer();
 757                         $dbname = $conn->getDBname();
 758                         $this->trxProfiler->recordConnection( $host, $dbname, $masterOnly );
 759                 }
 760
 761                 if ( $masterOnly ) {
 762                         # Make master-requested DB handles inherit any read-only mode setting
 763                         $conn->setLBInfo( 'readOnlyReason', $this->getReadOnlyReason( $domain, $conn ) );
 764                 }
 765
 766                 return $conn;
 767         }
 768
 769         public function reuseConnection( IDatabase $conn ) {
 770                 $serverIndex = $conn->getLBInfo( 'serverIndex' );
 771                 $refCount = $conn->getLBInfo( 'foreignPoolRefCount' );
 772                 if ( $serverIndex === null || $refCount === null ) {
 773                         /**
 774                          * This can happen in code like:
 775                          *   foreach ( $dbs as $db ) {
 776                          *     $conn = $lb->getConnection( $lb::DB_REPLICA, [], $db );
 777                          *     ...
 778                          *     $lb->reuseConnection( $conn );
 779                          *   }
 780                          * When a connection to the local DB is opened in this way, reuseConnection()
 781                          * should be ignored
 782                          */
 783                         return;
 784                 } elseif ( $conn instanceof DBConnRef ) {
 785                         // DBConnRef already handles calling reuseConnection() and only passes the live
 786                         // Database instance to this method. Any caller passing in a DBConnRef is broken.
 787                         $this->connLogger->error(
 788                                 __METHOD__ . ": got DBConnRef instance.\n" .
 789                                 ( new RuntimeException() )->getTraceAsString() );
 790
 791                         return;
 792                 }
 793
 794                 if ( $this->disabled ) {
 795                         return; // DBConnRef handle probably survived longer than the LoadBalancer
 796                 }
 797
 798                 if ( $conn->getLBInfo( 'autoCommitOnly' ) ) {
 799                         $connFreeKey = self::KEY_FOREIGN_FREE_NOROUND;
 800                         $connInUseKey = self::KEY_FOREIGN_INUSE_NOROUND;
 801                 } else {
 802                         $connFreeKey = self::KEY_FOREIGN_FREE;
 803                         $connInUseKey = self::KEY_FOREIGN_INUSE;
 804                 }
 805
 806                 $domain = $conn->getDomainID();
 807                 if ( !isset( $this->conns[$connInUseKey][$serverIndex][$domain] ) ) {
 808                         throw new InvalidArgumentException( __METHOD__ .
 809                                 ": connection $serverIndex/$domain not found; it may have already been freed." );
 810                 } elseif ( $this->conns[$connInUseKey][$serverIndex][$domain] !== $conn ) {
 811                         throw new InvalidArgumentException( __METHOD__ .
 812                                 ": connection $serverIndex/$domain mismatched; it may have already been freed." );
 813                 }
 814
 815                 $conn->setLBInfo( 'foreignPoolRefCount', --$refCount );
 816                 if ( $refCount <= 0 ) {
 817                         $this->conns[$connFreeKey][$serverIndex][$domain] = $conn;
 818                         unset( $this->conns[$connInUseKey][$serverIndex][$domain] );
 819                         if ( !$this->conns[$connInUseKey][$serverIndex] ) {
 820                                 unset( $this->conns[$connInUseKey][$serverIndex] ); // clean up
 821                         }
 822                         $this->connLogger->debug( __METHOD__ . ": freed connection $serverIndex/$domain" );
 823                 } else {
 824                         $this->connLogger->debug( __METHOD__ .
 825                                 ": reference count for $serverIndex/$domain reduced to $refCount" );
 826                 }
 827         }
 828
 829         public function getConnectionRef( $db, $groups = [], $domain = false, $flags = 0 ) {
 830                 $domain = ( $domain !== false ) ? $domain : $this->localDomain;
 831
 832                 return new DBConnRef( $this, $this->getConnection( $db, $groups, $domain, $flags ) );
 833         }
 834
 835         public function getLazyConnectionRef( $db, $groups = [], $domain = false, $flags = 0 ) {
 836                 $domain = ( $domain !== false ) ? $domain : $this->localDomain;
 837
 838                 return new DBConnRef( $this, [ $db, $groups, $domain, $flags ] );
 839         }
 840
 841         public function getMaintenanceConnectionRef( $db, $groups = [], $domain = false, $flags = 0 ) {
 842                 $domain = ( $domain !== false ) ? $domain : $this->localDomain;
 843
 844                 return new MaintainableDBConnRef(
 845                         $this, $this->getConnection( $db, $groups, $domain, $flags ) );
 846         }
 847
 848         public function openConnection( $i, $domain = false, $flags = 0 ) {
 849                 if ( $this->localDomain->equals( $domain ) || $domain === $this->localDomainIdAlias ) {
 850                         $domain = false; // local connection requested
 851                 }
 852
 853                 if ( !$this->connectionAttempted && $this->chronologyCallback ) {
 854                         $this->connLogger->debug( __METHOD__ . ': calling initLB() before first connection.' );
 855                         // Load any "waitFor" positions before connecting so that doWait() is triggered
 856                         $this->connectionAttempted = true;
 857                         call_user_func( $this->chronologyCallback, $this );
 858                 }
 859
 860                 // Check if an auto-commit connection is being requested. If so, it will not reuse the
 861                 // main set of DB connections but rather its own pool since:
 862                 // a) those are usually set to implicitly use transaction rounds via DBO_TRX
 863                 // b) those must support the use of explicit transaction rounds via beginMasterChanges()
 864                 $autoCommit = ( ( $flags & self::CONN_TRX_AUTOCOMMIT ) == self::CONN_TRX_AUTOCOMMIT );
 865
 866                 if ( $domain !== false ) {
 867                         // Connection is to a foreign domain
 868                         $conn = $this->openForeignConnection( $i, $domain, $flags );
 869                 } else {
 870                         // Connection is to the local domain
 871                         $connKey = $autoCommit ? self::KEY_LOCAL_NOROUND : self::KEY_LOCAL;
 872                         if ( isset( $this->conns[$connKey][$i][0] ) ) {
 873                                 $conn = $this->conns[$connKey][$i][0];
 874                         } else {
 875                                 if ( !isset( $this->servers[$i] ) || !is_array( $this->servers[$i] ) ) {
 876                                         throw new InvalidArgumentException( "No server with index '$i'." );
 877                                 }
 878                                 // Open a new connection
 879                                 $server = $this->servers[$i];
 880                                 $server['serverIndex'] = $i;
 881                                 $server['autoCommitOnly'] = $autoCommit;
 882                                 if ( $this->localDomain->getDatabase() !== null ) {
 883                                         // Use the local domain table prefix if the local domain is specified
 884                                         $server['tablePrefix'] = $this->localDomain->getTablePrefix();
 885                                 }
 886                                 $conn = $this->reallyOpenConnection( $server, $this->localDomain );
 887                                 $host = $this->getServerName( $i );
 888                                 if ( $conn->isOpen() ) {
 889                                         $this->connLogger->debug(
 890                                                 __METHOD__ . ": connected to database $i at '$host'." );
 891                                         $this->conns[$connKey][$i][0] = $conn;
 892                                 } else {
 893                                         $this->connLogger->warning(
 894                                                 __METHOD__ . ": failed to connect to database $i at '$host'." );
 895                                         $this->errorConnection = $conn;
 896                                         $conn = false;
 897                                 }
 898                         }
 899                 }
 900
 901                 if ( $conn instanceof IDatabase && !$conn->isOpen() ) {
 902                         // Connection was made but later unrecoverably lost for some reason.
 903                         // Do not return a handle that will just throw exceptions on use,
 904                         // but let the calling code (e.g. getReaderIndex) try another server.
 905                         // See DatabaseMyslBase::ping() for how this can happen.
 906                         $this->errorConnection = $conn;
 907                         $conn = false;
 908                 }
 909
 910                 if ( $autoCommit && $conn instanceof IDatabase ) {
 911                         $conn->clearFlag( $conn::DBO_TRX ); // auto-commit mode
 912                 }
 913
 914                 return $conn;
 915         }
 916
 917         /**
 918          * Open a connection to a foreign DB, or return one if it is already open.
 919          *
 920          * Increments a reference count on the returned connection which locks the
 921          * connection to the requested domain. This reference count can be
 922          * decremented by calling reuseConnection().
 923          *
 924          * If a connection is open to the appropriate server already, but with the wrong
 925          * database, it will be switched to the right database and returned, as long as
 926          * it has been freed first with reuseConnection().
 927          *
 928          * On error, returns false, and the connection which caused the
 929          * error will be available via $this->errorConnection.
 930          *
 931          * @note If disable() was called on this LoadBalancer, this method will throw a DBAccessError.
 932          *
 933          * @param int $i Server index
 934          * @param string $domain Domain ID to open
 935          * @param int $flags Class CONN_* constant bitfield
 936          * @return Database
 937          */
 938         private function openForeignConnection( $i, $domain, $flags = 0 ) {
 939                 $domainInstance = DatabaseDomain::newFromId( $domain );
 940                 $dbName = $domainInstance->getDatabase();
 941                 $prefix = $domainInstance->getTablePrefix();
 942                 $autoCommit = ( ( $flags & self::CONN_TRX_AUTOCOMMIT ) == self::CONN_TRX_AUTOCOMMIT );
 943
 944                 if ( $autoCommit ) {
 945                         $connFreeKey = self::KEY_FOREIGN_FREE_NOROUND;
 946                         $connInUseKey = self::KEY_FOREIGN_INUSE_NOROUND;
 947                 } else {
 948                         $connFreeKey = self::KEY_FOREIGN_FREE;
 949                         $connInUseKey = self::KEY_FOREIGN_INUSE;
 950                 }
 951
 952                 if ( isset( $this->conns[$connInUseKey][$i][$domain] ) ) {
 953                         // Reuse an in-use connection for the same domain
 954                         $conn = $this->conns[$connInUseKey][$i][$domain];
 955                         $this->connLogger->debug( __METHOD__ . ": reusing connection $i/$domain" );
 956                 } elseif ( isset( $this->conns[$connFreeKey][$i][$domain] ) ) {
 957                         // Reuse a free connection for the same domain
 958                         $conn = $this->conns[$connFreeKey][$i][$domain];
 959                         unset( $this->conns[$connFreeKey][$i][$domain] );
 960                         $this->conns[$connInUseKey][$i][$domain] = $conn;
 961                         $this->connLogger->debug( __METHOD__ . ": reusing free connection $i/$domain" );
 962                 } elseif ( !empty( $this->conns[$connFreeKey][$i] ) ) {
 963                         // Reuse a free connection from another domain
 964                         $conn = reset( $this->conns[$connFreeKey][$i] );
 965                         $oldDomain = key( $this->conns[$connFreeKey][$i] );
 966                         if ( strlen( $dbName ) && !$conn->selectDB( $dbName ) ) {
 967                                 $this->lastError = "Error selecting database '$dbName' on server " .
 968                                         $conn->getServer() . " from client host {$this->host}";
 969                                 $this->errorConnection = $conn;
 970                                 $conn = false;
 971                         } else {
 972                                 $conn->tablePrefix( $prefix );
 973                                 unset( $this->conns[$connFreeKey][$i][$oldDomain] );
 974                                 // Note that if $domain is an empty string, getDomainID() might not match it
 975                                 $this->conns[$connInUseKey][$i][$conn->getDomainId()] = $conn;
 976                                 $this->connLogger->debug( __METHOD__ .
 977                                         ": reusing free connection from $oldDomain for $domain" );
 978                         }
 979                 } else {
 980                         if ( !isset( $this->servers[$i] ) || !is_array( $this->servers[$i] ) ) {
 981                                 throw new InvalidArgumentException( "No server with index '$i'." );
 982                         }
 983                         // Open a new connection
 984                         $server = $this->servers[$i];
 985                         $server['serverIndex'] = $i;
 986                         $server['foreignPoolRefCount'] = 0;
 987                         $server['foreign'] = true;
 988                         $server['autoCommitOnly'] = $autoCommit;
 989                         $conn = $this->reallyOpenConnection( $server, $domainInstance );
 990                         if ( !$conn->isOpen() ) {
 991                                 $this->connLogger->warning( __METHOD__ . ": connection error for $i/$domain" );
 992                                 $this->errorConnection = $conn;
 993                                 $conn = false;
 994                         } else {
 995                                 $conn->tablePrefix( $prefix ); // as specified
 996                                 // Note that if $domain is an empty string, getDomainID() might not match it
 997                                 $this->conns[$connInUseKey][$i][$conn->getDomainID()] = $conn;
 998                                 $this->connLogger->debug( __METHOD__ . ": opened new connection for $i/$domain" );
 999                         }
1000                 }
1001
1002                 if ( $conn instanceof IDatabase ) {
1003                         // Final sanity check to make sure the right domain is selected
1004                         if ( !$domainInstance->isCompatible( $conn->getDomainID() ) ) {
1005                                 throw new UnexpectedValueException(
1006                                         "Got connection to '{$conn->getDomainID()}', but expected '$domain'." );
1007                         }
1008                         // Increment reference count
1009                         $refCount = $conn->getLBInfo( 'foreignPoolRefCount' );
1010                         $conn->setLBInfo( 'foreignPoolRefCount', $refCount + 1 );
1011                 }
1012
1013                 return $conn;
1014         }
1015
1016         public function getServerAttributes( $i ) {
1017                 return Database::attributesFromType(
1018                         $this->getServerType( $i ),
1019                         isset( $this->servers[$i]['driver'] ) ? $this->servers[$i]['driver'] : null
1020                 );
1021         }
1022
1023         /**
1024          * Test if the specified index represents an open connection
1025          *
1026          * @param int $index Server index
1027          * @access private
1028          * @return bool
1029          */
1030         private function isOpen( $index ) {
1031                 if ( !is_int( $index ) ) {
1032                         return false;
1033                 }
1034
1035                 return (bool)$this->getAnyOpenConnection( $index );
1036         }
1037
1038         /**
1039          * Open a new network connection to a server (uncached)
1040          *
1041          * Returns a Database object whether or not the connection was successful.
1042          *
1043          * @param array $server
1044          * @param DatabaseDomain $domainOverride Use an unspecified domain to not select any database
1045          * @return Database
1046          * @throws DBAccessError
1047          * @throws InvalidArgumentException
1048          */
1049         protected function reallyOpenConnection( array $server, DatabaseDomain $domainOverride ) {
1050                 if ( $this->disabled ) {
1051                         throw new DBAccessError();
1052                 }
1053
1054                 // Handle $domainOverride being a specified or an unspecified domain
1055                 if ( $domainOverride->getDatabase() === null ) {
1056                         // Normally, an RDBMS requires a DB name specified on connection and the $server
1057                         // configuration array is assumed to already specify an appropriate DB name.
1058                         if ( $server['type'] === 'mysql' ) {
1059                                 // For MySQL, DATABASE and SCHEMA are synonyms, connections need not specify a DB,
1060                                 // and the DB name in $server might not exist due to legacy reasons (the default
1061                                 // domain used to ignore the local LB domain, even when mismatched).
1062                                 $server['dbname'] = null;
1063                         }
1064                 } else {
1065                         $server['dbname'] = $domainOverride->getDatabase();
1066                         $server['schema'] = $domainOverride->getSchema();
1067                 }
1068
1069                 // Let the handle know what the cluster master is (e.g. "db1052")
1070                 $masterName = $this->getServerName( $this->getWriterIndex() );
1071                 $server['clusterMasterHost'] = $masterName;
1072
1073                 // Log when many connection are made on requests
1074                 if ( ++$this->connsOpened >= self::CONN_HELD_WARN_THRESHOLD ) {
1075                         $this->perfLogger->warning( __METHOD__ . ": " .
1076                                 "{$this->connsOpened}+ connections made (master=$masterName)" );
1077                 }
1078
1079                 $server['srvCache'] = $this->srvCache;
1080                 // Set loggers and profilers
1081                 $server['connLogger'] = $this->connLogger;
1082                 $server['queryLogger'] = $this->queryLogger;
1083                 $server['errorLogger'] = $this->errorLogger;
1084                 $server['deprecationLogger'] = $this->deprecationLogger;
1085                 $server['profiler'] = $this->profiler;
1086                 $server['trxProfiler'] = $this->trxProfiler;
1087                 // Use the same agent and PHP mode for all DB handles
1088                 $server['cliMode'] = $this->cliMode;
1089                 $server['agent'] = $this->agent;
1090                 // Use DBO_DEFAULT flags by default for LoadBalancer managed databases. Assume that the
1091                 // application calls LoadBalancer::commitMasterChanges() before the PHP script completes.
1092                 $server['flags'] = isset( $server['flags'] ) ? $server['flags'] : IDatabase::DBO_DEFAULT;
1093
1094                 // Create a live connection object
1095                 try {
1096                         $db = Database::factory( $server['type'], $server );
1097                 } catch ( DBConnectionError $e ) {
1098                         // FIXME: This is probably the ugliest thing I have ever done to
1099                         // PHP. I'm half-expecting it to segfault, just out of disgust. -- TS
1100                         $db = $e->db;
1101                 }
1102
1103                 $db->setLBInfo( $server );
1104                 $db->setLazyMasterHandle(
1105                         $this->getLazyConnectionRef( self::DB_MASTER, [], $db->getDomainID() )
1106                 );
1107                 $db->setTableAliases( $this->tableAliases );
1108                 $db->setIndexAliases( $this->indexAliases );
1109
1110                 if ( $server['serverIndex'] === $this->getWriterIndex() ) {
1111                         if ( $this->trxRoundId !== false ) {
1112                                 $this->applyTransactionRoundFlags( $db );
1113                         }
1114                         foreach ( $this->trxRecurringCallbacks as $name => $callback ) {
1115                                 $db->setTransactionListener( $name, $callback );
1116                         }
1117                 }
1118
1119                 return $db;
1120         }
1121
1122         /**
1123          * @throws DBConnectionError
1124          */
1125         private function reportConnectionError() {
1126                 $conn = $this->errorConnection; // the connection which caused the error
1127                 $context = [
1128                         'method' => __METHOD__,
1129                         'last_error' => $this->lastError,
1130                 ];
1131
1132                 if ( $conn instanceof IDatabase ) {
1133                         $context['db_server'] = $conn->getServer();
1134                         $this->connLogger->warning(
1135                                 __METHOD__ . ": connection error: {last_error} ({db_server})",
1136                                 $context
1137                         );
1138
1139                         // throws DBConnectionError
1140                         $conn->reportConnectionError( "{$this->lastError} ({$context['db_server']})" );
1141                 } else {
1142                         // No last connection, probably due to all servers being too busy
1143                         $this->connLogger->error(
1144                                 __METHOD__ .
1145                                 ": LB failure with no last connection. Connection error: {last_error}",
1146                                 $context
1147                         );
1148
1149                         // If all servers were busy, "lastError" will contain something sensible
1150                         throw new DBConnectionError( null, $this->lastError );
1151                 }
1152         }
1153
1154         public function getWriterIndex() {
1155                 return 0;
1156         }
1157
1158         public function haveIndex( $i ) {
1159                 return array_key_exists( $i, $this->servers );
1160         }
1161
1162         public function isNonZeroLoad( $i ) {
1163                 return array_key_exists( $i, $this->servers ) && $this->loads[$i] != 0;
1164         }
1165
1166         public function getServerCount() {
1167                 return count( $this->servers );
1168         }
1169
1170         public function getServerName( $i ) {
1171                 if ( isset( $this->servers[$i]['hostName'] ) ) {
1172                         $name = $this->servers[$i]['hostName'];
1173                 } elseif ( isset( $this->servers[$i]['host'] ) ) {
1174                         $name = $this->servers[$i]['host'];
1175                 } else {
1176                         $name = '';
1177                 }
1178
1179                 return ( $name != '' ) ? $name : 'localhost';
1180         }
1181
1182         public function getServerInfo( $i ) {
1183                 if ( isset( $this->servers[$i] ) ) {
1184                         return $this->servers[$i];
1185                 } else {
1186                         return false;
1187                 }
1188         }
1189
1190         public function getServerType( $i ) {
1191                 return isset( $this->servers[$i]['type'] ) ? $this->servers[$i]['type'] : 'unknown';
1192         }
1193
1194         public function getMasterPos() {
1195                 # If this entire request was served from a replica DB without opening a connection to the
1196                 # master (however unlikely that may be), then we can fetch the position from the replica DB.
1197                 $masterConn = $this->getAnyOpenConnection( $this->getWriterIndex() );
1198                 if ( !$masterConn ) {
1199                         $serverCount = count( $this->servers );
1200                         for ( $i = 1; $i < $serverCount; $i++ ) {
1201                                 $conn = $this->getAnyOpenConnection( $i );
1202                                 if ( $conn ) {
1203                                         return $conn->getReplicaPos();
1204                                 }
1205                         }
1206                 } else {
1207                         return $masterConn->getMasterPos();
1208                 }
1209
1210                 return false;
1211         }
1212
1213         public function disable() {
1214                 $this->closeAll();
1215                 $this->disabled = true;
1216         }
1217
1218         public function closeAll() {
1219                 $this->forEachOpenConnection( function ( IDatabase $conn ) {
1220                         $host = $conn->getServer();
1221                         $this->connLogger->debug(
1222                                 __METHOD__ . ": closing connection to database '$host'." );
1223                         $conn->close();
1224                 } );
1225
1226                 $this->conns = [
1227                         self::KEY_LOCAL => [],
1228                         self::KEY_FOREIGN_INUSE => [],
1229                         self::KEY_FOREIGN_FREE => [],
1230                         self::KEY_LOCAL_NOROUND => [],
1231                         self::KEY_FOREIGN_INUSE_NOROUND => [],
1232                         self::KEY_FOREIGN_FREE_NOROUND => []
1233                 ];
1234                 $this->connsOpened = 0;
1235         }
1236
1237         public function closeConnection( IDatabase $conn ) {
1238                 $serverIndex = $conn->getLBInfo( 'serverIndex' );
1239                 foreach ( $this->conns as $type => $connsByServer ) {
1240                         if ( !isset( $connsByServer[$serverIndex] ) ) {
1241                                 continue;
1242                         }
1243
1244                         foreach ( $connsByServer[$serverIndex] as $i => $trackedConn ) {
1245                                 if ( $conn === $trackedConn ) {
1246                                         $host = $this->getServerName( $i );
1247                                         $this->connLogger->debug(
1248                                                 __METHOD__ . ": closing connection to database $i at '$host'." );
1249                                         unset( $this->conns[$type][$serverIndex][$i] );
1250                                         --$this->connsOpened;
1251                                         break 2;
1252                                 }
1253                         }
1254                 }
1255
1256                 $conn->close();
1257         }
1258
1259         public function commitAll( $fname = __METHOD__ ) {
1260                 $failures = [];
1261
1262                 $restore = ( $this->trxRoundId !== false );
1263                 $this->trxRoundId = false;
1264                 $this->forEachOpenConnection(
1265                         function ( IDatabase $conn ) use ( $fname, $restore, &$failures ) {
1266                                 try {
1267                                         $conn->commit( $fname, $conn::FLUSHING_ALL_PEERS );
1268                                 } catch ( DBError $e ) {
1269                                         call_user_func( $this->errorLogger, $e );
1270                                         $failures[] = "{$conn->getServer()}: {$e->getMessage()}";
1271                                 }
1272                                 if ( $restore && $conn->getLBInfo( 'master' ) ) {
1273                                         $this->undoTransactionRoundFlags( $conn );
1274                                 }
1275                         }
1276                 );
1277
1278                 if ( $failures ) {
1279                         throw new DBExpectedError(
1280                                 null,
1281                                 "Commit failed on server(s) " . implode( "\n", array_unique( $failures ) )
1282                         );
1283                 }
1284         }
1285
1286         public function finalizeMasterChanges() {
1287                 $this->forEachOpenMasterConnection( function ( Database $conn ) {
1288                         // Any error should cause all DB transactions to be rolled back together
1289                         $conn->setTrxEndCallbackSuppression( false );
1290                         $conn->runOnTransactionPreCommitCallbacks();
1291                         // Defer post-commit callbacks until COMMIT finishes for all DBs
1292                         $conn->setTrxEndCallbackSuppression( true );
1293                 } );
1294         }
1295
1296         public function approveMasterChanges( array $options ) {
1297                 $limit = isset( $options['maxWriteDuration'] ) ? $options['maxWriteDuration'] : 0;
1298                 $this->forEachOpenMasterConnection( function ( IDatabase $conn ) use ( $limit ) {
1299                         // If atomic sections or explicit transactions are still open, some caller must have
1300                         // caught an exception but failed to properly rollback any changes. Detect that and
1301                         // throw and error (causing rollback).
1302                         if ( $conn->explicitTrxActive() ) {
1303                                 throw new DBTransactionError(
1304                                         $conn,
1305                                         "Explicit transaction still active. A caller may have caught an error."
1306                                 );
1307                         }
1308                         // Assert that the time to replicate the transaction will be sane.
1309                         // If this fails, then all DB transactions will be rollback back together.
1310                         $time = $conn->pendingWriteQueryDuration( $conn::ESTIMATE_DB_APPLY );
1311                         if ( $limit > 0 && $time > $limit ) {
1312                                 throw new DBTransactionSizeError(
1313                                         $conn,
1314                                         "Transaction spent $time second(s) in writes, exceeding the limit of $limit.",
1315                                         [ $time, $limit ]
1316                                 );
1317                         }
1318                         // If a connection sits idle while slow queries execute on another, that connection
1319                         // may end up dropped before the commit round is reached. Ping servers to detect this.
1320                         if ( $conn->writesOrCallbacksPending() && !$conn->ping() ) {
1321                                 throw new DBTransactionError(
1322                                         $conn,
1323                                         "A connection to the {$conn->getDBname()} database was lost before commit."
1324                                 );
1325                         }
1326                 } );
1327         }
1328
1329         public function beginMasterChanges( $fname = __METHOD__ ) {
1330                 if ( $this->trxRoundId !== false ) {
1331                         throw new DBTransactionError(
1332                                 null,
1333                                 "$fname: Transaction round '{$this->trxRoundId}' already started."
1334                         );
1335                 }
1336                 $this->trxRoundId = $fname;
1337
1338                 $failures = [];
1339                 $this->forEachOpenMasterConnection(
1340                         function ( Database $conn ) use ( $fname, &$failures ) {
1341                                 $conn->setTrxEndCallbackSuppression( true );
1342                                 try {
1343                                         $conn->flushSnapshot( $fname );
1344                                 } catch ( DBError $e ) {
1345                                         call_user_func( $this->errorLogger, $e );
1346                                         $failures[] = "{$conn->getServer()}: {$e->getMessage()}";
1347                                 }
1348                                 $conn->setTrxEndCallbackSuppression( false );
1349                                 $this->applyTransactionRoundFlags( $conn );
1350                         }
1351                 );
1352
1353                 if ( $failures ) {
1354                         throw new DBExpectedError(
1355                                 null,
1356                                 "$fname: Flush failed on server(s) " . implode( "\n", array_unique( $failures ) )
1357                         );
1358                 }
1359         }
1360
1361         public function commitMasterChanges( $fname = __METHOD__ ) {
1362                 $failures = [];
1363
1364                 /** @noinspection PhpUnusedLocalVariableInspection */
1365                 $scope = $this->getScopedPHPBehaviorForCommit(); // try to ignore client aborts
1366
1367                 $restore = ( $this->trxRoundId !== false );
1368                 $this->trxRoundId = false;
1369                 $this->forEachOpenMasterConnection(
1370                         function ( IDatabase $conn ) use ( $fname, $restore, &$failures ) {
1371                                 try {
1372                                         if ( $conn->writesOrCallbacksPending() ) {
1373                                                 $conn->commit( $fname, $conn::FLUSHING_ALL_PEERS );
1374                                         } elseif ( $restore ) {
1375                                                 $conn->flushSnapshot( $fname );
1376                                         }
1377                                 } catch ( DBError $e ) {
1378                                         call_user_func( $this->errorLogger, $e );
1379                                         $failures[] = "{$conn->getServer()}: {$e->getMessage()}";
1380                                 }
1381                                 if ( $restore ) {
1382                                         $this->undoTransactionRoundFlags( $conn );
1383                                 }
1384                         }
1385                 );
1386
1387                 if ( $failures ) {
1388                         throw new DBExpectedError(
1389                                 null,
1390                                 "$fname: Commit failed on server(s) " . implode( "\n", array_unique( $failures ) )
1391                         );
1392                 }
1393         }
1394
1395         public function runMasterPostTrxCallbacks( $type ) {
1396                 $e = null; // first exception
1397                 $this->forEachOpenMasterConnection( function ( Database $conn ) use ( $type, &$e ) {
1398                         $conn->setTrxEndCallbackSuppression( false );
1399                         // Callbacks run in AUTO-COMMIT mode, so make sure no transactions are pending...
1400                         if ( $conn->writesPending() ) {
1401                                 // This happens if onTransactionIdle() callbacks write to *other* handles
1402                                 // (which already finished their callbacks). Let any callbacks run in the final
1403                                 // commitMasterChanges() in LBFactory::shutdown(), when the transaction is gone.
1404                                 $this->queryLogger->warning( __METHOD__ . ": found writes pending." );
1405                                 return;
1406                         } elseif ( $conn->trxLevel() ) {
1407                                 // This happens for single-DB setups where DB_REPLICA uses the master DB,
1408                                 // thus leaving an implicit read-only transaction open at this point. It
1409                                 // also happens if onTransactionIdle() callbacks leave implicit transactions
1410                                 // open on *other* DBs (which is slightly improper). Let these COMMIT on the
1411                                 // next call to commitMasterChanges(), possibly in LBFactory::shutdown().
1412                                 return;
1413                         }
1414                         try {
1415                                 $conn->runOnTransactionIdleCallbacks( $type );
1416                         } catch ( Exception $ex ) {
1417                                 $e = $e ?: $ex;
1418                         }
1419                         try {
1420                                 $conn->runTransactionListenerCallbacks( $type );
1421                         } catch ( Exception $ex ) {
1422                                 $e = $e ?: $ex;
1423                         }
1424                 } );
1425
1426                 return $e;
1427         }
1428
1429         public function rollbackMasterChanges( $fname = __METHOD__ ) {
1430                 $restore = ( $this->trxRoundId !== false );
1431                 $this->trxRoundId = false;
1432                 $this->forEachOpenMasterConnection(
1433                         function ( IDatabase $conn ) use ( $fname, $restore ) {
1434                                 $conn->rollback( $fname, $conn::FLUSHING_ALL_PEERS );
1435                                 if ( $restore ) {
1436                                         $this->undoTransactionRoundFlags( $conn );
1437                                 }
1438                         }
1439                 );
1440         }
1441
1442         public function suppressTransactionEndCallbacks() {
1443                 $this->forEachOpenMasterConnection( function ( Database $conn ) {
1444                         $conn->setTrxEndCallbackSuppression( true );
1445                 } );
1446         }
1447
1448         /**
1449          * Make all DB servers with DBO_DEFAULT/DBO_TRX set join the transaction round
1450          *
1451          * Some servers may have neither flag enabled, meaning that they opt out of such
1452          * transaction rounds and remain in auto-commit mode. Such behavior might be desired
1453          * when a DB server is used for something like simple key/value storage.
1454          *
1455          * @param IDatabase $conn
1456          */
1457         private function applyTransactionRoundFlags( IDatabase $conn ) {
1458                 if ( $conn->getLBInfo( 'autoCommitOnly' ) ) {
1459                         return; // transaction rounds do not apply to these connections
1460                 }
1461
1462                 if ( $conn->getFlag( $conn::DBO_DEFAULT ) ) {
1463                         // DBO_TRX is controlled entirely by CLI mode presence with DBO_DEFAULT.
1464                         // Force DBO_TRX even in CLI mode since a commit round is expected soon.
1465                         $conn->setFlag( $conn::DBO_TRX, $conn::REMEMBER_PRIOR );
1466                 }
1467
1468                 if ( $conn->getFlag( $conn::DBO_TRX ) ) {
1469                         $conn->setLBInfo( 'trxRoundId', $this->trxRoundId );
1470                 }
1471         }
1472
1473         /**
1474          * @param IDatabase $conn
1475          */
1476         private function undoTransactionRoundFlags( IDatabase $conn ) {
1477                 if ( $conn->getLBInfo( 'autoCommitOnly' ) ) {
1478                         return; // transaction rounds do not apply to these connections
1479                 }
1480
1481                 if ( $conn->getFlag( $conn::DBO_TRX ) ) {
1482                         $conn->setLBInfo( 'trxRoundId', false );
1483                 }
1484
1485                 if ( $conn->getFlag( $conn::DBO_DEFAULT ) ) {
1486                         $conn->restoreFlags( $conn::RESTORE_PRIOR );
1487                 }
1488         }
1489
1490         public function flushReplicaSnapshots( $fname = __METHOD__ ) {
1491                 $this->forEachOpenReplicaConnection( function ( IDatabase $conn ) {
1492                         $conn->flushSnapshot( __METHOD__ );
1493                 } );
1494         }
1495
1496         public function hasMasterConnection() {
1497                 return $this->isOpen( $this->getWriterIndex() );
1498         }
1499
1500         public function hasMasterChanges() {
1501                 $pending = 0;
1502                 $this->forEachOpenMasterConnection( function ( IDatabase $conn ) use ( &$pending ) {
1503                         $pending |= $conn->writesOrCallbacksPending();
1504                 } );
1505
1506                 return (bool)$pending;
1507         }
1508
1509         public function lastMasterChangeTimestamp() {
1510                 $lastTime = false;
1511                 $this->forEachOpenMasterConnection( function ( IDatabase $conn ) use ( &$lastTime ) {
1512                         $lastTime = max( $lastTime, $conn->lastDoneWrites() );
1513                 } );
1514
1515                 return $lastTime;
1516         }
1517
1518         public function hasOrMadeRecentMasterChanges( $age = null ) {
1519                 $age = ( $age === null ) ? $this->waitTimeout : $age;
1520
1521                 return ( $this->hasMasterChanges()
1522                         || $this->lastMasterChangeTimestamp() > microtime( true ) - $age );
1523         }
1524
1525         public function pendingMasterChangeCallers() {
1526                 $fnames = [];
1527                 $this->forEachOpenMasterConnection( function ( IDatabase $conn ) use ( &$fnames ) {
1528                         $fnames = array_merge( $fnames, $conn->pendingWriteCallers() );
1529                 } );
1530
1531                 return $fnames;
1532         }
1533
1534         public function getLaggedReplicaMode( $domain = false ) {
1535                 // No-op if there is only one DB (also avoids recursion)
1536                 if ( !$this->laggedReplicaMode && $this->getServerCount() > 1 ) {
1537                         try {
1538                                 // See if laggedReplicaMode gets set
1539                                 $conn = $this->getConnection( self::DB_REPLICA, false, $domain );
1540                                 $this->reuseConnection( $conn );
1541                         } catch ( DBConnectionError $e ) {
1542                                 // Avoid expensive re-connect attempts and failures
1543                                 $this->allReplicasDownMode = true;
1544                                 $this->laggedReplicaMode = true;
1545                         }
1546                 }
1547
1548                 return $this->laggedReplicaMode;
1549         }
1550
1551         /**
1552          * @param bool $domain
1553          * @return bool
1554          * @deprecated 1.28; use getLaggedReplicaMode()
1555          */
1556         public function getLaggedSlaveMode( $domain = false ) {
1557                 return $this->getLaggedReplicaMode( $domain );
1558         }
1559
1560         public function laggedReplicaUsed() {
1561                 return $this->laggedReplicaMode;
1562         }
1563
1564         /**
1565          * @return bool
1566          * @since 1.27
1567          * @deprecated Since 1.28; use laggedReplicaUsed()
1568          */
1569         public function laggedSlaveUsed() {
1570                 return $this->laggedReplicaUsed();
1571         }
1572
1573         public function getReadOnlyReason( $domain = false, IDatabase $conn = null ) {
1574                 if ( $this->readOnlyReason !== false ) {
1575                         return $this->readOnlyReason;
1576                 } elseif ( $this->getLaggedReplicaMode( $domain ) ) {
1577                         if ( $this->allReplicasDownMode ) {
1578                                 return 'The database has been automatically locked ' .
1579                                         'until the replica database servers become available';
1580                         } else {
1581                                 return 'The database has been automatically locked ' .
1582                                         'while the replica database servers catch up to the master.';
1583                         }
1584                 } elseif ( $this->masterRunningReadOnly( $domain, $conn ) ) {
1585                         return 'The database master is running in read-only mode.';
1586                 }
1587
1588                 return false;
1589         }
1590
1591         /**
1592          * @param string $domain Domain ID, or false for the current domain
1593          * @param IDatabase|null $conn DB master connectionl used to avoid loops [optional]
1594          * @return bool
1595          */
1596         private function masterRunningReadOnly( $domain, IDatabase $conn = null ) {
1597                 $cache = $this->wanCache;
1598                 $masterServer = $this->getServerName( $this->getWriterIndex() );
1599
1600                 return (bool)$cache->getWithSetCallback(
1601                         $cache->makeGlobalKey( __CLASS__, 'server-read-only', $masterServer ),
1602                         self::TTL_CACHE_READONLY,
1603                         function () use ( $domain, $conn ) {
1604                                 $old = $this->trxProfiler->setSilenced( true );
1605                                 try {
1606                                         $dbw = $conn ?: $this->getConnection( self::DB_MASTER, [], $domain );
1607                                         $readOnly = (int)$dbw->serverIsReadOnly();
1608                                         if ( !$conn ) {
1609                                                 $this->reuseConnection( $dbw );
1610                                         }
1611                                 } catch ( DBError $e ) {
1612                                         $readOnly = 0;
1613                                 }
1614                                 $this->trxProfiler->setSilenced( $old );
1615                                 return $readOnly;
1616                         },
1617                         [ 'pcTTL' => $cache::TTL_PROC_LONG, 'busyValue' => 0 ]
1618                 );
1619         }
1620
1621         public function allowLagged( $mode = null ) {
1622                 if ( $mode === null ) {
1623                         return $this->allowLagged;
1624                 }
1625                 $this->allowLagged = $mode;
1626
1627                 return $this->allowLagged;
1628         }
1629
1630         public function pingAll() {
1631                 $success = true;
1632                 $this->forEachOpenConnection( function ( IDatabase $conn ) use ( &$success ) {
1633                         if ( !$conn->ping() ) {
1634                                 $success = false;
1635                         }
1636                 } );
1637
1638                 return $success;
1639         }
1640
1641         public function forEachOpenConnection( $callback, array $params = [] ) {
1642                 foreach ( $this->conns as $connsByServer ) {
1643                         foreach ( $connsByServer as $serverConns ) {
1644                                 foreach ( $serverConns as $conn ) {
1645                                         $mergedParams = array_merge( [ $conn ], $params );
1646                                         call_user_func_array( $callback, $mergedParams );
1647                                 }
1648                         }
1649                 }
1650         }
1651
1652         public function forEachOpenMasterConnection( $callback, array $params = [] ) {
1653                 $masterIndex = $this->getWriterIndex();
1654                 foreach ( $this->conns as $connsByServer ) {
1655                         if ( isset( $connsByServer[$masterIndex] ) ) {
1656                                 /** @var IDatabase $conn */
1657                                 foreach ( $connsByServer[$masterIndex] as $conn ) {
1658                                         $mergedParams = array_merge( [ $conn ], $params );
1659                                         call_user_func_array( $callback, $mergedParams );
1660                                 }
1661                         }
1662                 }
1663         }
1664
1665         public function forEachOpenReplicaConnection( $callback, array $params = [] ) {
1666                 foreach ( $this->conns as $connsByServer ) {
1667                         foreach ( $connsByServer as $i => $serverConns ) {
1668                                 if ( $i === $this->getWriterIndex() ) {
1669                                         continue; // skip master
1670                                 }
1671                                 foreach ( $serverConns as $conn ) {
1672                                         $mergedParams = array_merge( [ $conn ], $params );
1673                                         call_user_func_array( $callback, $mergedParams );
1674                                 }
1675                         }
1676                 }
1677         }
1678
1679         public function getMaxLag( $domain = false ) {
1680                 $maxLag = -1;
1681                 $host = '';
1682                 $maxIndex = 0;
1683
1684                 if ( $this->getServerCount() <= 1 ) {
1685                         return [ $host, $maxLag, $maxIndex ]; // no replication = no lag
1686                 }
1687
1688                 $lagTimes = $this->getLagTimes( $domain );
1689                 foreach ( $lagTimes as $i => $lag ) {
1690                         if ( $this->loads[$i] > 0 && $lag > $maxLag ) {
1691                                 $maxLag = $lag;
1692                                 $host = $this->servers[$i]['host'];
1693                                 $maxIndex = $i;
1694                         }
1695                 }
1696
1697                 return [ $host, $maxLag, $maxIndex ];
1698         }
1699
1700         public function getLagTimes( $domain = false ) {
1701                 if ( $this->getServerCount() <= 1 ) {
1702                         return [ $this->getWriterIndex() => 0 ]; // no replication = no lag
1703                 }
1704
1705                 $knownLagTimes = []; // map of (server index => 0 seconds)
1706                 $indexesWithLag = [];
1707                 foreach ( $this->servers as $i => $server ) {
1708                         if ( empty( $server['is static'] ) ) {
1709                                 $indexesWithLag[] = $i; // DB server might have replication lag
1710                         } else {
1711                                 $knownLagTimes[$i] = 0; // DB server is a non-replicating and read-only archive
1712                         }
1713                 }
1714
1715                 return $this->getLoadMonitor()->getLagTimes( $indexesWithLag, $domain ) + $knownLagTimes;
1716         }
1717
1718         public function safeGetLag( IDatabase $conn ) {
1719                 if ( $this->getServerCount() <= 1 ) {
1720                         return 0;
1721                 } else {
1722                         return $conn->getLag();
1723                 }
1724         }
1725
1726         /**
1727          * @param IDatabase $conn
1728          * @param DBMasterPos|bool $pos
1729          * @param int|null $timeout
1730          * @return bool
1731          */
1732         public function safeWaitForMasterPos( IDatabase $conn, $pos = false, $timeout = null ) {
1733                 $timeout = max( 1, $timeout ?: $this->waitTimeout );
1734
1735                 if ( $this->getServerCount() <= 1 || !$conn->getLBInfo( 'replica' ) ) {
1736                         return true; // server is not a replica DB
1737                 }
1738
1739                 if ( !$pos ) {
1740                         // Get the current master position, opening a connection if needed
1741                         $masterConn = $this->getAnyOpenConnection( $this->getWriterIndex() );
1742                         if ( $masterConn ) {
1743                                 $pos = $masterConn->getMasterPos();
1744                         } else {
1745                                 $masterConn = $this->openConnection( $this->getWriterIndex(), self::DOMAIN_ANY );
1746                                 $pos = $masterConn->getMasterPos();
1747                                 $this->closeConnection( $masterConn );
1748                         }
1749                 }
1750
1751                 if ( $pos instanceof DBMasterPos ) {
1752                         $result = $conn->masterPosWait( $pos, $timeout );
1753                         if ( $result == -1 || is_null( $result ) ) {
1754                                 $msg = __METHOD__ . ': timed out waiting on {host} pos {pos}';
1755                                 $this->replLogger->warning( $msg, [
1756                                         'host' => $conn->getServer(),
1757                                         'pos' => $pos,
1758                                         'trace' => ( new RuntimeException() )->getTraceAsString()
1759                                 ] );
1760                                 $ok = false;
1761                         } else {
1762                                 $this->replLogger->debug( __METHOD__ . ': done waiting' );
1763                                 $ok = true;
1764                         }
1765                 } else {
1766                         $ok = false; // something is misconfigured
1767                         $this->replLogger->error(
1768                                 __METHOD__ . ': could not get master pos for {host}',
1769                                 [
1770                                         'host' => $conn->getServer(),
1771                                         'trace' => ( new RuntimeException() )->getTraceAsString()
1772                                 ]
1773                         );
1774                 }
1775
1776                 return $ok;
1777         }
1778
1779         public function setTransactionListener( $name, callable $callback = null ) {
1780                 if ( $callback ) {
1781                         $this->trxRecurringCallbacks[$name] = $callback;
1782                 } else {
1783                         unset( $this->trxRecurringCallbacks[$name] );
1784                 }
1785                 $this->forEachOpenMasterConnection(
1786                         function ( IDatabase $conn ) use ( $name, $callback ) {
1787                                 $conn->setTransactionListener( $name, $callback );
1788                         }
1789                 );
1790         }
1791
1792         public function setTableAliases( array $aliases ) {
1793                 $this->tableAliases = $aliases;
1794         }
1795
1796         public function setIndexAliases( array $aliases ) {
1797                 $this->indexAliases = $aliases;
1798         }
1799
1800         public function setDomainPrefix( $prefix ) {
1801                 // Find connections to explicit foreign domains still marked as in-use...
1802                 $domainsInUse = [];
1803                 $this->forEachOpenConnection( function ( IDatabase $conn ) use ( &$domainsInUse ) {
1804                         // Once reuseConnection() is called on a handle, its reference count goes from 1 to 0.
1805                         // Until then, it is still in use by the caller (explicitly or via DBConnRef scope).
1806                         if ( $conn->getLBInfo( 'foreignPoolRefCount' ) > 0 ) {
1807                                 $domainsInUse[] = $conn->getDomainID();
1808                         }
1809                 } );
1810
1811                 // Do not switch connections to explicit foreign domains unless marked as safe
1812                 if ( $domainsInUse ) {
1813                         $domains = implode( ', ', $domainsInUse );
1814                         throw new DBUnexpectedError( null,
1815                                 "Foreign domain connections are still in use ($domains)." );
1816                 }
1817
1818                 $oldDomain = $this->localDomain->getId();
1819                 $this->setLocalDomain( new DatabaseDomain(
1820                         $this->localDomain->getDatabase(),
1821                         $this->localDomain->getSchema(),
1822                         $prefix
1823                 ) );
1824
1825                 $this->forEachOpenConnection( function ( IDatabase $db ) use ( $prefix, $oldDomain ) {
1826                         if ( !$db->getLBInfo( 'foreign' ) ) {
1827                                 $db->tablePrefix( $prefix );
1828                         }
1829                 } );
1830         }
1831
1832         /**
1833          * @param DatabaseDomain $domain
1834          */
1835         private function setLocalDomain( DatabaseDomain $domain ) {
1836                 $this->localDomain = $domain;
1837                 // In case a caller assumes that the domain ID is simply <db>-<prefix>, which is almost
1838                 // always true, gracefully handle the case when they fail to account for escaping.
1839                 if ( $this->localDomain->getTablePrefix() != '' ) {
1840                         $this->localDomainIdAlias =
1841                                 $this->localDomain->getDatabase() . '-' . $this->localDomain->getTablePrefix();
1842                 } else {
1843                         $this->localDomainIdAlias = $this->localDomain->getDatabase();
1844                 }
1845         }
1846
1847         /**
1848          * Make PHP ignore user aborts/disconnects until the returned
1849          * value leaves scope. This returns null and does nothing in CLI mode.
1850          *
1851          * @return ScopedCallback|null
1852          */
1853         final protected function getScopedPHPBehaviorForCommit() {
1854                 if ( PHP_SAPI != 'cli' ) { // https://bugs.php.net/bug.php?id=47540
1855                         $old = ignore_user_abort( true ); // avoid half-finished operations
1856                         return new ScopedCallback( function () use ( $old ) {
1857                                 ignore_user_abort( $old );
1858                         } );
1859                 }
1860
1861                 return null;
1862         }
1863
1864         function __destruct() {
1865                 // Avoid connection leaks for sanity
1866                 $this->disable();
1867         }
1868 }
1869
1870 class_alias( LoadBalancer::class, 'LoadBalancer' );