rdbms: make safeWaitForMasterPos() respect "waitTimeout"
[lhc/web/wiklou.git] / includes / libs / rdbms / loadbalancer / LoadBalancer.php
index 864e6f0..0af0e6e 100644 (file)
@@ -49,7 +49,7 @@ class LoadBalancer implements ILoadBalancer {
        /** @var bool Whether to disregard replica DB lag as a factor in replica DB selection */
        private $mAllowLagged;
        /** @var int Seconds to spend waiting on replica DB lag to resolve */
-       private $mWaitTimeout;
+       private $waitTimeout;
        /** @var array The LoadMonitor configuration */
        private $loadMonitorConfig;
        /** @var array[] $aliases Map of (table => (dbname, schema, prefix) map) */
@@ -122,6 +122,8 @@ class LoadBalancer implements ILoadBalancer {
 
        /** @var int Default 'maxLag' when unspecified */
        const MAX_LAG_DEFAULT = 10;
+       /** @var int Default 'waitTimeout' when unspecified */
+       const MAX_WAIT_DEFAULT = 10;
        /** @var int Seconds to cache master server read-only status */
        const TTL_CACHE_READONLY = 5;
 
@@ -151,7 +153,9 @@ class LoadBalancer implements ILoadBalancer {
                        : DatabaseDomain::newUnspecified();
                $this->setLocalDomain( $localDomain );
 
-               $this->mWaitTimeout = isset( $params['waitTimeout'] ) ? $params['waitTimeout'] : 10;
+               $this->waitTimeout = isset( $params['waitTimeout'] )
+                       ? $params['waitTimeout']
+                       : self::MAX_WAIT_DEFAULT;
 
                $this->mReadIndex = -1;
                $this->mConns = [
@@ -511,7 +515,7 @@ class LoadBalancer implements ILoadBalancer {
        }
 
        public function waitForAll( $pos, $timeout = null ) {
-               $timeout = $timeout ?: $this->mWaitTimeout;
+               $timeout = $timeout ?: $this->waitTimeout;
 
                $oldPos = $this->mWaitForPos;
                try {
@@ -522,7 +526,7 @@ class LoadBalancer implements ILoadBalancer {
                        for ( $i = 1; $i < $serverCount; $i++ ) {
                                if ( $this->mLoads[$i] > 0 ) {
                                        $start = microtime( true );
-                                       $ok = $this->doWait( $i, true, max( 1, (int)$timeout ) ) && $ok;
+                                       $ok = $this->doWait( $i, true, $timeout ) && $ok;
                                        $timeout -= ( microtime( true ) - $start );
                                        if ( $timeout <= 0 ) {
                                                break; // timeout reached
@@ -571,11 +575,11 @@ class LoadBalancer implements ILoadBalancer {
         * Wait for a given replica DB to catch up to the master pos stored in $this
         * @param int $index Server index
         * @param bool $open Check the server even if a new connection has to be made
-        * @param int $timeout Max seconds to wait; default is mWaitTimeout
+        * @param int $timeout Max seconds to wait; default is "waitTimeout" given to __construct()
         * @return bool
         */
        protected function doWait( $index, $open = false, $timeout = null ) {
-               $close = false; // close the connection afterwards
+               $timeout = max( 1, $timeout ?: $this->waitTimeout );
 
                // Check if we already know that the DB has reached this point
                $server = $this->getServerName( $index );
@@ -586,25 +590,32 @@ class LoadBalancer implements ILoadBalancer {
                        $knownReachedPos instanceof DBMasterPos &&
                        $knownReachedPos->hasReached( $this->mWaitForPos )
                ) {
-                       $this->replLogger->debug( __METHOD__ .
+                       $this->replLogger->debug(
+                               __METHOD__ .
                                ': replica DB {dbserver} known to be caught up (pos >= $knownReachedPos).',
-                               [ 'dbserver' => $server ] );
+                               [ 'dbserver' => $server ]
+                       );
                        return true;
                }
 
                // Find a connection to wait on, creating one if needed and allowed
+               $close = false; // close the connection afterwards
                $conn = $this->getAnyOpenConnection( $index );
                if ( !$conn ) {
                        if ( !$open ) {
-                               $this->replLogger->debug( __METHOD__ . ': no connection open for {dbserver}',
-                                       [ 'dbserver' => $server ] );
+                               $this->replLogger->debug(
+                                       __METHOD__ . ': no connection open for {dbserver}',
+                                       [ 'dbserver' => $server ]
+                               );
 
                                return false;
                        } else {
                                $conn = $this->openConnection( $index, self::DOMAIN_ANY );
                                if ( !$conn ) {
-                                       $this->replLogger->warning( __METHOD__ . ': failed to connect to {dbserver}',
-                                               [ 'dbserver' => $server ] );
+                                       $this->replLogger->warning(
+                                               __METHOD__ . ': failed to connect to {dbserver}',
+                                               [ 'dbserver' => $server ]
+                                       );
 
                                        return false;
                                }
@@ -614,16 +625,32 @@ class LoadBalancer implements ILoadBalancer {
                        }
                }
 
-               $this->replLogger->info( __METHOD__ . ': Waiting for replica DB {dbserver} to catch up...',
-                       [ 'dbserver' => $server ] );
-               $timeout = $timeout ?: $this->mWaitTimeout;
+               $this->replLogger->info(
+                       __METHOD__ .
+                       ': Waiting for replica DB {dbserver} to catch up...',
+                       [ 'dbserver' => $server ]
+               );
+
                $result = $conn->masterPosWait( $this->mWaitForPos, $timeout );
 
-               if ( $result == -1 || is_null( $result ) ) {
-                       // Timed out waiting for replica DB, use master instead
+               if ( $result === null ) {
+                       $this->replLogger->warning(
+                               __METHOD__ . ': Errored out waiting on {host} pos {pos}',
+                               [
+                                       'host' => $server,
+                                       'pos' => $this->mWaitForPos,
+                                       'trace' => ( new RuntimeException() )->getTraceAsString()
+                               ]
+                       );
+                       $ok = false;
+               } elseif ( $result == -1 ) {
                        $this->replLogger->warning(
                                __METHOD__ . ': Timed out waiting on {host} pos {pos}',
-                               [ 'host' => $server, 'pos' => $this->mWaitForPos ]
+                               [
+                                       'host' => $server,
+                                       'pos' => $this->mWaitForPos,
+                                       'trace' => ( new RuntimeException() )->getTraceAsString()
+                               ]
                        );
                        $ok = false;
                } else {
@@ -1411,7 +1438,7 @@ class LoadBalancer implements ILoadBalancer {
        }
 
        public function hasOrMadeRecentMasterChanges( $age = null ) {
-               $age = ( $age === null ) ? $this->mWaitTimeout : $age;
+               $age = ( $age === null ) ? $this->waitTimeout : $age;
 
                return ( $this->hasMasterChanges()
                        || $this->lastMasterChangeTimestamp() > microtime( true ) - $age );
@@ -1621,10 +1648,12 @@ class LoadBalancer implements ILoadBalancer {
        /**
         * @param IDatabase $conn
         * @param DBMasterPos|bool $pos
-        * @param int $timeout
+        * @param int|null $timeout
         * @return bool
         */
-       public function safeWaitForMasterPos( IDatabase $conn, $pos = false, $timeout = 10 ) {
+       public function safeWaitForMasterPos( IDatabase $conn, $pos = false, $timeout = null ) {
+               $timeout = max( 1, $timeout ?: $this->waitTimeout );
+
                if ( $this->getServerCount() <= 1 || !$conn->getLBInfo( 'replica' ) ) {
                        return true; // server is not a replica DB
                }
@@ -1645,8 +1674,11 @@ class LoadBalancer implements ILoadBalancer {
                        $result = $conn->masterPosWait( $pos, $timeout );
                        if ( $result == -1 || is_null( $result ) ) {
                                $msg = __METHOD__ . ': Timed out waiting on {host} pos {pos}';
-                               $this->replLogger->warning( $msg,
-                                       [ 'host' => $conn->getServer(), 'pos' => $pos ] );
+                               $this->replLogger->warning( $msg, [
+                                       'host' => $conn->getServer(),
+                                       'pos' => $pos,
+                                       'trace' => ( new RuntimeException() )->getTraceAsString()
+                               ] );
                                $ok = false;
                        } else {
                                $this->replLogger->info( __METHOD__ . ': Done' );
@@ -1654,8 +1686,13 @@ class LoadBalancer implements ILoadBalancer {
                        }
                } else {
                        $ok = false; // something is misconfigured
-                       $this->replLogger->error( 'Could not get master pos for {host}',
-                               [ 'host' => $conn->getServer() ] );
+                       $this->replLogger->error(
+                               __METHOD__ . ': could not get master pos for {host}',
+                               [
+                                       'host' => $conn->getServer(),
+                                       'trace' => ( new RuntimeException() )->getTraceAsString()
+                               ]
+                       );
                }
 
                return $ok;