Merge "[LockManager] Memc lockmanager improvements."

diff --combined includes/filerepo/backend/lockmanager/MemcLockManager.php
index 79696b2,1a6045b..9e81dbf
--- 1/includes/filerepo/backend/lockmanager/MemcLockManager.php
--- 2/includes/filerepo/backend/lockmanager/MemcLockManager.php
+++ b/includes/filerepo/backend/lockmanager/MemcLockManager.php
@@@ -56,12 -56,12 +56,12 @@@ class MemcLockManager extends QuorumLoc
         * Construct a new instance from configuration.
         *
         * $config paramaters include:
 -       *     'lockServers'  : Associative array of server names to <IP>:<port> strings.
 -       *     'srvsByBucket' : Array of 1-16 consecutive integer keys, starting from 0,
 +       *   - 'lockServers'  : Associative array of server names to "<IP>:<port>" strings.
 +       *   - 'srvsByBucket' : Array of 1-16 consecutive integer keys, starting from 0,
         *                      each having an odd-numbered list of server names (peers) as values.
 -       *     'memcConfig'   : Configuration array for ObjectCache::newFromParams. [optional]
 +       *   - 'memcConfig'   : Configuration array for ObjectCache::newFromParams. [optional]
         *                      If set, this must use one of the memcached classes.
 -       *     'wikiId'       : Wiki ID string that all resources are relative to. [optional]
 +       *   - 'wikiId'       : Wiki ID string that all resources are relative to. [optional]
         *
         * @param Array $config
         */
@@@ -92,7 -92,7 +92,7 @@@
                $met = ini_get( 'max_execution_time' ); // this is 0 in CLI mode
                $this->lockExpiry = $met ? 2*(int)$met : 2*3600;
  
-               $this->session = wfRandomString( 31 );
+               $this->session = wfRandomString( 32 );
        }
  
        /**
@@@ -264,11 -264,24 +264,24 @@@
        protected function acquireMutexes( MemcachedBagOStuff $memc, array $keys ) {
                $lockedKeys = array();
  
+               // Acquire the keys in lexicographical order, to avoid deadlock problems.
+               // If P1 is waiting to acquire a key P2 has, P2 can't also be waiting for a key P1 has.
+               sort( $keys );
+ 
+               // Try to quickly loop to acquire the keys, but back off after a few rounds.
+               // This reduces memcached spam, especially in the rare case where a server acquires
+               // some lock keys and dies without releasing them. Lock keys expire after a few minutes.
+               $rounds = 0;
                $start = microtime( true );
                do {
+                       if ( ( ++$rounds % 4 ) == 0 ) {
+                               usleep( 1000*50 ); // 50 ms
+                       }
                        foreach ( array_diff( $keys, $lockedKeys ) as $key ) {
                                if ( $memc->add( "$key:mutex", 1, 180 ) ) { // lock record
                                        $lockedKeys[] = $key;
+                               } else {
+                                       continue; // acquire in order
                                }
                        }
                } while ( count( $lockedKeys ) < count( $keys ) && ( microtime( true ) - $start ) <= 6 );