'PathRouter' => 'includes/PathRouter.php',
'PathRouterPatternReplacer' => 'includes/PathRouter.php',
'PhpHttpRequest' => 'includes/HttpFunctions.php',
- 'PoolCounter' => 'includes/PoolCounter.php',
- 'PoolCounter_Stub' => 'includes/PoolCounter.php',
- 'PoolCounterRedis' => 'includes/PoolCounterRedis.php',
- 'PoolCounterWork' => 'includes/PoolCounter.php',
- 'PoolCounterWorkViaCallback' => 'includes/PoolCounter.php',
+ 'PoolCounter' => 'includes/poolcounter/PoolCounter.php',
+ 'PoolCounter_Stub' => 'includes/poolcounter/PoolCounter.php',
+ 'PoolCounterRedis' => 'includes/poolcounter/PoolCounterRedis.php',
+ 'PoolCounterWork' => 'includes/poolcounter/PoolCounterWork.php',
+ 'PoolCounterWorkViaCallback' => 'includes/poolcounter/PoolCounterWork.php',
'PoolWorkArticleView' => 'includes/WikiPage.php',
'Preferences' => 'includes/Preferences.php',
'PreferencesForm' => 'includes/Preferences.php',
+++ /dev/null
-<?php
-/**
- * Provides of semaphore semantics for restricting the number
- * of workers that may be concurrently performing the same task.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- * http://www.gnu.org/copyleft/gpl.html
- *
- * @file
- */
-
-/**
- * When you have many workers (threads/servers) giving service, and a
- * cached item expensive to produce expires, you may get several workers
- * doing the job at the same time.
- *
- * Given enough requests and the item expiring fast (non-cacheable,
- * lots of edits...) that single work can end up unfairly using most (all)
- * of the cpu of the pool. This is also known as 'Michael Jackson effect'
- * since this effect triggered on the english wikipedia on the day Michael
- * Jackson died, the biographical article got hit with several edits per
- * minutes and hundreds of read hits.
- *
- * The PoolCounter provides semaphore semantics for restricting the number
- * of workers that may be concurrently performing such single task.
- *
- * By default PoolCounter_Stub is used, which provides no locking. You
- * can get a useful one in the PoolCounter extension.
- */
-abstract class PoolCounter {
- /* Return codes */
- const LOCKED = 1; /* Lock acquired */
- const RELEASED = 2; /* Lock released */
- const DONE = 3; /* Another worker did the work for you */
-
- const ERROR = -1; /* Indeterminate error */
- const NOT_LOCKED = -2; /* Called release() with no lock held */
- const QUEUE_FULL = -3; /* There are already maxqueue workers on this lock */
- const TIMEOUT = -4; /* Timeout exceeded */
- const LOCK_HELD = -5; /* Cannot acquire another lock while you have one lock held */
-
- /** @var string All workers with the same key share the lock */
- protected $key;
- /** @var integer Maximum number of workers doing the task simultaneously */
- protected $workers;
- /** @var integer If this number of workers are already working/waiting, fail instead of wait */
- protected $maxqueue;
- /** @var float Maximum time in seconds to wait for the lock */
- protected $timeout;
-
- /**
- * @param array $conf
- * @param string $type
- * @param string $key
- */
- protected function __construct( $conf, $type, $key ) {
- $this->key = $key;
- $this->workers = $conf['workers'];
- $this->maxqueue = $conf['maxqueue'];
- $this->timeout = $conf['timeout'];
- }
-
- /**
- * Create a Pool counter. This should only be called from the PoolWorks.
- *
- * @param $type
- * @param $key
- *
- * @return PoolCounter
- */
- public static function factory( $type, $key ) {
- global $wgPoolCounterConf;
- if ( !isset( $wgPoolCounterConf[$type] ) ) {
- return new PoolCounter_Stub;
- }
- $conf = $wgPoolCounterConf[$type];
- $class = $conf['class'];
-
- return new $class( $conf, $type, $key );
- }
-
- /**
- * @return string
- */
- public function getKey() {
- return $this->key;
- }
-
- /**
- * I want to do this task and I need to do it myself.
- *
- * @return Status Value is one of Locked/Error
- */
- abstract public function acquireForMe();
-
- /**
- * I want to do this task, but if anyone else does it
- * instead, it's also fine for me. I will read its cached data.
- *
- * @return Status Value is one of Locked/Done/Error
- */
- abstract public function acquireForAnyone();
-
- /**
- * I have successfully finished my task.
- * Lets another one grab the lock, and returns the workers
- * waiting on acquireForAnyone()
- *
- * @return Status value is one of Released/NotLocked/Error
- */
- abstract public function release();
-}
-
-class PoolCounter_Stub extends PoolCounter {
- public function __construct() {
- /* No parameters needed */
- }
-
- public function acquireForMe() {
- return Status::newGood( PoolCounter::LOCKED );
- }
-
- public function acquireForAnyone() {
- return Status::newGood( PoolCounter::LOCKED );
- }
-
- public function release() {
- return Status::newGood( PoolCounter::RELEASED );
- }
-}
-
-/**
- * Class for dealing with PoolCounters using class members
- */
-abstract class PoolCounterWork {
- protected $cacheable = false; //Does this override getCachedWork() ?
-
- /**
- * @param string $type The type of PoolCounter to use
- * @param string $key Key that identifies the queue this work is placed on
- */
- public function __construct( $type, $key ) {
- $this->poolCounter = PoolCounter::factory( $type, $key );
- }
-
- /**
- * Actually perform the work, caching it if needed
- * @return mixed work result or false
- */
- abstract public function doWork();
-
- /**
- * Retrieve the work from cache
- * @return mixed work result or false
- */
- public function getCachedWork() {
- return false;
- }
-
- /**
- * A work not so good (eg. expired one) but better than an error
- * message.
- * @return mixed work result or false
- */
- public function fallback() {
- return false;
- }
-
- /**
- * Do something with the error, like showing it to the user.
- * @return bool
- */
- public function error( $status ) {
- return false;
- }
-
- /**
- * Log an error
- *
- * @param $status Status
- * @return void
- */
- public function logError( $status ) {
- $key = $this->poolCounter->getKey();
-
- wfDebugLog( 'poolcounter', "Pool key '$key': "
- . $status->getMessage()->inLanguage( 'en' )->useDatabase( false )->text() );
- }
-
- /**
- * Get the result of the work (whatever it is), or the result of the error() function.
- * This returns the result of the first applicable method that returns a non-false value,
- * where the methods are checked in the following order:
- * - a) doWork() : Applies if the work is exclusive or no another process
- * is doing it, and on the condition that either this process
- * successfully entered the pool or the pool counter is down.
- * - b) doCachedWork() : Applies if the work is cacheable and this blocked on another
- * process which finished the work.
- * - c) fallback() : Applies for all remaining cases.
- * If these all fall through (by returning false), then the result of error() is returned.
- *
- * @param $skipcache bool
- * @return mixed
- */
- public function execute( $skipcache = false ) {
- if ( $this->cacheable && !$skipcache ) {
- $status = $this->poolCounter->acquireForAnyone();
- } else {
- $status = $this->poolCounter->acquireForMe();
- }
-
- if ( !$status->isOK() ) {
- // Respond gracefully to complete server breakage: just log it and do the work
- $this->logError( $status );
- return $this->doWork();
- }
-
- switch ( $status->value ) {
- case PoolCounter::LOCKED:
- $result = $this->doWork();
- $this->poolCounter->release();
- return $result;
-
- case PoolCounter::DONE:
- $result = $this->getCachedWork();
- if ( $result === false ) {
- /* That someone else work didn't serve us.
- * Acquire the lock for me
- */
- return $this->execute( true );
- }
- return $result;
-
- case PoolCounter::QUEUE_FULL:
- case PoolCounter::TIMEOUT:
- $result = $this->fallback();
-
- if ( $result !== false ) {
- return $result;
- }
- /* no break */
-
- /* These two cases should never be hit... */
- case PoolCounter::ERROR:
- default:
- $errors = array(
- PoolCounter::QUEUE_FULL => 'pool-queuefull',
- PoolCounter::TIMEOUT => 'pool-timeout' );
-
- $status = Status::newFatal( isset( $errors[$status->value] )
- ? $errors[$status->value]
- : 'pool-errorunknown' );
- $this->logError( $status );
- return $this->error( $status );
- }
- }
-}
-
-/**
- * Convenience class for dealing with PoolCounters using callbacks
- * @since 1.22
- */
-class PoolCounterWorkViaCallback extends PoolCounterWork {
- /** @var callable */
- protected $doWork;
- /** @var callable|null */
- protected $doCachedWork;
- /** @var callable|null */
- protected $fallback;
- /** @var callable|null */
- protected $error;
-
- /**
- * Build a PoolCounterWork class from a type, key, and callback map.
- *
- * The callback map must at least have a callback for the 'doWork' method.
- * Additionally, callbacks can be provided for the 'doCachedWork', 'fallback',
- * and 'error' methods. Methods without callbacks will be no-ops that return false.
- * If a 'doCachedWork' callback is provided, then execute() may wait for any prior
- * process in the pool to finish and reuse its cached result.
- *
- * @param string $type
- * @param string $key
- * @param array $callbacks Map of callbacks
- * @throws MWException
- */
- public function __construct( $type, $key, array $callbacks ) {
- parent::__construct( $type, $key );
- foreach ( array( 'doWork', 'doCachedWork', 'fallback', 'error' ) as $name ) {
- if ( isset( $callbacks[$name] ) ) {
- if ( !is_callable( $callbacks[$name] ) ) {
- throw new MWException( "Invalid callback provided for '$name' function." );
- }
- $this->$name = $callbacks[$name];
- }
- }
- if ( !isset( $this->doWork ) ) {
- throw new MWException( "No callback provided for 'doWork' function." );
- }
- $this->cacheable = isset( $this->doCachedWork );
- }
-
- public function doWork() {
- return call_user_func_array( $this->doWork, array() );
- }
-
- public function getCachedWork() {
- if ( $this->doCachedWork ) {
- return call_user_func_array( $this->doCachedWork, array() );
- }
- return false;
- }
-
- public function fallback() {
- if ( $this->fallback ) {
- return call_user_func_array( $this->fallback, array() );
- }
- return false;
- }
-
- public function error( $status ) {
- if ( $this->error ) {
- return call_user_func_array( $this->error, array( $status ) );
- }
- return false;
- }
-}
+++ /dev/null
-<?php
-/**
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- * http://www.gnu.org/copyleft/gpl.html
- *
- * @file
- * @author Aaron Schulz
- */
-
-/**
- * Version of PoolCounter that uses Redis
- *
- * There are four main redis keys used to track each pool counter key:
- * - poolcounter:l-slots-* : A list of available slot IDs for a pool.
- * - poolcounter:z-renewtime-* : A sorted set of (slot ID, UNIX timestamp as score)
- * used for tracking the next time a slot should be
- * released. This is -1 when a slot is created, and is
- * set when released (expired), locked, and unlocked.
- * - poolcounter:z-wait-* : A sorted set of (slot ID, UNIX timestamp as score)
- * used for tracking waiting processes (and wait time).
- * - poolcounter:l-wakeup-* : A list pushed to for the sake of waking up processes
- * when a any process in the pool finishes (lasts for 1ms).
- * For a given pool key, all the redis keys start off non-existing and are deleted if not
- * used for a while to prevent garbage from building up on the server. They are atomically
- * re-initialized as needed. The "z-renewtime" key is used for detecting sessions which got
- * slots but then disappeared. Stale entries from there have their timestamp updated and the
- * corresponding slots freed up. The "z-wait" key is used for detecting processes registered
- * as waiting but that disappeared. Stale entries from there are deleted and the corresponding
- * slots are freed up. The worker count is included in all the redis key names as it does not
- * vary within each $wgPoolCounterConf type and doing so handles configuration changes.
- *
- * This class requires Redis 2.6 as it makes use Lua scripts for fast atomic operations.
- * Also this should be on a server plenty of RAM for the working set to avoid evictions.
- * Evictions could temporarily allow wait queues to double in size or temporarily cause
- * pools to appear as full when they are not. Using volatile-ttl and bumping memory-samples
- * and redis.conf can be helpful otherwise.
- *
- * @ingroup Redis
- * @since 1.23
- */
-class PoolCounterRedis extends PoolCounter {
- /** @var HashRing */
- protected $ring;
- /** @var RedisConnectionPool */
- protected $pool;
- /** @var array (server label => host) map */
- protected $serversByLabel;
- /** @var string SHA-1 of the key */
- protected $keySha1;
- /** @var integer TTL for locks to expire (work should finish in this time) */
- protected $lockTTL;
-
- /** @var RedisConnRef */
- protected $conn;
- /** @var string Pool slot value */
- protected $slot;
- /** @var integer AWAKE_* constant */
- protected $onRelease;
- /** @var string Unique string to identify this process */
- protected $session;
- /** @var integer UNIX timestamp */
- protected $slotTime;
-
- const AWAKE_ONE = 1; // wake-up if when a slot can be taken from an existing process
- const AWAKE_ALL = 2; // wake-up if an existing process finishes and wake up such others
-
- /** @var Array List of active PoolCounterRedis objects in this script */
- protected static $active = null;
-
- function __construct( $conf, $type, $key ) {
- parent::__construct( $conf, $type, $key );
-
- $this->serversByLabel = $conf['servers'];
- $this->ring = new HashRing( array_fill_keys( array_keys( $conf['servers'] ), 100 ) );
-
- $conf['redisConfig']['serializer'] = 'none'; // for use with Lua
- $this->pool = RedisConnectionPool::singleton( $conf['redisConfig'] );
-
- $this->keySha1 = sha1( $this->key );
- $met = ini_get( 'max_execution_time' ); // usually 0 in CLI mode
- $this->lockTTL = $met ? 2*$met : 3600;
-
- if ( self::$active === null ) {
- self::$active = array();
- register_shutdown_function( array( __CLASS__, 'releaseAll' ) );
- }
- }
-
- /**
- * @return Status Uses RediConnRef as value on success
- */
- protected function getConnection() {
- if ( !isset( $this->conn ) ) {
- $conn = false;
- $servers = $this->ring->getLocations( $this->key, 3 );
- ArrayUtils::consistentHashSort( $servers, $this->key );
- foreach ( $servers as $server ) {
- $conn = $this->pool->getConnection( $this->serversByLabel[$server] );
- if ( $conn ) {
- break;
- }
- }
- if ( !$conn ) {
- return Status::newFatal( 'pool-servererror', implode( ', ', $servers ) );
- }
- $this->conn = $conn;
- }
- return Status::newGood( $this->conn );
- }
-
- function acquireForMe() {
- $section = new ProfileSection( __METHOD__ );
-
- return $this->waitForSlotOrNotif( self::AWAKE_ONE );
- }
-
- function acquireForAnyone() {
- $section = new ProfileSection( __METHOD__ );
-
- return $this->waitForSlotOrNotif( self::AWAKE_ALL );
- }
-
- function release() {
- $section = new ProfileSection( __METHOD__ );
-
- if ( $this->slot === null ) {
- return Status::newGood( PoolCounter::NOT_LOCKED ); // not locked
- }
-
- $status = $this->getConnection();
- if ( !$status->isOK() ) {
- return $status;
- }
- $conn = $status->value;
-
- static $script =
-<<<LUA
- local kSlots,kSlotsNextRelease,kWakeup,kWaiting = unpack(KEYS)
- local rMaxWorkers,rExpiry,rSlot,rSlotTime,rAwakeAll,rTime = unpack(ARGV)
- -- Add the slots back to the list (if rSlot is "w" then it is not a slot).
- -- Treat the list as expired if the "next release" time sorted-set is missing.
- if rSlot ~= 'w' and redis.call('exists',kSlotsNextRelease) == 1 then
- if 1*redis.call('zScore',kSlotsNextRelease,rSlot) ~= (rSlotTime + rExpiry) then
- -- Slot lock expired and was released already
- elseif redis.call('lLen',kSlots) >= (1*rMaxWorkers - 1) then
- -- Clear list to save space; it will re-init as needed
- redis.call('del',kSlots,kSlotsNextRelease)
- else
- -- Add slot back to pool and update the "next release" time
- redis.call('rPush',kSlots,rSlot)
- redis.call('zAdd',kSlotsNextRelease,rTime + 30,rSlot)
- -- Always keep renewing the expiry on use
- redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
- redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
- end
- end
- -- Update an ephemeral list to wake up other clients that can
- -- reuse any cached work from this process. Only do this if no
- -- slots are currently free (e.g. clients could be waiting).
- if 1*rAwakeAll == 1 then
- local count = redis.call('zCard',kWaiting)
- for i = 1,count do
- redis.call('rPush',kWakeup,'w')
- end
- redis.call('pexpire',kWakeup,1)
- end
- return 1
-LUA;
- try {
- $res = $conn->luaEval( $script,
- array(
- $this->getSlotListKey(),
- $this->getSlotRTimeSetKey(),
- $this->getWakeupListKey(),
- $this->getWaitSetKey(),
- $this->workers,
- $this->lockTTL,
- $this->slot,
- $this->slotTime, // used for CAS-style sanity check
- ( $this->onRelease === self::AWAKE_ALL ) ? 1 : 0,
- microtime( true )
- ),
- 4 # number of first argument(s) that are keys
- );
- } catch ( RedisException $e ) {
- return Status::newFatal( 'pool-error-unknown', $e->getMessage() );
- }
-
- $this->slot = null;
- $this->slotTime = null;
- $this->onRelease = null;
- unset( self::$active[$this->session] );
-
- return Status::newGood( PoolCounter::RELEASED );
- }
-
- /**
- * @param int $doWakeup AWAKE_* constant
- * @return Status
- */
- protected function waitForSlotOrNotif( $doWakeup ) {
- if ( $this->slot !== null ) {
- return Status::newGood( PoolCounter::LOCK_HELD ); // already acquired
- }
-
- $status = $this->getConnection();
- if ( !$status->isOK() ) {
- return $status;
- }
- $conn = $status->value;
-
- $now = microtime( true );
- try {
- $slot = $this->initAndPopPoolSlotList( $conn, $now );
- if ( ctype_digit( $slot ) ) {
- // Pool slot acquired by this process
- $slotTime = $now;
- } elseif ( $slot === 'QUEUE_FULL' ) {
- // Too many processes are waiting for pooled processes to finish
- return Status::newGood( PoolCounter::QUEUE_FULL );
- } elseif ( $slot === 'QUEUE_WAIT' ) {
- // This process is now registered as waiting
- $keys = ( $doWakeup == self::AWAKE_ALL )
- // Wait for an open slot or wake-up signal (preferring the later)
- ? array( $this->getWakeupListKey(), $this->getSlotListKey() )
- // Just wait for an actual pool slot
- : array( $this->getSlotListKey() );
-
- $res = $conn->blPop( $keys, $this->timeout );
- if ( $res === array() ) {
- $conn->zRem( $this->getWaitSetKey(), $this->session ); // no longer waiting
- return Status::newGood( PoolCounter::TIMEOUT );
- }
-
- $slot = $res[1]; // pool slot or "w" for wake-up notifications
- $slotTime = microtime( true ); // last microtime() was a few RTTs ago
- // Unregister this process as waiting and bump slot "next release" time
- $this->registerAcquisitionTime( $conn, $slot, $slotTime );
- } else {
- return Status::newFatal( 'pool-error-unknown', "Server gave slot '$slot'." );
- }
- } catch ( RedisException $e ) {
- return Status::newFatal( 'pool-error-unknown', $e->getMessage() );
- }
-
- if ( $slot !== 'w' ) {
- $this->slot = $slot;
- $this->slotTime = $slotTime;
- $this->onRelease = $doWakeup;
- self::$active[$this->session] = $this;
- }
-
- return Status::newGood( $slot === 'w' ? PoolCounter::DONE : PoolCounter::LOCKED );
- }
-
- /**
- * @param RedisConnRef $conn
- * @param float UNIX timestamp
- * @return string|bool False on failure
- */
- protected function initAndPopPoolSlotList( RedisConnRef $conn, $now ) {
- static $script =
-<<<LUA
- local kSlots,kSlotsNextRelease,kSlotWaits = unpack(KEYS)
- local rMaxWorkers,rMaxQueue,rTimeout,rExpiry,rSess,rTime = unpack(ARGV)
- -- Initialize if the "next release" time sorted-set is empty. The slot key
- -- itself is empty if all slots are busy or when nothing is initialized.
- -- If the list is empty but the set is not, then it is the later case.
- -- For sanity, if the list exists but not the set, then reset everything.
- if redis.call('exists',kSlotsNextRelease) == 0 then
- redis.call('del',kSlots)
- for i = 1,1*rMaxWorkers do
- redis.call('rPush',kSlots,i)
- redis.call('zAdd',kSlotsNextRelease,-1,i)
- end
- -- Otherwise do maintenance to clean up after network partitions
- else
- -- Find stale slot locks and add free them (avoid duplicates for sanity)
- local staleLocks = redis.call('zRangeByScore',kSlotsNextRelease,0,rTime)
- for k,slot in ipairs(staleLocks) do
- redis.call('lRem',kSlots,0,slot)
- redis.call('rPush',kSlots,slot)
- redis.call('zAdd',kSlotsNextRelease,rTime + 30,slot)
- end
- -- Find stale wait slot entries and remove them
- redis.call('zRemRangeByScore',kSlotWaits,0,rTime - 2*rTimeout)
- end
- local slot
- -- Try to acquire a slot if possible now
- if redis.call('lLen',kSlots) > 0 then
- slot = redis.call('lPop',kSlots)
- -- Update the slot "next release" time
- redis.call('zAdd',kSlotsNextRelease,rTime + rExpiry,slot)
- elseif redis.call('zCard',kSlotWaits) >= 1*rMaxQueue then
- slot = 'QUEUE_FULL'
- else
- slot = 'QUEUE_WAIT'
- -- Register this process as waiting
- redis.call('zAdd',kSlotWaits,rTime,rSess)
- redis.call('expireAt',kSlotWaits,math.ceil(rTime + 2*rTimeout))
- end
- -- Always keep renewing the expiry on use
- redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
- redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
- return slot
-LUA;
- return $conn->luaEval( $script,
- array(
- $this->getSlotListKey(),
- $this->getSlotRTimeSetKey(),
- $this->getWaitSetKey(),
- $this->workers,
- $this->maxqueue,
- $this->timeout,
- $this->lockTTL,
- $this->session,
- $now
- ),
- 3 # number of first argument(s) that are keys
- );
- }
-
- /**
- * @param RedisConnRef $conn
- * @param string $slot
- * @param float $now
- * @return int|bool False on failure
- */
- protected function registerAcquisitionTime( RedisConnRef $conn, $slot, $now ) {
- static $script =
-<<<LUA
- local kSlots,kSlotsNextRelease,kSlotWaits = unpack(KEYS)
- local rSlot,rExpiry,rSess,rTime = unpack(ARGV)
- -- If rSlot is 'w' then the client was told to wake up but got no slot
- if rSlot ~= 'w' then
- -- Update the slot "next release" time
- redis.call('zAdd',kSlotsNextRelease,rTime + rExpiry,rSlot)
- -- Always keep renewing the expiry on use
- redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
- redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
- end
- -- Unregister this process as waiting
- redis.call('zRem',kSlotWaits,rSess)
- return 1
-LUA;
- return $conn->luaEval( $script,
- array(
- $this->getSlotListKey(),
- $this->getSlotRTimeSetKey(),
- $this->getWaitSetKey(),
- $slot,
- $this->lockTTL,
- $this->session,
- $now
- ),
- 3 # number of first argument(s) that are keys
- );
- }
-
- /**
- * @return string
- */
- protected function getSlotListKey() {
- return "poolcounter:l-slots-{$this->keySha1}-{$this->workers}";
- }
-
- /**
- * @return string
- */
- protected function getSlotRTimeSetKey() {
- return "poolcounter:z-renewtime-{$this->keySha1}-{$this->workers}";
- }
-
- /**
- * @return string
- */
- protected function getWaitSetKey() {
- return "poolcounter:z-wait-{$this->keySha1}-{$this->workers}";
- }
-
- /**
- * @return string
- */
- protected function getWakeupListKey() {
- return "poolcounter:l-wakeup-{$this->keySha1}-{$this->workers}";
- }
-
- /**
- * Try to make sure that locks get released (even with exceptions and fatals)
- */
- public static function releaseAll() {
- foreach ( self::$active as $poolCounter ) {
- try {
- if ( $poolCounter->slot !== null ) {
- $poolCounter->release();
- }
- } catch ( Exception $e ) {}
- }
- }
-}
--- /dev/null
+<?php
+/**
+ * Provides of semaphore semantics for restricting the number
+ * of workers that may be concurrently performing the same task.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+/**
+ * When you have many workers (threads/servers) giving service, and a
+ * cached item expensive to produce expires, you may get several workers
+ * doing the job at the same time.
+ *
+ * Given enough requests and the item expiring fast (non-cacheable,
+ * lots of edits...) that single work can end up unfairly using most (all)
+ * of the cpu of the pool. This is also known as 'Michael Jackson effect'
+ * since this effect triggered on the english wikipedia on the day Michael
+ * Jackson died, the biographical article got hit with several edits per
+ * minutes and hundreds of read hits.
+ *
+ * The PoolCounter provides semaphore semantics for restricting the number
+ * of workers that may be concurrently performing such single task.
+ *
+ * By default PoolCounter_Stub is used, which provides no locking. You
+ * can get a useful one in the PoolCounter extension.
+ */
+abstract class PoolCounter {
+ /* Return codes */
+ const LOCKED = 1; /* Lock acquired */
+ const RELEASED = 2; /* Lock released */
+ const DONE = 3; /* Another worker did the work for you */
+
+ const ERROR = -1; /* Indeterminate error */
+ const NOT_LOCKED = -2; /* Called release() with no lock held */
+ const QUEUE_FULL = -3; /* There are already maxqueue workers on this lock */
+ const TIMEOUT = -4; /* Timeout exceeded */
+ const LOCK_HELD = -5; /* Cannot acquire another lock while you have one lock held */
+
+ /** @var string All workers with the same key share the lock */
+ protected $key;
+ /** @var integer Maximum number of workers doing the task simultaneously */
+ protected $workers;
+ /** @var integer If this number of workers are already working/waiting, fail instead of wait */
+ protected $maxqueue;
+ /** @var float Maximum time in seconds to wait for the lock */
+ protected $timeout;
+
+ /**
+ * @param array $conf
+ * @param string $type
+ * @param string $key
+ */
+ protected function __construct( $conf, $type, $key ) {
+ $this->key = $key;
+ $this->workers = $conf['workers'];
+ $this->maxqueue = $conf['maxqueue'];
+ $this->timeout = $conf['timeout'];
+ }
+
+ /**
+ * Create a Pool counter. This should only be called from the PoolWorks.
+ *
+ * @param $type
+ * @param $key
+ *
+ * @return PoolCounter
+ */
+ public static function factory( $type, $key ) {
+ global $wgPoolCounterConf;
+ if ( !isset( $wgPoolCounterConf[$type] ) ) {
+ return new PoolCounter_Stub;
+ }
+ $conf = $wgPoolCounterConf[$type];
+ $class = $conf['class'];
+
+ return new $class( $conf, $type, $key );
+ }
+
+ /**
+ * @return string
+ */
+ public function getKey() {
+ return $this->key;
+ }
+
+ /**
+ * I want to do this task and I need to do it myself.
+ *
+ * @return Status Value is one of Locked/Error
+ */
+ abstract public function acquireForMe();
+
+ /**
+ * I want to do this task, but if anyone else does it
+ * instead, it's also fine for me. I will read its cached data.
+ *
+ * @return Status Value is one of Locked/Done/Error
+ */
+ abstract public function acquireForAnyone();
+
+ /**
+ * I have successfully finished my task.
+ * Lets another one grab the lock, and returns the workers
+ * waiting on acquireForAnyone()
+ *
+ * @return Status value is one of Released/NotLocked/Error
+ */
+ abstract public function release();
+}
+
+class PoolCounter_Stub extends PoolCounter {
+ public function __construct() {
+ /* No parameters needed */
+ }
+
+ public function acquireForMe() {
+ return Status::newGood( PoolCounter::LOCKED );
+ }
+
+ public function acquireForAnyone() {
+ return Status::newGood( PoolCounter::LOCKED );
+ }
+
+ public function release() {
+ return Status::newGood( PoolCounter::RELEASED );
+ }
+}
--- /dev/null
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @author Aaron Schulz
+ */
+
+/**
+ * Version of PoolCounter that uses Redis
+ *
+ * There are four main redis keys used to track each pool counter key:
+ * - poolcounter:l-slots-* : A list of available slot IDs for a pool.
+ * - poolcounter:z-renewtime-* : A sorted set of (slot ID, UNIX timestamp as score)
+ * used for tracking the next time a slot should be
+ * released. This is -1 when a slot is created, and is
+ * set when released (expired), locked, and unlocked.
+ * - poolcounter:z-wait-* : A sorted set of (slot ID, UNIX timestamp as score)
+ * used for tracking waiting processes (and wait time).
+ * - poolcounter:l-wakeup-* : A list pushed to for the sake of waking up processes
+ * when a any process in the pool finishes (lasts for 1ms).
+ * For a given pool key, all the redis keys start off non-existing and are deleted if not
+ * used for a while to prevent garbage from building up on the server. They are atomically
+ * re-initialized as needed. The "z-renewtime" key is used for detecting sessions which got
+ * slots but then disappeared. Stale entries from there have their timestamp updated and the
+ * corresponding slots freed up. The "z-wait" key is used for detecting processes registered
+ * as waiting but that disappeared. Stale entries from there are deleted and the corresponding
+ * slots are freed up. The worker count is included in all the redis key names as it does not
+ * vary within each $wgPoolCounterConf type and doing so handles configuration changes.
+ *
+ * This class requires Redis 2.6 as it makes use Lua scripts for fast atomic operations.
+ * Also this should be on a server plenty of RAM for the working set to avoid evictions.
+ * Evictions could temporarily allow wait queues to double in size or temporarily cause
+ * pools to appear as full when they are not. Using volatile-ttl and bumping memory-samples
+ * and redis.conf can be helpful otherwise.
+ *
+ * @ingroup Redis
+ * @since 1.23
+ */
+class PoolCounterRedis extends PoolCounter {
+ /** @var HashRing */
+ protected $ring;
+ /** @var RedisConnectionPool */
+ protected $pool;
+ /** @var array (server label => host) map */
+ protected $serversByLabel;
+ /** @var string SHA-1 of the key */
+ protected $keySha1;
+ /** @var integer TTL for locks to expire (work should finish in this time) */
+ protected $lockTTL;
+
+ /** @var RedisConnRef */
+ protected $conn;
+ /** @var string Pool slot value */
+ protected $slot;
+ /** @var integer AWAKE_* constant */
+ protected $onRelease;
+ /** @var string Unique string to identify this process */
+ protected $session;
+ /** @var integer UNIX timestamp */
+ protected $slotTime;
+
+ const AWAKE_ONE = 1; // wake-up if when a slot can be taken from an existing process
+ const AWAKE_ALL = 2; // wake-up if an existing process finishes and wake up such others
+
+ /** @var Array List of active PoolCounterRedis objects in this script */
+ protected static $active = null;
+
+ function __construct( $conf, $type, $key ) {
+ parent::__construct( $conf, $type, $key );
+
+ $this->serversByLabel = $conf['servers'];
+ $this->ring = new HashRing( array_fill_keys( array_keys( $conf['servers'] ), 100 ) );
+
+ $conf['redisConfig']['serializer'] = 'none'; // for use with Lua
+ $this->pool = RedisConnectionPool::singleton( $conf['redisConfig'] );
+
+ $this->keySha1 = sha1( $this->key );
+ $met = ini_get( 'max_execution_time' ); // usually 0 in CLI mode
+ $this->lockTTL = $met ? 2*$met : 3600;
+
+ if ( self::$active === null ) {
+ self::$active = array();
+ register_shutdown_function( array( __CLASS__, 'releaseAll' ) );
+ }
+ }
+
+ /**
+ * @return Status Uses RediConnRef as value on success
+ */
+ protected function getConnection() {
+ if ( !isset( $this->conn ) ) {
+ $conn = false;
+ $servers = $this->ring->getLocations( $this->key, 3 );
+ ArrayUtils::consistentHashSort( $servers, $this->key );
+ foreach ( $servers as $server ) {
+ $conn = $this->pool->getConnection( $this->serversByLabel[$server] );
+ if ( $conn ) {
+ break;
+ }
+ }
+ if ( !$conn ) {
+ return Status::newFatal( 'pool-servererror', implode( ', ', $servers ) );
+ }
+ $this->conn = $conn;
+ }
+ return Status::newGood( $this->conn );
+ }
+
+ function acquireForMe() {
+ $section = new ProfileSection( __METHOD__ );
+
+ return $this->waitForSlotOrNotif( self::AWAKE_ONE );
+ }
+
+ function acquireForAnyone() {
+ $section = new ProfileSection( __METHOD__ );
+
+ return $this->waitForSlotOrNotif( self::AWAKE_ALL );
+ }
+
+ function release() {
+ $section = new ProfileSection( __METHOD__ );
+
+ if ( $this->slot === null ) {
+ return Status::newGood( PoolCounter::NOT_LOCKED ); // not locked
+ }
+
+ $status = $this->getConnection();
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ $conn = $status->value;
+
+ static $script =
+<<<LUA
+ local kSlots,kSlotsNextRelease,kWakeup,kWaiting = unpack(KEYS)
+ local rMaxWorkers,rExpiry,rSlot,rSlotTime,rAwakeAll,rTime = unpack(ARGV)
+ -- Add the slots back to the list (if rSlot is "w" then it is not a slot).
+ -- Treat the list as expired if the "next release" time sorted-set is missing.
+ if rSlot ~= 'w' and redis.call('exists',kSlotsNextRelease) == 1 then
+ if 1*redis.call('zScore',kSlotsNextRelease,rSlot) ~= (rSlotTime + rExpiry) then
+ -- Slot lock expired and was released already
+ elseif redis.call('lLen',kSlots) >= (1*rMaxWorkers - 1) then
+ -- Clear list to save space; it will re-init as needed
+ redis.call('del',kSlots,kSlotsNextRelease)
+ else
+ -- Add slot back to pool and update the "next release" time
+ redis.call('rPush',kSlots,rSlot)
+ redis.call('zAdd',kSlotsNextRelease,rTime + 30,rSlot)
+ -- Always keep renewing the expiry on use
+ redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
+ redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
+ end
+ end
+ -- Update an ephemeral list to wake up other clients that can
+ -- reuse any cached work from this process. Only do this if no
+ -- slots are currently free (e.g. clients could be waiting).
+ if 1*rAwakeAll == 1 then
+ local count = redis.call('zCard',kWaiting)
+ for i = 1,count do
+ redis.call('rPush',kWakeup,'w')
+ end
+ redis.call('pexpire',kWakeup,1)
+ end
+ return 1
+LUA;
+ try {
+ $res = $conn->luaEval( $script,
+ array(
+ $this->getSlotListKey(),
+ $this->getSlotRTimeSetKey(),
+ $this->getWakeupListKey(),
+ $this->getWaitSetKey(),
+ $this->workers,
+ $this->lockTTL,
+ $this->slot,
+ $this->slotTime, // used for CAS-style sanity check
+ ( $this->onRelease === self::AWAKE_ALL ) ? 1 : 0,
+ microtime( true )
+ ),
+ 4 # number of first argument(s) that are keys
+ );
+ } catch ( RedisException $e ) {
+ return Status::newFatal( 'pool-error-unknown', $e->getMessage() );
+ }
+
+ $this->slot = null;
+ $this->slotTime = null;
+ $this->onRelease = null;
+ unset( self::$active[$this->session] );
+
+ return Status::newGood( PoolCounter::RELEASED );
+ }
+
+ /**
+ * @param int $doWakeup AWAKE_* constant
+ * @return Status
+ */
+ protected function waitForSlotOrNotif( $doWakeup ) {
+ if ( $this->slot !== null ) {
+ return Status::newGood( PoolCounter::LOCK_HELD ); // already acquired
+ }
+
+ $status = $this->getConnection();
+ if ( !$status->isOK() ) {
+ return $status;
+ }
+ $conn = $status->value;
+
+ $now = microtime( true );
+ try {
+ $slot = $this->initAndPopPoolSlotList( $conn, $now );
+ if ( ctype_digit( $slot ) ) {
+ // Pool slot acquired by this process
+ $slotTime = $now;
+ } elseif ( $slot === 'QUEUE_FULL' ) {
+ // Too many processes are waiting for pooled processes to finish
+ return Status::newGood( PoolCounter::QUEUE_FULL );
+ } elseif ( $slot === 'QUEUE_WAIT' ) {
+ // This process is now registered as waiting
+ $keys = ( $doWakeup == self::AWAKE_ALL )
+ // Wait for an open slot or wake-up signal (preferring the later)
+ ? array( $this->getWakeupListKey(), $this->getSlotListKey() )
+ // Just wait for an actual pool slot
+ : array( $this->getSlotListKey() );
+
+ $res = $conn->blPop( $keys, $this->timeout );
+ if ( $res === array() ) {
+ $conn->zRem( $this->getWaitSetKey(), $this->session ); // no longer waiting
+ return Status::newGood( PoolCounter::TIMEOUT );
+ }
+
+ $slot = $res[1]; // pool slot or "w" for wake-up notifications
+ $slotTime = microtime( true ); // last microtime() was a few RTTs ago
+ // Unregister this process as waiting and bump slot "next release" time
+ $this->registerAcquisitionTime( $conn, $slot, $slotTime );
+ } else {
+ return Status::newFatal( 'pool-error-unknown', "Server gave slot '$slot'." );
+ }
+ } catch ( RedisException $e ) {
+ return Status::newFatal( 'pool-error-unknown', $e->getMessage() );
+ }
+
+ if ( $slot !== 'w' ) {
+ $this->slot = $slot;
+ $this->slotTime = $slotTime;
+ $this->onRelease = $doWakeup;
+ self::$active[$this->session] = $this;
+ }
+
+ return Status::newGood( $slot === 'w' ? PoolCounter::DONE : PoolCounter::LOCKED );
+ }
+
+ /**
+ * @param RedisConnRef $conn
+ * @param float UNIX timestamp
+ * @return string|bool False on failure
+ */
+ protected function initAndPopPoolSlotList( RedisConnRef $conn, $now ) {
+ static $script =
+<<<LUA
+ local kSlots,kSlotsNextRelease,kSlotWaits = unpack(KEYS)
+ local rMaxWorkers,rMaxQueue,rTimeout,rExpiry,rSess,rTime = unpack(ARGV)
+ -- Initialize if the "next release" time sorted-set is empty. The slot key
+ -- itself is empty if all slots are busy or when nothing is initialized.
+ -- If the list is empty but the set is not, then it is the later case.
+ -- For sanity, if the list exists but not the set, then reset everything.
+ if redis.call('exists',kSlotsNextRelease) == 0 then
+ redis.call('del',kSlots)
+ for i = 1,1*rMaxWorkers do
+ redis.call('rPush',kSlots,i)
+ redis.call('zAdd',kSlotsNextRelease,-1,i)
+ end
+ -- Otherwise do maintenance to clean up after network partitions
+ else
+ -- Find stale slot locks and add free them (avoid duplicates for sanity)
+ local staleLocks = redis.call('zRangeByScore',kSlotsNextRelease,0,rTime)
+ for k,slot in ipairs(staleLocks) do
+ redis.call('lRem',kSlots,0,slot)
+ redis.call('rPush',kSlots,slot)
+ redis.call('zAdd',kSlotsNextRelease,rTime + 30,slot)
+ end
+ -- Find stale wait slot entries and remove them
+ redis.call('zRemRangeByScore',kSlotWaits,0,rTime - 2*rTimeout)
+ end
+ local slot
+ -- Try to acquire a slot if possible now
+ if redis.call('lLen',kSlots) > 0 then
+ slot = redis.call('lPop',kSlots)
+ -- Update the slot "next release" time
+ redis.call('zAdd',kSlotsNextRelease,rTime + rExpiry,slot)
+ elseif redis.call('zCard',kSlotWaits) >= 1*rMaxQueue then
+ slot = 'QUEUE_FULL'
+ else
+ slot = 'QUEUE_WAIT'
+ -- Register this process as waiting
+ redis.call('zAdd',kSlotWaits,rTime,rSess)
+ redis.call('expireAt',kSlotWaits,math.ceil(rTime + 2*rTimeout))
+ end
+ -- Always keep renewing the expiry on use
+ redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
+ redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
+ return slot
+LUA;
+ return $conn->luaEval( $script,
+ array(
+ $this->getSlotListKey(),
+ $this->getSlotRTimeSetKey(),
+ $this->getWaitSetKey(),
+ $this->workers,
+ $this->maxqueue,
+ $this->timeout,
+ $this->lockTTL,
+ $this->session,
+ $now
+ ),
+ 3 # number of first argument(s) that are keys
+ );
+ }
+
+ /**
+ * @param RedisConnRef $conn
+ * @param string $slot
+ * @param float $now
+ * @return int|bool False on failure
+ */
+ protected function registerAcquisitionTime( RedisConnRef $conn, $slot, $now ) {
+ static $script =
+<<<LUA
+ local kSlots,kSlotsNextRelease,kSlotWaits = unpack(KEYS)
+ local rSlot,rExpiry,rSess,rTime = unpack(ARGV)
+ -- If rSlot is 'w' then the client was told to wake up but got no slot
+ if rSlot ~= 'w' then
+ -- Update the slot "next release" time
+ redis.call('zAdd',kSlotsNextRelease,rTime + rExpiry,rSlot)
+ -- Always keep renewing the expiry on use
+ redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry))
+ redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry))
+ end
+ -- Unregister this process as waiting
+ redis.call('zRem',kSlotWaits,rSess)
+ return 1
+LUA;
+ return $conn->luaEval( $script,
+ array(
+ $this->getSlotListKey(),
+ $this->getSlotRTimeSetKey(),
+ $this->getWaitSetKey(),
+ $slot,
+ $this->lockTTL,
+ $this->session,
+ $now
+ ),
+ 3 # number of first argument(s) that are keys
+ );
+ }
+
+ /**
+ * @return string
+ */
+ protected function getSlotListKey() {
+ return "poolcounter:l-slots-{$this->keySha1}-{$this->workers}";
+ }
+
+ /**
+ * @return string
+ */
+ protected function getSlotRTimeSetKey() {
+ return "poolcounter:z-renewtime-{$this->keySha1}-{$this->workers}";
+ }
+
+ /**
+ * @return string
+ */
+ protected function getWaitSetKey() {
+ return "poolcounter:z-wait-{$this->keySha1}-{$this->workers}";
+ }
+
+ /**
+ * @return string
+ */
+ protected function getWakeupListKey() {
+ return "poolcounter:l-wakeup-{$this->keySha1}-{$this->workers}";
+ }
+
+ /**
+ * Try to make sure that locks get released (even with exceptions and fatals)
+ */
+ public static function releaseAll() {
+ foreach ( self::$active as $poolCounter ) {
+ try {
+ if ( $poolCounter->slot !== null ) {
+ $poolCounter->release();
+ }
+ } catch ( Exception $e ) {}
+ }
+ }
+}
--- /dev/null
+<?php
+/**
+ * Provides of semaphore semantics for restricting the number
+ * of workers that may be concurrently performing the same task.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+/**
+ * Class for dealing with PoolCounters using class members
+ */
+abstract class PoolCounterWork {
+ protected $cacheable = false; //Does this override getCachedWork() ?
+
+ /**
+ * @param string $type The type of PoolCounter to use
+ * @param string $key Key that identifies the queue this work is placed on
+ */
+ public function __construct( $type, $key ) {
+ $this->poolCounter = PoolCounter::factory( $type, $key );
+ }
+
+ /**
+ * Actually perform the work, caching it if needed
+ * @return mixed work result or false
+ */
+ abstract public function doWork();
+
+ /**
+ * Retrieve the work from cache
+ * @return mixed work result or false
+ */
+ public function getCachedWork() {
+ return false;
+ }
+
+ /**
+ * A work not so good (eg. expired one) but better than an error
+ * message.
+ * @return mixed work result or false
+ */
+ public function fallback() {
+ return false;
+ }
+
+ /**
+ * Do something with the error, like showing it to the user.
+ * @return bool
+ */
+ public function error( $status ) {
+ return false;
+ }
+
+ /**
+ * Log an error
+ *
+ * @param $status Status
+ * @return void
+ */
+ public function logError( $status ) {
+ $key = $this->poolCounter->getKey();
+
+ wfDebugLog( 'poolcounter', "Pool key '$key': "
+ . $status->getMessage()->inLanguage( 'en' )->useDatabase( false )->text() );
+ }
+
+ /**
+ * Get the result of the work (whatever it is), or the result of the error() function.
+ * This returns the result of the first applicable method that returns a non-false value,
+ * where the methods are checked in the following order:
+ * - a) doWork() : Applies if the work is exclusive or no another process
+ * is doing it, and on the condition that either this process
+ * successfully entered the pool or the pool counter is down.
+ * - b) doCachedWork() : Applies if the work is cacheable and this blocked on another
+ * process which finished the work.
+ * - c) fallback() : Applies for all remaining cases.
+ * If these all fall through (by returning false), then the result of error() is returned.
+ *
+ * @param $skipcache bool
+ * @return mixed
+ */
+ public function execute( $skipcache = false ) {
+ if ( $this->cacheable && !$skipcache ) {
+ $status = $this->poolCounter->acquireForAnyone();
+ } else {
+ $status = $this->poolCounter->acquireForMe();
+ }
+
+ if ( !$status->isOK() ) {
+ // Respond gracefully to complete server breakage: just log it and do the work
+ $this->logError( $status );
+ return $this->doWork();
+ }
+
+ switch ( $status->value ) {
+ case PoolCounter::LOCKED:
+ $result = $this->doWork();
+ $this->poolCounter->release();
+ return $result;
+
+ case PoolCounter::DONE:
+ $result = $this->getCachedWork();
+ if ( $result === false ) {
+ /* That someone else work didn't serve us.
+ * Acquire the lock for me
+ */
+ return $this->execute( true );
+ }
+ return $result;
+
+ case PoolCounter::QUEUE_FULL:
+ case PoolCounter::TIMEOUT:
+ $result = $this->fallback();
+
+ if ( $result !== false ) {
+ return $result;
+ }
+ /* no break */
+
+ /* These two cases should never be hit... */
+ case PoolCounter::ERROR:
+ default:
+ $errors = array(
+ PoolCounter::QUEUE_FULL => 'pool-queuefull',
+ PoolCounter::TIMEOUT => 'pool-timeout' );
+
+ $status = Status::newFatal( isset( $errors[$status->value] )
+ ? $errors[$status->value]
+ : 'pool-errorunknown' );
+ $this->logError( $status );
+ return $this->error( $status );
+ }
+ }
+}
+
+/**
+ * Convenience class for dealing with PoolCounters using callbacks
+ * @since 1.22
+ */
+class PoolCounterWorkViaCallback extends PoolCounterWork {
+ /** @var callable */
+ protected $doWork;
+ /** @var callable|null */
+ protected $doCachedWork;
+ /** @var callable|null */
+ protected $fallback;
+ /** @var callable|null */
+ protected $error;
+
+ /**
+ * Build a PoolCounterWork class from a type, key, and callback map.
+ *
+ * The callback map must at least have a callback for the 'doWork' method.
+ * Additionally, callbacks can be provided for the 'doCachedWork', 'fallback',
+ * and 'error' methods. Methods without callbacks will be no-ops that return false.
+ * If a 'doCachedWork' callback is provided, then execute() may wait for any prior
+ * process in the pool to finish and reuse its cached result.
+ *
+ * @param string $type
+ * @param string $key
+ * @param array $callbacks Map of callbacks
+ * @throws MWException
+ */
+ public function __construct( $type, $key, array $callbacks ) {
+ parent::__construct( $type, $key );
+ foreach ( array( 'doWork', 'doCachedWork', 'fallback', 'error' ) as $name ) {
+ if ( isset( $callbacks[$name] ) ) {
+ if ( !is_callable( $callbacks[$name] ) ) {
+ throw new MWException( "Invalid callback provided for '$name' function." );
+ }
+ $this->$name = $callbacks[$name];
+ }
+ }
+ if ( !isset( $this->doWork ) ) {
+ throw new MWException( "No callback provided for 'doWork' function." );
+ }
+ $this->cacheable = isset( $this->doCachedWork );
+ }
+
+ public function doWork() {
+ return call_user_func_array( $this->doWork, array() );
+ }
+
+ public function getCachedWork() {
+ if ( $this->doCachedWork ) {
+ return call_user_func_array( $this->doCachedWork, array() );
+ }
+ return false;
+ }
+
+ public function fallback() {
+ if ( $this->fallback ) {
+ return call_user_func_array( $this->fallback, array() );
+ }
+ return false;
+ }
+
+ public function error( $status ) {
+ if ( $this->error ) {
+ return call_user_func_array( $this->error, array( $status ) );
+ }
+ return false;
+ }
+}
dépôts / lhc / web / wiklou.git / commitdiff