From: Aaron Schulz Date: Mon, 4 Feb 2013 00:28:52 +0000 (-0800) Subject: [JobQueue] Use target wiki configuration for some key functions. X-Git-Tag: 1.31.0-rc.0~20614 X-Git-Url: http://git.cyclocoop.org/url?a=commitdiff_plain;h=04e0d75f86b189a389e055b4764a951ba836ca82;p=lhc%2Fweb%2Fwiklou.git [JobQueue] Use target wiki configuration for some key functions. * getQueueTypes() now gets the config of the target wiki. Previously, for WMF, if an extension using jobs was not on aawiki, those job queues would not be seen by nextJobDB. * Also fixed nextJobDB.php so that it no longer only considers jobs types known to aawiki for picking a DB for default jobs. * Note that $wgJobTypesExcludedFromDefaultQueue should be global. * This adds a SiteConfiguration::getConfig() function, which calls a new getConfiguration.php script. Change-Id: I7e6904ead17efa407291f423a2b18e3c866d55fd --- diff --git a/includes/SiteConfiguration.php b/includes/SiteConfiguration.php index 7e96d45e42..11d7fd606a 100644 --- a/includes/SiteConfiguration.php +++ b/includes/SiteConfiguration.php @@ -161,6 +161,12 @@ class SiteConfiguration { */ public $siteParamsCallback = null; + /** + * Configuration cache for getConfig() + * @var array + */ + protected $cfgCache = array(); + /** * Retrieves a configuration setting for a given wiki. * @param $settingName String ID of the setting name to retrieve @@ -486,6 +492,67 @@ class SiteConfiguration { return array( $site, $lang ); } + /** + * Get the resolved (post-setup) configuration of a potentially foreign wiki. + * For foreign wikis, this is expensive, and only works if maintenance + * scripts are setup to handle the --wiki parameter such as in wiki farms. + * + * @param string $wiki + * @param array|string $settings A setting name or array of setting names + * @return Array|mixed Array if $settings is an array, otherwise the value + * @throws MWException + * @since 1.21 + */ + public function getConfig( $wiki, $settings ) { + global $IP; + + $multi = is_array( $settings ); + $settings = (array)$settings; + if ( $wiki === wfWikiID() ) { // $wiki is this wiki + $res = array(); + foreach ( $settings as $name ) { + if ( !preg_match( '/^wg[A-Z]/', $name ) ) { + throw new MWException( "Variable '$name' does start with 'wg'." ); + } elseif ( !isset( $GLOBALS[$name] ) ) { + throw new MWException( "Variable '$name' is not set." ); + } + $res[$name] = $GLOBALS[$name]; + } + } else { // $wiki is a foreign wiki + if ( isset( $this->cfgCache[$wiki] ) ) { + $res = array_intersect_key( $this->cfgCache[$wiki], array_flip( $settings ) ); + if ( count( $res ) == count( $settings ) ) { + return $res; // cache hit + } + } elseif ( !in_array( $wiki, $this->wikis ) ) { + throw new MWException( "No such wiki '$wiki'." ); + } else { + $this->cfgCache[$wiki] = array(); + } + $retVal = 1; + $cmd = wfShellWikiCmd( + "$IP/maintenance/getConfiguration.php", + array( + '--wiki', $wiki, + '--settings', implode( ' ', $settings ), + '--format', 'PHP' + ) + ); + // ulimit5.sh breaks this call + $data = trim( wfShellExec( $cmd, $retVal, array(), array( 'memory' => 0 ) ) ); + if ( $retVal != 0 || !strlen( $data ) ) { + throw new MWException( "Failed to run getConfiguration.php." ); + } + $res = unserialize( $data ); + if ( !is_array( $res ) ) { + throw new MWException( "Failed to unserialize configuration array." ); + } + $this->cfgCache[$wiki] = $this->cfgCache[$wiki] + $res; + } + + return $multi ? $res : current( $res ); + } + /** * Returns true if the given vhost is handled locally. * @param $vhost String diff --git a/includes/job/JobQueueGroup.php b/includes/job/JobQueueGroup.php index 0118853d5c..32c881f8ee 100644 --- a/includes/job/JobQueueGroup.php +++ b/includes/job/JobQueueGroup.php @@ -200,9 +200,7 @@ class JobQueueGroup { * @return array List of strings */ public function getQueueTypes() { - global $wgJobClasses; - - return array_keys( $wgJobClasses ); + return array_keys( $this->getCachedConfigVar( 'wgJobClasses' ) ); } /** @@ -284,4 +282,23 @@ class JobQueueGroup { return $count; } + + private function getCachedConfigVar( $name ) { + global $wgConf, $wgMemc; + + if ( $this->wiki === wfWikiID() ) { + return $GLOBALS[$name]; // common case + } else { + list( $db, $prefix ) = wfSplitWikiID( $this->wiki ); + $key = wfForeignMemcKey( $db, $prefix, 'configvalue', $name ); + $value = $wgMemc->get( $key ); // ('v' => ...) or false + if ( is_array( $value ) ) { + return $value['v']; + } else { + $value = $wgConf->getConfig( $this->wiki, $name ); + $wgMemc->set( $key, array( 'v' => $value ), 86400 + mt_rand( 0, 86400 ) ); + return $value; + } + } + } } diff --git a/maintenance/getConfiguration.php b/maintenance/getConfiguration.php new file mode 100644 index 0000000000..83b5b029cc --- /dev/null +++ b/maintenance/getConfiguration.php @@ -0,0 +1,89 @@ +mDescription = "Get serialized MediaWiki site configuration"; + $this->addOption( 'settings', 'Space-separated list of wg* variables', true, true ); + $this->addOption( 'format', 'PHP or JSON', true, true ); + $this->addOption( 'wiki', 'Wiki ID', true, true ); + } + + public function execute() { + $res = array(); + foreach ( explode( ' ', $this->getOption( 'settings' ) ) as $name ) { + if ( !preg_match( '/^wg[A-Z]/', $name ) ) { + throw new MWException( "Variable '$name' does start with 'wg'." ); + } elseif ( !isset( $GLOBALS[$name] ) ) { + throw new MWException( "Variable '$name' is not set." ); + } elseif ( !$this->isAllowedVariable( $GLOBALS[$name] ) ) { + throw new MWException( "Variable '$name' includes non-array, non-scalar, items." ); + } + $res[$name] = $GLOBALS[$name]; + } + + $out = null; + switch( $this->getOption( 'format' ) ) { + case 'PHP': + $out = serialize( $res ); + break; + case 'JSON': + $out = FormatJson::encode( $res ); + break; + default: + throw new MWException( "Invalid serialization format given." ); + } + if ( !is_string( $out ) ) { + throw new MWException( "Failed to serialize the requested settings." ); + } + + $this->output( $out . "\n" ); + } + + private function isAllowedVariable( $value ) { + if ( is_array( $value ) ) { + foreach ( $value as $k => $v ) { + if ( !$this->isAllowedVariable( $v ) ) { + return false; + } + } + return true; + } elseif ( is_scalar( $value ) ) { + return true; + } + return false; + } +} + +$maintClass = "GetConfiguration"; +require_once( RUN_MAINTENANCE_IF_MAIN ); diff --git a/maintenance/nextJobDB.php b/maintenance/nextJobDB.php index cf90cf66f1..6cc8566b45 100644 --- a/maintenance/nextJobDB.php +++ b/maintenance/nextJobDB.php @@ -37,15 +37,16 @@ class nextJobDB extends Maintenance { } public function execute() { - global $wgMemc; + global $wgMemc, $wgJobTypesExcludedFromDefaultQueue; $type = false; // job type required/picked + if ( $this->hasOption( 'types' ) ) { $types = explode( ' ', $this->getOption( 'types' ) ); } elseif ( $this->hasOption( 'type' ) ) { $types = array( $this->getOption( 'type' ) ); } else { - $types = JobQueueGroup::singleton()->getDefaultQueueTypes(); + $types = false; } // Handle any required periodic queue maintenance @@ -64,7 +65,10 @@ class nextJobDB extends Maintenance { // Flatten the tree of candidates into a flat list so that a random // item can be selected, weighing each queue (type/db tuple) equally. foreach ( $pendingDBs as $type => $dbs ) { - if ( in_array( $type, $types ) ) { + if ( + ( is_array( $types ) && in_array( $type, $types ) ) || + ( $types === false && !in_array( $type, $wgJobTypesExcludedFromDefaultQueue ) ) + ) { foreach ( $dbs as $db ) { $candidates[] = array( $type, $db ); }