From cc21627b4dbe1b56d663c901fa1c066c49499aba Mon Sep 17 00:00:00 2001 From: Krinkle Date: Tue, 26 Jul 2011 21:10:34 +0000 Subject: [PATCH] [ResourceLoader 2]: Add support for multiple loadScript sources Front-end: * New mw.loader method: addSource(). Call with two arguments or an object as first argument for multiple registrations * New property in module registry: "source". Optional for local modules (falls back to 'local'). When loading/using one or more modules, the worker will group the request by source and make separate requests to the sources as needed. * Re-arranging object properties in mw.loader.register to match the same order all other code parts use. * Adding documentation for 'source' and where missing updating it to include 'group' as well. * Refactor of mw.loader.work() by Roan Kattouw and Timo Tijhof:' -- Additional splitting layer by source (in addition to splitting by group), renamed 'groups' to 'splits' -- Clean up of the loop, and removing a no longer needed loop after the for-in-loop -- Much more function documentation in mw.loader.work() -- Moved caching of wgResourceLoaderMaxQueryLength out of the loop and renamed 'limit' to 'maxQueryLength Back-end changed provided through patch by Roan Kattouw (to avoid broken code between commits): * New method in ResourceLoader: addSource(). During construction of ResourceLoader this will be called by default for 'local' with loadScript property set to $wgLoadScript. Additional sources can be registered through $wgResourceLoaderSources (empty array by default) * Calling mw.loader.addSource from the startup module * Passing source to mw.loader.register from startup module * Some new static helper methods Use: * By default nothing should change in core, all modules simply default to 'local'. This info originates from the getSource()-method of the ResourceLoaderModule class, which is inherited to all core ResourceLoaderModule-implementations (none override it) * Third-party users and/or extensions can create new classes extending ResourceLoaderModule, re-implementing the getSource-method to return something else. Basic example: $wgResourceLoaderSources['mywiki'] = array( 'loadScript' => 'http://example.org/w/load.php' ); class MyCentralWikiModule extends ResourceLoaderModule { function getSource(){ return 'mywiki'; } } $wgResourceModules['cool.stuff'] => array( 'class' => 'MyCentralWikiModule' ); More complicated example // imagine some stuff with a ForeignGadgetRepo class, putting stuff in $wgResourceLoaderSources in the __construct() method class ForeignGadgetRepoGadget extends ResourceLoaderModule { function getSource(){ return $this->source; } } Loading: Loading is completely transparent, stuff like $wgOut->addModules() or mw.loader.loader/using both take it as any other module and load from the right source accordingly. -- This commit is part of the ResourceLoader 2 project. --- includes/DefaultSettings.php | 10 + includes/resourceloader/ResourceLoader.php | 99 ++++++- .../resourceloader/ResourceLoaderModule.php | 10 + .../ResourceLoaderStartUpModule.php | 26 +- resources/mediawiki/mediawiki.js | 243 ++++++++++++------ 5 files changed, 288 insertions(+), 100 deletions(-) diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index c976171f30..0e796035af 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -2438,6 +2438,16 @@ $wgBetterDirectionality = true; */ $wgResourceModules = array(); +/** + * Extensions should register foreign module sources here. 'local' is a + * built-in source that is not in this array, but defined by + * ResourceLoader::__construct() so that it cannot be unset. + * + * Example: + * $wgResourceLoaderSources['foo'] = array( 'loadScript' => 'http://example.org/w/load.php' ); + */ +$wgResourceLoaderSources = array(); + /** * Maximum time in seconds to cache resources served by the resource loader */ diff --git a/includes/resourceloader/ResourceLoader.php b/includes/resourceloader/ResourceLoader.php index 2f513b770d..af75a9f052 100644 --- a/includes/resourceloader/ResourceLoader.php +++ b/includes/resourceloader/ResourceLoader.php @@ -30,12 +30,17 @@ class ResourceLoader { /* Protected Static Members */ protected static $filterCacheVersion = 4; + protected static $requiredSourceProperties = array( 'loadScript' ); /** Array: List of module name/ResourceLoaderModule object pairs */ protected $modules = array(); + /** Associative array mapping module name to info associative array */ protected $moduleInfos = array(); + /** array( 'source-id' => array( 'loadScript' => 'http://.../load.php' ) ) **/ + protected $sources = array(); + /* Protected Methods */ /** @@ -178,10 +183,16 @@ class ResourceLoader { * Registers core modules and runs registration hooks. */ public function __construct() { - global $IP, $wgResourceModules; + global $IP, $wgResourceModules, $wgResourceLoaderSources, $wgLoadScript; wfProfileIn( __METHOD__ ); + // Add 'local' source first + $this->addSource( 'local', array( 'loadScript' => $wgLoadScript ) ); + + // Add other sources + $this->addSource( $wgResourceLoaderSources ); + // Register core modules $this->register( include( "$IP/resources/Resources.php" ) ); // Register extension modules @@ -250,7 +261,43 @@ class ResourceLoader { wfProfileOut( __METHOD__ ); } - /** + /** + * Add a foreign source of modules. + * + * Source properties: + * 'loadScript': URL (either fully-qualified or protocol-relative) of load.php for this source + * + * @param $id Mixed: source ID (string), or array( id1 => props1, id2 => props2, ... ) + * @param $properties Array: source properties + */ + public function addSource( $id, $properties = null) { + // Allow multiple sources to be registered in one call + if ( is_array( $id ) ) { + foreach ( $id as $key => $value ) { + $this->addSource( $key, $value ); + } + return; + } + + // Disallow duplicates + if ( isset( $this->sources[$id] ) ) { + throw new MWException( + 'ResourceLoader duplicate source addition error. ' . + 'Another source has already been registered as ' . $id + ); + } + + // Validate properties + foreach ( self::$requiredSourceProperties as $prop ) { + if ( !isset( $properties[$prop] ) ) { + throw new MWException( "Required property $prop missing from source ID $id" ); + } + } + + $this->sources[$id] = $properties; + } + + /** * Get a list of module names * * @return Array: List of module names @@ -291,6 +338,15 @@ class ResourceLoader { return $this->modules[$name]; } + /** + * Get the list of sources + * + * @return Array: array( id => array of properties, .. ) + */ + public function getSources() { + return $this->sources; + } + /** * Outputs a response to a resource load-request, including a content-type header. * @@ -660,30 +716,31 @@ class ResourceLoader { * @param $version Integer: Module version number as a timestamp * @param $dependencies Array: List of module names on which this module depends * @param $group String: Group which the module is in. + * @param $source String: Source of the module, or 'local' if not foreign. * @param $script String: JavaScript code * * @return string */ - public static function makeCustomLoaderScript( $name, $version, $dependencies, $group, $script ) { + public static function makeCustomLoaderScript( $name, $version, $dependencies, $group, $source, $script ) { $script = str_replace( "\n", "\n\t", trim( $script ) ); return Xml::encodeJsCall( - "( function( name, version, dependencies, group ) {\n\t$script\n} )", - array( $name, $version, $dependencies, $group ) ); + "( function( name, version, dependencies, group, source ) {\n\t$script\n} )", + array( $name, $version, $dependencies, $group, $source ) ); } /** * Returns JS code which calls mw.loader.register with the given * parameters. Has three calling conventions: * - * - ResourceLoader::makeLoaderRegisterScript( $name, $version, $dependencies, $group ): + * - ResourceLoader::makeLoaderRegisterScript( $name, $version, $dependencies, $group, $source ): * Register a single module. * * - ResourceLoader::makeLoaderRegisterScript( array( $name1, $name2 ) ): * Register modules with the given names. * * - ResourceLoader::makeLoaderRegisterScript( array( - * array( $name1, $version1, $dependencies1, $group1 ), - * array( $name2, $version2, $dependencies1, $group2 ), + * array( $name1, $version1, $dependencies1, $group1, $source1 ), + * array( $name2, $version2, $dependencies1, $group2, $source2 ), * ... * ) ): * Registers modules with the given names and parameters. @@ -692,18 +749,40 @@ class ResourceLoader { * @param $version Integer: Module version number as a timestamp * @param $dependencies Array: List of module names on which this module depends * @param $group String: group which the module is in. + * @param $source String: source of the module, or 'local' if not foreign * * @return string */ public static function makeLoaderRegisterScript( $name, $version = null, - $dependencies = null, $group = null ) + $dependencies = null, $group = null, $source = null ) { if ( is_array( $name ) ) { return Xml::encodeJsCall( 'mw.loader.register', array( $name ) ); } else { $version = (int) $version > 1 ? (int) $version : 1; return Xml::encodeJsCall( 'mw.loader.register', - array( $name, $version, $dependencies, $group ) ); + array( $name, $version, $dependencies, $group, $source ) ); + } + } + + /** + * Returns JS code which calls mw.loader.addSource() with the given + * parameters. Has two calling conventions: + * + * - ResourceLoader::makeLoaderSourcesScript( $id, $properties ): + * Register a single source + * + * - ResourceLoader::makeLoaderSourcesScript( array( $id1 => $props1, $id2 => $props2, ... ) ); + * Register sources with the given IDs and properties. + * + * @param $id String: source ID + * @param $properties Array: source properties (see addSource()) + */ + public static function makeLoaderSourcesScript( $id, $properties = null ) { + if ( is_array( $id ) ) { + return Xml::encodeJsCall( 'mw.loader.addSource', array( $id ) ); + } else { + return Xml::encodeJsCall( 'mw.loader.addSource', array( $id, $properties ) ); } } diff --git a/includes/resourceloader/ResourceLoaderModule.php b/includes/resourceloader/ResourceLoaderModule.php index 5af7c5fedc..49e9ef60da 100644 --- a/includes/resourceloader/ResourceLoaderModule.php +++ b/includes/resourceloader/ResourceLoaderModule.php @@ -159,6 +159,16 @@ abstract class ResourceLoaderModule { // Stub, override expected return null; } + + /** + * Get the origin of this module. Should only be overridden for foreign modules. + * + * @return String: Origin name, 'local' for local modules + */ + public function getSource() { + // Stub, override expected + return 'local'; + } /** * Where on the HTML page should this module's JS be loaded? diff --git a/includes/resourceloader/ResourceLoaderStartUpModule.php b/includes/resourceloader/ResourceLoaderStartUpModule.php index 870b4ec232..436a569d44 100644 --- a/includes/resourceloader/ResourceLoaderStartUpModule.php +++ b/includes/resourceloader/ResourceLoaderStartUpModule.php @@ -137,6 +137,11 @@ class ResourceLoaderStartUpModule extends ResourceLoaderModule { $out = ''; $registrations = array(); $resourceLoader = $context->getResourceLoader(); + + // Register sources + $out .= ResourceLoader::makeLoaderSourcesScript( $resourceLoader->getSources() ); + + // Register modules foreach ( $resourceLoader->getModuleNames() as $name ) { $module = $resourceLoader->getModule( $name ); // Support module loader scripts @@ -144,9 +149,10 @@ class ResourceLoaderStartUpModule extends ResourceLoaderModule { if ( $loader !== false ) { $deps = $module->getDependencies(); $group = $module->getGroup(); + $source = $module->getSource(); $version = wfTimestamp( TS_ISO_8601_BASIC, $module->getModifiedTime( $context ) ); - $out .= ResourceLoader::makeCustomLoaderScript( $name, $version, $deps, $group, $loader ); + $out .= ResourceLoader::makeCustomLoaderScript( $name, $version, $deps, $group, $source, $loader ); } // Automatically register module else { @@ -154,23 +160,29 @@ class ResourceLoaderStartUpModule extends ResourceLoaderModule { // seem to do that, and custom implementations might forget. Coerce it to TS_UNIX $moduleMtime = wfTimestamp( TS_UNIX, $module->getModifiedTime( $context ) ); $mtime = max( $moduleMtime, wfTimestamp( TS_UNIX, $wgCacheEpoch ) ); - // Modules without dependencies or a group pass two arguments (name, timestamp) to + // Modules without dependencies, a group or a foreign source pass two arguments (name, timestamp) to // mw.loader.register() - if ( !count( $module->getDependencies() && $module->getGroup() === null ) ) { + if ( !count( $module->getDependencies() && $module->getGroup() === null && $module->getSource() === 'local' ) ) { $registrations[] = array( $name, $mtime ); } - // Modules with dependencies but no group pass three arguments + // Modules with dependencies but no group or foreign source pass three arguments // (name, timestamp, dependencies) to mw.loader.register() - elseif ( $module->getGroup() === null ) { + elseif ( $module->getGroup() === null && $module->getSource() === 'local' ) { $registrations[] = array( $name, $mtime, $module->getDependencies() ); } - // Modules with dependencies pass four arguments (name, timestamp, dependencies, group) + // Modules with a group but no foreign source pass four arguments (name, timestamp, dependencies, group) // to mw.loader.register() - else { + else if ( $module->getSource() === 'local' ) { $registrations[] = array( $name, $mtime, $module->getDependencies(), $module->getGroup() ); } + // Modules with a foreign source pass five arguments (name, timestamp, dependencies, group, source) + // to mw.loader.register() + else { + $registrations[] = array( + $name, $mtime, $module->getDependencies(), $module->getGroup(), $module->getSource() ); + } } } $out .= ResourceLoader::makeLoaderRegisterScript( $registrations ); diff --git a/resources/mediawiki/mediawiki.js b/resources/mediawiki/mediawiki.js index 00b8218c61..e0cab820dc 100644 --- a/resources/mediawiki/mediawiki.js +++ b/resources/mediawiki/mediawiki.js @@ -299,18 +299,34 @@ window.mw = window.mediaWiki = new ( function( $ ) { * making it impossible to hold back registration of jquery until after * mediawiki. * + * For exact details on support for script, style and messages, look at + * mw.loader.implement. + * * Format: * { * 'moduleName': { - * 'dependencies': ['required module', 'required module', ...], (or) function() {} - * 'state': 'registered', 'loading', 'loaded', 'ready', or 'error' - * 'script': function() {}, - * 'style': 'css code string', - * 'messages': { 'key': 'value' }, - * 'version': ############## (unix timestamp) + * 'version': ############## (unix timestamp), + * 'dependencies': ['required.foo', 'bar.also', ...], (or) function() {} + * 'group': 'somegroup', (or) null, + * 'source': 'local', 'someforeignwiki', (or) null + * 'state': 'registered', 'loading', 'loaded', 'ready', or 'error' + * 'script': ..., + * 'style': ..., + * 'messages': { 'key': 'value' }, + * } * } */ var registry = {}, + /** + * Mapping of sources, keyed by source-id, values are objects. + * Format: + * { + * 'sourceId': { + * 'loadScript': 'http://foo.bar/w/load.php' + * } + * } + */ + sources = {}, // List of modules which will be loaded as when ready batch = [], // List of modules to be loaded @@ -724,24 +740,41 @@ window.mw = window.mediaWiki = new ( function( $ ) { } } + /** + * Asynchronously append a script tag to the end of the body + * that invokes load.php + * @param moduleMap {Object}: Module map, see buildModulesString() + * @param currReqBase {Object}: Object with other parameters (other than 'modules') to use in the request + * @param sourceLoadScript {String}: URL of load.php + */ + function doRequest( moduleMap, currReqBase, sourceLoadScript ) { + var request = $.extend( + { 'modules': buildModulesString( moduleMap ) }, + currReqBase + ); + request = sortQuery( request ); + // Asynchronously append a script tag to the end of the body + // Append &* to avoid triggering the IE6 extension check + addScript( sourceLoadScript + '?' + $.param( request ) + '&*' ); + } + /* Public Methods */ /** * Requests dependencies from server, loading and executing when things when ready. */ this.work = function() { - // Build a list of request parameters - var base = { - 'skin': mw.config.get( 'skin' ), - 'lang': mw.config.get( 'wgUserLanguage' ), - 'debug': mw.config.get( 'debug' ) + // Build a list of request parameters common to all requests. + var reqBase = { + skin: mw.config.get( 'skin' ), + lang: mw.config.get( 'wgUserLanguage' ), + debug: mw.config.get( 'debug' ) }, - // Extend request parameters with a list of modules in the batch - requests = [], - // Split into groups - groups = {}; + // Split module batch by source and by group. + splits = {}, + maxQueryLength = mw.config.get( 'wgResourceLoaderMaxQueryLength', -1 ); - // Appends a list of modules to the batch + // Appends a list of modules from the queue to the batch for ( var q = 0; q < queue.length; q++ ) { // Only request modules which are undefined or registered if ( !( queue[q] in registry ) || registry[queue[q]].state === 'registered' ) { @@ -755,85 +788,125 @@ window.mw = window.mediaWiki = new ( function( $ ) { } } } - // Early exit if there's nothing to load + // Early exit if there's nothing to load... if ( !batch.length ) { return; } - // Clean up the queue + + // The queue has been processed into the batch, clear up the queue. queue = []; + // Always order modules alphabetically to help reduce cache - // misses for otherwise identical content + // misses for otherwise identical content. batch.sort(); + + // Split batch by source and by group. for ( var b = 0; b < batch.length; b++ ) { - var bGroup = registry[batch[b]].group; - if ( !( bGroup in groups ) ) { - groups[bGroup] = []; + var bSource = registry[batch[b]].source, + bGroup = registry[batch[b]].group; + if ( !( bSource in splits ) ) { + splits[bSource] = {}; } - groups[bGroup][groups[bGroup].length] = batch[b]; - } - for ( var group in groups ) { - // Calculate the highest timestamp - var version = 0; - for ( var g = 0; g < groups[group].length; g++ ) { - if ( registry[groups[group][g]].version > version ) { - version = registry[groups[group][g]].version; - } + if ( !( bGroup in splits[bSource] ) ) { + splits[bSource][bGroup] = []; } + var bSourceGroup = splits[bSource][bGroup]; + bSourceGroup[bSourceGroup.length] = batch[b]; + } + + // Clear the batch - this MUST happen before we append any + // script elements to the body or it's possible that a script + // will be locally cached, instantly load, and work the batch + // again, all before we've cleared it causing each request to + // include modules which are already loaded. + batch = []; + + var source, group, modules, maxVersion, sourceLoadScript; + + for ( source in splits ) { + + sourceLoadScript = sources[source].loadScript; + + for ( group in splits[source] ) { + + // Cache access to currently selected list of + // modules for this group from this source. + modules = splits[source][group]; + + // Calculate the highest timestamp + maxVersion = 0; + for ( var g = 0; g < modules.length; g++ ) { + if ( registry[modules[g]].version > maxVersion ) { + maxVersion = registry[modules[g]].version; + } + } - var reqBase = $.extend( { 'version': formatVersionNumber( version ) }, base ), - reqBaseLength = $.param( reqBase ).length, - reqs = [], - limit = mw.config.get( 'wgResourceLoaderMaxQueryLength', -1 ), - // We may need to split up the request to honor the query string length limit, - // so build it piece by piece. - l = reqBaseLength + 9, // '&modules='.length == 9 - r = 0; - - reqs[0] = {}; // { prefix: [ suffixes ] } - - for ( var i = 0; i < groups[group].length; i++ ) { - // Determine how many bytes this module would add to the query string - var lastDotIndex = groups[group][i].lastIndexOf( '.' ), - // Note that these substr() calls work even if lastDotIndex == -1 - prefix = groups[group][i].substr( 0, lastDotIndex ), - suffix = groups[group][i].substr( lastDotIndex + 1 ), - bytesAdded = prefix in reqs[r] - ? suffix.length + 3 // '%2C'.length == 3 - : groups[group][i].length + 3; // '%7C'.length == 3 - - // If the request would become too long, create a new one, - // but don't create empty requests - if ( limit > 0 && !$.isEmptyObject( reqs[r] ) && l + bytesAdded > limit ) { - // This request would become too long, create a new one - r++; - reqs[r] = {}; - l = reqBaseLength + 9; + var currReqBase = $.extend( { 'version': formatVersionNumber( maxVersion ) }, reqBase ), + currReqBaseLength = $.param( currReqBase ).length, + moduleMap = {}, + // We may need to split up the request to honor the query string length limit, + // so build it piece by piece. + l = currReqBaseLength + 9; // '&modules='.length == 9 + + moduleMap = {}; // { prefix: [ suffixes ] } + + for ( var i = 0; i < modules.length; i++ ) { + // Determine how many bytes this module would add to the query string + var lastDotIndex = modules[i].lastIndexOf( '.' ), + // Note that these substr() calls work even if lastDotIndex == -1 + prefix = modules[i].substr( 0, lastDotIndex ), + suffix = modules[i].substr( lastDotIndex + 1 ), + bytesAdded = prefix in moduleMap + ? suffix.length + 3 // '%2C'.length == 3 + : modules[i].length + 3; // '%7C'.length == 3 + + // If the request would become too long, create a new one, + // but don't create empty requests + if ( maxQueryLength > 0 && !$.isEmptyObject( moduleMap ) && l + bytesAdded > maxQueryLength ) { + // This request would become too long, create a new one + // and fire off the old one + doRequest( moduleMap, currReqBase, sourceLoadScript ); + moduleMap = {}; + l = currReqBaseLength + 9; + } + if ( !( prefix in moduleMap ) ) { + moduleMap[prefix] = []; + } + moduleMap[prefix].push( suffix ); + l += bytesAdded; } - if ( !( prefix in reqs[r] ) ) { - reqs[r][prefix] = []; + // If there's anything left in moduleMap, request that too + if ( !$.isEmptyObject( moduleMap ) ) { + doRequest( moduleMap, currReqBase, sourceLoadScript ); } - reqs[r][prefix].push( suffix ); - l += bytesAdded; } - for ( var r = 0; r < reqs.length; r++ ) { - requests[requests.length] = $.extend( - { 'modules': buildModulesString( reqs[r] ) }, reqBase - ); + } + }; + + /** + * Register a source. + * + * @param id {String}: Short lowercase a-Z string representing a source, only used internally. + * @param props {Object}: Object containing only the loadScript property which is a url to + * the load.php location of the source. + * @return {Boolean} + */ + this.addSource = function( id, props ) { + // Allow multiple additions + if ( typeof id === 'object' ) { + for ( var source in id ) { + mw.loader.addSource( source, id[source] ); } + return true; } - // Clear the batch - this MUST happen before we append the - // script element to the body or it's possible that the script - // will be locally cached, instantly load, and work the batch - // again, all before we've cleared it causing each request to - // include modules which are already loaded - batch = []; - // Asynchronously append a script tag to the end of the body - for ( var r = 0; r < requests.length; r++ ) { - requests[r] = sortQuery( requests[r] ); - // Append &* to avoid triggering the IE6 extension check - var src = mw.config.get( 'wgLoadScript' ) + '?' + $.param( requests[r] ) + '&*'; - addScript( src ); + + if ( sources[id] !== undefined ) { + throw new Error( 'source already registered: ' + id ); } + + sources[id] = props; + + return true; }; /** @@ -845,13 +918,16 @@ window.mw = window.mediaWiki = new ( function( $ ) { * @param dependencies {String|Array|Function}: One string or array of strings of module * names on which this module depends, or a function that returns that array. * @param group {String}: Group which the module is in (optional, defaults to null) + * @param source {String}: Name of the source. Defaults to local. */ - this.register = function( module, version, dependencies, group ) { + this.register = function( module, version, dependencies, group, source ) { // Allow multiple registration if ( typeof module === 'object' ) { for ( var m = 0; m < module.length; m++ ) { + // module is an array of module names if ( typeof module[m] === 'string' ) { mw.loader.register( module[m] ); + // module is an array of arrays } else if ( typeof module[m] === 'object' ) { mw.loader.register.apply( mw.loader, module[m] ); } @@ -867,10 +943,11 @@ window.mw = window.mediaWiki = new ( function( $ ) { } // List the module as registered registry[module] = { - 'state': 'registered', - 'group': typeof group === 'string' ? group : null, + 'version': version !== undefined ? parseInt( version, 10 ) : 0, 'dependencies': [], - 'version': version !== undefined ? parseInt( version, 10 ) : 0 + 'group': typeof group === 'string' ? group : null, + 'source': typeof source === 'string' ? source: 'local', + 'state': 'registered' }; if ( typeof dependencies === 'string' ) { // Allow dependencies to be given as a single module name -- 2.20.1