Cache ResourceLoader modules in localStorage
authorOri Livneh <ori@wikimedia.org>
Tue, 1 Oct 2013 15:52:43 +0000 (08:52 -0700)
committerOri Livneh <ori@wikimedia.org>
Mon, 28 Oct 2013 21:14:35 +0000 (14:14 -0700)
To minimize the number of discrete requests that the browser has to make in
order to render the page, ResourceLoader tries to condense as many modules as
possible into each request. To ensure that the response is not stale,
ResourceLoader tacks the modification time of the most recently modified module
to the request. This behavior makes poor use of locality: an update to a single
module will change the URL that is used to retrieve it and a number of
unrelated modules, causing those modules to be re-retrieved even though they
have not changed since they were last retrieved. This is because the browser
cache is not aware that the response from load.php is a composite of modules
that should be versioned separately.

This patch adds mw.loader.store. On browsers that implement the localStorage
API, the module store serves as a smart complement to the browser cache. Unlike
the browser cache, the module store can slice a concatenated response from
ResourceLoader into its constituent modules and cache each of them separately,
using each module's versioning scheme to determine when the cache should be
invalidated.

Because the localStorage API is synchronous and slower than memory access,
modules are cached as a single JSON blob. At the beginning of the page load
process, the entire blob is loaded from storage into memory. During the load
process, any required modules that are not in the store are fetched from the
server and set in the in-memory store. When the DOM is complete, the store is
synced back to localStorage in a single operation.

* NOTE: The module store is enabled only if $wgResourceLoaderStorageEnabled is
  set to true; it is false by default. We can change the default if / when we
  establish conclusively that the feature is beneficial and stable.

Change-Id: If2ad2d80db2336fb447817f5c56599667141ec66

RELEASE-NOTES-1.23
includes/DefaultSettings.php
includes/resourceloader/ResourceLoaderStartUpModule.php
maintenance/jsduck/categories.json
resources/mediawiki/mediawiki.js

index ec7b898..6c78253 100644 (file)
@@ -11,6 +11,13 @@ production.
 === Configuration changes in 1.23 ===
 
 === New features in 1.23 ===
+* ResourceLoader can utilize the Web Storage API to cache modules client-side.
+  Compared to the browser cache, caching in Web Storage allows ResourceLoader
+  to be more granular about evicting stale modules from the cache while
+  retaining the ability to retrieve multiple modules in a single HTTP request.
+  This capability can be enabled by setting $wgResourceLoaderStorageEnabled to
+  true. This feature is currently considered experimental and should only be
+  enabled with care.
 
 === Bug fixes in 1.23 ===
 * (bug 41759) The "updated since last visit" markers (on history pages, recent
index dbdd89e..9b879a6 100644 (file)
@@ -3322,6 +3322,22 @@ $wgResourceLoaderLESSImportPaths = array(
        "$IP/resources/mediawiki.less/",
 );
 
+/**
+ * Whether ResourceLoader should attempt to persist modules in localStorage on
+ * browsers that support the Web Storage API.
+ *
+ * @since 1.23 - Client-side module persistence is experimental. Exercise care.
+ */
+$wgResourceLoaderStorageEnabled = false;
+
+/**
+ * Cache version for client-side ResourceLoader module storage. You can trigger
+ * invalidation of the contents of the module store by incrementing this value.
+ *
+ * @since 1.23
+ */
+$wgResourceLoaderStorageVersion = 1;
+
 /** @} */ # End of resource loader settings }
 
 /*************************************************************************//**
index 20f6e0b..b38f448 100644 (file)
@@ -41,7 +41,8 @@ class ResourceLoaderStartUpModule extends ResourceLoaderModule {
                        $wgVariantArticlePath, $wgActionPaths, $wgVersion,
                        $wgEnableAPI, $wgEnableWriteAPI, $wgDBname,
                        $wgSitename, $wgFileExtensions, $wgExtensionAssetsPath,
-                       $wgCookiePrefix, $wgResourceLoaderMaxQueryLength;
+                       $wgCookiePrefix, $wgResourceLoaderMaxQueryLength,
+                       $wgResourceLoaderStorageEnabled, $wgResourceLoaderStorageVersion;
 
                $mainPage = Title::newMainPage();
 
@@ -96,6 +97,8 @@ class ResourceLoaderStartUpModule extends ResourceLoaderModule {
                        'wgResourceLoaderMaxQueryLength' => $wgResourceLoaderMaxQueryLength,
                        'wgCaseSensitiveNamespaces' => $caseSensitiveNamespaces,
                        'wgLegalTitleChars' => Title::convertByteClassToUnicodeClass( Title::legalChars() ),
+                       'wgResourceLoaderStorageVersion' => $wgResourceLoaderStorageVersion,
+                       'wgResourceLoaderStorageEnabled' => $wgResourceLoaderStorageEnabled,
                );
 
                wfRunHooks( 'ResourceLoaderGetConfigVars', array( &$vars ) );
index f96902d..c595980 100644 (file)
@@ -9,6 +9,7 @@
                                        "mw.Map",
                                        "mw.Message",
                                        "mw.loader",
+                                       "mw.loader.store",
                                        "mw.log",
                                        "mw.html",
                                        "mw.html.Cdata",
index 03248ba..acda5d6 100644 (file)
@@ -848,8 +848,9 @@ var mw = ( function ( $, undefined ) {
                                }
 
                                if ( registry[module].state === 'ready' ) {
-                                       // The current module became 'ready'. Recursively execute all dependent modules that are loaded
-                                       // and now have all dependencies satisfied.
+                                       // The current module became 'ready'. Set it in the module store, and recursively execute all
+                                       // dependent modules that are loaded and now have all dependencies satisfied.
+                                       mw.loader.store.set( module, registry[module] );
                                        for ( m in registry ) {
                                                if ( registry[m].state === 'loaded' && allReady( registry[m].dependencies ) ) {
                                                        execute( m );
@@ -1203,7 +1204,7 @@ var mw = ( function ( $, undefined ) {
                                addScript( sourceLoadScript + '?' + $.param( request ) + '&*', null, async );
                        }
 
-                       /* Public Methods */
+                       /* Public Members */
                        return {
                                /**
                                 * The module registry is exposed as an aid for debugging and inspecting page
@@ -1252,6 +1253,19 @@ var mw = ( function ( $, undefined ) {
                                                        }
                                                }
                                        }
+
+                                       mw.loader.store.init();
+                                       if ( mw.loader.store.enabled ) {
+                                               batch = $.grep( batch, function ( module ) {
+                                                       var source = mw.loader.store.get( module );
+                                                       if ( source ) {
+                                                               $.globalEval( source );
+                                                               return false; // Don't fetch
+                                                       }
+                                                       return true; // Fetch
+                                               } );
+                                       }
+
                                        // Early exit if there's nothing to load...
                                        if ( !batch.length ) {
                                                return;
@@ -1703,8 +1717,237 @@ var mw = ( function ( $, undefined ) {
                                        mw.loader.using( 'mediawiki.inspect', function () {
                                                mw.inspect.runReports.apply( mw.inspect, args );
                                        } );
-                               }
+                               },
+
+                               /**
+                                * On browsers that implement the localStorage API, the module store serves as a
+                                * smart complement to the browser cache. Unlike the browser cache, the module store
+                                * can slice a concatenated response from ResourceLoader into its constituent
+                                * modules and cache each of them separately, using each module's versioning scheme
+                                * to determine when the cache should be invalidated.
+                                *
+                                * @singleton
+                                * @class mw.loader.store
+                                */
+                               store: {
+                                       // Whether the store is in use on this page.
+                                       enabled: null,
 
+                                       // The contents of the store, mapping '[module name]@[version]' keys
+                                       // to module implementations.
+                                       items: {},
+
+                                       // Cache hit stats
+                                       stats: { hits: 0, misses: 0, expired: 0 },
+
+                                       /**
+                                        * Construct a JSON-serializable object representing the content of the store.
+                                        * @return {Object} Module store contents.
+                                        */
+                                       toJSON: function () {
+                                               return { items: mw.loader.store.items, vary: mw.loader.store.getVary() };
+                                       },
+
+                                       /**
+                                        * Get the localStorage key for the entire module store. The key references
+                                        * $wgDBname to prevent clashes between wikis which share a common host.
+                                        *
+                                        * @return {string} localStorage item key
+                                        */
+                                       getStoreKey: function () {
+                                               return 'MediaWikiModuleStore:' + mw.config.get( 'wgDBname' );
+                                       },
+
+                                       /**
+                                        * Get a string key on which to vary the module cache.
+                                        * @return {string} String of concatenated vary conditions.
+                                        */
+                                       getVary: function () {
+                                               return [
+                                                       mw.config.get( 'skin' ),
+                                                       mw.config.get( 'wgResourceLoaderStorageVersion' ),
+                                                       mw.config.get( 'wgUserLanguage' )
+                                               ].join(':');
+                                       },
+
+                                       /**
+                                        * Get a string key for a specific module. The key format is '[name]@[version]'.
+                                        *
+                                        * @param {string} module Module name
+                                        * @return {string|null} Module key or null if module does not exist
+                                        */
+                                       getModuleKey: function ( module ) {
+                                               return typeof registry[module] === 'object' ?
+                                                       ( module + '@' + registry[module].version ) : null;
+                                       },
+
+                                       /**
+                                        * Initialize the store by retrieving it from localStorage and (if successfully
+                                        * retrieved) decoding the stored JSON value to a plain object.
+                                        *
+                                        * The try / catch block is used for JSON & localStorage feature detection.
+                                        * See the in-line documentation for Modernizr's localStorage feature detection
+                                        * code for a full account of why we need a try / catch: <http://git.io/4NEwKg>.
+                                        */
+                                       init: function () {
+                                               var raw, data;
+
+                                               if ( mw.loader.store.enabled !== null ) {
+                                                       // #init already ran.
+                                                       return;
+                                               }
+
+                                               if ( !mw.config.get( 'wgResourceLoaderStorageEnabled' ) || mw.config.get( 'debug' ) ) {
+                                                       // Disabled by configuration, or because debug mode is set.
+                                                       mw.loader.store.enabled = false;
+                                                       return;
+                                               }
+
+                                               try {
+                                                       raw = localStorage.getItem( mw.loader.store.getStoreKey() );
+                                                       // If we get here, localStorage is available; mark enabled.
+                                                       mw.loader.store.enabled = true;
+                                                       data = JSON.parse( raw );
+                                                       if ( data && typeof data.items === 'object' && data.vary === mw.loader.store.getVary() ) {
+                                                               mw.loader.store.items = data.items;
+                                                               return;
+                                                       }
+                                               } catch (e) {}
+
+                                               if ( raw === undefined ) {
+                                                       mw.loader.store.enabled = false;  // localStorage failed; disable store.
+                                               } else {
+                                                       mw.loader.store.update();
+                                               }
+                                       },
+
+                                       /**
+                                        * Retrieve a module from the store and update cache hit stats.
+                                        *
+                                        * @param {string} module Module name
+                                        * @return {string|boolean} Module implementation or false if unavailable
+                                        */
+                                       get: function ( module ) {
+                                               var key;
+
+                                               if ( mw.loader.store.enabled !== true ) {
+                                                       return false;
+                                               }
+
+                                               key = mw.loader.store.getModuleKey( module );
+                                               if ( key in mw.loader.store.items ) {
+                                                       mw.loader.store.stats.hits++;
+                                                       return mw.loader.store.items[key];
+                                               }
+                                               mw.loader.store.stats.misses++;
+                                               return false;
+                                       },
+
+                                       /**
+                                        * Stringify a module and queue it for storage.
+                                        *
+                                        * @param {string} module Module name
+                                        * @param {Object} descriptor The module's descriptor as set in the registry
+                                        */
+                                       set: function ( module, descriptor ) {
+                                               var args, key;
+
+                                               if ( mw.loader.store.enabled !== true ) {
+                                                       return false;
+                                               }
+
+                                               key = mw.loader.store.getModuleKey( module );
+
+                                               if ( key in mw.loader.store.items ) {
+                                                       // Already set; decline to store.
+                                                       return false;
+                                               }
+
+                                               if ( descriptor.state !== 'ready' ) {
+                                                       // Module failed to load; decline to store.
+                                                       return false;
+                                               }
+
+                                               if ( !descriptor.version || $.inArray( descriptor.group, [ 'private', 'user', 'site' ] ) !== -1 ) {
+                                                       // Unversioned, private, or site-/user-specific; decline to store.
+                                                       return false;
+                                               }
+
+                                               if ( $.inArray( undefined, [ descriptor.script, descriptor.style, descriptor.messages ] ) !== -1 ) {
+                                                       // Partial descriptor; decline to store.
+                                                       return false;
+                                               }
+
+                                               try {
+                                                       args = [
+                                                               JSON.stringify( module ),
+                                                               typeof descriptor.script === 'function' ?
+                                                                       String( descriptor.script ) : JSON.stringify( descriptor.script ),
+                                                               JSON.stringify( descriptor.style ),
+                                                               JSON.stringify( descriptor.messages )
+                                                       ];
+                                               } catch (e) {
+                                                       return;
+                                               }
+                                               mw.loader.store.items[key] = 'mw.loader.implement(' + args.join(',') + ');';
+                                               mw.loader.store.update();
+                                       },
+
+                                       /**
+                                        * Iterate through the module store, removing any item that does not correspond
+                                        * (in name and version) to an item in the module registry.
+                                        */
+                                       prune: function () {
+                                               var key, module;
+
+                                               if ( mw.loader.store.enabled !== true ) {
+                                                       return false;
+                                               }
+
+                                               for ( key in mw.loader.store.items ) {
+                                                       module = key.substring( 0, key.indexOf( '@' ) );
+                                                       if ( mw.loader.store.getModuleKey( module ) !== key ) {
+                                                               mw.loader.store.stats.expired++;
+                                                               delete mw.loader.store.items[key];
+                                                       }
+                                               }
+                                       },
+
+                                       /**
+                                        * Sync modules to localStorage.
+                                        *
+                                        * This function debounces localStorage updates. When called multiple times in
+                                        * quick succession, the calls are coalesced into a single update operation.
+                                        * This allows us to call #update without having to consider the module load
+                                        * queue; the call to localStorage.setItem will be naturally deferred until the
+                                        * page is quiescent.
+                                        *
+                                        * Because localStorage is shared by all pages with the same origin, if multiple
+                                        * pages are loaded with different module sets, the possibility exists that
+                                        * modules saved by one page will be clobbered by another. But the impact would
+                                        * be minor and the problem would be corrected by subsequent page views.
+                                        */
+                                       update: ( function () {
+                                               var timer;
+
+                                               function flush() {
+                                                       var data;
+                                                       if ( mw.loader.store.enabled !== true ) {
+                                                               return false;
+                                                       }
+                                                       mw.loader.store.prune();
+                                                       try {
+                                                               data = JSON.stringify( mw.loader.store );
+                                                               localStorage.setItem( mw.loader.store.getStoreKey(), data );
+                                                       } catch (e) {}
+                                               }
+
+                                               return function () {
+                                                       clearTimeout( timer );
+                                                       timer = setTimeout( flush, 2000 );
+                                               };
+                                       }() )
+                               }
                        };
                }() ),