From: Trevor Parscal Date: Thu, 23 Sep 2010 21:23:51 +0000 (+0000) Subject: Improved the performance of ResourceLoader by pre-loading module information in a... X-Git-Tag: 1.31.0-rc.0~34813 X-Git-Url: https://git.cyclocoop.org//%22?a=commitdiff_plain;h=389c3722dda207517431f7b2ed4859e9cb9643a1;p=lhc%2Fweb%2Fwiklou.git Improved the performance of ResourceLoader by pre-loading module information in a batch query. This was mostly code written by catrope and patched in / gotten working by me. --- diff --git a/includes/ResourceLoader.php b/includes/ResourceLoader.php index 8a79cc226a..bb958c3879 100644 --- a/includes/ResourceLoader.php +++ b/includes/ResourceLoader.php @@ -33,7 +33,7 @@ class ResourceLoader { /* Protected Static Methods */ - /* + /** * Registers core modules and runs registration hooks */ protected static function initialize() { @@ -50,6 +50,55 @@ class ResourceLoader { wfProfileOut( __METHOD__ ); } } + + protected static function preloadModuleInfo( $modules, ResourceLoaderContext $context ) { + $dbr = wfGetDb( DB_SLAVE ); + $skin = $context->getSkin(); + $lang = $context->getLanguage(); + + // Get file dependency information + $res = $dbr->select( 'module_deps', array( 'md_module', 'md_deps' ), array( + 'md_module' => $modules, + 'md_skin' => $context->getSkin() + ), __METHOD__ + ); + + $modulesWithDeps = array(); + foreach ( $res as $row ) { + self::$modules[$row->md_module]->setFileDependencies( $skin, + FormatJson::decode( $row->md_deps, true ) + ); + $modulesWithDeps[] = $row->md_module; + } + // Register the absence of a dependencies row too + foreach ( array_diff( $modules, $modulesWithDeps ) as $name ) { + self::$modules[$name]->setFileDependencies( $skin, array() ); + } + + // Get message blob mtimes. Only do this for modules with messages + $modulesWithMessages = array(); + $modulesWithoutMessages = array(); + foreach ( $modules as $name ) { + if ( count( self::$modules[$name]->getMessages() ) ) { + $modulesWithMessages[] = $name; + } else { + $modulesWithoutMessages[] = $name; + } + } + if ( count( $modulesWithMessages ) ) { + $res = $dbr->select( 'msg_resource', array( 'mr_resource', 'mr_timestamp' ), array( + 'mr_resource' => $modulesWithMessages, + 'mr_lang' => $lang + ), __METHOD__ + ); + foreach ( $res as $row ) { + self::$modules[$row->mr_resource]->setMsgBlobMtime( $lang, $row->mr_timestamp ); + } + } + foreach ( $modulesWithoutMessages as $name ) { + self::$modules[$name]->setMsgBlobMtime( $lang, 0 ); + } + } /** * Runs text through a filter, caching the filtered result for future calls @@ -279,6 +328,9 @@ class ResourceLoader { $smaxage = $wgResourceLoaderMaxage['versioned']['server']; } + // Preload information needed to the mtime calculation below + self::preloadModuleInfo( $modules, $context ); + // To send Last-Modified and support If-Modified-Since, we need to detect // the last modified time wfProfileIn( __METHOD__.'-getModifiedTime' ); @@ -307,7 +359,7 @@ class ResourceLoader { // Pre-fetch blobs $blobs = $context->shouldIncludeMessages() ? - MessageBlobStore::get( $modules, $context->getLanguage() ) : array(); + MessageBlobStore::get( $modules, $context->getLanguage() ) : array(); // Generate output foreach ( $modules as $name ) { diff --git a/includes/ResourceLoaderModule.php b/includes/ResourceLoaderModule.php index d7c8a8311c..546602bc50 100644 --- a/includes/ResourceLoaderModule.php +++ b/includes/ResourceLoaderModule.php @@ -27,6 +27,11 @@ abstract class ResourceLoaderModule { /* Protected Members */ protected $name = null; + + // In-object cache for file dependencies + protected $fileDeps = array(); + // In-object cache for message blob mtime + protected $msgBlobMtime = array(); /* Methods */ @@ -131,7 +136,73 @@ abstract class ResourceLoaderModule { // Stub, override expected return array(); } + + /** + * Get the files this module depends on indirectly for a given skin. + * Currently these are only image files referenced by the module's CSS. + * + * @param $skin String: skin name + * @return array of files + */ + public function getFileDependencies( $skin ) { + // Try in-object cache first + if ( isset( $this->fileDeps[$skin] ) ) { + return $this->fileDeps[$skin]; + } + $dbr = wfGetDB( DB_SLAVE ); + $deps = $dbr->selectField( 'module_deps', 'md_deps', array( + 'md_module' => $this->getName(), + 'md_skin' => $skin, + ), __METHOD__ + ); + if ( !is_null( $deps ) ) { + $this->fileDeps[$skin] = (array) FormatJson::decode( $deps, true ); + } + return $this->fileDeps[$skin]; + } + + /** + * Set preloaded file dependency information. Used so we can load this + * information for all modules at once. + * @param $skin string Skin name + * @param $deps array Array of file names + */ + public function setFileDependencies( $skin, $deps ) { + $this->fileDeps[$skin] = $deps; + } + + /** + * Get the last modification timestamp of the message blob for this + * module in a given language. + * @param $lang string Language code + * @return int UNIX timestamp, or 0 if no blob found + */ + public function getMsgBlobMtime( $lang ) { + if ( !count( $this->getMessages() ) ) + return 0; + + $dbr = wfGetDB( DB_SLAVE ); + $msgBlobMtime = $dbr->selectField( 'msg_resource', 'mr_timestamp', array( + 'mr_resource' => $this->getName(), + 'mr_lang' => $lang + ), __METHOD__ + ); + $this->msgBlobMtime[$lang] = $msgBlobMtime ? wfTimestamp( TS_UNIX, $msgBlobMtime ) : 0; + return $this->msgBlobMtime[$lang]; + } + + /** + * Set a preloaded message blob last modification timestamp. Used so we + * can load this information for all modules at once. + * @param $lang string Language code + * @param $mtime int UNIX timestamp or 0 if there is no such blob + */ + public function setMsgBlobMtime( $lang, $mtime ) { + $this->msgBlobMtime[$lang] = $mtime; + } + + /* Abstract Methods */ /** @@ -483,24 +554,11 @@ class ResourceLoaderFileModule extends ResourceLoaderModule { $this->loaders, $this->getFileDependencies( $context->getSkin() ) ); + wfProfileIn( __METHOD__.'-filemtime' ); $filesMtime = max( array_map( 'filemtime', array_map( array( __CLASS__, 'remapFilename' ), $files ) ) ); wfProfileOut( __METHOD__.'-filemtime' ); - // Only get the message timestamp if there are messages in the module - $msgBlobMtime = 0; - if ( count( $this->messages ) ) { - // Get the mtime of the message blob - // TODO: This timestamp is queried a lot and queried separately for each module. - // Maybe it should be put in memcached? - $dbr = wfGetDB( DB_SLAVE ); - $msgBlobMtime = $dbr->selectField( 'msg_resource', 'mr_timestamp', array( - 'mr_resource' => $this->getName(), - 'mr_lang' => $context->getLanguage() - ), __METHOD__ - ); - $msgBlobMtime = $msgBlobMtime ? wfTimestamp( TS_UNIX, $msgBlobMtime ) : 0; - } - $this->modifiedTime[$context->getHash()] = max( $filesMtime, $msgBlobMtime ); + $this->modifiedTime[$context->getHash()] = max( $filesMtime, $this->getMsgBlobMtime( $context->getLanguage() ) ); wfProfileOut( __METHOD__ ); return $this->modifiedTime[$context->getHash()]; } @@ -589,43 +647,6 @@ class ResourceLoaderFileModule extends ResourceLoaderModule { return $retval; } - /** - * Get the files this module depends on indirectly for a given skin. - * Currently these are only image files referenced by the module's CSS. - * - * @param $skin String: skin name - * @return array of files - */ - protected function getFileDependencies( $skin ) { - // Try in-object cache first - if ( isset( $this->fileDeps[$skin] ) ) { - return $this->fileDeps[$skin]; - } - - // Now try memcached - global $wgMemc; - - $key = wfMemcKey( 'resourceloader', 'module_deps', $this->getName(), $skin ); - $deps = $wgMemc->get( $key ); - - if ( !$deps ) { - $dbr = wfGetDB( DB_SLAVE ); - $deps = $dbr->selectField( 'module_deps', 'md_deps', array( - 'md_module' => $this->getName(), - 'md_skin' => $skin, - ), __METHOD__ - ); - if ( !$deps ) { - $deps = '[]'; // Empty array so we can do negative caching - } - $wgMemc->set( $key, $deps ); - } - - $this->fileDeps = FormatJson::decode( $deps, true ); - - return $this->fileDeps; - } - /** * Get the contents of a set of files and concatenate them, with * newlines in between. Each file is used only once.