From 12afb3607de5df533a761faf75666672ae5a7ab2 Mon Sep 17 00:00:00 2001 From: Ori Livneh Date: Wed, 23 Sep 2015 00:47:40 -0700 Subject: [PATCH] resourceloader: Improve caching for LESS file compilation Caching the output of a LESS compiler is tricky, because a LESS file may include additional LESS files via @imports, in which case the cache needs to vary as the contents of those files vary (and not just the contents of the primary LESS file). To solve this, we first introduce a utility class, FileContentsHasher. This class is essentially a smart version of md5_file() -- given one or more file names, it computes a hash digest of their contents. It tries to avoid re-reading files by caching the hash digest in APC and re-using it as long as the files' mtimes have not changed. This is the same approach I used in I5ceb8537c. Next, we use this class in ResourceLoaderFileModule in the following way: whenever we compile a LESS file, we cache the result as an associative array with the following keys: * `files` : the list of files whose contents influenced the compiled CSS. * `hash` : a hash digest of the combined contents of those files. * `css` : the CSS output of the compiler itself. Before using a cached value, we verify that it is still current by asking FileContentHasher for a hash of the combined contents of all referenced files, and we compare that against the value of the `hash` key of the cached entry. Bug: T112035 Change-Id: I1ff61153ddb95ed17e543bd4af7dd13fa3352861 --- autoload.php | 1 + includes/FileContentsHasher.php | 111 ++++++++++++++++++ .../ResourceLoaderFileModule.php | 38 +++++- 3 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 includes/FileContentsHasher.php diff --git a/autoload.php b/autoload.php index 4bed0141bf..4ad921a509 100644 --- a/autoload.php +++ b/autoload.php @@ -436,6 +436,7 @@ $wgAutoloadLocalClasses = array( 'FileBackendStoreShardListIterator' => __DIR__ . '/includes/filebackend/FileBackendStore.php', 'FileBasedSiteLookup' => __DIR__ . '/includes/site/FileBasedSiteLookup.php', 'FileCacheBase' => __DIR__ . '/includes/cache/FileCacheBase.php', + 'FileContentsHasher' => __DIR__ . '/includes/FileContentsHasher.php', 'FileDeleteForm' => __DIR__ . '/includes/FileDeleteForm.php', 'FileDependency' => __DIR__ . '/includes/cache/CacheDependency.php', 'FileDuplicateSearchPage' => __DIR__ . '/includes/specials/SpecialFileDuplicateSearch.php', diff --git a/includes/FileContentsHasher.php b/includes/FileContentsHasher.php new file mode 100644 index 0000000000..67eb9d29f0 --- /dev/null +++ b/includes/FileContentsHasher.php @@ -0,0 +1,111 @@ +cache = ObjectCache::newAccelerator( 'hash' ); + } + + /** + * Get the singleton instance of this class. + * + * @return FileContentsHasher + */ + public static function singleton() { + if ( !self::$instance ) { + self::$instance = new self; + } + + return self::$instance; + } + + /** + * Get a hash of a file's contents, either by retrieving a previously- + * computed hash from the cache, or by computing a hash from the file. + * + * @private + * @param string $filePath Full path to the file. + * @param string $algo Name of selected hashing algorithm. + * @return string|bool Hash of file contents, or false if the file could not be read. + */ + public function getFileContentsHashInternal( $filePath, $algo = 'md4' ) { + $mtime = MediaWiki\quietCall( 'filemtime', $filePath ); + if ( $mtime === false ) { + return false; + } + + $cacheKey = wfGlobalCacheKey( __CLASS__, $filePath, $mtime, $algo ); + $hash = $this->cache->get( $cacheKey ); + + if ( $hash ) { + return $hash; + } + + $contents = MediaWiki\quietCall( 'file_get_contents', $filePath ); + if ( $contents === false ) { + return false; + } + + $hash = hash( $algo, $contents ); + $this->cache->set( $cacheKey, $hash, 60 * 60 * 24 ); // 24h + + return $hash; + } + + /** + * Get a hash of the combined contents of one or more files, either by + * retrieving a previously-computed hash from the cache, or by computing + * a hash from the files. + * + * @param string|string[] $filePaths One or more file paths. + * @param string $algo Name of selected hashing algorithm. + * @return string|bool Hash of files' contents, or false if no file could not be read. + */ + public static function getFileContentsHash( $filePaths, $algo = 'md4' ) { + $instance = self::singleton(); + + if ( !is_array( $filePaths ) ) { + $filePaths = (array) $filePaths; + } + + if ( count( $filePaths ) === 1 ) { + return $instance->getFileContentsHashInternal( $filePaths[0], $algo ); + } + + sort( $filePaths ); + $hashes = array_map( function ( $filePath ) use ( $instance, $algo ) { + return $instance->getFileContentsHashInternal( $filePath, $algo ) ?: ''; + }, $filePaths ); + + $hashes = implode( '', $hashes ); + return $hashes ? hash( $algo, $hashes ) : false; + } +} diff --git a/includes/resourceloader/ResourceLoaderFileModule.php b/includes/resourceloader/ResourceLoaderFileModule.php index 7fbc1cb40e..51118b1f9f 100644 --- a/includes/resourceloader/ResourceLoaderFileModule.php +++ b/includes/resourceloader/ResourceLoaderFileModule.php @@ -966,12 +966,44 @@ class ResourceLoaderFileModule extends ResourceLoaderModule { * @return string CSS source */ protected function compileLessFile( $fileName, $compiler = null ) { + static $cache; + + if ( !$cache ) { + $cache = ObjectCache::newAccelerator( CACHE_ANYTHING ); + } + + // Construct a cache key from the LESS file name and a hash digest + // of the LESS variables used for compilation. + $varsHash = hash( 'md4', serialize( ResourceLoader::getLessVars( $this->getConfig() ) ) ); + $cacheKey = wfGlobalCacheKey( 'LESS', $fileName, $varsHash ); + $cachedCompile = $cache->get( $cacheKey ); + + // If we got a cached value, we have to validate it by getting a + // checksum of all the files that were loaded by the parser and + // ensuring it matches the cached entry's. + if ( isset( $cachedCompile['hash'] ) ) { + $contentHash = FileContentsHasher::getFileContentsHash( $cachedCompile['files'] ); + if ( $contentHash === $cachedCompile['hash'] ) { + $this->localFileRefs += $cachedCompile['files']; + return $cachedCompile['css']; + } + } + if ( !$compiler ) { $compiler = $this->getLessCompiler(); } - $result = $compiler->parseFile( $fileName )->getCss(); - $this->localFileRefs += array_keys( $compiler->AllParsedFiles() ); - return $result; + + $css = $compiler->parseFile( $fileName )->getCss(); + $files = $compiler->AllParsedFiles(); + $this->localFileRefs = array_merge( $this->localFileRefs, $files ); + + $cache->set( $cacheKey, array( + 'css' => $css, + 'files' => $files, + 'hash' => FileContentsHasher::getFileContentsHash( $files ), + ), 60 * 60 * 24 ); // 86400 seconds, or 24 hours. + + return $css; } /** -- 2.20.1