'FileBackendStoreShardListIterator' => __DIR__ . '/includes/filebackend/FileBackendStore.php',
'FileBasedSiteLookup' => __DIR__ . '/includes/site/FileBasedSiteLookup.php',
'FileCacheBase' => __DIR__ . '/includes/cache/FileCacheBase.php',
+ 'FileContentsHasher' => __DIR__ . '/includes/FileContentsHasher.php',
'FileDeleteForm' => __DIR__ . '/includes/FileDeleteForm.php',
'FileDependency' => __DIR__ . '/includes/cache/CacheDependency.php',
'FileDuplicateSearchPage' => __DIR__ . '/includes/specials/SpecialFileDuplicateSearch.php',
--- /dev/null
+<?php
+/**
+ * Generate hash digests of file contents to help with cache invalidation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+class FileContentsHasher {
+
+ /** @var BagOStuff */
+ protected $cache;
+
+ /** @var FileContentsHasher */
+ private static $instance;
+
+ /**
+ * Constructor.
+ */
+ public function __construct() {
+ $this->cache = ObjectCache::newAccelerator( 'hash' );
+ }
+
+ /**
+ * Get the singleton instance of this class.
+ *
+ * @return FileContentsHasher
+ */
+ public static function singleton() {
+ if ( !self::$instance ) {
+ self::$instance = new self;
+ }
+
+ return self::$instance;
+ }
+
+ /**
+ * Get a hash of a file's contents, either by retrieving a previously-
+ * computed hash from the cache, or by computing a hash from the file.
+ *
+ * @private
+ * @param string $filePath Full path to the file.
+ * @param string $algo Name of selected hashing algorithm.
+ * @return string|bool Hash of file contents, or false if the file could not be read.
+ */
+ public function getFileContentsHashInternal( $filePath, $algo = 'md4' ) {
+ $mtime = MediaWiki\quietCall( 'filemtime', $filePath );
+ if ( $mtime === false ) {
+ return false;
+ }
+
+ $cacheKey = wfGlobalCacheKey( __CLASS__, $filePath, $mtime, $algo );
+ $hash = $this->cache->get( $cacheKey );
+
+ if ( $hash ) {
+ return $hash;
+ }
+
+ $contents = MediaWiki\quietCall( 'file_get_contents', $filePath );
+ if ( $contents === false ) {
+ return false;
+ }
+
+ $hash = hash( $algo, $contents );
+ $this->cache->set( $cacheKey, $hash, 60 * 60 * 24 ); // 24h
+
+ return $hash;
+ }
+
+ /**
+ * Get a hash of the combined contents of one or more files, either by
+ * retrieving a previously-computed hash from the cache, or by computing
+ * a hash from the files.
+ *
+ * @param string|string[] $filePaths One or more file paths.
+ * @param string $algo Name of selected hashing algorithm.
+ * @return string|bool Hash of files' contents, or false if no file could not be read.
+ */
+ public static function getFileContentsHash( $filePaths, $algo = 'md4' ) {
+ $instance = self::singleton();
+
+ if ( !is_array( $filePaths ) ) {
+ $filePaths = (array) $filePaths;
+ }
+
+ if ( count( $filePaths ) === 1 ) {
+ return $instance->getFileContentsHashInternal( $filePaths[0], $algo );
+ }
+
+ sort( $filePaths );
+ $hashes = array_map( function ( $filePath ) use ( $instance, $algo ) {
+ return $instance->getFileContentsHashInternal( $filePath, $algo ) ?: '';
+ }, $filePaths );
+
+ $hashes = implode( '', $hashes );
+ return $hashes ? hash( $algo, $hashes ) : false;
+ }
+}
const LOCK_TTL = 5;
/** Default remaining TTL at which to consider pre-emptive regeneration */
const LOW_TTL = 10;
- /** Default TTL for temporarily caching tombstoned keys */
- const TEMP_TTL = 5;
+ /** Default time-since-expiry on a miss that makes a key "hot" */
+ const LOCK_TSE = 1;
/** Idiom for set()/getWithSetCallback() TTL */
const TTL_NONE = 0;
/** Idiom for getWithSetCallback() callbacks to avoid calling set() */
const TTL_UNCACHEABLE = -1;
+ /** Idiom for getWithSetCallback() callbacks to 'lockTSE' logic */
+ const TSE_NONE = -1;
/** Cache format version number */
const VERSION = 1;
* - lowTTL : consider pre-emptive updates when the current TTL (sec)
* of the key is less than this. It becomes more likely
* over time, becoming a certainty once the key is expired.
+ * [Default: WANObjectCache::LOW_TTL seconds]
* - lockTSE : if the key is tombstoned or expired (by $checkKeys) less
* than this many seconds ago, then try to have a single
* thread handle cache regeneration at any given time.
* If, on miss, the time since expiration is low, the assumption
* is that the key is hot and that a stampede is worth avoiding.
* Setting this above WANObjectCache::HOLDOFF_TTL makes no difference.
- * - tempTTL : TTL of the temp key used to cache values while a key is tombstoned.
- * This avoids excessive regeneration of hot keys on delete() but may
- * result in stale values.
+ * The higher this is set, the higher the worst-case staleness can be.
+ * Use WANObjectCache::TSE_NONE to disable this logic.
+ * [Default: WANObjectCache::TSE_NONE]
* @return mixed Value to use for the key
*/
final public function getWithSetCallback(
$key, $callback, $ttl, array $checkKeys = array(), array $opts = array()
) {
$lowTTL = isset( $opts['lowTTL'] ) ? $opts['lowTTL'] : min( self::LOW_TTL, $ttl );
- $lockTSE = isset( $opts['lockTSE'] ) ? $opts['lockTSE'] : -1;
- $tempTTL = isset( $opts['tempTTL'] ) ? $opts['tempTTL'] : self::TEMP_TTL;
+ $lockTSE = isset( $opts['lockTSE'] ) ? $opts['lockTSE'] : self::TSE_NONE;
// Get the current key value
$curTTL = null;
$isTombstone = ( $curTTL !== null && $value === false );
// Assume a key is hot if requested soon after invalidation
$isHot = ( $curTTL !== null && $curTTL <= 0 && abs( $curTTL ) <= $lockTSE );
+ // Decide whether a single thread should handle regenerations.
+ // This avoids stampedes when $checkKeys are bumped and when preemptive
+ // renegerations take too long. It also reduces regenerations while $key
+ // is tombstoned. This balances cache freshness with avoiding DB load.
+ $useMutex = ( $isHot || ( $isTombstone && $lockTSE > 0 ) );
$lockAcquired = false;
- if ( $isHot ) {
+ if ( $useMutex ) {
// Acquire a cluster-local non-blocking lock
if ( $this->cache->lock( $key, 0, self::LOCK_TTL ) ) {
// Lock acquired; this thread should update the key
} elseif ( $value !== false ) {
// If it cannot be acquired; then the stale value can be used
return $value;
- }
- }
-
- if ( !$lockAcquired && ( $isTombstone || $isHot ) ) {
- // Use the stash value for tombstoned keys to reduce regeneration load.
- // For hot keys, either another thread has the lock or the lock failed;
- // use the stash value from the last thread that regenerated it.
- $value = $this->cache->get( self::STASH_KEY_PREFIX . $key );
- if ( $value !== false ) {
- return $value;
+ } else {
+ // Use the stash value for tombstoned keys to reduce regeneration load.
+ // For hot keys, either another thread has the lock or the lock failed;
+ // use the stash value from the last thread that regenerated it.
+ $value = $this->cache->get( self::STASH_KEY_PREFIX . $key );
+ if ( $value !== false ) {
+ return $value;
+ }
}
}
$value = call_user_func_array( $callback, array( $cValue, &$ttl ) );
// When delete() is called, writes are write-holed by the tombstone,
// so use a special stash key to pass the new value around threads.
- if ( $value !== false && ( $isHot || $isTombstone ) && $ttl >= 0 ) {
+ if ( $useMutex && $value !== false && $ttl >= 0 ) {
+ $tempTTL = max( 1, (int)$lockTSE ); // set() expects seconds
$this->cache->set( self::STASH_KEY_PREFIX . $key, $value, $tempTTL );
}
return $wgLang->commaList( $info );
}
+
+ /**
+ * Return the duration of the GIF file.
+ *
+ * Shown in the &query=imageinfo&iiprop=size api query.
+ *
+ * @param $file File
+ * @return float The duration of the file.
+ */
+ public function getLength( $file ) {
+ $serMeta = $file->getMetadata();
+ MediaWiki\suppressWarnings();
+ $metadata = unserialize( $serMeta );
+ MediaWiki\restoreWarnings();
+
+ if ( !$metadata || !isset( $metadata['duration'] ) || !$metadata['duration'] ) {
+ return 0.0;
+ } else {
+ return (float)$metadata['duration'];
+ }
+ }
}
return $wgLang->commaList( $info );
}
+ /**
+ * Return the duration of an APNG file.
+ *
+ * Shown in the &query=imageinfo&iiprop=size api query.
+ *
+ * @param $file File
+ * @return float The duration of the file.
+ */
+ public function getLength( $file ) {
+ $serMeta = $file->getMetadata();
+ MediaWiki\suppressWarnings();
+ $metadata = unserialize( $serMeta );
+ MediaWiki\restoreWarnings();
+
+ if ( !$metadata || !isset( $metadata['duration'] ) || !$metadata['duration'] ) {
+ return 0.0;
+ } else {
+ return (float)$metadata['duration'];
+ }
+ }
+
// PNGs should be easy to support, but it will need some sharpening applied
// and another user test to check if the perceived quality change is noticeable
-
public function supportsBucketing() {
return false;
}
* @return string CSS source
*/
protected function compileLessFile( $fileName, $compiler = null ) {
+ static $cache;
+
+ if ( !$cache ) {
+ $cache = ObjectCache::newAccelerator( CACHE_ANYTHING );
+ }
+
+ // Construct a cache key from the LESS file name and a hash digest
+ // of the LESS variables used for compilation.
+ $varsHash = hash( 'md4', serialize( ResourceLoader::getLessVars( $this->getConfig() ) ) );
+ $cacheKey = wfGlobalCacheKey( 'LESS', $fileName, $varsHash );
+ $cachedCompile = $cache->get( $cacheKey );
+
+ // If we got a cached value, we have to validate it by getting a
+ // checksum of all the files that were loaded by the parser and
+ // ensuring it matches the cached entry's.
+ if ( isset( $cachedCompile['hash'] ) ) {
+ $contentHash = FileContentsHasher::getFileContentsHash( $cachedCompile['files'] );
+ if ( $contentHash === $cachedCompile['hash'] ) {
+ $this->localFileRefs += $cachedCompile['files'];
+ return $cachedCompile['css'];
+ }
+ }
+
if ( !$compiler ) {
$compiler = $this->getLessCompiler();
}
- $result = $compiler->parseFile( $fileName )->getCss();
- $this->localFileRefs += array_keys( $compiler->AllParsedFiles() );
- return $result;
+
+ $css = $compiler->parseFile( $fileName )->getCss();
+ $files = $compiler->AllParsedFiles();
+ $this->localFileRefs = array_merge( $this->localFileRefs, $files );
+
+ $cache->set( $cacheKey, array(
+ 'css' => $css,
+ 'files' => $files,
+ 'hash' => FileContentsHasher::getFileContentsHash( $files ),
+ ), 60 * 60 * 24 ); // 86400 seconds, or 24 hours.
+
+ return $css;
}
/**
* @return string Hash
*/
protected static function safeFileHash( $filePath ) {
- static $cache;
-
- if ( !$cache ) {
- $cache = ObjectCache::newAccelerator( CACHE_NONE );
- }
-
- MediaWiki\suppressWarnings();
- $mtime = filemtime( $filePath );
- MediaWiki\restoreWarnings();
- if ( !$mtime ) {
- return '';
- }
-
- $cacheKey = wfGlobalCacheKey( 'resourceloader', __METHOD__, $filePath );
- $cachedHash = $cache->get( $cacheKey );
- if ( isset( $cachedHash['mtime'] ) && $cachedHash['mtime'] === $mtime ) {
- return $cachedHash['hash'];
- }
-
- MediaWiki\suppressWarnings();
- $contents = file_get_contents( $filePath );
- MediaWiki\restoreWarnings();
- if ( !$contents ) {
- return '';
- }
-
- $hash = hash( 'md4', $contents );
- $cache->set( $cacheKey, array( 'mtime' => $mtime, 'hash' => $hash ), 60 * 60 * 24 );
-
- return $hash;
+ return FileContentsHasher::getFileContentsHash( $filePath );
}
}
),
);
}
+
+ /**
+ * @param $filename string
+ * @param $expectedLength float
+ * @dataProvider provideGetLength
+ */
+ public function testGetLength( $filename, $expectedLength ) {
+ $file = $this->dataFile( $filename, 'image/gif' );
+ $actualLength = $file->getLength();
+ $this->assertEquals( $expectedLength, $actualLength, '', 0.00001 );
+ }
+
+ public function provideGetLength() {
+ return array(
+ array( 'animated.gif', 2.4 ),
+ array( 'animated-xmp.gif', 2.4 ),
+ array( 'nonanimated', 0.0 ),
+ array( 'Bishzilla_blink.gif', 1.4 ),
+ );
+ }
}
),
);
}
+
+ /**
+ * @param $filename string
+ * @param $expectedLength float
+ * @dataProvider provideGetLength
+ */
+ public function testGetLength( $filename, $expectedLength ) {
+ $file = $this->dataFile( $filename, 'image/png' );
+ $actualLength = $file->getLength();
+ $this->assertEquals( $expectedLength, $actualLength, '', 0.00001 );
+ }
+
+ public function provideGetLength() {
+ return array(
+ array( 'Animated_PNG_example_bouncing_beach_ball.png', 1.5 ),
+ array( 'Png-native-test.png', 0.0 ),
+ array( 'greyscale-png.png', 0.0 ),
+ array( '1bit-png.png', 0.0 ),
+ );
+ }
}
dépôts / lhc / web / wiklou.git / commitdiff