From: Sam Reed Date: Mon, 25 Apr 2011 21:33:38 +0000 (+0000) Subject: Move 3 files into cache directory X-Git-Tag: 1.31.0-rc.0~30555 X-Git-Url: https://git.cyclocoop.org/%7B%24www_url%7Dadmin/compta/banques/ajouter.php?a=commitdiff_plain;h=8e0ac8db6311a8c4b57dc62917e70cde9ec2fa4c;p=lhc%2Fweb%2Fwiklou.git Move 3 files into cache directory Move in AutoLoader to cache section also --- diff --git a/includes/AutoLoader.php b/includes/AutoLoader.php index b56929501a..a52454386a 100644 --- a/includes/AutoLoader.php +++ b/includes/AutoLoader.php @@ -27,7 +27,6 @@ $wgAutoloadLocalClasses = array( 'BadTitle' => 'includes/Title.php', 'BaseTemplate' => 'includes/SkinTemplate.php', 'Block' => 'includes/Block.php', - 'CacheDependency' => 'includes/CacheDependency.php', 'Category' => 'includes/Category.php', 'Categoryfinder' => 'includes/Categoryfinder.php', 'CategoryPage' => 'includes/CategoryPage.php', @@ -48,11 +47,9 @@ $wgAutoloadLocalClasses = array( 'ConfEditor' => 'includes/ConfEditor.php', 'ConfEditorParseError' => 'includes/ConfEditor.php', 'ConfEditorToken' => 'includes/ConfEditor.php', - 'ConstantDependency' => 'includes/CacheDependency.php', 'Cookie' => 'includes/Cookie.php', 'CookieJar' => 'includes/Cookie.php', 'CreativeCommonsRdf' => 'includes/Metadata.php', - 'DependencyWrapper' => 'includes/CacheDependency.php', 'DiffHistoryBlob' => 'includes/HistoryBlob.php', 'DjVuImage' => 'includes/DjVuImage.php', 'DoubleReplacer' => 'includes/StringUtils.php', @@ -87,13 +84,11 @@ $wgAutoloadLocalClasses = array( 'FeedItem' => 'includes/Feed.php', 'FeedUtils' => 'includes/FeedUtils.php', 'FileDeleteForm' => 'includes/FileDeleteForm.php', - 'FileDependency' => 'includes/CacheDependency.php', 'FileRevertForm' => 'includes/FileRevertForm.php', 'ForkController' => 'includes/ForkController.php', 'FormOptions' => 'includes/FormOptions.php', 'FormSpecialPage' => 'includes/SpecialPage.php', 'GenderCache' => 'includes/GenderCache.php', - 'GlobalDependency' => 'includes/CacheDependency.php', 'HashtableReplacer' => 'includes/StringUtils.php', 'HistoryBlob' => 'includes/HistoryBlob.php', 'HistoryBlobCurStub' => 'includes/HistoryBlob.php', @@ -102,11 +97,8 @@ $wgAutoloadLocalClasses = array( 'HistoryPager' => 'includes/HistoryPage.php', 'Hooks' => 'includes/Hooks.php', 'Html' => 'includes/Html.php', - 'HTMLCacheUpdate' => 'includes/HTMLCacheUpdate.php', - 'HTMLCacheUpdateJob' => 'includes/HTMLCacheUpdate.php', 'HTMLCheckField' => 'includes/HTMLForm.php', 'HTMLEditTools' => 'includes/HTMLForm.php', - 'HTMLFileCache' => 'includes/HTMLFileCache.php', 'HTMLFloatField' => 'includes/HTMLForm.php', 'HTMLForm' => 'includes/HTMLForm.php', 'HTMLFormField' => 'includes/HTMLForm.php', @@ -229,8 +221,6 @@ $wgAutoloadLocalClasses = array( 'Title' => 'includes/Title.php', 'TitleArray' => 'includes/TitleArray.php', 'TitleArrayFromResult' => 'includes/TitleArray.php', - 'TitleDependency' => 'includes/CacheDependency.php', - 'TitleListDependency' => 'includes/CacheDependency.php', 'ThrottledError' => 'includes/Exception.php', 'UnlistedSpecialPage' => 'includes/SpecialPage.php', 'UppercaseCollation' => 'includes/Collation.php', @@ -356,6 +346,18 @@ $wgAutoloadLocalClasses = array( 'ApiUserrights' => 'includes/api/ApiUserrights.php', 'ApiWatch' => 'includes/api/ApiWatch.php', + # includes/cache + 'CacheDependency' => 'includes/cache/CacheDependency.php', + 'ConstantDependency' => 'includes/cache/CacheDependency.php', + 'DependencyWrapper' => 'includes/cache/CacheDependency.php', + 'FileDependency' => 'includes/cache/CacheDependency.php', + 'GlobalDependency' => 'includes/cache/CacheDependency.php', + 'HTMLCacheUpdate' => 'includes/cache/HTMLCacheUpdate.php', + 'HTMLCacheUpdateJob' => 'includes/cache/HTMLCacheUpdate.php', + 'HTMLFileCache' => 'includes/cache/HTMLFileCache.php', + 'TitleDependency' => 'includes/cache/CacheDependency.php', + 'TitleListDependency' => 'includes/cache/CacheDependency.php', + 'UsageException' => 'includes/api/ApiMain.php', # includes/db diff --git a/includes/CacheDependency.php b/includes/CacheDependency.php deleted file mode 100644 index 74ca2864e2..0000000000 --- a/includes/CacheDependency.php +++ /dev/null @@ -1,367 +0,0 @@ -value = $value; - - if ( !is_array( $deps ) ) { - $deps = array( $deps ); - } - - $this->deps = $deps; - } - - /** - * Returns true if any of the dependencies have expired - */ - function isExpired() { - foreach ( $this->deps as $dep ) { - if ( $dep->isExpired() ) { - return true; - } - } - - return false; - } - - /** - * Initialise dependency values in preparation for storing. This must be - * called before serialization. - */ - function initialiseDeps() { - foreach ( $this->deps as $dep ) { - $dep->loadDependencyValues(); - } - } - - /** - * Get the user-defined value - */ - function getValue() { - return $this->value; - } - - /** - * Store the wrapper to a cache - */ - function storeToCache( $cache, $key, $expiry = 0 ) { - $this->initialiseDeps(); - $cache->set( $key, $this, $expiry ); - } - - /** - * Attempt to get a value from the cache. If the value is expired or missing, - * it will be generated with the callback function (if present), and the newly - * calculated value will be stored to the cache in a wrapper. - * - * @param $cache Object: a cache object such as $wgMemc - * @param $key String: the cache key - * @param $expiry Integer: the expiry timestamp or interval in seconds - * @param $callback Mixed: the callback for generating the value, or false - * @param $callbackParams Array: the function parameters for the callback - * @param $deps Array: the dependencies to store on a cache miss. Note: these - * are not the dependencies used on a cache hit! Cache hits use the stored - * dependency array. - * - * @return mixed The value, or null if it was not present in the cache and no - * callback was defined. - */ - static function getValueFromCache( $cache, $key, $expiry = 0, $callback = false, - $callbackParams = array(), $deps = array() ) - { - $obj = $cache->get( $key ); - - if ( is_object( $obj ) && $obj instanceof DependencyWrapper && !$obj->isExpired() ) { - $value = $obj->value; - } elseif ( $callback ) { - $value = call_user_func_array( $callback, $callbackParams ); - # Cache the newly-generated value - $wrapper = new DependencyWrapper( $value, $deps ); - $wrapper->storeToCache( $cache, $key, $expiry ); - } else { - $value = null; - } - - return $value; - } -} - -/** - * @ingroup Cache - */ -abstract class CacheDependency { - /** - * Returns true if the dependency is expired, false otherwise - */ - abstract function isExpired(); - - /** - * Hook to perform any expensive pre-serialize loading of dependency values. - */ - function loadDependencyValues() { } -} - -/** - * @ingroup Cache - */ -class FileDependency extends CacheDependency { - var $filename, $timestamp; - - /** - * Create a file dependency - * - * @param $filename String: the name of the file, preferably fully qualified - * @param $timestamp Mixed: the unix last modified timestamp, or false if the - * file does not exist. If omitted, the timestamp will be loaded from - * the file. - * - * A dependency on a nonexistent file will be triggered when the file is - * created. A dependency on an existing file will be triggered when the - * file is changed. - */ - function __construct( $filename, $timestamp = null ) { - $this->filename = $filename; - $this->timestamp = $timestamp; - } - - function __sleep() { - $this->loadDependencyValues(); - return array( 'filename', 'timestamp' ); - } - - function loadDependencyValues() { - if ( is_null( $this->timestamp ) ) { - if ( !file_exists( $this->filename ) ) { - # Dependency on a non-existent file - # This is a valid concept! - $this->timestamp = false; - } else { - $this->timestamp = filemtime( $this->filename ); - } - } - } - - function isExpired() { - if ( !file_exists( $this->filename ) ) { - if ( $this->timestamp === false ) { - # Still nonexistent - return false; - } else { - # Deleted - wfDebug( "Dependency triggered: {$this->filename} deleted.\n" ); - return true; - } - } else { - $lastmod = filemtime( $this->filename ); - if ( $lastmod > $this->timestamp ) { - # Modified or created - wfDebug( "Dependency triggered: {$this->filename} changed.\n" ); - return true; - } else { - # Not modified - return false; - } - } - } -} - -/** - * @ingroup Cache - */ -class TitleDependency extends CacheDependency { - var $titleObj; - var $ns, $dbk; - var $touched; - - /** - * Construct a title dependency - * @param $title Title - */ - function __construct( Title $title ) { - $this->titleObj = $title; - $this->ns = $title->getNamespace(); - $this->dbk = $title->getDBkey(); - } - - function loadDependencyValues() { - $this->touched = $this->getTitle()->getTouched(); - } - - /** - * Get rid of bulky Title object for sleep - */ - function __sleep() { - return array( 'ns', 'dbk', 'touched' ); - } - - function getTitle() { - if ( !isset( $this->titleObj ) ) { - $this->titleObj = Title::makeTitle( $this->ns, $this->dbk ); - } - - return $this->titleObj; - } - - function isExpired() { - $touched = $this->getTitle()->getTouched(); - - if ( $this->touched === false ) { - if ( $touched === false ) { - # Still missing - return false; - } else { - # Created - return true; - } - } elseif ( $touched === false ) { - # Deleted - return true; - } elseif ( $touched > $this->touched ) { - # Updated - return true; - } else { - # Unmodified - return false; - } - } -} - -/** - * @ingroup Cache - */ -class TitleListDependency extends CacheDependency { - var $linkBatch; - var $timestamps; - - /** - * Construct a dependency on a list of titles - */ - function __construct( LinkBatch $linkBatch ) { - $this->linkBatch = $linkBatch; - } - - function calculateTimestamps() { - # Initialise values to false - $timestamps = array(); - - foreach ( $this->getLinkBatch()->data as $ns => $dbks ) { - if ( count( $dbks ) > 0 ) { - $timestamps[$ns] = array(); - - foreach ( $dbks as $dbk => $value ) { - $timestamps[$ns][$dbk] = false; - } - } - } - - # Do the query - if ( count( $timestamps ) ) { - $dbr = wfGetDB( DB_SLAVE ); - $where = $this->getLinkBatch()->constructSet( 'page', $dbr ); - $res = $dbr->select( - 'page', - array( 'page_namespace', 'page_title', 'page_touched' ), - $where, - __METHOD__ - ); - - foreach ( $res as $row ) { - $timestamps[$row->page_namespace][$row->page_title] = $row->page_touched; - } - } - - return $timestamps; - } - - function loadDependencyValues() { - $this->timestamps = $this->calculateTimestamps(); - } - - function __sleep() { - return array( 'timestamps' ); - } - - function getLinkBatch() { - if ( !isset( $this->linkBatch ) ) { - $this->linkBatch = new LinkBatch; - $this->linkBatch->setArray( $this->timestamps ); - } - return $this->linkBatch; - } - - function isExpired() { - $newTimestamps = $this->calculateTimestamps(); - - foreach ( $this->timestamps as $ns => $dbks ) { - foreach ( $dbks as $dbk => $oldTimestamp ) { - $newTimestamp = $newTimestamps[$ns][$dbk]; - - if ( $oldTimestamp === false ) { - if ( $newTimestamp === false ) { - # Still missing - } else { - # Created - return true; - } - } elseif ( $newTimestamp === false ) { - # Deleted - return true; - } elseif ( $newTimestamp > $oldTimestamp ) { - # Updated - return true; - } else { - # Unmodified - } - } - } - - return false; - } -} - -/** - * @ingroup Cache - */ -class GlobalDependency extends CacheDependency { - var $name, $value; - - function __construct( $name ) { - $this->name = $name; - $this->value = $GLOBALS[$name]; - } - - function isExpired() { - return $GLOBALS[$this->name] != $this->value; - } -} - -/** - * @ingroup Cache - */ -class ConstantDependency extends CacheDependency { - var $name, $value; - - function __construct( $name ) { - $this->name = $name; - $this->value = constant( $name ); - } - - function isExpired() { - return constant( $this->name ) != $this->value; - } -} diff --git a/includes/HTMLCacheUpdate.php b/includes/HTMLCacheUpdate.php deleted file mode 100644 index ca5fc2ec96..0000000000 --- a/includes/HTMLCacheUpdate.php +++ /dev/null @@ -1,237 +0,0 @@ -mTitle = $titleTo; - $this->mTable = $table; - $this->mStart = $start; - $this->mEnd = $end; - $this->mRowsPerJob = $wgUpdateRowsPerJob; - $this->mRowsPerQuery = $wgUpdateRowsPerQuery; - $this->mCache = $this->mTitle->getBacklinkCache(); - } - - public function doUpdate() { - if ( $this->mStart || $this->mEnd ) { - $this->doPartialUpdate(); - return; - } - - # Get an estimate of the number of rows from the BacklinkCache - $numRows = $this->mCache->getNumLinks( $this->mTable ); - if ( $numRows > $this->mRowsPerJob * 2 ) { - # Do fast cached partition - $this->insertJobs(); - } else { - # Get the links from the DB - $titleArray = $this->mCache->getLinks( $this->mTable ); - # Check if the row count estimate was correct - if ( $titleArray->count() > $this->mRowsPerJob * 2 ) { - # Not correct, do accurate partition - wfDebug( __METHOD__.": row count estimate was incorrect, repartitioning\n" ); - $this->insertJobsFromTitles( $titleArray ); - } else { - $this->invalidateTitles( $titleArray ); - } - } - } - - /** - * Update some of the backlinks, defined by a page ID range - */ - protected function doPartialUpdate() { - $titleArray = $this->mCache->getLinks( $this->mTable, $this->mStart, $this->mEnd ); - if ( $titleArray->count() <= $this->mRowsPerJob * 2 ) { - # This partition is small enough, do the update - $this->invalidateTitles( $titleArray ); - } else { - # Partitioning was excessively inaccurate. Divide the job further. - # This can occur when a large number of links are added in a short - # period of time, say by updating a heavily-used template. - $this->insertJobsFromTitles( $titleArray ); - } - } - - /** - * Partition the current range given by $this->mStart and $this->mEnd, - * using a pre-calculated title array which gives the links in that range. - * Queue the resulting jobs. - */ - protected function insertJobsFromTitles( $titleArray ) { - # We make subpartitions in the sense that the start of the first job - # will be the start of the parent partition, and the end of the last - # job will be the end of the parent partition. - $jobs = array(); - $start = $this->mStart; # start of the current job - $numTitles = 0; - foreach ( $titleArray as $title ) { - $id = $title->getArticleID(); - # $numTitles is now the number of titles in the current job not - # including the current ID - if ( $numTitles >= $this->mRowsPerJob ) { - # Add a job up to but not including the current ID - $params = array( - 'table' => $this->mTable, - 'start' => $start, - 'end' => $id - 1 - ); - $jobs[] = new HTMLCacheUpdateJob( $this->mTitle, $params ); - $start = $id; - $numTitles = 0; - } - $numTitles++; - } - # Last job - $params = array( - 'table' => $this->mTable, - 'start' => $start, - 'end' => $this->mEnd - ); - $jobs[] = new HTMLCacheUpdateJob( $this->mTitle, $params ); - wfDebug( __METHOD__.": repartitioning into " . count( $jobs ) . " jobs\n" ); - - if ( count( $jobs ) < 2 ) { - # I don't think this is possible at present, but handling this case - # makes the code a bit more robust against future code updates and - # avoids a potential infinite loop of repartitioning - wfDebug( __METHOD__.": repartitioning failed!\n" ); - $this->invalidateTitles( $titleArray ); - return; - } - - Job::batchInsert( $jobs ); - } - - protected function insertJobs() { - $batches = $this->mCache->partition( $this->mTable, $this->mRowsPerJob ); - if ( !$batches ) { - return; - } - $jobs = array(); - foreach ( $batches as $batch ) { - $params = array( - 'table' => $this->mTable, - 'start' => $batch[0], - 'end' => $batch[1], - ); - $jobs[] = new HTMLCacheUpdateJob( $this->mTitle, $params ); - } - Job::batchInsert( $jobs ); - } - - /** - * Invalidate a range of pages, right now - * @deprecated - */ - public function invalidate( $startId = false, $endId = false ) { - $titleArray = $this->mCache->getLinks( $this->mTable, $startId, $endId ); - $this->invalidateTitles( $titleArray ); - } - - /** - * Invalidate an array (or iterator) of Title objects, right now - */ - protected function invalidateTitles( $titleArray ) { - global $wgUseFileCache, $wgUseSquid; - - $dbw = wfGetDB( DB_MASTER ); - $timestamp = $dbw->timestamp(); - - # Get all IDs in this query into an array - $ids = array(); - foreach ( $titleArray as $title ) { - $ids[] = $title->getArticleID(); - } - - if ( !$ids ) { - return; - } - - # Update page_touched - $batches = array_chunk( $ids, $this->mRowsPerQuery ); - foreach ( $batches as $batch ) { - $dbw->update( 'page', - array( 'page_touched' => $timestamp ), - array( 'page_id IN (' . $dbw->makeList( $batch ) . ')' ), - __METHOD__ - ); - } - - # Update squid - if ( $wgUseSquid ) { - $u = SquidUpdate::newFromTitles( $titleArray ); - $u->doUpdate(); - } - - # Update file cache - if ( $wgUseFileCache ) { - foreach ( $titleArray as $title ) { - HTMLFileCache::clearFileCache( $title ); - } - } - } - -} - -/** - * Job wrapper for HTMLCacheUpdate. Gets run whenever a related - * job gets called from the queue. - * - * @ingroup JobQueue - */ -class HTMLCacheUpdateJob extends Job { - var $table, $start, $end; - - /** - * Construct a job - * @param $title Title: the title linked to - * @param $params Array: job parameters (table, start and end page_ids) - * @param $id Integer: job id - */ - function __construct( $title, $params, $id = 0 ) { - parent::__construct( 'htmlCacheUpdate', $title, $params, $id ); - $this->table = $params['table']; - $this->start = $params['start']; - $this->end = $params['end']; - } - - public function run() { - $update = new HTMLCacheUpdate( $this->title, $this->table, $this->start, $this->end ); - $update->doUpdate(); - return true; - } -} diff --git a/includes/HTMLFileCache.php b/includes/HTMLFileCache.php deleted file mode 100644 index 949ed36f5c..0000000000 --- a/includes/HTMLFileCache.php +++ /dev/null @@ -1,250 +0,0 @@ -mTitle = $title; - $this->mType = ($type == 'raw' || $type == 'view' ) ? $type : false; - $this->fileCacheName(); // init name - } - - public function fileCacheName() { - if( !$this->mFileCache ) { - global $wgCacheDirectory, $wgFileCacheDirectory, $wgFileCacheDepth; - - if ( $wgFileCacheDirectory ) { - $dir = $wgFileCacheDirectory; - } elseif ( $wgCacheDirectory ) { - $dir = "$wgCacheDirectory/html"; - } else { - throw new MWException( 'Please set $wgCacheDirectory in LocalSettings.php if you wish to use the HTML file cache' ); - } - - # Store raw pages (like CSS hits) elsewhere - $subdir = ($this->mType === 'raw') ? 'raw/' : ''; - - $key = $this->mTitle->getPrefixedDbkey(); - if ( $wgFileCacheDepth > 0 ) { - $hash = md5( $key ); - for ( $i = 1; $i <= $wgFileCacheDepth; $i++ ) { - $subdir .= substr( $hash, 0, $i ) . '/'; - } - } - # Avoid extension confusion - $key = str_replace( '.', '%2E', urlencode( $key ) ); - $this->mFileCache = "{$dir}/{$subdir}{$key}.html"; - - if( $this->useGzip() ) { - $this->mFileCache .= '.gz'; - } - - wfDebug( __METHOD__ . ": {$this->mFileCache}\n" ); - } - return $this->mFileCache; - } - - public function isFileCached() { - if( $this->mType === false ) { - return false; - } - return file_exists( $this->fileCacheName() ); - } - - public function fileCacheTime() { - return wfTimestamp( TS_MW, filemtime( $this->fileCacheName() ) ); - } - - /** - * Check if pages can be cached for this request/user - * @return bool - */ - public static function useFileCache() { - global $wgUser, $wgUseFileCache, $wgShowIPinHeader, $wgRequest, $wgLang, $wgContLang; - if( !$wgUseFileCache ) { - return false; - } - // Get all query values - $queryVals = $wgRequest->getValues(); - foreach( $queryVals as $query => $val ) { - if( $query == 'title' || $query == 'curid' ) { - continue; - } - // Normal page view in query form can have action=view. - // Raw hits for pages also stored, like .css pages for example. - else if( $query == 'action' && ($val == 'view' || $val == 'raw') ) { - continue; - } else if( $query == 'usemsgcache' && $val == 'yes' ) { - continue; - } - // Below are header setting params - else if( $query == 'maxage' || $query == 'smaxage' || $query == 'ctype' || $query == 'gen' ) { - continue; - } else { - return false; - } - } - // Check for non-standard user language; this covers uselang, - // and extensions for auto-detecting user language. - $ulang = $wgLang->getCode(); - $clang = $wgContLang->getCode(); - // Check that there are no other sources of variation - return !$wgShowIPinHeader && !$wgUser->getId() && !$wgUser->getNewtalk() && $ulang == $clang; - } - - /* - * Check if up to date cache file exists - * @param $timestamp string - */ - public function isFileCacheGood( $timestamp = '' ) { - global $wgCacheEpoch; - - if( !$this->isFileCached() ) { - return false; - } - - $cachetime = $this->fileCacheTime(); - $good = $timestamp <= $cachetime && $wgCacheEpoch <= $cachetime; - - wfDebug( __METHOD__ . ": cachetime $cachetime, touched '{$timestamp}' epoch {$wgCacheEpoch}, good $good\n"); - return $good; - } - - public function useGzip() { - global $wgUseGzip; - return $wgUseGzip; - } - - /* In handy string packages */ - public function fetchRawText() { - return file_get_contents( $this->fileCacheName() ); - } - - public function fetchPageText() { - if( $this->useGzip() ) { - /* Why is there no gzfile_get_contents() or gzdecode()? */ - return implode( '', gzfile( $this->fileCacheName() ) ); - } else { - return $this->fetchRawText(); - } - } - - /* Working directory to/from output */ - public function loadFromFileCache() { - global $wgOut, $wgMimeType, $wgOutputEncoding, $wgLanguageCode; - wfDebug( __METHOD__ . "()\n"); - $filename = $this->fileCacheName(); - // Raw pages should handle cache control on their own, - // even when using file cache. This reduces hits from clients. - if( $this->mType !== 'raw' ) { - $wgOut->sendCacheControl(); - header( "Content-Type: $wgMimeType; charset={$wgOutputEncoding}" ); - header( "Content-Language: $wgLanguageCode" ); - } - - if( $this->useGzip() ) { - if( wfClientAcceptsGzip() ) { - header( 'Content-Encoding: gzip' ); - } else { - /* Send uncompressed */ - readgzfile( $filename ); - return; - } - } - readfile( $filename ); - $wgOut->disable(); // tell $wgOut that output is taken care of - } - - protected function checkCacheDirs() { - $filename = $this->fileCacheName(); - $mydir2 = substr($filename,0,strrpos($filename,'/')); # subdirectory level 2 - $mydir1 = substr($mydir2,0,strrpos($mydir2,'/')); # subdirectory level 1 - - wfMkdirParents( $mydir1 ); - wfMkdirParents( $mydir2 ); - } - - public function saveToFileCache( $text ) { - global $wgUseFileCache; - if( !$wgUseFileCache || strlen( $text ) < 512 ) { - // Disabled or empty/broken output (OOM and PHP errors) - return $text; - } - - wfDebug( __METHOD__ . "()\n", false); - - $this->checkCacheDirs(); - - $f = fopen( $this->fileCacheName(), 'w' ); - if($f) { - $now = wfTimestampNow(); - if( $this->useGzip() ) { - $rawtext = str_replace( '', - '\n", - $text ); - $text = gzencode( $rawtext ); - } else { - $text = str_replace( '', - '\n", - $text ); - } - fwrite( $f, $text ); - fclose( $f ); - if( $this->useGzip() ) { - if( wfClientAcceptsGzip() ) { - header( 'Content-Encoding: gzip' ); - return $text; - } else { - return $rawtext; - } - } else { - return $text; - } - } - return $text; - } - - public static function clearFileCache( $title ) { - global $wgUseFileCache; - - if ( !$wgUseFileCache ) { - return false; - } - - wfSuppressWarnings(); - - $fc = new self( $title, 'view' ); - unlink( $fc->fileCacheName() ); - - $fc = new self( $title, 'raw' ); - unlink( $fc->fileCacheName() ); - - wfRestoreWarnings(); - - return true; - } -} diff --git a/includes/cache/CacheDependency.php b/includes/cache/CacheDependency.php new file mode 100644 index 0000000000..74ca2864e2 --- /dev/null +++ b/includes/cache/CacheDependency.php @@ -0,0 +1,367 @@ +value = $value; + + if ( !is_array( $deps ) ) { + $deps = array( $deps ); + } + + $this->deps = $deps; + } + + /** + * Returns true if any of the dependencies have expired + */ + function isExpired() { + foreach ( $this->deps as $dep ) { + if ( $dep->isExpired() ) { + return true; + } + } + + return false; + } + + /** + * Initialise dependency values in preparation for storing. This must be + * called before serialization. + */ + function initialiseDeps() { + foreach ( $this->deps as $dep ) { + $dep->loadDependencyValues(); + } + } + + /** + * Get the user-defined value + */ + function getValue() { + return $this->value; + } + + /** + * Store the wrapper to a cache + */ + function storeToCache( $cache, $key, $expiry = 0 ) { + $this->initialiseDeps(); + $cache->set( $key, $this, $expiry ); + } + + /** + * Attempt to get a value from the cache. If the value is expired or missing, + * it will be generated with the callback function (if present), and the newly + * calculated value will be stored to the cache in a wrapper. + * + * @param $cache Object: a cache object such as $wgMemc + * @param $key String: the cache key + * @param $expiry Integer: the expiry timestamp or interval in seconds + * @param $callback Mixed: the callback for generating the value, or false + * @param $callbackParams Array: the function parameters for the callback + * @param $deps Array: the dependencies to store on a cache miss. Note: these + * are not the dependencies used on a cache hit! Cache hits use the stored + * dependency array. + * + * @return mixed The value, or null if it was not present in the cache and no + * callback was defined. + */ + static function getValueFromCache( $cache, $key, $expiry = 0, $callback = false, + $callbackParams = array(), $deps = array() ) + { + $obj = $cache->get( $key ); + + if ( is_object( $obj ) && $obj instanceof DependencyWrapper && !$obj->isExpired() ) { + $value = $obj->value; + } elseif ( $callback ) { + $value = call_user_func_array( $callback, $callbackParams ); + # Cache the newly-generated value + $wrapper = new DependencyWrapper( $value, $deps ); + $wrapper->storeToCache( $cache, $key, $expiry ); + } else { + $value = null; + } + + return $value; + } +} + +/** + * @ingroup Cache + */ +abstract class CacheDependency { + /** + * Returns true if the dependency is expired, false otherwise + */ + abstract function isExpired(); + + /** + * Hook to perform any expensive pre-serialize loading of dependency values. + */ + function loadDependencyValues() { } +} + +/** + * @ingroup Cache + */ +class FileDependency extends CacheDependency { + var $filename, $timestamp; + + /** + * Create a file dependency + * + * @param $filename String: the name of the file, preferably fully qualified + * @param $timestamp Mixed: the unix last modified timestamp, or false if the + * file does not exist. If omitted, the timestamp will be loaded from + * the file. + * + * A dependency on a nonexistent file will be triggered when the file is + * created. A dependency on an existing file will be triggered when the + * file is changed. + */ + function __construct( $filename, $timestamp = null ) { + $this->filename = $filename; + $this->timestamp = $timestamp; + } + + function __sleep() { + $this->loadDependencyValues(); + return array( 'filename', 'timestamp' ); + } + + function loadDependencyValues() { + if ( is_null( $this->timestamp ) ) { + if ( !file_exists( $this->filename ) ) { + # Dependency on a non-existent file + # This is a valid concept! + $this->timestamp = false; + } else { + $this->timestamp = filemtime( $this->filename ); + } + } + } + + function isExpired() { + if ( !file_exists( $this->filename ) ) { + if ( $this->timestamp === false ) { + # Still nonexistent + return false; + } else { + # Deleted + wfDebug( "Dependency triggered: {$this->filename} deleted.\n" ); + return true; + } + } else { + $lastmod = filemtime( $this->filename ); + if ( $lastmod > $this->timestamp ) { + # Modified or created + wfDebug( "Dependency triggered: {$this->filename} changed.\n" ); + return true; + } else { + # Not modified + return false; + } + } + } +} + +/** + * @ingroup Cache + */ +class TitleDependency extends CacheDependency { + var $titleObj; + var $ns, $dbk; + var $touched; + + /** + * Construct a title dependency + * @param $title Title + */ + function __construct( Title $title ) { + $this->titleObj = $title; + $this->ns = $title->getNamespace(); + $this->dbk = $title->getDBkey(); + } + + function loadDependencyValues() { + $this->touched = $this->getTitle()->getTouched(); + } + + /** + * Get rid of bulky Title object for sleep + */ + function __sleep() { + return array( 'ns', 'dbk', 'touched' ); + } + + function getTitle() { + if ( !isset( $this->titleObj ) ) { + $this->titleObj = Title::makeTitle( $this->ns, $this->dbk ); + } + + return $this->titleObj; + } + + function isExpired() { + $touched = $this->getTitle()->getTouched(); + + if ( $this->touched === false ) { + if ( $touched === false ) { + # Still missing + return false; + } else { + # Created + return true; + } + } elseif ( $touched === false ) { + # Deleted + return true; + } elseif ( $touched > $this->touched ) { + # Updated + return true; + } else { + # Unmodified + return false; + } + } +} + +/** + * @ingroup Cache + */ +class TitleListDependency extends CacheDependency { + var $linkBatch; + var $timestamps; + + /** + * Construct a dependency on a list of titles + */ + function __construct( LinkBatch $linkBatch ) { + $this->linkBatch = $linkBatch; + } + + function calculateTimestamps() { + # Initialise values to false + $timestamps = array(); + + foreach ( $this->getLinkBatch()->data as $ns => $dbks ) { + if ( count( $dbks ) > 0 ) { + $timestamps[$ns] = array(); + + foreach ( $dbks as $dbk => $value ) { + $timestamps[$ns][$dbk] = false; + } + } + } + + # Do the query + if ( count( $timestamps ) ) { + $dbr = wfGetDB( DB_SLAVE ); + $where = $this->getLinkBatch()->constructSet( 'page', $dbr ); + $res = $dbr->select( + 'page', + array( 'page_namespace', 'page_title', 'page_touched' ), + $where, + __METHOD__ + ); + + foreach ( $res as $row ) { + $timestamps[$row->page_namespace][$row->page_title] = $row->page_touched; + } + } + + return $timestamps; + } + + function loadDependencyValues() { + $this->timestamps = $this->calculateTimestamps(); + } + + function __sleep() { + return array( 'timestamps' ); + } + + function getLinkBatch() { + if ( !isset( $this->linkBatch ) ) { + $this->linkBatch = new LinkBatch; + $this->linkBatch->setArray( $this->timestamps ); + } + return $this->linkBatch; + } + + function isExpired() { + $newTimestamps = $this->calculateTimestamps(); + + foreach ( $this->timestamps as $ns => $dbks ) { + foreach ( $dbks as $dbk => $oldTimestamp ) { + $newTimestamp = $newTimestamps[$ns][$dbk]; + + if ( $oldTimestamp === false ) { + if ( $newTimestamp === false ) { + # Still missing + } else { + # Created + return true; + } + } elseif ( $newTimestamp === false ) { + # Deleted + return true; + } elseif ( $newTimestamp > $oldTimestamp ) { + # Updated + return true; + } else { + # Unmodified + } + } + } + + return false; + } +} + +/** + * @ingroup Cache + */ +class GlobalDependency extends CacheDependency { + var $name, $value; + + function __construct( $name ) { + $this->name = $name; + $this->value = $GLOBALS[$name]; + } + + function isExpired() { + return $GLOBALS[$this->name] != $this->value; + } +} + +/** + * @ingroup Cache + */ +class ConstantDependency extends CacheDependency { + var $name, $value; + + function __construct( $name ) { + $this->name = $name; + $this->value = constant( $name ); + } + + function isExpired() { + return constant( $this->name ) != $this->value; + } +} diff --git a/includes/cache/HTMLCacheUpdate.php b/includes/cache/HTMLCacheUpdate.php new file mode 100644 index 0000000000..ca5fc2ec96 --- /dev/null +++ b/includes/cache/HTMLCacheUpdate.php @@ -0,0 +1,237 @@ +mTitle = $titleTo; + $this->mTable = $table; + $this->mStart = $start; + $this->mEnd = $end; + $this->mRowsPerJob = $wgUpdateRowsPerJob; + $this->mRowsPerQuery = $wgUpdateRowsPerQuery; + $this->mCache = $this->mTitle->getBacklinkCache(); + } + + public function doUpdate() { + if ( $this->mStart || $this->mEnd ) { + $this->doPartialUpdate(); + return; + } + + # Get an estimate of the number of rows from the BacklinkCache + $numRows = $this->mCache->getNumLinks( $this->mTable ); + if ( $numRows > $this->mRowsPerJob * 2 ) { + # Do fast cached partition + $this->insertJobs(); + } else { + # Get the links from the DB + $titleArray = $this->mCache->getLinks( $this->mTable ); + # Check if the row count estimate was correct + if ( $titleArray->count() > $this->mRowsPerJob * 2 ) { + # Not correct, do accurate partition + wfDebug( __METHOD__.": row count estimate was incorrect, repartitioning\n" ); + $this->insertJobsFromTitles( $titleArray ); + } else { + $this->invalidateTitles( $titleArray ); + } + } + } + + /** + * Update some of the backlinks, defined by a page ID range + */ + protected function doPartialUpdate() { + $titleArray = $this->mCache->getLinks( $this->mTable, $this->mStart, $this->mEnd ); + if ( $titleArray->count() <= $this->mRowsPerJob * 2 ) { + # This partition is small enough, do the update + $this->invalidateTitles( $titleArray ); + } else { + # Partitioning was excessively inaccurate. Divide the job further. + # This can occur when a large number of links are added in a short + # period of time, say by updating a heavily-used template. + $this->insertJobsFromTitles( $titleArray ); + } + } + + /** + * Partition the current range given by $this->mStart and $this->mEnd, + * using a pre-calculated title array which gives the links in that range. + * Queue the resulting jobs. + */ + protected function insertJobsFromTitles( $titleArray ) { + # We make subpartitions in the sense that the start of the first job + # will be the start of the parent partition, and the end of the last + # job will be the end of the parent partition. + $jobs = array(); + $start = $this->mStart; # start of the current job + $numTitles = 0; + foreach ( $titleArray as $title ) { + $id = $title->getArticleID(); + # $numTitles is now the number of titles in the current job not + # including the current ID + if ( $numTitles >= $this->mRowsPerJob ) { + # Add a job up to but not including the current ID + $params = array( + 'table' => $this->mTable, + 'start' => $start, + 'end' => $id - 1 + ); + $jobs[] = new HTMLCacheUpdateJob( $this->mTitle, $params ); + $start = $id; + $numTitles = 0; + } + $numTitles++; + } + # Last job + $params = array( + 'table' => $this->mTable, + 'start' => $start, + 'end' => $this->mEnd + ); + $jobs[] = new HTMLCacheUpdateJob( $this->mTitle, $params ); + wfDebug( __METHOD__.": repartitioning into " . count( $jobs ) . " jobs\n" ); + + if ( count( $jobs ) < 2 ) { + # I don't think this is possible at present, but handling this case + # makes the code a bit more robust against future code updates and + # avoids a potential infinite loop of repartitioning + wfDebug( __METHOD__.": repartitioning failed!\n" ); + $this->invalidateTitles( $titleArray ); + return; + } + + Job::batchInsert( $jobs ); + } + + protected function insertJobs() { + $batches = $this->mCache->partition( $this->mTable, $this->mRowsPerJob ); + if ( !$batches ) { + return; + } + $jobs = array(); + foreach ( $batches as $batch ) { + $params = array( + 'table' => $this->mTable, + 'start' => $batch[0], + 'end' => $batch[1], + ); + $jobs[] = new HTMLCacheUpdateJob( $this->mTitle, $params ); + } + Job::batchInsert( $jobs ); + } + + /** + * Invalidate a range of pages, right now + * @deprecated + */ + public function invalidate( $startId = false, $endId = false ) { + $titleArray = $this->mCache->getLinks( $this->mTable, $startId, $endId ); + $this->invalidateTitles( $titleArray ); + } + + /** + * Invalidate an array (or iterator) of Title objects, right now + */ + protected function invalidateTitles( $titleArray ) { + global $wgUseFileCache, $wgUseSquid; + + $dbw = wfGetDB( DB_MASTER ); + $timestamp = $dbw->timestamp(); + + # Get all IDs in this query into an array + $ids = array(); + foreach ( $titleArray as $title ) { + $ids[] = $title->getArticleID(); + } + + if ( !$ids ) { + return; + } + + # Update page_touched + $batches = array_chunk( $ids, $this->mRowsPerQuery ); + foreach ( $batches as $batch ) { + $dbw->update( 'page', + array( 'page_touched' => $timestamp ), + array( 'page_id IN (' . $dbw->makeList( $batch ) . ')' ), + __METHOD__ + ); + } + + # Update squid + if ( $wgUseSquid ) { + $u = SquidUpdate::newFromTitles( $titleArray ); + $u->doUpdate(); + } + + # Update file cache + if ( $wgUseFileCache ) { + foreach ( $titleArray as $title ) { + HTMLFileCache::clearFileCache( $title ); + } + } + } + +} + +/** + * Job wrapper for HTMLCacheUpdate. Gets run whenever a related + * job gets called from the queue. + * + * @ingroup JobQueue + */ +class HTMLCacheUpdateJob extends Job { + var $table, $start, $end; + + /** + * Construct a job + * @param $title Title: the title linked to + * @param $params Array: job parameters (table, start and end page_ids) + * @param $id Integer: job id + */ + function __construct( $title, $params, $id = 0 ) { + parent::__construct( 'htmlCacheUpdate', $title, $params, $id ); + $this->table = $params['table']; + $this->start = $params['start']; + $this->end = $params['end']; + } + + public function run() { + $update = new HTMLCacheUpdate( $this->title, $this->table, $this->start, $this->end ); + $update->doUpdate(); + return true; + } +} diff --git a/includes/cache/HTMLFileCache.php b/includes/cache/HTMLFileCache.php new file mode 100644 index 0000000000..949ed36f5c --- /dev/null +++ b/includes/cache/HTMLFileCache.php @@ -0,0 +1,250 @@ +mTitle = $title; + $this->mType = ($type == 'raw' || $type == 'view' ) ? $type : false; + $this->fileCacheName(); // init name + } + + public function fileCacheName() { + if( !$this->mFileCache ) { + global $wgCacheDirectory, $wgFileCacheDirectory, $wgFileCacheDepth; + + if ( $wgFileCacheDirectory ) { + $dir = $wgFileCacheDirectory; + } elseif ( $wgCacheDirectory ) { + $dir = "$wgCacheDirectory/html"; + } else { + throw new MWException( 'Please set $wgCacheDirectory in LocalSettings.php if you wish to use the HTML file cache' ); + } + + # Store raw pages (like CSS hits) elsewhere + $subdir = ($this->mType === 'raw') ? 'raw/' : ''; + + $key = $this->mTitle->getPrefixedDbkey(); + if ( $wgFileCacheDepth > 0 ) { + $hash = md5( $key ); + for ( $i = 1; $i <= $wgFileCacheDepth; $i++ ) { + $subdir .= substr( $hash, 0, $i ) . '/'; + } + } + # Avoid extension confusion + $key = str_replace( '.', '%2E', urlencode( $key ) ); + $this->mFileCache = "{$dir}/{$subdir}{$key}.html"; + + if( $this->useGzip() ) { + $this->mFileCache .= '.gz'; + } + + wfDebug( __METHOD__ . ": {$this->mFileCache}\n" ); + } + return $this->mFileCache; + } + + public function isFileCached() { + if( $this->mType === false ) { + return false; + } + return file_exists( $this->fileCacheName() ); + } + + public function fileCacheTime() { + return wfTimestamp( TS_MW, filemtime( $this->fileCacheName() ) ); + } + + /** + * Check if pages can be cached for this request/user + * @return bool + */ + public static function useFileCache() { + global $wgUser, $wgUseFileCache, $wgShowIPinHeader, $wgRequest, $wgLang, $wgContLang; + if( !$wgUseFileCache ) { + return false; + } + // Get all query values + $queryVals = $wgRequest->getValues(); + foreach( $queryVals as $query => $val ) { + if( $query == 'title' || $query == 'curid' ) { + continue; + } + // Normal page view in query form can have action=view. + // Raw hits for pages also stored, like .css pages for example. + else if( $query == 'action' && ($val == 'view' || $val == 'raw') ) { + continue; + } else if( $query == 'usemsgcache' && $val == 'yes' ) { + continue; + } + // Below are header setting params + else if( $query == 'maxage' || $query == 'smaxage' || $query == 'ctype' || $query == 'gen' ) { + continue; + } else { + return false; + } + } + // Check for non-standard user language; this covers uselang, + // and extensions for auto-detecting user language. + $ulang = $wgLang->getCode(); + $clang = $wgContLang->getCode(); + // Check that there are no other sources of variation + return !$wgShowIPinHeader && !$wgUser->getId() && !$wgUser->getNewtalk() && $ulang == $clang; + } + + /* + * Check if up to date cache file exists + * @param $timestamp string + */ + public function isFileCacheGood( $timestamp = '' ) { + global $wgCacheEpoch; + + if( !$this->isFileCached() ) { + return false; + } + + $cachetime = $this->fileCacheTime(); + $good = $timestamp <= $cachetime && $wgCacheEpoch <= $cachetime; + + wfDebug( __METHOD__ . ": cachetime $cachetime, touched '{$timestamp}' epoch {$wgCacheEpoch}, good $good\n"); + return $good; + } + + public function useGzip() { + global $wgUseGzip; + return $wgUseGzip; + } + + /* In handy string packages */ + public function fetchRawText() { + return file_get_contents( $this->fileCacheName() ); + } + + public function fetchPageText() { + if( $this->useGzip() ) { + /* Why is there no gzfile_get_contents() or gzdecode()? */ + return implode( '', gzfile( $this->fileCacheName() ) ); + } else { + return $this->fetchRawText(); + } + } + + /* Working directory to/from output */ + public function loadFromFileCache() { + global $wgOut, $wgMimeType, $wgOutputEncoding, $wgLanguageCode; + wfDebug( __METHOD__ . "()\n"); + $filename = $this->fileCacheName(); + // Raw pages should handle cache control on their own, + // even when using file cache. This reduces hits from clients. + if( $this->mType !== 'raw' ) { + $wgOut->sendCacheControl(); + header( "Content-Type: $wgMimeType; charset={$wgOutputEncoding}" ); + header( "Content-Language: $wgLanguageCode" ); + } + + if( $this->useGzip() ) { + if( wfClientAcceptsGzip() ) { + header( 'Content-Encoding: gzip' ); + } else { + /* Send uncompressed */ + readgzfile( $filename ); + return; + } + } + readfile( $filename ); + $wgOut->disable(); // tell $wgOut that output is taken care of + } + + protected function checkCacheDirs() { + $filename = $this->fileCacheName(); + $mydir2 = substr($filename,0,strrpos($filename,'/')); # subdirectory level 2 + $mydir1 = substr($mydir2,0,strrpos($mydir2,'/')); # subdirectory level 1 + + wfMkdirParents( $mydir1 ); + wfMkdirParents( $mydir2 ); + } + + public function saveToFileCache( $text ) { + global $wgUseFileCache; + if( !$wgUseFileCache || strlen( $text ) < 512 ) { + // Disabled or empty/broken output (OOM and PHP errors) + return $text; + } + + wfDebug( __METHOD__ . "()\n", false); + + $this->checkCacheDirs(); + + $f = fopen( $this->fileCacheName(), 'w' ); + if($f) { + $now = wfTimestampNow(); + if( $this->useGzip() ) { + $rawtext = str_replace( '', + '\n", + $text ); + $text = gzencode( $rawtext ); + } else { + $text = str_replace( '', + '\n", + $text ); + } + fwrite( $f, $text ); + fclose( $f ); + if( $this->useGzip() ) { + if( wfClientAcceptsGzip() ) { + header( 'Content-Encoding: gzip' ); + return $text; + } else { + return $rawtext; + } + } else { + return $text; + } + } + return $text; + } + + public static function clearFileCache( $title ) { + global $wgUseFileCache; + + if ( !$wgUseFileCache ) { + return false; + } + + wfSuppressWarnings(); + + $fc = new self( $title, 'view' ); + unlink( $fc->fileCacheName() ); + + $fc = new self( $title, 'raw' ); + unlink( $fc->fileCacheName() ); + + wfRestoreWarnings(); + + return true; + } +}