Merge "Send job ACKs to statsd"

diff --git a/.gitignore b/.gitignore
index b1649df..01a11bf 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -26,6 +26,7 @@ sftp-config.json
 /docs/js
 /images/[0-9a-f]
 /images/archive
+/images/cache
 /images/deleted
 /images/lockdir
 /images/temp

diff --git a/includes/jobqueue/JobRunner.php b/includes/jobqueue/JobRunner.php
index 0948092..f413701 100644 (file)
--- a/includes/jobqueue/JobRunner.php
+++ b/includes/jobqueue/JobRunner.php
@@ -175,7 +175,7 @@ class JobRunner implements LoggerAwareInterface {
                                }
 
                                $msg = $job->toString() . " STARTING";
-                               $this->logger->info( $msg );
+                               $this->logger->debug( $msg );
                                $this->debugCallback( $msg );
 
                                // Run the job...

diff --git a/includes/jobqueue/jobs/RefreshLinksJob.php b/includes/jobqueue/jobs/RefreshLinksJob.php
index 1252b0b..749913a 100644 (file)
--- a/includes/jobqueue/jobs/RefreshLinksJob.php
+++ b/includes/jobqueue/jobs/RefreshLinksJob.php
@@ -37,6 +37,8 @@
 class RefreshLinksJob extends Job {
        const PARSE_THRESHOLD_SEC = 1.0;
 
+       const CLOCK_FUDGE = 10;
+
        function __construct( $title, $params = '' ) {
                parent::__construct( 'refreshLinks', $title, $params );
                // A separate type is used just for cascade-protected backlinks
@@ -140,22 +142,38 @@ class RefreshLinksJob extends Job {
 
                $parserOutput = false;
                $parserOptions = $page->makeParserOptions( 'canonical' );
-               // If page_touched changed after this root job (with a good slave lag skew factor),
-               // then it is likely that any views of the pages already resulted in re-parses which
-               // are now in cache. This can be reused to avoid expensive parsing in some cases.
+               // If page_touched changed after this root job, then it is likely that
+               // any views of the pages already resulted in re-parses which are now in
+               // cache. The cache can be reused to avoid expensive parsing in some cases.
                if ( isset( $this->params['rootJobTimestamp'] ) ) {
-                       $skewedTimestamp = wfTimestamp( TS_UNIX, $this->params['rootJobTimestamp'] ) + 5;
-                       if ( $page->getLinksTimestamp() > wfTimestamp( TS_MW, $skewedTimestamp ) ) {
+                       $opportunistic = !empty( $this->params['isOpportunistic'] );
+
+                       $skewedTimestamp = $this->params['rootJobTimestamp'];
+                       if ( $opportunistic ) {
+                               // Neither clock skew nor DB snapshot/slave lag matter much for such
+                               // updates; focus on reusing the (often recently updated) cache
+                       } else {
+                               // For transclusion updates, the template changes must be reflected
+                               $skewedTimestamp = wfTimestamp( TS_MW,
+                                       wfTimestamp( TS_UNIX, $skewedTimestamp ) + self::CLOCK_FUDGE
+                               );
+                       }
+
+                       if ( $page->getLinksTimestamp() > $skewedTimestamp ) {
                                // Something already updated the backlinks since this job was made
                                return true;
                        }
-                       if ( $page->getTouched() > wfTimestamp( TS_MW, $skewedTimestamp ) ) {
+
+                       if ( $page->getTouched() >= $skewedTimestamp || $opportunistic ) {
+                               // Something bumped page_touched since this job was made
+                               // or the cache is otherwise suspected to be up-to-date
                                $parserOutput = ParserCache::singleton()->getDirty( $page, $parserOptions );
-                               if ( $parserOutput && $parserOutput->getCacheTime() <= $skewedTimestamp ) {
+                               if ( $parserOutput && $parserOutput->getCacheTime() < $skewedTimestamp ) {
                                        $parserOutput = false; // too stale
                                }
                        }
                }
+
                // Fetch the current revision and parse it if necessary...
                if ( $parserOutput == false ) {
                        $start = microtime( true );

diff --git a/includes/libs/objectcache/WANObjectCache.php b/includes/libs/objectcache/WANObjectCache.php
index 92bd0bd..3099dce 100755 (executable)
--- a/includes/libs/objectcache/WANObjectCache.php
+++ b/includes/libs/objectcache/WANObjectCache.php
@@ -263,6 +263,8 @@ class WANObjectCache {
        /**
         * Fetch the value of a timestamp "check" key
         *
+        * Note that "check" keys won't collide with other regular keys
+        *
         * @param string $key
         * @return float|bool TS_UNIX timestamp of the key; false if not present
         */
@@ -283,6 +285,8 @@ class WANObjectCache {
         * avoid race conditions where dependent keys get updated with a
         * stale value (e.g. from a DB slave).
         *
+        * Note that "check" keys won't collide with other regular keys
+        *
         * @see WANObjectCache::get()
         *
         * @param string $key Cache key

diff --git a/includes/media/DjVu.php b/includes/media/DjVu.php
index 5b57952..749ef23 100644 (file)
--- a/includes/media/DjVu.php
+++ b/includes/media/DjVu.php
@@ -376,29 +376,55 @@ class DjVuHandler extends ImageHandler {
        }
 
        function pageCount( $image ) {
-               $tree = $this->getMetaTree( $image );
-               if ( !$tree ) {
-                       return false;
+               global $wgMemc;
+
+               $key = wfMemcKey( 'file-djvu', 'pageCount', $image->getSha1() );
+
+               $count = $wgMemc->get( $key );
+               if ( $count === false ) {
+                       $tree = $this->getMetaTree( $image );
+                       if ( !$tree ) {
+                               return false;
+                       }
+                       $count = count( $tree->xpath( '//OBJECT' ) );
+                       $wgMemc->set( $key, $count );
                }
 
-               return count( $tree->xpath( '//OBJECT' ) );
+               return $count;
        }
 
        function getPageDimensions( $image, $page ) {
-               $tree = $this->getMetaTree( $image );
-               if ( !$tree ) {
-                       return false;
-               }
+               global $wgMemc;
 
-               $o = $tree->BODY[0]->OBJECT[$page - 1];
-               if ( $o ) {
-                       return array(
-                               'width' => intval( $o['width'] ),
-                               'height' => intval( $o['height'] )
-                       );
-               } else {
-                       return false;
+               $key = wfMemcKey( 'file-djvu', 'dimensions', $image->getSha1() );
+
+               $dimsByPage = $wgMemc->get( $key );
+               if ( !is_array( $dimsByPage ) ) {
+                       $tree = $this->getMetaTree( $image );
+                       if ( !$tree ) {
+                               return false;
+                       }
+
+                       $dimsByPage = array();
+                       $count = count( $tree->xpath( '//OBJECT' ) );
+                       for ( $i = 0; $i < $count; ++$i ) {
+                               $o = $tree->BODY[0]->OBJECT[$i];
+                               if ( $o ) {
+                                       $dimsByPage[$i] = array(
+                                               'width' => (int)$o['width'],
+                                               'height' => (int)$o['height']
+                                       );
+                               } else {
+                                       $dimsByPage[$i] = false;
+                               }
+                       }
+
+                       $wgMemc->set( $key, $dimsByPage );
                }
+
+               $index = $page - 1; // MW starts pages at 1
+
+               return isset( $dimsByPage[$index] ) ? $dimsByPage[$index] : false;
        }
 
        /**

diff --git a/includes/page/WikiPage.php b/includes/page/WikiPage.php
index cc182a4..7b33b02 100644 (file)
--- a/includes/page/WikiPage.php
+++ b/includes/page/WikiPage.php
@@ -3411,6 +3411,9 @@ class WikiPage implements Page, IDBAccessObject {
 
                // Check if the last link refresh was before page_touched
                if ( $this->getLinksTimestamp() < $this->getTouched() ) {
+                       $params['isOpportunistic'] = true;
+                       $params['rootJobTimestamp'] = $parserOutput->getCacheTime();
+
                        JobQueueGroup::singleton()->push( EnqueueJob::newFromLocalJobs(
                                new JobSpecification( 'refreshLinks', $params,
                                        array( 'removeDuplicates' => true ), $this->mTitle )

diff --git a/includes/resourceloader/ResourceLoader.php b/includes/resourceloader/ResourceLoader.php
index ce18c32..8c9c130 100644 (file)
--- a/includes/resourceloader/ResourceLoader.php
+++ b/includes/resourceloader/ResourceLoader.php
@@ -805,14 +805,14 @@ class ResourceLoader {
                        if ( $this->tryRespondLastModified( $context, $ts ) ) {
                                return false; // output handled (buffers cleared)
                        }
+                       // Send content type and cache headers
+                       $this->sendResponseHeaders( $context, $ts, false );
+                       $response = $fileCache->fetchText();
                        // Capture any PHP warnings from the output buffer and append them to the
                        // response in a comment if we're in debug mode.
                        if ( $context->getDebug() && strlen( $warnings = ob_get_contents() ) ) {
-                               $response = "/*\n$warnings\n*/\n" . $response;
+                               $response = self::makeComment( $warnings ) . $response;
                        }
-                       // Send content type and cache headers
-                       $this->sendResponseHeaders( $context, $ts, false );
-                       $response = $fileCache->fetchText();
                        // Remove the output buffer and output the response
                        ob_end_clean();
                        echo $response . "\n/* Cached {$ts} */";

diff --git a/languages/messages/MessagesIt.php b/languages/messages/MessagesIt.php
index 27bbe78..a369d1d 100644 (file)
--- a/languages/messages/MessagesIt.php
+++ b/languages/messages/MessagesIt.php
@@ -215,7 +215,6 @@ $magicWords = array(
        'displaytitle'              => array( '1', 'MOSTRATITOLO', 'DISPLAYTITLE' ),
        'language'                  => array( '0', '#LINGUA', '#LANGUAGE:' ),
        'numberofadmins'            => array( '1', 'NUMEROADMIN', 'NUMBEROFADMINS' ),
-       'special'                   => array( '0', 'speciale', 'special' ),
        'tag'                       => array( '0', 'etichetta', 'tag' ),
        'pagesincategory'           => array( '1', 'PAGINEINCAT', 'PAGESINCATEGORY', 'PAGESINCAT' ),
        'pagesize'                  => array( '1', 'DIMENSIONEPAGINA', 'PESOPAGINA', 'PAGESIZE' ),

diff --git a/languages/messages/MessagesSq.php b/languages/messages/MessagesSq.php
index 987464a..71180c3 100644 (file)
--- a/languages/messages/MessagesSq.php
+++ b/languages/messages/MessagesSq.php
@@ -195,7 +195,6 @@ $magicWords = array(
        'fullurl'                   => array( '0', 'URLEPLOTË', 'FULLURL:' ),
        'language'                  => array( '0', '#GJUHA:', '#LANGUAGE:' ),
        'numberofadmins'            => array( '1', 'NUMRIIADMINISTRUESVE', 'NUMBEROFADMINS' ),
-       'special'                   => array( '0', 'speciale', 'special' ),
        'hiddencat'                 => array( '1', '__KATEGORIEFSHEHUR__', '__HIDDENCAT__' ),
        'pagesize'                  => array( '1', 'MADHËSIAEFAQES', 'PAGESIZE' ),
 );