From: Aaron Schulz <aschulz@wikimedia.org>
Date: Fri, 13 Nov 2015 06:41:37 +0000 (-0800)
Subject: Add $wgCdnReboundPurgeDelay for more consistent CDN purges
X-Git-Tag: 1.31.0-rc.0~8738
X-Git-Url: http://git.cyclocoop.org/data/%24self?a=commitdiff_plain;h=01c2b0a4255f90805bee878bdd0432e5e3733d57;p=lhc%2Fweb%2Fwiklou.git

Add $wgCdnReboundPurgeDelay for more consistent CDN purges

Bug: T113192
Change-Id: I89deb4f8143c1cc6154cdd05bcee1f49d3e3a75a
---

diff --git a/RELEASE-NOTES-1.27 b/RELEASE-NOTES-1.27
index b41b8c2ae0..ab136b1f93 100644
--- a/RELEASE-NOTES-1.27
+++ b/RELEASE-NOTES-1.27
@@ -97,6 +97,8 @@ production.
 * $wgMaxUserDBWriteDuration added to limit huge user-generated transactions.
   Regular web request transactions that takes longer than this are aborted.
 * Added a new hook, 'TitleMoveCompleting', which runs before a page move is committed.
+* $wgCdnReboundPurgeDelay was added to provide secondary delayed purges of URLs
+  from CDN to mitigate DB replication lag and WAN cache purge lag.
 
 === External library changes in 1.27 ===
 ==== Upgraded external libraries ====
diff --git a/autoload.php b/autoload.php
index fce9cd4135..dea31abe19 100644
--- a/autoload.php
+++ b/autoload.php
@@ -200,6 +200,7 @@ $wgAutoloadLocalClasses = array(
 	'CdbReader' => __DIR__ . '/includes/compat/CdbCompat.php',
 	'CdbWriter' => __DIR__ . '/includes/compat/CdbCompat.php',
 	'CdnCacheUpdate' => __DIR__ . '/includes/deferred/CdnCacheUpdate.php',
+	'CdnPurgeJob' => __DIR__ . '/includes/jobqueue/jobs/CdnPurgeJob.php',
 	'CentralIdLookup' => __DIR__ . '/includes/user/CentralIdLookup.php',
 	'CgzCopyTransaction' => __DIR__ . '/maintenance/storage/recompressTracked.php',
 	'ChangePassword' => __DIR__ . '/maintenance/changePassword.php',
diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php
index 236a3f9bdb..bf8567882c 100644
--- a/includes/DefaultSettings.php
+++ b/includes/DefaultSettings.php
@@ -2600,6 +2600,15 @@ $wgSquidMaxage = 18000;
  */
 $wgCdnMaxageLagged = 30;
 
+/**
+ * If set, any SquidPurge call on a URL or URLs will send a second purge no less than
+ * this many seconds later via the job queue. This requires delayed job support.
+ * This should be safely higher than the 'max lag' value in $wgLBFactoryConf.
+ *
+ * @since 1.27
+ */
+$wgCdnReboundPurgeDelay = 0;
+
 /**
  * Default maximum age for raw CSS/JS accesses
  *
@@ -6689,10 +6698,11 @@ $wgJobClasses = array(
 	'PublishStashedFile' => 'PublishStashedFileJob',
 	'ThumbnailRender' => 'ThumbnailRenderJob',
 	'recentChangesUpdate' => 'RecentChangesUpdateJob',
-	'refreshLinksPrioritized' => 'RefreshLinksJob', // for cascading protection
-	'refreshLinksDynamic' => 'RefreshLinksJob', // for pages with dynamic content
+	'refreshLinksPrioritized' => 'RefreshLinksJob',
+	'refreshLinksDynamic' => 'RefreshLinksJob',
 	'activityUpdateJob' => 'ActivityUpdateJob',
 	'categoryMembershipChange' => 'CategoryMembershipChangeJob',
+	'cdnPurge' => 'CdnPurgeJob',
 	'enqueue' => 'EnqueueJob', // local queue for multi-DC setups
 	'null' => 'NullJob'
 );
diff --git a/includes/deferred/CdnCacheUpdate.php b/includes/deferred/CdnCacheUpdate.php
index bc2b05cd98..25c27e3fb8 100644
--- a/includes/deferred/CdnCacheUpdate.php
+++ b/includes/deferred/CdnCacheUpdate.php
@@ -38,6 +38,13 @@ class CdnCacheUpdate implements DeferrableUpdate, MergeableUpdate {
 		$this->urls = $urlArr;
 	}
 
+	public function merge( MergeableUpdate $update ) {
+		/** @var CdnCacheUpdate $update */
+		Assert::parameterType( __CLASS__, $update, '$update' );
+
+		$this->urls = array_merge( $this->urls, $update->urls );
+	}
+
 	/**
 	 * Create an update object from an array of Title objects, or a TitleArray object
 	 *
@@ -67,14 +74,19 @@ class CdnCacheUpdate implements DeferrableUpdate, MergeableUpdate {
 	 * Purges the list of URLs passed to the constructor.
 	 */
 	public function doUpdate() {
-		self::purge( $this->urls );
-	}
+		global $wgCdnReboundPurgeDelay;
 
-	public function merge( MergeableUpdate $update ) {
-		/** @var CdnCacheUpdate $update */
-		Assert::parameterType( __CLASS__, $update, '$update' );
+		self::purge( $this->urls );
 
-		$this->urls = array_merge( $this->urls, $update->urls );
+		if ( $wgCdnReboundPurgeDelay > 0 ) {
+			JobQueueGroup::singleton()->lazyPush( new CdnPurgeJob(
+				Title::makeTitle( NS_SPECIAL, 'Badtitle/' . __CLASS__ ),
+				array(
+					'urls' => $this->urls,
+					'jobReleaseTimestamp' => time() + $wgCdnReboundPurgeDelay
+				)
+			) );
+		}
 	}
 
 	/**
diff --git a/includes/jobqueue/jobs/CdnPurgeJob.php b/includes/jobqueue/jobs/CdnPurgeJob.php
new file mode 100644
index 0000000000..356eebab6c
--- /dev/null
+++ b/includes/jobqueue/jobs/CdnPurgeJob.php
@@ -0,0 +1,46 @@
+<?php
+/**
+ * Job to purge a set of URLs from CDN
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup JobQueue
+ */
+
+/**
+ * Job to purge a set of URLs from CDN
+ *
+ * @ingroup JobQueue
+ * @since 1.27
+ */
+class CdnPurgeJob extends Job {
+	/**
+	 * @param Title $title
+	 * @param array $params Job parameters (urls)
+	 */
+	function __construct( Title $title, array $params ) {
+		parent::__construct( 'cdnPurge', $title, $params );
+		$this->removeDuplicates = false; // delay semantics are critical
+	}
+
+	public function run() {
+		// Use purge() directly to avoid infinite recursion
+		CdnCacheUpdate::purge( $this->params['urls'] );
+
+		return true;
+	}
+}