From: Aaron Schulz <aschulz@wikimedia.org>
Date: Mon, 30 Nov 2015 22:02:53 +0000 (-0800)
Subject: Add pre-send update support to DeferredUpdates
X-Git-Tag: 1.31.0-rc.0~8816^2
X-Git-Url: http://git.cyclocoop.org//%22%22.url_de_base%28%29.%22/%22?a=commitdiff_plain;h=34dd248e285abbfab09f124ab7803b40101c5505;p=lhc%2Fweb%2Fwiklou.git

Add pre-send update support to DeferredUpdates

* PRESEND/POSTSEND constants can now be used in addUpdate()
  and addCallableUpdate() to control when the update runs.
  This is useful for updates that may report errors the client
  should see or to just get a head start on queued or pubsub
  based updates like CDN purges. The OutputPage::output() method
  can easily take a few 100ms.
* Removed some argument b/c code from doUpdates().
* Also moved DeferrableUpdate to a separate file.

Change-Id: I9831fe890f9f68f9ad8c4f4bba6921a8f29ba666
---

diff --git a/autoload.php b/autoload.php
index 84bfe0b562..f79e58b4c2 100644
--- a/autoload.php
+++ b/autoload.php
@@ -164,8 +164,8 @@ $wgAutoloadLocalClasses = array(
 	'BenchIfSwitch' => __DIR__ . '/maintenance/benchmarks/bench_if_switch.php',
 	'BenchStrtrStrReplace' => __DIR__ . '/maintenance/benchmarks/bench_strtr_str_replace.php',
 	'BenchUtf8TitleCheck' => __DIR__ . '/maintenance/benchmarks/bench_utf8_title_check.php',
-	'BenchWikimediaBaseConvert' => __DIR__ . '/maintenance/benchmarks/bench_Wikimedia_base_convert.php',
 	'BenchWfIsWindows' => __DIR__ . '/maintenance/benchmarks/bench_wfIsWindows.php',
+	'BenchWikimediaBaseConvert' => __DIR__ . '/maintenance/benchmarks/bench_Wikimedia_base_convert.php',
 	'BenchmarkDeleteTruncate' => __DIR__ . '/maintenance/benchmarks/bench_delete_truncate.php',
 	'BenchmarkHooks' => __DIR__ . '/maintenance/benchmarks/benchmarkHooks.php',
 	'BenchmarkParse' => __DIR__ . '/maintenance/benchmarks/benchmarkParse.php',
@@ -307,7 +307,7 @@ $wgAutoloadLocalClasses = array(
 	'DateFormats' => __DIR__ . '/maintenance/language/date-formats.php',
 	'DateFormatter' => __DIR__ . '/includes/parser/DateFormatter.php',
 	'DeadendPagesPage' => __DIR__ . '/includes/specials/SpecialDeadendpages.php',
-	'DeferrableUpdate' => __DIR__ . '/includes/deferred/DeferredUpdates.php',
+	'DeferrableUpdate' => __DIR__ . '/includes/deferred/DeferrableUpdate.php',
 	'DeferredStringifier' => __DIR__ . '/includes/libs/DeferredStringifier.php',
 	'DeferredUpdates' => __DIR__ . '/includes/deferred/DeferredUpdates.php',
 	'DeleteAction' => __DIR__ . '/includes/actions/DeleteAction.php',
diff --git a/includes/MediaWiki.php b/includes/MediaWiki.php
index 38d9e476f4..ecfd8f87aa 100644
--- a/includes/MediaWiki.php
+++ b/includes/MediaWiki.php
@@ -509,8 +509,10 @@ class MediaWiki {
 		$factory = wfGetLBFactory();
 		$factory->commitMasterChanges();
 		$factory->shutdown();
+		wfDebug( __METHOD__ . ': all transactions committed' );
 
-		wfDebug( __METHOD__ . ' completed; all transactions committed' );
+		DeferredUpdates::doUpdates( 'enqueue', DeferredUpdates::PRESEND );
+		wfDebug( __METHOD__ . ': pre-send deferred updates completed' );
 
 		// Set a cookie to tell all CDN edge nodes to "stick" the user to the
 		// DC that handles this POST request (e.g. the "master" data center)
diff --git a/includes/deferred/DeferrableUpdate.php b/includes/deferred/DeferrableUpdate.php
new file mode 100644
index 0000000000..5f4d821060
--- /dev/null
+++ b/includes/deferred/DeferrableUpdate.php
@@ -0,0 +1,14 @@
+<?php
+
+/**
+ * Interface that deferrable updates should implement. Basically required so we
+ * can validate input on DeferredUpdates::addUpdate()
+ *
+ * @since 1.19
+ */
+interface DeferrableUpdate {
+	/**
+	 * Perform the actual work
+	 */
+	function doUpdate();
+}
diff --git a/includes/deferred/DeferredUpdates.php b/includes/deferred/DeferredUpdates.php
index 8eec202c9d..3323c04d4f 100644
--- a/includes/deferred/DeferredUpdates.php
+++ b/includes/deferred/DeferredUpdates.php
@@ -20,50 +20,89 @@
  * @file
  */
 
-/**
- * Interface that deferrable updates should implement. Basically required so we
- * can validate input on DeferredUpdates::addUpdate()
- *
- * @since 1.19
- */
-interface DeferrableUpdate {
-	/**
-	 * Perform the actual work
-	 */
-	function doUpdate();
-}
-
 /**
  * Class for managing the deferred updates
  *
- * Deferred updates can be run at the end of the request,
- * after the HTTP response has been sent. In CLI mode, updates
- * are only deferred until there is no local master DB transaction.
- * When updates are deferred, they go into a simple FIFO queue.
+ * In web request mode, deferred updates can be run at the end of the request, either before or
+ * after the HTTP response has been sent. In either case, they run after the DB commit step. If
+ * an update runs after the response is sent, it will not block clients. If sent before, it will
+ * run synchronously. If such an update works via queueing, it will be more likely to complete by
+ * the time the client makes their next request after this one.
+ *
+ * In CLI mode, updates are only deferred until the current wiki has no DB write transaction
+ * active within this request.
+ *
+ * When updates are deferred, they use a FIFO queue (one for pre-send and one for post-send).
  *
  * @since 1.19
  */
 class DeferredUpdates {
-	/** @var DeferrableUpdate[] Updates to be deferred until the end of the request */
-	private static $updates = array();
+	/** @var DeferrableUpdate[] Updates to be deferred until before request end */
+	private static $preSendUpdates = array();
+	/** @var DeferrableUpdate[] Updates to be deferred until after request end */
+	private static $postSendUpdates = array();
+
 	/** @var bool Defer updates fully even in CLI mode */
 	private static $forceDeferral = false;
 
+	const ALL = 0; // all updates
+	const PRESEND = 1; // for updates that should run before flushing output buffer
+	const POSTSEND = 2; // for updates that should run after flushing output buffer
+
 	/**
 	 * Add an update to the deferred list
+	 *
 	 * @param DeferrableUpdate $update Some object that implements doUpdate()
+	 * @param integer $type DeferredUpdates constant (PRESEND or POSTSEND) (since 1.27)
 	 */
-	public static function addUpdate( DeferrableUpdate $update ) {
+	public static function addUpdate( DeferrableUpdate $update, $type = self::POSTSEND ) {
+		if ( $type === self::PRESEND ) {
+			self::push( self::$preSendUpdates, $update );
+		} else {
+			self::push( self::$postSendUpdates, $update );
+		}
+	}
+
+	/**
+	 * Add a callable update.  In a lot of cases, we just need a callback/closure,
+	 * defining a new DeferrableUpdate object is not necessary
+	 *
+	 * @see MWCallableUpdate::__construct()
+	 *
+	 * @param callable $callable
+	 * @param integer $type DeferredUpdates constant (PRESEND or POSTSEND) (since 1.27)
+	 */
+	public static function addCallableUpdate( $callable, $type = self::POSTSEND ) {
+		self::addUpdate( new MWCallableUpdate( $callable ), $type );
+	}
+
+	/**
+	 * Do any deferred updates and clear the list
+	 *
+	 * @param string $mode Use "enqueue" to use the job queue when possible [Default: "run"]
+	 * @param integer $type DeferredUpdates constant (PRESEND, POSTSEND, or ALL) (since 1.27)
+	 */
+	public static function doUpdates( $mode = 'run', $type = self::ALL ) {
+		if ( $type === self::ALL || $type == self::PRESEND ) {
+			self::execute( self::$preSendUpdates, $mode );
+		}
+
+		if ( $type === self::ALL || $type == self::POSTSEND ) {
+			self::execute( self::$postSendUpdates, $mode );
+		}
+	}
+
+	private static function push( array &$queue, DeferrableUpdate $update ) {
 		global $wgCommandLineMode;
 
-		array_push( self::$updates, $update );
+		array_push( $queue, $update );
 		if ( self::$forceDeferral ) {
-			return;
+			return; // do not run
 		}
 
 		// CLI scripts may forget to periodically flush these updates,
-		// so try to handle that rather than OOMing and losing them.
-		// Try to run the updates as soon as there is no local transaction.
+		// so try to handle that rather than OOMing and losing them entirely.
+		// Try to run the updates as soon as there is no current wiki transaction.
 		static $waitingOnTrx = false; // de-duplicate callback
 		if ( $wgCommandLineMode && !$waitingOnTrx ) {
 			$lb = wfGetLB();
@@ -81,34 +120,12 @@ class DeferredUpdates {
 		}
 	}
 
-	/**
-	 * Add a callable update.  In a lot of cases, we just need a callback/closure,
-	 * defining a new DeferrableUpdate object is not necessary
-	 * @see MWCallableUpdate::__construct()
-	 * @param callable $callable
-	 */
-	public static function addCallableUpdate( $callable ) {
-		self::addUpdate( new MWCallableUpdate( $callable ) );
-	}
-
-	/**
-	 * Do any deferred updates and clear the list
-	 *
-	 * @param string $mode Use "enqueue" to use the job queue when possible [Default: run]
-	 *   prevent lock contention
-	 * @param string $oldMode Unused
-	 */
-	public static function doUpdates( $mode = 'run', $oldMode = '' ) {
-		// B/C for ( $commit, $mode ) args
-		$mode = $oldMode ?: $mode;
-		if ( $mode === 'commit' ) {
-			$mode = 'run';
-		}
-
-		$updates = self::$updates;
+	public static function execute( array &$queue, $mode ) {
+		$updates = $queue; // snapshot of queue
 
+		// Keep doing rounds of updates until none get enqueued
 		while ( count( $updates ) ) {
-			self::clearPendingUpdates();
+			$queue = array(); // clear the queue
 			/** @var DataUpdate[] $dataUpdates */
 			$dataUpdates = array();
 			/** @var DeferrableUpdate[] $otherUpdates */
@@ -140,7 +157,7 @@ class DeferredUpdates {
 				}
 			}
 
-			$updates = self::$updates;
+			$updates = $queue; // new snapshot of queue (check for new entries)
 		}
 	}
 
@@ -149,7 +166,8 @@ class DeferredUpdates {
 	 * want or need to call this. Unit tests need it though.
 	 */
 	public static function clearPendingUpdates() {
-		self::$updates = array();
+		self::$preSendUpdates = array();
+		self::$postSendUpdates = array();
 	}
 
 	/**
diff --git a/tests/phpunit/includes/deferred/DeferredUpdatesTest.php b/tests/phpunit/includes/deferred/DeferredUpdatesTest.php
index df4213abdf..42e48ff933 100644
--- a/tests/phpunit/includes/deferred/DeferredUpdatesTest.php
+++ b/tests/phpunit/includes/deferred/DeferredUpdatesTest.php
@@ -1,5 +1,8 @@
 <?php
 
+/**
+ * @group DeferredUpdates
+ */
 class DeferredUpdatesTest extends MediaWikiTestCase {
 	public function testDoUpdatesWeb() {
 		$this->setMwGlobals( 'wgCommandLineMode', false );
@@ -34,6 +37,31 @@ class DeferredUpdatesTest extends MediaWikiTestCase {
 		$this->expectOutputString( implode( '', $updates ) );
 
 		DeferredUpdates::doUpdates();
+
+		$x = null;
+		$y = null;
+		DeferredUpdates::addCallableUpdate(
+			function () use ( &$x ) {
+				$x = 'Sherity';
+			},
+			DeferredUpdates::PRESEND
+		);
+		DeferredUpdates::addCallableUpdate(
+			function () use ( &$y ) {
+				$y = 'Marychu';
+			},
+			DeferredUpdates::POSTSEND
+		);
+
+		$this->assertNull( $x, "Update not run yet" );
+		$this->assertNull( $y, "Update not run yet" );
+
+		DeferredUpdates::doUpdates( 'run', DeferredUpdates::PRESEND );
+		$this->assertEquals( "Sherity", $x, "PRESEND update ran" );
+		$this->assertNull( $y, "POSTSEND update not run yet" );
+
+		DeferredUpdates::doUpdates( 'run', DeferredUpdates::POSTSEND );
+		$this->assertEquals( "Marychu", $y, "POSTSEND update ran" );
 	}
 
 	public function testDoUpdatesCLI() {