From 8e3721a2b95c83fd7dd2c4026bc12e6ecc0c711f Mon Sep 17 00:00:00 2001
From: Aaron Schulz <aschulz@wikimedia.org>
Date: Wed, 4 Mar 2015 13:53:40 -0800
Subject: [PATCH] Added EnqueueJob class that handles routing jobs to queues

Bug: T89308
Change-Id: Iadb34f24d8bbe94c0f9f119e530c0bbe1060df0a
---
 autoload.php                                  |  1 +
 includes/DefaultSettings.php                  |  1 +
 includes/jobqueue/JobSpecification.php        | 36 +++++++-
 includes/jobqueue/jobs/DuplicateJob.php       |  2 +-
 includes/jobqueue/jobs/EnqueueJob.php         | 88 +++++++++++++++++++
 includes/jobqueue/jobs/HTMLCacheUpdateJob.php |  1 +
 includes/jobqueue/jobs/NullJob.php            |  2 +-
 .../jobqueue/jobs/PublishStashedFileJob.php   |  2 +
 .../jobqueue/jobs/RecentChangesUpdateJob.php  |  1 +
 9 files changed, 128 insertions(+), 6 deletions(-)
 create mode 100755 includes/jobqueue/jobs/EnqueueJob.php

diff --git a/autoload.php b/autoload.php
index 998deb92b7..faf8252481 100644
--- a/autoload.php
+++ b/autoload.php
@@ -370,6 +370,7 @@ $wgAutoloadLocalClasses = array(
 	'EncryptedPassword' => __DIR__ . '/includes/password/EncryptedPassword.php',
 	'EnhancedChangesList' => __DIR__ . '/includes/changes/EnhancedChangesList.php',
 	'EnotifNotifyJob' => __DIR__ . '/includes/jobqueue/jobs/EnotifNotifyJob.php',
+	'EnqueueJob' => __DIR__ . '/includes/jobqueue/jobs/EnqueueJob.php',
 	'EraseArchivedFile' => __DIR__ . '/maintenance/eraseArchivedFile.php',
 	'ErrorPageError' => __DIR__ . '/includes/exception/ErrorPageError.php',
 	'Exif' => __DIR__ . '/includes/media/Exif.php',
diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php
index 89cc1fd4f7..bafec8d212 100644
--- a/includes/DefaultSettings.php
+++ b/includes/DefaultSettings.php
@@ -6425,6 +6425,7 @@ $wgJobClasses = array(
 	'ThumbnailRender' => 'ThumbnailRenderJob',
 	'recentChangesUpdate' => 'RecentChangesUpdateJob',
 	'refreshLinksPrioritized' => 'RefreshLinksJob', // for cascading protection
+	'enqueue' => 'EnqueueJob',
 	'null' => 'NullJob'
 );
 
diff --git a/includes/jobqueue/JobSpecification.php b/includes/jobqueue/JobSpecification.php
index 9fa7747ffc..42d2a39b8e 100644
--- a/includes/jobqueue/JobSpecification.php
+++ b/includes/jobqueue/JobSpecification.php
@@ -91,8 +91,8 @@ class JobSpecification implements IJobSpecification {
 	/** @var Title */
 	protected $title;
 
-	/** @var bool Expensive jobs may set this to true */
-	protected $ignoreDuplicates;
+	/** @var array */
+	protected $opts;
 
 	/**
 	 * @param string $type
@@ -104,11 +104,12 @@ class JobSpecification implements IJobSpecification {
 		$type, array $params, array $opts = array(), Title $title = null
 	) {
 		$this->validateParams( $params );
+		$this->validateParams( $opts );
 
 		$this->type = $type;
 		$this->params = $params;
 		$this->title = $title ?: Title::newMainPage();
-		$this->ignoreDuplicates = !empty( $opts['removeDuplicates'] );
+		$this->opts = $opts;
 	}
 
 	/**
@@ -158,7 +159,7 @@ class JobSpecification implements IJobSpecification {
 	 * @return bool Whether only one of each identical set of jobs should be run
 	 */
 	public function ignoreDuplicates() {
-		return $this->ignoreDuplicates;
+		return !empty( $this->opts['removeDuplicates'] );
 	}
 
 	/**
@@ -186,4 +187,31 @@ class JobSpecification implements IJobSpecification {
 
 		return $info;
 	}
+
+	/**
+	 * @return array Field/value map that can immediately be serialized
+	 * @since 1.25
+	 */
+	public function toSerializableArray() {
+		return array(
+			'type'   => $this->type,
+			'params' => $this->params,
+			'opts'   => $this->opts,
+			'title'  => array(
+				'ns'  => $this->title->getNamespace(),
+				'key' => $this->title->getDbKey()
+			)
+		);
+	}
+
+	/**
+	 * @param array $map Field/value map
+	 * @return JobSpecification
+	 * @since 1.25
+	 */
+	public static function newFromArray( array $map ) {
+		$title = Title::makeTitle( $map['title']['ns'], $map['title']['key'] );
+
+		return new self( $map['type'], $map['params'], $map['opts'], $title );
+	}
 }
diff --git a/includes/jobqueue/jobs/DuplicateJob.php b/includes/jobqueue/jobs/DuplicateJob.php
index 1fa6cefedf..c5e3a23401 100644
--- a/includes/jobqueue/jobs/DuplicateJob.php
+++ b/includes/jobqueue/jobs/DuplicateJob.php
@@ -18,7 +18,7 @@
  * http://www.gnu.org/copyleft/gpl.html
  *
  * @file
- * @ingroup Cache
+ * @ingroup JobQueue
  */
 
 /**
diff --git a/includes/jobqueue/jobs/EnqueueJob.php b/includes/jobqueue/jobs/EnqueueJob.php
new file mode 100755
index 0000000000..46fb2aa74e
--- /dev/null
+++ b/includes/jobqueue/jobs/EnqueueJob.php
@@ -0,0 +1,88 @@
+<?php
+/**
+ * Router job that takes jobs and enqueues them.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup JobQueue
+ */
+
+/**
+ * Router job that takes jobs and enqueues them to their proper queues
+ *
+ * This can be used for several things:
+ *   - a) Making multi-job enqueues more robust by atomically enqueueing
+ *        a single job that pushes the actual jobs (with retry logic)
+ *   - b) Masking the latency of pushing jobs to different queues/wikis
+ *   - c) Low-latency enqueues to push jobs from warm to hot datacenters
+ *
+ * @ingroup JobQueue
+ * @since 1.25
+ */
+final class EnqueueJob extends Job {
+	/**
+	 * Callers should use the factory methods instead
+	 *
+	 * @param Title $title
+	 * @param array $params Job parameters
+	 */
+	function __construct( $title, $params ) {
+		parent::__construct( 'enqueue', $title, $params );
+	}
+
+	/**
+	 * @param Job|JobSpecification|array $jobs
+	 * @return JobRouteJob
+	 */
+	public static function newFromLocalJobs( $jobs ) {
+		$jobs = is_array( $jobs ) ? $jobs : array( $jobs );
+
+		return self::newFromJobsByWiki( array( wfWikiID() => $jobs ) );
+	}
+
+	/**
+	 * @param array $jobsByWiki Map of (wiki => JobSpecification list)
+	 * @return JobRouteJob
+	 */
+	public static function newFromJobsByWiki( array $jobsByWiki ) {
+		$jobMapsByWiki = array();
+		foreach ( $jobsByWiki as $wiki => $jobs ) {
+			$jobMapsByWiki[$wiki] = array();
+			foreach ( $jobs as $job ) {
+				if ( $job instanceof JobSpecification ) {
+					$jobMapsByWiki[$wiki][] = $job->toSerializableArray();
+				} else {
+					throw new InvalidArgumentException( "Jobs must be of type JobSpecification." );
+				}
+			}
+		}
+
+		return new self( Title::newMainPage(), array( 'jobsByWiki' => $jobMapsByWiki ) );
+	}
+
+	public function run() {
+		foreach ( $this->params['jobsByWiki'] as $wiki => $jobMaps ) {
+			$jobSpecs = array();
+			foreach ( $jobMaps as $jobMap ) {
+				$jobSpecs[] = JobSpecification::newFromArray( $jobMap );
+			}
+			JobQueueGroup::singleton( $wiki )->push( $jobSpecs );
+		}
+
+		return true;
+	}
+}
diff --git a/includes/jobqueue/jobs/HTMLCacheUpdateJob.php b/includes/jobqueue/jobs/HTMLCacheUpdateJob.php
index 9d91565ceb..e5e521c3c0 100644
--- a/includes/jobqueue/jobs/HTMLCacheUpdateJob.php
+++ b/includes/jobqueue/jobs/HTMLCacheUpdateJob.php
@@ -18,6 +18,7 @@
  * http://www.gnu.org/copyleft/gpl.html
  *
  * @file
+ * @ingroup JobQueue
  * @ingroup Cache
  */
 
diff --git a/includes/jobqueue/jobs/NullJob.php b/includes/jobqueue/jobs/NullJob.php
index 66291e9d8d..f94d6ebc85 100644
--- a/includes/jobqueue/jobs/NullJob.php
+++ b/includes/jobqueue/jobs/NullJob.php
@@ -18,7 +18,7 @@
  * http://www.gnu.org/copyleft/gpl.html
  *
  * @file
- * @ingroup Cache
+ * @ingroup JobQueue
  */
 
 /**
diff --git a/includes/jobqueue/jobs/PublishStashedFileJob.php b/includes/jobqueue/jobs/PublishStashedFileJob.php
index 3d4cfae7fe..a922dd3d11 100644
--- a/includes/jobqueue/jobs/PublishStashedFileJob.php
+++ b/includes/jobqueue/jobs/PublishStashedFileJob.php
@@ -19,12 +19,14 @@
  *
  * @file
  * @ingroup Upload
+ * @ingroup JobQueue
  */
 
 /**
  * Upload a file from the upload stash into the local file repo.
  *
  * @ingroup Upload
+ * @ingroup JobQueue
  */
 class PublishStashedFileJob extends Job {
 	public function __construct( $title, $params ) {
diff --git a/includes/jobqueue/jobs/RecentChangesUpdateJob.php b/includes/jobqueue/jobs/RecentChangesUpdateJob.php
index fb10c15467..b550f50b2c 100644
--- a/includes/jobqueue/jobs/RecentChangesUpdateJob.php
+++ b/includes/jobqueue/jobs/RecentChangesUpdateJob.php
@@ -17,6 +17,7 @@
  *
  * @file
  * @author Aaron Schulz
+ * @ingroup JobQueue
  */
 
 /**
-- 
2.20.1