From 6477026675a3da444017e4ca4464638791be7e3c Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Fri, 9 Dec 2016 23:26:34 -0800 Subject: [PATCH] Back off from job types longer for DB read-only errors Such error are likely to persist longer than other random exceptions. In that case, it is better to avoid burning through the job retry count. Change-Id: I6785bd608856f98d21e0b0b05d3899a7081c38e2 --- includes/jobqueue/JobRunner.php | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/includes/jobqueue/JobRunner.php b/includes/jobqueue/JobRunner.php index 990f112a60..cacccbec08 100644 --- a/includes/jobqueue/JobRunner.php +++ b/includes/jobqueue/JobRunner.php @@ -46,6 +46,7 @@ class JobRunner implements LoggerAwareInterface { const MAX_ALLOWED_LAG = 3; // abort if more than this much DB lag is present const LAG_CHECK_PERIOD = 1.0; // check replica DB lag this many seconds const ERROR_BACKOFF_TTL = 1; // seconds to back off a queue due to errors + const READONLY_BACKOFF_TTL = 30; // seconds to back off a queue due to read-only errors /** * @param callable $debug Optional debug output handler @@ -190,7 +191,7 @@ class JobRunner implements LoggerAwareInterface { // Back off of certain jobs for a while (for throttling and for errors) if ( $info['status'] === false && mt_rand( 0, 49 ) == 0 ) { - $ttw = max( $ttw, self::ERROR_BACKOFF_TTL ); // too many errors + $ttw = max( $ttw, $this->getErrorBackoffTTL( $info['error'] ) ); $backoffDeltas[$jType] = isset( $backoffDeltas[$jType] ) ? $backoffDeltas[$jType] + $ttw : $ttw; @@ -253,6 +254,16 @@ class JobRunner implements LoggerAwareInterface { return $response; } + /** + * @param string $error + * @return int TTL in seconds + */ + private function getErrorBackoffTTL( $error ) { + return strpos( $error, 'DBReadOnlyError' ) !== false + ? self::READONLY_BACKOFF_TTL + : self::ERROR_BACKOFF_TTL; + } + /** * @param Job $job * @param LBFactory $lbFactory -- 2.20.1