From d9bf616a444d0d33dcf336cc91d3ce5262d7800c Mon Sep 17 00:00:00 2001 From: Tim Starling Date: Fri, 27 Feb 2009 04:18:34 +0000 Subject: [PATCH] * Added restart-on-error feature to ForkController, changed the interface to suit. * Close some sockets before pcntl_exec() in an attempt to delay file handle exhaustion. --- includes/ForkController.php | 140 ++++++++++++++++++-------- maintenance/gearman/gearman.inc | 5 + maintenance/gearman/gearmanWorker.php | 5 +- maintenance/runJobs.php | 5 +- 4 files changed, 107 insertions(+), 48 deletions(-) diff --git a/includes/ForkController.php b/includes/ForkController.php index 689b9c8de2..b5f1f18583 100644 --- a/includes/ForkController.php +++ b/includes/ForkController.php @@ -4,15 +4,110 @@ * Class for managing forking command line scripts. * Currently just does forking and process control, but it could easily be extended * to provide IPC and job dispatch. + * + * This class requires the posix and pcntl extensions. */ class ForkController { var $children = array(); var $termReceived = false; + var $flags = 0, $procsToStart = 0; - public function __construct() { + static $restartableSignals = array( + SIGFPE, + SIGILL, + SIGSEGV, + SIGBUS, + SIGABRT, + SIGSYS, + SIGPIPE, + SIGXCPU, + SIGXFSZ, + ); + + /** + * Pass this flag to __construct() to cause the class to automatically restart + * workers that exit with non-zero exit status or a signal such as SIGSEGV. + */ + const RESTART_ON_ERROR = 1; + + public function __construct( $numProcs, $flags = 0 ) { if ( php_sapi_name() != 'cli' ) { throw new MWException( "MultiProcess cannot be used from the web." ); } + $this->procsToStart = $numProcs; + $this->flags = $flags; + } + + /** + * Start the child processes. + * + * This should only be called from the command line. It should be called + * as early as possible during execution. + * + * This will return 'child' in the child processes. In the parent process, + * it will run until all the child processes exit or a TERM signal is + * received. It will then return 'done'. + */ + public function start() { + // Trap SIGTERM + pcntl_signal( SIGTERM, array( $this, 'handleTermSignal' ), false ); + + do { + // Start child processes + if ( $this->procsToStart ) { + if ( $this->forkWorkers( $this->procsToStart ) == 'child' ) { + return 'child'; + } + $this->procsToStart = 0; + } + + // Check child status + $status = false; + $deadPid = pcntl_wait( $status ); + + if ( $deadPid > 0 ) { + // Respond to child process termination + unset( $this->children[$deadPid] ); + if ( $this->flags & self::RESTART_ON_ERROR ) { + if ( pcntl_wifsignaled( $status ) ) { + // Restart if the signal was abnormal termination + // Don't restart if it was deliberately killed + $signal = pcntl_wtermsig( $status ); + if ( in_array( $signal, self::$restartableSignals ) ) { + echo "Worker exited with signal $signal, restarting\n"; + $this->procsToStart++; + } + } elseif ( pcntl_wifexited( $status ) ) { + // Restart on non-zero exit status + $exitStatus = pcntl_wexitstatus( $status ); + if ( $exitStatus > 0 ) { + echo "Worker exited with status $exitStatus, restarting\n"; + $this->procsToStart++; + } + } + } + // Throttle restarts + if ( $this->procsToStart ) { + usleep( 500000 ); + } + } + + // Run signal handlers + if ( function_exists( 'pcntl_signal_dispatch' ) ) { + pcntl_signal_dispatch(); + } else { + declare (ticks=1) { $status = $status; } + } + // Respond to TERM signal + if ( $this->termReceived ) { + foreach ( $this->children as $childPid => $unused ) { + posix_kill( $childPid, SIGTERM ); + } + $this->termReceived = false; + } + } while ( count( $this->children ) ); + pcntl_signal( SIGTERM, SIG_DFL ); + return 'done'; } protected function prepareEnvironment() { @@ -25,15 +120,8 @@ class ForkController { /** * Fork a number of worker processes. - * - * This should only be called from the command line. It should be called - * as early as possible during execution. It will return 'child' in the - * child processes and 'parent' in the parent process. The parent process - * should then call monitor(). - * - * This function requires the posix and pcntl extensions. */ - public function forkWorkers( $numProcs ) { + protected function forkWorkers( $numProcs ) { global $wgMemc, $wgCaches, $wgMainCacheType; $this->prepareEnvironment(); @@ -49,7 +137,6 @@ class ForkController { if ( !$pid ) { $this->initChild(); - $this->children = null; return 'child'; } else { // This is the parent process @@ -60,41 +147,10 @@ class ForkController { return 'parent'; } - /** - * The parent process main loop - */ - public function runParent() { - // Trap SIGTERM - pcntl_signal( SIGTERM, array( $this, 'handleTermSignal' ), false ); - - do { - $status = false; - $deadPid = pcntl_wait( $status ); - - if ( $deadPid > 0 ) { - unset( $this->children[$deadPid] ); - } - - // Run signal handlers - if ( function_exists( 'pcntl_signal_dispatch' ) ) { - pcntl_signal_dispatch(); - } else { - declare (ticks=1) { $status = $status; } - } - // Respond to TERM signal - if ( $this->termReceived ) { - foreach ( $this->children as $childPid => $unused ) { - posix_kill( $childPid, SIGTERM ); - } - $this->termReceived = false; - } - } while ( count( $this->children ) ); - pcntl_signal( SIGTERM, SIG_DFL ); - } - protected function initChild() { global $wgMemc, $wgMainCacheType; $wgMemc = wfGetCache( $wgMainCacheType ); + $this->children = null; } protected function handleTermSignal( $signal ) { diff --git a/maintenance/gearman/gearman.inc b/maintenance/gearman/gearman.inc index 0678b825f3..a2a4018aa3 100644 --- a/maintenance/gearman/gearman.inc +++ b/maintenance/gearman/gearman.inc @@ -11,6 +11,11 @@ class MWGearmanJob extends Net_Gearman_Job_Common { $this->complete( array( 'result' => true ) ); socket_close( $this->conn ); + # Close some more sockets + wfGetLBFactory()->shutdown(); + global $wgMemc; + $wgMemc->disconnect_all(); + # Find PHP $php = readlink( '/proc/' . posix_getpid() . '/exe' ); diff --git a/maintenance/gearman/gearmanWorker.php b/maintenance/gearman/gearmanWorker.php index f800b67a72..0b26ff9f17 100644 --- a/maintenance/gearman/gearmanWorker.php +++ b/maintenance/gearman/gearmanWorker.php @@ -10,9 +10,8 @@ if ( isset( $options['procs'] ) ) { echo "Invalid number of processes, please specify a number between 1 and 1000\n"; exit( 1 ); } - $fc = new ForkController; - if ( $fc->forkWorkers( $procs ) == 'parent' ) { - $fc->runParent(); + $fc = new ForkController( $procs, ForkController::RESTART_ON_ERROR ); + if ( $fc->start() != 'child' ) { exit( 0 ); } } diff --git a/maintenance/runJobs.php b/maintenance/runJobs.php index 63dddeba1e..e5cc6a7bd7 100644 --- a/maintenance/runJobs.php +++ b/maintenance/runJobs.php @@ -20,9 +20,8 @@ if ( isset( $options['procs'] ) ) { echo "Invalid argument to --procs\n"; exit( 1 ); } - $fc = new ForkController; - if ( $fc->forkWorkers( $procs ) == 'parent' ) { - $fc->runParent(); + $fc = new ForkController( $procs ); + if ( $fc->start( $procs ) != 'child' ) { exit( 0 ); } } -- 2.20.1