* Added restart-on-error feature to ForkController, changed the interface to suit.

* Close some sockets before pcntl_exec() in an attempt to delay file handle exhaustion.

diff --git a/includes/ForkController.php b/includes/ForkController.php
index 689b9c8..b5f1f18 100644 (file)
--- a/includes/ForkController.php
+++ b/includes/ForkController.php
@@ -4,15 +4,110 @@
  * Class for managing forking command line scripts.
  * Currently just does forking and process control, but it could easily be extended 
  * to provide IPC and job dispatch.
+ *
+ * This class requires the posix and pcntl extensions.
  */
 class ForkController {
        var $children = array();
        var $termReceived = false;
+       var $flags = 0, $procsToStart = 0;
 
-       public function __construct() {
+       static $restartableSignals = array(
+               SIGFPE,
+               SIGILL,
+               SIGSEGV,
+               SIGBUS,
+               SIGABRT,
+               SIGSYS,
+               SIGPIPE,
+               SIGXCPU,
+               SIGXFSZ,
+       );
+
+       /**
+        * Pass this flag to __construct() to cause the class to automatically restart
+        * workers that exit with non-zero exit status or a signal such as SIGSEGV.
+        */
+       const RESTART_ON_ERROR = 1;
+
+       public function __construct( $numProcs, $flags = 0 ) {
                if ( php_sapi_name() != 'cli' ) {
                        throw new MWException( "MultiProcess cannot be used from the web." );
                }
+               $this->procsToStart = $numProcs;
+               $this->flags = $flags;
+       }
+
+       /**
+        * Start the child processes. 
+        *
+        * This should only be called from the command line. It should be called 
+        * as early as possible during execution.
+        *
+        * This will return 'child' in the child processes. In the parent process, 
+        * it will run until all the child processes exit or a TERM signal is 
+        * received. It will then return 'done'.
+        */
+       public function start() {
+               // Trap SIGTERM
+               pcntl_signal( SIGTERM, array( $this, 'handleTermSignal' ), false );
+
+               do {
+                       // Start child processes
+                       if ( $this->procsToStart ) {
+                               if ( $this->forkWorkers( $this->procsToStart ) == 'child' ) {
+                                       return 'child';
+                               }
+                               $this->procsToStart = 0;
+                       }
+
+                       // Check child status
+                       $status = false;
+                       $deadPid = pcntl_wait( $status );
+
+                       if ( $deadPid > 0 ) {
+                               // Respond to child process termination
+                               unset( $this->children[$deadPid] );
+                               if ( $this->flags & self::RESTART_ON_ERROR ) {
+                                       if ( pcntl_wifsignaled( $status ) ) {
+                                               // Restart if the signal was abnormal termination
+                                               // Don't restart if it was deliberately killed
+                                               $signal = pcntl_wtermsig( $status );
+                                               if ( in_array( $signal, self::$restartableSignals ) ) { 
+                                                       echo "Worker exited with signal $signal, restarting\n";
+                                                       $this->procsToStart++;
+                                               }
+                                       } elseif ( pcntl_wifexited( $status ) ) {
+                                               // Restart on non-zero exit status
+                                               $exitStatus = pcntl_wexitstatus( $status );
+                                               if ( $exitStatus > 0 ) {
+                                                       echo "Worker exited with status $exitStatus, restarting\n";
+                                                       $this->procsToStart++;
+                                               }
+                                       }
+                               }
+                               // Throttle restarts
+                               if ( $this->procsToStart ) {
+                                       usleep( 500000 );
+                               }
+                       }
+
+                       // Run signal handlers
+                       if ( function_exists( 'pcntl_signal_dispatch' ) ) {
+                               pcntl_signal_dispatch();
+                       } else {
+                               declare (ticks=1) { $status = $status; } 
+                       }
+                       // Respond to TERM signal
+                       if ( $this->termReceived ) {
+                               foreach ( $this->children as $childPid => $unused ) {
+                                       posix_kill( $childPid, SIGTERM );
+                               }
+                               $this->termReceived = false;
+                       }
+               } while ( count( $this->children ) );
+               pcntl_signal( SIGTERM, SIG_DFL );
+               return 'done';
        }
 
        protected function prepareEnvironment() {
@@ -25,15 +120,8 @@ class ForkController {
 
        /**
         * Fork a number of worker processes.
-        *
-        * This should only be called from the command line. It should be called 
-        * as early as possible during execution. It will return 'child' in the 
-        * child processes and 'parent' in the parent process. The parent process
-        * should then call monitor(). 
-        *
-        * This function requires the posix and pcntl extensions.
         */
-       public function forkWorkers( $numProcs ) {
+       protected function forkWorkers( $numProcs ) {
                global $wgMemc, $wgCaches, $wgMainCacheType;
        
                $this->prepareEnvironment();
@@ -49,7 +137,6 @@ class ForkController {
 
                        if ( !$pid ) {
                                $this->initChild();
-                               $this->children = null;
                                return 'child';
                        } else {
                                // This is the parent process
@@ -60,41 +147,10 @@ class ForkController {
                return 'parent';
        }
 
-       /**
-        * The parent process main loop
-        */
-       public function runParent() {
-               // Trap SIGTERM
-               pcntl_signal( SIGTERM, array( $this, 'handleTermSignal' ), false );
-
-               do {
-                       $status = false;
-                       $deadPid = pcntl_wait( $status );
-
-                       if ( $deadPid > 0 ) {
-                               unset( $this->children[$deadPid] );
-                       }
-
-                       // Run signal handlers
-                       if ( function_exists( 'pcntl_signal_dispatch' ) ) {
-                               pcntl_signal_dispatch();
-                       } else {
-                               declare (ticks=1) { $status = $status; } 
-                       }
-                       // Respond to TERM signal
-                       if ( $this->termReceived ) {
-                               foreach ( $this->children as $childPid => $unused ) {
-                                       posix_kill( $childPid, SIGTERM );
-                               }
-                               $this->termReceived = false;
-                       }
-               } while ( count( $this->children ) );
-               pcntl_signal( SIGTERM, SIG_DFL );
-       }
-
        protected function initChild() {
                global $wgMemc, $wgMainCacheType;
                $wgMemc = wfGetCache( $wgMainCacheType );
+               $this->children = null;
        }
 
        protected function handleTermSignal( $signal ) {

diff --git a/maintenance/gearman/gearman.inc b/maintenance/gearman/gearman.inc
index 0678b82..a2a4018 100644 (file)
--- a/maintenance/gearman/gearman.inc
+++ b/maintenance/gearman/gearman.inc
@@ -11,6 +11,11 @@ class MWGearmanJob extends Net_Gearman_Job_Common {
                $this->complete( array( 'result' => true ) );
                socket_close( $this->conn );
 
+               # Close some more sockets
+               wfGetLBFactory()->shutdown();
+               global $wgMemc;
+               $wgMemc->disconnect_all();
+
                # Find PHP
                $php = readlink( '/proc/' . posix_getpid() . '/exe' );
                

diff --git a/maintenance/gearman/gearmanWorker.php b/maintenance/gearman/gearmanWorker.php
index f800b67..0b26ff9 100644 (file)
--- a/maintenance/gearman/gearmanWorker.php
+++ b/maintenance/gearman/gearmanWorker.php
@@ -10,9 +10,8 @@ if ( isset( $options['procs'] ) ) {
                echo "Invalid number of processes, please specify a number between 1 and 1000\n";
                exit( 1 );
        }
-       $fc = new ForkController;
-       if ( $fc->forkWorkers( $procs ) == 'parent' ) {
-               $fc->runParent();
+       $fc = new ForkController( $procs, ForkController::RESTART_ON_ERROR );
+       if ( $fc->start() != 'child' ) {
                exit( 0 );
        }
 }

diff --git a/maintenance/runJobs.php b/maintenance/runJobs.php
index 63dddeb..e5cc6a7 100644 (file)
--- a/maintenance/runJobs.php
+++ b/maintenance/runJobs.php
@@ -20,9 +20,8 @@ if ( isset( $options['procs'] ) ) {
                echo "Invalid argument to --procs\n";
                exit( 1 );
        }
-       $fc = new ForkController;
-       if ( $fc->forkWorkers( $procs ) == 'parent' ) {
-               $fc->runParent();
+       $fc = new ForkController( $procs );
+       if ( $fc->start( $procs ) != 'child' ) {
                exit( 0 );
        }
 }