maintenance/hiphop/make

   1 #!/usr/bin/hphpi -f
   2 <?php
   3
   4 require( dirname( __FILE__ ) . '/../Maintenance.php' );
   5
   6 class MakeHipHop extends Maintenance {
   7         function execute() {
   8                 $startTime = time();
   9
  10                 $sourceDir = realpath( dirname( __FILE__ ) );
  11                 $IP = realpath( "$sourceDir/../.." );
  12                 $buildDir = "$sourceDir/build";
  13                 $outDir = "$buildDir/hiphop-output";
  14                 $persistentDir = "$buildDir/persistent" ;
  15
  16                 if ( !is_dir( $buildDir ) ) {
  17                         mkdir( $buildDir, 0777, true );
  18                 }
  19                 if ( !is_dir( $persistentDir ) ) {
  20                         mkdir( $persistentDir, 0777, true );
  21                 }
  22
  23                 # With the CentOS RPMs, you just get g++44, no g++, so we have to
  24                 # use the environment
  25                 if ( isset( $_ENV['CXX'] ) ) {
  26                         $cxx = $_ENV['CXX'];
  27                 } else {
  28                         $cxx = 'g++';
  29                 }
  30
  31                 # Create a function that provides the HipHop compiler version, and
  32                 # doesn't exist when MediaWiki is invoked in interpreter mode.
  33                 $version = str_replace( PHP_EOL, ' ', trim( `hphp --version` ) );
  34                 file_put_contents(
  35                         "$buildDir/HipHopCompilerVersion.php",
  36                         "<" . "?php\n" .
  37                         "function wfHipHopCompilerVersion() {\n" .
  38                         "return " . var_export( $version, true ) . ";\n" .
  39                         "}\n"
  40                 );
  41
  42                 # Generate the file list from the autoloader
  43                 global $wgAutoloadLocalClasses;
  44                 $files = array_merge(
  45                         array_values( $wgAutoloadLocalClasses ),
  46                         array_map( 'trim', file( "$sourceDir/extra-files" ) )
  47                 );
  48                 $files = array_unique( $files );
  49                 file_put_contents(
  50                         "$buildDir/file-list",
  51                         implode( "\n", $files ) . "\n" );
  52
  53                 # Generate the C++
  54                 passthru(
  55                         'hphp' .
  56                         ' --target=cpp' .
  57                         ' --format=file' .
  58                         ' --input-dir=' . wfEscapeShellArg( $IP ) .
  59                         ' --input-list=' . wfEscapeShellArg( "$buildDir/file-list" ) .
  60                         ' --inputs=' . wfEscapeShellArg( "$buildDir/HipHopCompilerVersion.php" ) .
  61                         ' -c ' . wfEscapeShellArg( "$sourceDir/compiler.conf" ) .
  62                         ' --parse-on-demand=false' .
  63                         ' --program=mediawiki-hphp' .
  64                         ' --output-dir=' . wfEscapeShellArg( $outDir ) .
  65                         ' --log=3', $ret );
  66
  67                 if ( $ret ) {
  68                         $this->error( "hphp hit an error. Stopping build.\n" );
  69                         exit( 1 );
  70                 }
  71
  72                 # Sanity check, quickly make sure we've got an output directory
  73                 if( !is_dir( $outDir ) ) {
  74                         $this->error( "No output directory", true );
  75                 }
  76
  77                 # Warn about volatile classes
  78                 $this->checkVolatileClasses( $outDir );
  79
  80                 # Copy the generated C++ files into the source directory for cmake
  81                 $iter = new RecursiveIteratorIterator(
  82                         new RecursiveDirectoryIterator( $outDir ),
  83                         RecursiveIteratorIterator::SELF_FIRST );
  84                 $sourceFiles = array();
  85                 $regenerateMakefile = false;
  86                 $numFiles = 0;
  87                 $numFilesChanged = 0;
  88                 foreach ( $iter as $sourcePath => $file ) {
  89                         $name = substr( $sourcePath, strlen( $outDir ) + 1 );
  90                         $sourceFiles[$name] = true;
  91                         $destPath = "$persistentDir/$name";
  92                         if ( $file->isDir() ) {
  93                                 if ( !is_dir( $destPath ) ) {
  94                                         mkdir( $destPath );
  95                                 }
  96                                 continue;
  97                         }
  98
  99                         $numFiles++;
 100                         # Remove any files that weren't touched, these may have been removed
 101                         # from file-list, we should not compile them
 102                         if ( $file->getMTime() < $startTime ) {
 103                                 if ( file_exists( $destPath ) ) {
 104                                         unlink( $destPath );
 105                                         # Files removed, regenerate the makefile
 106                                         $regenerateMakefile = true;
 107                                 }
 108                                 unlink( $sourcePath );
 109                                 $numFilesChanged++;
 110                                 continue;
 111                         }
 112
 113                         if ( file_exists( $destPath ) ) {
 114                                 $sourceHash = md5( file_get_contents( $sourcePath ) );
 115                                 $destHash = md5( file_get_contents( $destPath ) );
 116                                 if ( $sourceHash == $destHash ) {
 117                                         continue;
 118                                 }
 119                         } else {
 120                                 # New files added, regenerate the makefile
 121                                 $regenerateMakefile = true;
 122                         }
 123                         $numFilesChanged++;
 124                         copy( $sourcePath, $destPath );
 125                 }
 126
 127                 echo "MediaWiki: $numFilesChanged files changed out of $numFiles\n";
 128
 129                 if ( !file_exists( "$persistentDir/CMakeLists.txt" ) ) {
 130                         # Run cmake for the first time
 131                         $regenerateMakefile = true;
 132                 }
 133
 134                 # Do our own version of $HPHP_HOME/bin/run.sh, which isn't so broken.
 135                 # HipHop's RELEASE mode seems to be stuck always on, so symbols get
 136                 # stripped. Also we will try keeping the generated .o files instead of
 137                 # throwing away hours of CPU time every time you make a typo.
 138
 139                 chdir( $persistentDir );
 140
 141                 if ( $regenerateMakefile ) {
 142                         copy( $_ENV['HPHP_HOME'] . '/bin/CMakeLists.base.txt',
 143                                 "$persistentDir/CMakeLists.txt" );
 144
 145                         if ( file_exists( "$persistentDir/CMakeCache.txt" ) ) {
 146                                 unlink( "$persistentDir/CMakeCache.txt" );
 147                         }
 148
 149                         $cmd = 'cmake' .
 150                                 " -D CMAKE_BUILD_TYPE:string=" . wfEscapeShellArg( $GLOBALS['wgHipHopBuildType'] ) .
 151                                 ' -D PROGRAM_NAME:string=mediawiki-hphp';
 152
 153                         if ( file_exists( '/usr/bin/ccache' ) ) {
 154                                 $cmd .= ' -D CMAKE_CXX_COMPILER:string=ccache' .
 155                                         ' -D CMAKE_CXX_COMPILER_ARG1:string=' . wfEscapeShellArg( $cxx );
 156                         }
 157
 158                         $cmd .= ' .';
 159                         echo "$cmd\n";
 160                         passthru( $cmd );
 161                 }
 162
 163                 # Determine appropriate make concurrency
 164                 # Compilation can take a lot of memory, let's assume that that is limiting.
 165                 $procs = $this->getNumProcs();
 166
 167                 # Run make. This is the slow step.
 168                 passthru( 'make -j' . wfEscapeShellArg( $procs ) );
 169
 170                 $elapsed = time() - $startTime;
 171
 172                 echo "Completed in ";
 173                 if ( $elapsed >= 3600 ) {
 174                         $hours = floor( $elapsed / 3600 );
 175                         echo $hours . 'h ';
 176                         $elapsed -= $hours * 3600;
 177                 }
 178                 if ( $elapsed >= 60 ) {
 179                         $minutes = floor( $elapsed / 60 );
 180                         echo $minutes . 'm ';
 181                         $elapsed -= $minutes * 60;
 182                 }
 183                 echo $elapsed . "s\n";
 184                 echo "The MediaWiki executable is at build/persistent/mediawiki-hphp\n";
 185         }
 186
 187         function checkVolatileClasses( $dir ) {
 188                 $lines = file( "$dir/sys/dynamic_table_class.cpp" );
 189                 $classes = array();
 190                 foreach ( $lines as $line ) {
 191                         if ( preg_match( '/^\s+\(const char \*\)"([^"]*)", \(const char \*\)-1/', $line, $m ) ) {
 192                                 $classes[] = $m[1];
 193                         }
 194                 }
 195                 if ( !count( $classes ) ) {
 196                         print "No volatile classes found\n";
 197                         return;
 198                 }
 199                 sort( $classes );
 200                 $classes = array_unique( $classes );
 201                 print "WARNING: The following classes are volatile: " . implode( ', ', $classes ) . "\n";
 202         }
 203
 204         function getNumProcs() {
 205                 global $wgHipHopCompilerProcs;
 206                 if ( $wgHipHopCompilerProcs !== 'detect' ) {
 207                         return intval( $wgHipHopCompilerProcs );
 208                 }
 209
 210                 if ( !file_exists( '/proc/meminfo' ) ) {
 211                         return 1;
 212                 }
 213                 $mem = false;
 214                 foreach ( file( '/proc/meminfo' ) as $line ) {
 215                         if ( preg_match( '/^MemTotal:\s+(\d+)\s+kB/', $line, $m ) ) {
 216                                 $mem = intval( $m[1] );
 217                                 break;
 218                         }
 219                 }
 220                 if ( $mem ) {
 221                         // At least one process
 222                         return max( 1, floor( $mem / 1000000 ) );
 223                 } else {
 224                         return 1;
 225                 }
 226         }
 227 }
 228
 229 $maintClass = 'MakeHipHop';
 230 require_once( RUN_MAINTENANCE_IF_MAIN );