From 535501ed62853f54a4a849efeb1a35bb23b31724 Mon Sep 17 00:00:00 2001 From: Tim Starling Date: Mon, 15 Aug 2016 15:40:52 +1000 Subject: [PATCH] Add a script to build an HHVM RepoAuthoritative bytecode file This script compiles a list of known MW sources, including core and extensions, and generates a RepoAuthoritative bytecode file. Some ideas were taken from Bryan Davis's scap-hhvm-compile shell script. Also, renamed the old maintenance/hiphop directory to maintenance/hhvm, respecting the rename of the upstream project. Change-Id: I55798729d0553d2840e8099e81d604a814e8664c --- autoload.php | 1 + maintenance/Maintenance.php | 2 +- maintenance/hhvm/makeRepo.php | 161 +++++++++++++++++++++++ maintenance/{hiphop => hhvm}/run-server | 0 maintenance/{hiphop => hhvm}/server.conf | 0 5 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 maintenance/hhvm/makeRepo.php rename maintenance/{hiphop => hhvm}/run-server (100%) rename maintenance/{hiphop => hhvm}/server.conf (100%) diff --git a/autoload.php b/autoload.php index f6edd94b7d..fc6577ef14 100644 --- a/autoload.php +++ b/autoload.php @@ -513,6 +513,7 @@ $wgAutoloadLocalClasses = [ 'GitInfo' => __DIR__ . '/includes/GitInfo.php', 'GlobalDependency' => __DIR__ . '/includes/cache/CacheDependency.php', 'GlobalVarConfig' => __DIR__ . '/includes/config/GlobalVarConfig.php', + 'HHVMMakeRepo' => __DIR__ . '/maintenance/hhvm/makeRepo.php', 'HTMLApiField' => __DIR__ . '/includes/htmlform/fields/HTMLApiField.php', 'HTMLAutoCompleteSelectField' => __DIR__ . '/includes/htmlform/fields/HTMLAutoCompleteSelectField.php', 'HTMLButtonField' => __DIR__ . '/includes/htmlform/fields/HTMLButtonField.php', diff --git a/maintenance/Maintenance.php b/maintenance/Maintenance.php index 8368aabebb..ab316c0632 100644 --- a/maintenance/Maintenance.php +++ b/maintenance/Maintenance.php @@ -907,7 +907,7 @@ abstract class Maintenance { // Description ... if ( $this->mDescription ) { - $this->output( "\n" . $this->mDescription . "\n" ); + $this->output( "\n" . wordwrap( $this->mDescription, $screenWidth ) . "\n" ); } $output = "\nUsage: php " . basename( $this->mSelf ); diff --git a/maintenance/hhvm/makeRepo.php b/maintenance/hhvm/makeRepo.php new file mode 100644 index 0000000000..a0fe381c79 --- /dev/null +++ b/maintenance/hhvm/makeRepo.php @@ -0,0 +1,161 @@ +addDescription( 'Compile PHP sources for this MediaWiki instance, ' . + 'and generate an HHVM bytecode file to be used with HHVM\'s ' . + 'RepoAuthoritative mode. The MediaWiki core installation path and ' . + 'all registered extensions are automatically searched for the file ' . + 'extensions *.php, *.inc, *.php5 and *.phtml.' ); + $this->addOption( 'output', 'Output filename', true, true, 'o' ); + $this->addOption( 'input-dir', 'Add an input directory. ' . + 'This can be specified multiple times.', false, true, 'd', true ); + $this->addOption( 'exclude-dir', 'Directory to exclude. ' . + 'This can be specified multiple times.', false, true, false, true ); + $this->addOption( 'extension', 'Extra file extension', false, true, false, true ); + $this->addOption( 'hhvm', 'Location of HHVM binary', false, true ); + $this->addOption( 'base-dir', 'The root of all source files. ' . + 'This must match hhvm.server.source_root in the server\'s configuration file. ' . + 'By default, the MW core install path will be used.', + false, true ); + $this->addOption( 'verbose', 'Log level 0-3', false, true, 'v' ); + } + + private static function startsWith( $subject, $search ) { + return substr( $subject, 0, strlen( $search ) === $search ); + } + + function execute() { + global $wgExtensionCredits, $IP; + + $dirs = [ $IP ]; + + foreach ( $wgExtensionCredits as $type => $extensions ) { + foreach ( $extensions as $extension ) { + if ( isset( $extension['path'] ) + && !self::startsWith( $extension['path'], $IP ) + ) { + $dirs[] = dirname( $extension['path'] ); + } + } + } + + $dirs = array_merge( $dirs, $this->getOption( 'input-dir', [] ) ); + $fileExts = + [ + 'php' => true, + 'inc' => true, + 'php5' => true, + 'phtml' => true + ] + + array_flip( $this->getOption( 'extension', [] ) ); + + $dirs = array_unique( $dirs ); + + $baseDir = $this->getOption( 'base-dir', $IP ); + $excludeDirs = array_map( 'realpath', $this->getOption( 'exclude-dir', [] ) ); + + if ( $baseDir !== '' && substr( $baseDir, -1 ) !== '/' ) { + $baseDir .= '/'; + } + + $unfilteredFiles = [ "$IP/LocalSettings.php" ]; + foreach ( $dirs as $dir ) { + $this->appendDir( $unfilteredFiles, $dir ); + } + + $files = []; + foreach ( $unfilteredFiles as $file ) { + $dotPos = strrpos( $file, '.' ); + $slashPos = strrpos( $file, '/' ); + if ( $dotPos === false || $slashPos === false || $dotPos < $slashPos ) { + continue; + } + $extension = substr( $file, $dotPos + 1 ); + if ( !isset( $fileExts[$extension] ) ) { + continue; + } + $canonical = realpath( $file ); + foreach ( $excludeDirs as $excluded ) { + if ( self::startsWith( $canonical, $excluded ) ) { + continue 2; + } + } + if ( self::startsWith( $file, $baseDir ) ) { + $file = substr( $file, strlen( $baseDir ) ); + } + $files[] = $file; + } + + $files = array_unique( $files ); + + print "Found " . count( $files ) . " files in " . + count( $dirs ) . " directories\n"; + + $tmpDir = wfTempDir() . '/mw-make-repo' . mt_rand( 0, 1<<31 ); + if ( !mkdir( $tmpDir ) ) { + $this->error( 'Unable to create temporary directory', 1 ); + } + file_put_contents( "$tmpDir/file-list", implode( "\n", $files ) ); + + $hhvm = $this->getOption( 'hhvm', 'hhvm' ); + $verbose = $this->getOption( 'verbose', 3 ); + $cmd = wfEscapeShellArg( + $hhvm, + '--hphp', + '--target', 'hhbc', + '--format', 'binary', + '--force', '1', + '--keep-tempdir', '1', + '--log', $verbose, + '-v', 'AllVolatile=true', + '--input-dir', $baseDir, + '--input-list', "$tmpDir/file-list", + '--output-dir', $tmpDir ); + print "$cmd\n"; + passthru( $cmd, $ret ); + if ( $ret ) { + $this->cleanupTemp( $tmpDir ); + $this->error( "Error: HHVM returned error code $ret", 1 ); + } + if ( !rename( "$tmpDir/hhvm.hhbc", $this->getOption( 'output' ) ) ) { + $this->cleanupTemp( $tmpDir ); + $this->error( "Error: unable to rename output file", 1 ); + } + $this->cleanupTemp( $tmpDir ); + return 0; + } + + private function cleanupTemp( $tmpDir ) { + if ( file_exists( "$tmpDir/hhvm.hhbc" ) ) { + unlink( "$tmpDir/hhvm.hhbc" ); + } + if ( file_exists( "$tmpDir/Stats.js" ) ) { + unlink( "$tmpDir/Stats.js" ); + } + + unlink( "$tmpDir/file-list" ); + rmdir( $tmpDir ); + } + + private function appendDir( &$files, $dir ) { + $iter = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator( + $dir, + FilesystemIterator::UNIX_PATHS + ), + RecursiveIteratorIterator::LEAVES_ONLY + ); + foreach ( $iter as $file => $fileInfo ) { + if ( $fileInfo->isFile() ) { + $files[] = $file; + } + } + } +} + +$maintClass = 'HHVMMakeRepo'; +require RUN_MAINTENANCE_IF_MAIN; diff --git a/maintenance/hiphop/run-server b/maintenance/hhvm/run-server similarity index 100% rename from maintenance/hiphop/run-server rename to maintenance/hhvm/run-server diff --git a/maintenance/hiphop/server.conf b/maintenance/hhvm/server.conf similarity index 100% rename from maintenance/hiphop/server.conf rename to maintenance/hhvm/server.conf -- 2.20.1