From ff1dc8a17591c1e540d6f7e2ccb5c1ba0dec2e34 Mon Sep 17 00:00:00 2001 From: Tim Starling Date: Mon, 30 May 2011 13:49:09 +0000 Subject: [PATCH] HipHop improvements: * Added the ability to compile extensions. The build process is bootstrapped by running MediaWiki in interpreted mode. Extension setup file inclusions are slightly modified in a way that makes them register themselves for compilation. Then the same LocalSettings.php uses the compiled extension setup file when the compiled binary runs. * Tested with Cite and ParserFunctions. The code which lets you have an extensions directory in a place other than $IP/../extensions is untested. * Simplified WebStart.php slightly by using a custom $_SERVER variable to mark compiled mode. It will break if you don't use the supplied server.conf, but that will break a lot of things so don't do that. * Fixed the core web entry points to include WebStart.php in compiled mode instead of interpreted. * Made the build directory configurable. This is mostly so that I can grep the source tree without seeing loads of generated C++. * In server.conf, added a rewrite rule allowing a /wiki/$1 article path. * Removed server.conf log file location "/dev/stdout", breaks when you switch user * Disable static content cache, breaks horribly when you set SourceRoot to a directory containing 7GB of files. * Rewrote the run-server script in PHP, mostly to support the configurable build directory feature. * Added an option to the run-server script to allow running in interpreted (hphpi) mode. --- api.php | 6 +- img_auth.php | 6 +- includes/DefaultSettings.php | 31 ++++++++++ includes/Init.php | 52 +++++++++++++++- includes/WebStart.php | 22 +------ index.php | 6 +- load.php | 7 ++- maintenance/doMaintenance.php | 16 ++--- maintenance/hiphop/make | 106 ++++++++++++++++++++++++++++----- maintenance/hiphop/run-server | 76 ++++++++++++++++++++--- maintenance/hiphop/server.conf | 21 ++++++- mw-config/index.php | 6 +- profileinfo.php | 6 +- redirect.php | 6 +- thumb.php | 6 +- trackback.php | 6 +- 16 files changed, 313 insertions(+), 66 deletions(-) diff --git a/api.php b/api.php index f55f85361a..978e8d4e94 100644 --- a/api.php +++ b/api.php @@ -51,7 +51,11 @@ if( !function_exists( 'version_compare' ) || version_compare( phpversion(), '5.2 } // Initialise common code. -require ( dirname( __FILE__ ) . '/includes/WebStart.php' ); +if ( isset( $_SERVER['MW_COMPILED'] ) ) { + require ( 'phase3/includes/WebStart.php' ); +} else { + require ( dirname( __FILE__ ) . '/includes/WebStart.php' ); +} wfProfileIn( 'api.php' ); $starttime = microtime( true ); diff --git a/img_auth.php b/img_auth.php index b059be31bf..5ac56999f5 100644 --- a/img_auth.php +++ b/img_auth.php @@ -26,7 +26,11 @@ **/ define( 'MW_NO_OUTPUT_COMPRESSION', 1 ); -require_once( dirname( __FILE__ ) . '/includes/WebStart.php' ); +if ( isset( $_SERVER['MW_COMPILED'] ) ) { + require ( 'phase3/includes/WebStart.php' ); +} else { + require ( dirname( __FILE__ ) . '/includes/WebStart.php' ); +} wfProfileIn( 'img_auth.php' ); require_once( dirname( __FILE__ ) . '/includes/StreamFile.php' ); diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index 60b896490b..628a56ec82 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -5313,6 +5313,12 @@ $wgUpdateRowsPerQuery = 100; * @{ */ +/** + * The build directory for HipHop compilation. + * Defaults to $IP/maintenance/hiphop/build. + */ +$wgHipHopBuildDirectory = false; + /** * The HipHop build type. Can be either "Debug" or "Release". */ @@ -5324,6 +5330,31 @@ $wgHipHopBuildType = 'Debug'; */ $wgHipHopCompilerProcs = 'detect'; +/** + * Filesystem extensions directory. Defaults to $IP/../extensions. + * + * To compile extensions with HipHop, set $wgExtensionsDirectory correctly, + * and use code like: + * + * require( MWInit::extensionSetupPath( 'Extension/Extension.php' ) ); + * + * to include the extension setup file from LocalSettings.php. It is not + * necessary to set this variable unless you use MWInit::extensionSetupPath(). + */ +$wgExtensionsDirectory = false; + +/** + * A list of files that should be compiled into a HipHop build, in addition to + * those listed in $wgAutoloadClasses. Add to this array in an extension setup + * file in order to add files to the build. + * + * The files listed here must either be either absolute paths under $IP or + * under $wgExtensionsDirectory, or paths relative to the virtual source root + * "$IP/..", i.e. starting with "phase3" for core files, and "extensions" for + * extension files. + */ +$wgCompiledFiles = array(); + /** @} */ # End of HipHop compilation } diff --git a/includes/Init.php b/includes/Init.php index 40a9e3093d..660bb29711 100644 --- a/includes/Init.php +++ b/includes/Init.php @@ -61,12 +61,62 @@ class MWInit { global $IP; if ( defined( 'MW_COMPILED' ) ) { - return $file; + return "phase3/$file"; } else { return "$IP/$file"; } } + /** + * The equivalent of MWInit::interpretedPath() but for files relative to the + * extensions directory. + */ + static function extInterpretedPath( $file ) { + return getExtensionsDirectory() . '/' . $file; + } + + /** + * The equivalent of MWInit::compiledPath() but for files relative to the + * extensions directory. Any files referenced in this way must be registered + * for compilation by including them in $wgCompiledFiles. + */ + static function extCompiledPath( $file ) { + if ( defined( 'MW_COMPILED' ) ) { + return "extensions/$file"; + } else { + return getExtensionsDirectory() . '/' . $file; + } + } + + /** + * Register an extension setup file and return its path for compiled + * inclusion. Use this function in LocalSettings.php to add extensions + * to the build. For example: + * + * require( MWInit::extSetupPath( 'ParserFunctions/ParserFunctions.php' ) ); + * + * @param $path The path relative to the extensions directory, as defined by + * $wgExtensionsDirectory. + */ + static function extSetupPath( $extRel ) { + $baseRel = "extensions/$extRel"; + if ( defined( 'MW_COMPILED' ) ) { + return $baseRel; + } else { + global $wgCompiledFiles; + $wgCompiledFiles[] = $baseRel; + return self::getExtensionsDirectory() . '/' . $extRel; + } + } + + static function getExtensionsDirectory() { + global $wgExtensionsDirectory, $IP; + if ( $wgExtensionsDirectory === false ) { + $wgExtensionsDirectory = "$IP/../extensions"; + } + return $wgExtensionsDirectory; + } + /** * Determine whether a class exists, using a method which works under HipHop. * diff --git a/includes/WebStart.php b/includes/WebStart.php index feb80398a0..ca46b8979f 100644 --- a/includes/WebStart.php +++ b/includes/WebStart.php @@ -8,22 +8,6 @@ * @file */ -/** - * Detect compiled mode by looking for a function that only exists if compiled - * in. Note that we can't use function_exists(), because it is terribly broken - * under HipHop due to the "volatile" feature. - * - * @return bool - */ -function wfDetectCompiledMode() { - try { - $r = new ReflectionFunction( 'wfHipHopCompilerVersion' ); - } catch ( ReflectionException $e ) { - $r = false; - } - return $r !== false; -} - # Protect against register_globals # This must be done before any globals are set by the code if ( ini_get( 'register_globals' ) ) { @@ -88,11 +72,9 @@ if ( $IP === false ) { $IP = realpath( '.' ); } -if ( wfDetectCompiledMode() ) { +if ( isset( $_SERVER['MW_COMPILED'] ) ) { define( 'MW_COMPILED', 1 ); -} - -if ( !defined( 'MW_COMPILED' ) ) { +} else { # Get MWInit class require_once( "$IP/includes/Init.php" ); diff --git a/index.php b/index.php index 4f8950c885..3b7195c6aa 100644 --- a/index.php +++ b/index.php @@ -64,7 +64,11 @@ ENDL; # Initialise common code. This gives us access to GlobalFunctions, the AutoLoader, and # the globals $wgRequest, $wgOut, $wgUser, $wgLang and $wgContLang, amongst others; it # does *not* load $wgTitle -require ( dirname( __FILE__ ) . '/includes/WebStart.php' ); +if ( isset( $_SERVER['MW_COMPILED'] ) ) { + require ( 'phase3/includes/WebStart.php' ); +} else { + require ( dirname( __FILE__ ) . '/includes/WebStart.php' ); +} try { wfIndexMain(); diff --git a/load.php b/load.php index 7fff7c4991..2fc48fc950 100644 --- a/load.php +++ b/load.php @@ -36,7 +36,12 @@ if( !function_exists( 'version_compare' ) || version_compare( phpversion(), '5.2 wfDie( "MediaWiki $version requires at least PHP version 5.2.3." ); } -require ( dirname( __FILE__ ) . '/includes/WebStart.php' ); +if ( isset( $_SERVER['MW_COMPILED'] ) ) { + require ( 'phase3/includes/WebStart.php' ); +} else { + require ( dirname( __FILE__ ) . '/includes/WebStart.php' ); +} + wfProfileIn( 'load.php' ); // URL safety checks diff --git a/maintenance/doMaintenance.php b/maintenance/doMaintenance.php index d3cf30ccfe..12fbb49a9a 100644 --- a/maintenance/doMaintenance.php +++ b/maintenance/doMaintenance.php @@ -54,18 +54,10 @@ $maintenance->setup(); $self = $maintenance->getName(); // Detect compiled mode -try { - $r = new ReflectionFunction( 'wfHipHopCompilerVersion' ); -} catch ( ReflectionException $e ) { - $r = false; -} - -if ( $r ) { +if ( isset( $_SERVER['MW_COMPILED'] ) ) { define( 'MW_COMPILED', 1 ); -} - -# Get the MWInit class -if ( !defined( 'MW_COMPILED' ) ) { +} else { + # Get the MWInit class require_once( "$IP/includes/Init.php" ); require_once( "$IP/includes/AutoLoader.php" ); } @@ -77,7 +69,7 @@ require_once( MWInit::compiledPath( 'includes/profiler/Profiler.php' ) ); if ( !defined( 'MW_COMPILED' ) ) { require_once( "$IP/includes/Defines.php" ); } -require_once( "$IP/includes/DefaultSettings.php" ); +require_once( MWInit::compiledPath( 'includes/DefaultSettings.php' ) ); if ( defined( 'MW_CONFIG_CALLBACK' ) ) { # Use a callback function to configure MediaWiki diff --git a/maintenance/hiphop/make b/maintenance/hiphop/make index bfeb960a59..e792e08b5c 100755 --- a/maintenance/hiphop/make +++ b/maintenance/hiphop/make @@ -5,13 +5,20 @@ require( dirname( __FILE__ ) . '/../Maintenance.php' ); class MakeHipHop extends Maintenance { function execute() { + global $wgHipHopBuildDirectory; + $startTime = time(); - $sourceDir = realpath( dirname( __FILE__ ) ); - $IP = realpath( "$sourceDir/../.." ); - $buildDir = "$sourceDir/build"; + $thisDir = realpath( dirname( __FILE__ ) ); + $IP = realpath( "$thisDir/../.." ); + if ( strval( $wgHipHopBuildDirectory ) !== '' ) { + $buildDir = $wgHipHopBuildDirectory; + } else { + $buildDir = "$thisDir/build"; + } + $extensionsDir = realpath( MWInit::getExtensionsDirectory() ); $outDir = "$buildDir/hiphop-output"; - $persistentDir = "$buildDir/persistent" ; + $persistentDir = "$buildDir/persistent"; if ( !is_dir( $buildDir ) ) { mkdir( $buildDir, 0777, true ); @@ -20,6 +27,17 @@ class MakeHipHop extends Maintenance { mkdir( $persistentDir, 0777, true ); } + if ( realpath( "$IP/../phase3" ) !== $IP + || realpath( "$IP/../extensions" ) !== $extensionsDir ) + { + # Set up a fake source directory with the correct layout + $sourceBase = "$buildDir/source"; + $this->setupFakeSourceBase( $IP, $extensionsDir, $sourceBase ); + } else { + $sourceBase = realpath( "$IP/.." ); + unlink( "$buildDir/source" ); + } + # With the CentOS RPMs, you just get g++44, no g++, so we have to # use the environment if ( isset( $_ENV['CXX'] ) ) { @@ -39,13 +57,8 @@ class MakeHipHop extends Maintenance { "}\n" ); - # Generate the file list from the autoloader - global $wgAutoloadLocalClasses; - $files = array_merge( - array_values( $wgAutoloadLocalClasses ), - array_map( 'trim', file( "$sourceDir/extra-files" ) ) - ); - $files = array_unique( $files ); + # Generate the file list + $files = $this->getFileList(); file_put_contents( "$buildDir/file-list", implode( "\n", $files ) . "\n" ); @@ -55,10 +68,10 @@ class MakeHipHop extends Maintenance { 'hphp' . ' --target=cpp' . ' --format=file' . - ' --input-dir=' . wfEscapeShellArg( $IP ) . + ' --input-dir=' . wfEscapeShellArg( $sourceBase ) . ' --input-list=' . wfEscapeShellArg( "$buildDir/file-list" ) . ' --inputs=' . wfEscapeShellArg( "$buildDir/HipHopCompilerVersion.php" ) . - ' -c ' . wfEscapeShellArg( "$sourceDir/compiler.conf" ) . + ' -c ' . wfEscapeShellArg( "$thisDir/compiler.conf" ) . ' --parse-on-demand=false' . ' --program=mediawiki-hphp' . ' --output-dir=' . wfEscapeShellArg( $outDir ) . @@ -181,7 +194,7 @@ class MakeHipHop extends Maintenance { $elapsed -= $minutes * 60; } echo $elapsed . "s\n"; - echo "The MediaWiki executable is at build/persistent/mediawiki-hphp\n"; + echo "The MediaWiki executable is at $buildDir/persistent/mediawiki-hphp\n"; } function checkVolatileClasses( $dir ) { @@ -224,6 +237,71 @@ class MakeHipHop extends Maintenance { return 1; } } + + function setupFakeSourceBase( $phase3, $extensions, $dest ) { + if ( !file_exists( $dest ) ) { + mkdir( $dest, 0777, true ); + } + + $this->forceCreateLink( "$dest/phase3", $phase3 ); + $this->forceCreateLink( "$dest/extensions", $extensions ); + } + + function forceCreateLink( $target, $link ) { + if ( file_exists( $target ) ) { + if ( readlink( $target ) === $link ) { + return; + } + unlink( $target ); + } + symlink( $target, $link ); + } + + function getFileList() { + global $wgAutoloadClasses, $wgAutoloadLocalClasses, $wgCompiledFiles; + $inputFiles = array_merge( + array_values( $wgAutoloadClasses ), + array_values( $wgAutoloadLocalClasses ), + $wgCompiledFiles + ); + $processedFiles = array(); + foreach ( $inputFiles as $file ) { + if ( substr( $file, 0, 1 ) === '/' ) { + $processedFiles[] = $this->absoluteToRelative( $file ); + } elseif ( preg_match( '/^extensions/', $file ) ) { + $processedFiles[] = $file; + } else { + $processedFiles[] = "phase3/$file"; + } + } + + $extraCoreFiles = array_map( 'trim', file( dirname( __FILE__ ) . '/extra-files' ) ); + foreach ( $extraCoreFiles as $file ) { + if ( $file === '' ) { + continue; + } + $processedFiles[] = "phase3/$file"; + } + return array_unique( $processedFiles ); + } + + function absoluteToRelative( $file ) { + global $IP; + + $coreBase = realpath( $IP ) . '/'; + $extBase = realpath( MWInit::getExtensionsDirectory() ) . '/'; + $file = realpath( $file ); + + if ( substr( $file, 0, strlen( $extBase ) ) === $extBase ) { + return 'extensions/' . substr( $file, strlen( $extBase ) ); + } elseif ( substr( $file, 0, strlen( $coreBase ) ) === $coreBase ) { + return 'phase3/' . substr( $file, strlen( $coreBase ) ); + } else { + $this->error( "The following file is registered for compilation but is not in \$IP or " . + "\$wgExtensionsDirectory: $file \n" ); + exit( 1 ); + } + } } $maintClass = 'MakeHipHop'; diff --git a/maintenance/hiphop/run-server b/maintenance/hiphop/run-server index 415bf64034..3201689911 100755 --- a/maintenance/hiphop/run-server +++ b/maintenance/hiphop/run-server @@ -1,12 +1,72 @@ -#!/bin/bash +#!/usr/bin/hphpi -f +addOption( 'interpret', 'Run in interpreted mode' ); + } + function execute() { + if ( $this->hasOption( 'interpret' ) ) { + $this->runInterpreted(); + } else { + $this->runCompiled(); + } + } + function runCompiled() { + global $wgHipHopBuildDirectory; + $thisDir = realpath( dirname( __FILE__ ) ); + $IP = realpath( "$thisDir/../.." ); + if ( strval( $wgHipHopBuildDirectory ) !== '' ) { + $buildDir = $wgHipHopBuildDirectory; + } else { + $buildDir = "$thisDir/build"; + } + + if ( file_exists( "$buildDir/source" ) ) { + $sourceBase = "$buildDir/source"; + } else { + $sourceBase = realpath( "$IP/.." ); + } + + passthru( + 'cd ' . wfEscapeShellArg( $sourceBase ) . " && " . + 'MW_INSTALL_PATH=' . wfEscapeShellArg( $IP ) . ' ' . + wfEscapeShellArg( + "$buildDir/persistent/mediawiki-hphp", + '-c', "$thisDir/server.conf", + '-v', "Server.SourceRoot=$sourceBase", + '-v', "Server.IncludeSearchPaths.0=$sourceBase", + '--mode=server', + '--port=8080' + ), + $ret + ); + exit( $ret ); + } + + function runInterpreted() { + $thisDir = realpath( dirname( __FILE__ ) ); + $IP = realpath( "$thisDir/../.." ); + $sourceBase = realpath( "$IP/.." ); + + passthru( + wfEscapeShellArg( + 'hphpi', + '-c', "$thisDir/server.conf", + '-v', "Server.SourceRoot=$sourceBase", + '-v', "Server.IncludeSearchPaths.0=$sourceBase", + '--mode=server', + '--port=8080' + ), + $ret + ); + exit( $ret ); + } +} +$maintClass = 'RunHipHopServer'; +require_once( RUN_MAINTENANCE_IF_MAIN ); diff --git a/maintenance/hiphop/server.conf b/maintenance/hiphop/server.conf index 8c53ab5c0c..e0ea149a76 100644 --- a/maintenance/hiphop/server.conf +++ b/maintenance/hiphop/server.conf @@ -1,7 +1,6 @@ Log { Level = Warning UseLogFile = true - File = /dev/stdout NativeStackTrace = true InjectedStackTrace = true } @@ -12,5 +11,23 @@ Debug { TranslateSource = true } Server { - EnableStaticContentFromDisk = true + EnableStaticContentCache = false + EnableStaticContentFromDisk = false + AlwaysUseRelativePath = true } +ServerVariables { + MW_COMPILED = 1 +} +VirtualHost { + * { + ServerName = localhost + Pattern = . + RewriteRules { + * { + pattern = ^/wiki/(.*)$ + to = /phase3/index.php?title=$1 + } + } + } +} + diff --git a/mw-config/index.php b/mw-config/index.php index c00a0d0135..1e6d0e4f3f 100644 --- a/mw-config/index.php +++ b/mw-config/index.php @@ -9,7 +9,11 @@ define( 'MW_CONFIG_CALLBACK', 'Installer::overrideConfig' ); define( 'MEDIAWIKI_INSTALL', true ); chdir( dirname( dirname( __FILE__ ) ) ); -require( dirname( dirname( __FILE__ ) ) . '/includes/WebStart.php' ); +if ( isset( $_SERVER['MW_COMPILED'] ) ) { + require ( 'phase3/includes/WebStart.php' ); +} else { + require( dirname( dirname( __FILE__ ) ) . '/includes/WebStart.php' ); +} wfInstallerMain(); diff --git a/profileinfo.php b/profileinfo.php index ecf69e50e1..3117db31fd 100644 --- a/profileinfo.php +++ b/profileinfo.php @@ -28,8 +28,12 @@ ini_set( 'zlib.output_compression', 'off' ); $wgEnableProfileInfo = $wgProfileToDatabase = false; +if ( isset( $_SERVER['MW_COMPILED'] ) ) { + require ( 'phase3/includes/WebStart.php' ); +} else { + require ( dirname( __FILE__ ) . '/includes/WebStart.php' ); +} -require_once( './includes/WebStart.php' ); header( 'Content-Type: text/html; charset=utf-8' ); diff --git a/redirect.php b/redirect.php index 699926ec30..c0a1602155 100644 --- a/redirect.php +++ b/redirect.php @@ -6,8 +6,12 @@ * * @file */ +if ( isset( $_SERVER['MW_COMPILED'] ) ) { + require ( 'phase3/includes/WebStart.php' ); +} else { + require ( dirname( __FILE__ ) . '/includes/WebStart.php' ); +} -require_once( './includes/WebStart.php' ); global $wgArticlePath; $page = $wgRequest->getVal( 'wpDropdown' ); diff --git a/thumb.php b/thumb.php index 30b61592b4..1fe564d8fa 100644 --- a/thumb.php +++ b/thumb.php @@ -7,7 +7,11 @@ * @ingroup Media */ define( 'MW_NO_OUTPUT_COMPRESSION', 1 ); -require_once( './includes/WebStart.php' ); +if ( isset( $_SERVER['MW_COMPILED'] ) ) { + require ( 'phase3/includes/WebStart.php' ); +} else { + require ( dirname( __FILE__ ) . '/includes/WebStart.php' ); +} $wgTrivialMimeDetection = true; //don't use fancy mime detection, just check the file extension for jpg/gif/png. diff --git a/trackback.php b/trackback.php index f673c50877..0e2036a92c 100644 --- a/trackback.php +++ b/trackback.php @@ -5,7 +5,11 @@ * @ingroup SpecialPage */ -require_once( './includes/WebStart.php' ); +if ( isset( $_SERVER['MW_COMPILED'] ) ) { + require ( 'phase3/includes/WebStart.php' ); +} else { + require ( dirname( __FILE__ ) . '/includes/WebStart.php' ); +} class TrackBack { -- 2.20.1