HipHop build support files.

* Added a script called "make" which works around some of the inadequacies in the hphp build system:
** Allow incremental rebuilds. CMake is very smart and can do this with very little prompting, including header dependencies. This cuts down the build time for certain types of small change in the PHP source.
** Use ccache if it is present.
** Use debug mode, don't strip symbols.

* Added a script which runs the compiled program in server mode
* Added a PHP file list which is greatly cut down to reduce build time. Suitable only for initial testing. Current MediaWiki build time is about 36 minutes on my laptop.

diff --git a/maintenance/hiphop/compiler.conf b/maintenance/hiphop/compiler.conf
new file mode 100644 (file)
index 0000000..122b7b5
--- /dev/null
+++ b/maintenance/hiphop/compiler.conf
@@ -0,0 +1,3 @@
+GenerateSourceInfo = true
+EnableEval = 2
+AllDynamic = true

diff --git a/maintenance/hiphop/file-list.small b/maintenance/hiphop/file-list.small
new file mode 100644 (file)
index 0000000..84acdda
--- /dev/null
+++ b/maintenance/hiphop/file-list.small
@@ -0,0 +1,286 @@
+img_auth.php
+includes/AjaxDispatcher.php
+includes/AjaxFunctions.php
+includes/AjaxResponse.php
+includes/Article.php
+includes/AuthPlugin.php
+includes/AutoLoader.php
+includes/Autopromote.php
+includes/BacklinkCache.php
+includes/Block.php
+includes/CacheDependency.php
+includes/Category.php
+includes/CategoryPage.php
+includes/Categoryfinder.php
+includes/Cdb.php
+includes/Cdb_PHP.php
+includes/ChangeTags.php
+includes/ChangesFeed.php
+includes/ChangesList.php
+includes/Collation.php
+includes/ConfEditor.php
+includes/Cookie.php
+includes/Credits.php
+includes/DefaultSettings.php
+includes/Defines.php
+includes/DjVuImage.php
+includes/EditPage.php
+includes/Exception.php
+includes/Exif.php
+includes/Export.php
+includes/ExternalEdit.php
+includes/ExternalStore.php
+includes/ExternalStoreDB.php
+includes/ExternalStoreHttp.php
+includes/ExternalUser.php
+includes/FakeTitle.php
+includes/Fallback.php
+includes/Feed.php
+includes/FeedUtils.php
+includes/FileDeleteForm.php
+includes/FileRevertForm.php
+includes/ForkController.php
+includes/FormOptions.php
+includes/GenderCache.php
+includes/GlobalFunctions.php
+includes/HTMLCacheUpdate.php
+includes/HTMLFileCache.php
+includes/HTMLForm.php
+includes/HistoryBlob.php
+includes/HistoryPage.php
+includes/Hooks.php
+includes/Html.php
+includes/HttpFunctions.old.php
+includes/HttpFunctions.php
+includes/IP.php
+includes/ImageFunctions.php
+includes/ImageGallery.php
+includes/ImagePage.php
+includes/ImageQueryPage.php
+includes/Import.php
+includes/Init.php
+includes/Interwiki.php
+includes/Licenses.php
+includes/LinkBatch.php
+includes/LinkCache.php
+includes/LinkFilter.php
+includes/Linker.php
+includes/LinksUpdate.php
+includes/LocalisationCache.php
+includes/LogEventsList.php
+includes/LogPage.php
+includes/MWFunction.php
+includes/MacBinary.php
+includes/MagicWord.php
+includes/Math.php
+includes/MemcachedSessions.php
+includes/Message.php
+includes/MessageBlobStore.php
+includes/MessageCache.php
+includes/Metadata.php
+includes/MimeMagic.php
+includes/Namespace.php
+includes/OutputHandler.php
+includes/OutputPage.php
+includes/PageQueryPage.php
+includes/Pager.php
+includes/PatrolLog.php
+includes/PoolCounter.php
+includes/Preferences.php
+includes/PrefixSearch.php
+includes/ProfilerStub.php
+includes/ProtectionForm.php
+includes/ProxyTools.php
+includes/QueryPage.php
+includes/RawPage.php
+includes/RecentChange.php
+includes/RequestContext.php
+includes/Revision.php
+includes/Sanitizer.php
+includes/SeleniumWebSettings.php
+includes/Setup.php
+includes/SiteConfiguration.php
+includes/SiteStats.php
+includes/Skin.php
+includes/SkinLegacy.php
+includes/SkinTemplate.php
+includes/SpecialPage.php
+includes/SquidPurgeClient.php
+includes/SquidUpdate.php
+includes/Status.php
+includes/StreamFile.php
+includes/StringUtils.php
+includes/StubObject.php
+includes/Title.php
+includes/TitleArray.php
+includes/User.php
+includes/UserArray.php
+includes/UserMailer.php
+includes/UserRightsProxy.php
+includes/ViewCountUpdate.php
+includes/WatchedItem.php
+includes/WebRequest.php
+includes/WebResponse.php
+includes/WebStart.php
+includes/Wiki.php
+includes/WikiError.php
+includes/WikiMap.php
+includes/Xml.php
+includes/XmlTypeCheck.php
+includes/ZipDirectoryReader.php
+includes/db/CloneDatabase.php
+includes/db/Database.php
+includes/db/DatabaseMysql.php
+includes/db/LBFactory.php
+includes/db/LBFactory_Multi.php
+includes/db/LBFactory_Single.php
+includes/db/LoadBalancer.php
+includes/db/LoadMonitor.php
+includes/diff/DairikiDiff.php
+includes/diff/DifferenceEngine.php
+includes/diff/WikiDiff3.php
+includes/extauth/Hardcoded.php
+includes/extauth/MediaWiki.php
+includes/extauth/vB.php
+includes/filerepo/ArchivedFile.php
+includes/filerepo/FSRepo.php
+includes/filerepo/File.php
+includes/filerepo/FileRepo.php
+includes/filerepo/FileRepoStatus.php
+includes/filerepo/ForeignAPIFile.php
+includes/filerepo/ForeignAPIRepo.php
+includes/filerepo/ForeignDBFile.php
+includes/filerepo/ForeignDBRepo.php
+includes/filerepo/ForeignDBViaLBRepo.php
+includes/filerepo/LocalFile.php
+includes/filerepo/LocalRepo.php
+includes/filerepo/NullRepo.php
+includes/filerepo/OldLocalFile.php
+includes/filerepo/RepoGroup.php
+includes/filerepo/UnregisteredLocalFile.php
+includes/job/DoubleRedirectJob.php
+includes/job/EmaillingJob.php
+includes/job/EnotifNotifyJob.php
+includes/job/JobQueue.php
+includes/job/RefreshLinksJob.php
+includes/job/UploadFromUrlJob.php
+includes/json/FormatJson.php
+includes/json/Services_JSON.php
+includes/libs/CSSJanus.php
+includes/libs/CSSMin.php
+includes/libs/IEContentAnalyzer.php
+includes/libs/JavaScriptDistiller.php
+includes/libs/JavaScriptMinifier.php
+includes/libs/spyc.php
+includes/media/BMP.php
+includes/media/Bitmap.php
+includes/media/Bitmap_ClientOnly.php
+includes/media/DjVu.php
+includes/media/GIF.php
+includes/media/GIFMetadataExtractor.php
+includes/media/Generic.php
+includes/media/MediaTransformOutput.php
+includes/media/PNG.php
+includes/media/PNGMetadataExtractor.php
+includes/media/SVG.php
+includes/media/SVGMetadataExtractor.php
+includes/media/Tiff.php
+includes/normal/UtfNormal.php
+includes/normal/UtfNormalDefines.php
+includes/normal/UtfNormalUtil.php
+includes/objectcache/APCBagOStuff.php
+includes/objectcache/BagOStuff.php
+includes/objectcache/DBABagOStuff.php
+includes/objectcache/EhcacheBagOStuff.php
+includes/objectcache/EmptyBagOStuff.php
+includes/objectcache/HashBagOStuff.php
+includes/objectcache/MemcachedClient.php
+includes/objectcache/MemcachedPhpBagOStuff.php
+includes/objectcache/MultiWriteBagOStuff.php
+includes/objectcache/ObjectCache.php
+includes/objectcache/SqlBagOStuff.php
+includes/parser/CoreLinkFunctions.php
+includes/parser/CoreParserFunctions.php
+includes/parser/CoreTagHooks.php
+includes/parser/DateFormatter.php
+includes/parser/LinkHolderArray.php
+includes/parser/Parser.php
+includes/parser/ParserCache.php
+includes/parser/ParserOptions.php
+includes/parser/ParserOutput.php
+includes/parser/Parser_DiffTest.php
+includes/parser/Parser_LinkHooks.php
+includes/parser/Preprocessor.php
+includes/parser/Preprocessor_DOM.php
+includes/parser/Preprocessor_Hash.php
+includes/parser/StripState.php
+includes/parser/Tidy.php
+includes/proxy_check.php
+includes/resourceloader/ResourceLoader.php
+includes/resourceloader/ResourceLoaderContext.php
+includes/resourceloader/ResourceLoaderFileModule.php
+includes/resourceloader/ResourceLoaderModule.php
+includes/resourceloader/ResourceLoaderNoscriptModule.php
+includes/resourceloader/ResourceLoaderSiteModule.php
+includes/resourceloader/ResourceLoaderStartUpModule.php
+includes/resourceloader/ResourceLoaderUserGroupsModule.php
+includes/resourceloader/ResourceLoaderUserModule.php
+includes/resourceloader/ResourceLoaderUserOptionsModule.php
+includes/resourceloader/ResourceLoaderWikiModule.php
+includes/revisiondelete/RevisionDelete.php
+includes/revisiondelete/RevisionDeleteAbstracts.php
+includes/revisiondelete/RevisionDeleteUser.php
+includes/revisiondelete/RevisionDeleter.php
+includes/search/SearchEngine.php
+includes/search/SearchMySQL.php
+includes/search/SearchUpdate.php
+includes/specials/SpecialBlankpage.php
+includes/specials/SpecialCategories.php
+includes/specials/SpecialContributions.php
+includes/specials/SpecialEditWatchlist.php
+includes/specials/SpecialEmailuser.php
+includes/specials/SpecialExport.php
+includes/specials/SpecialImport.php
+includes/specials/SpecialLog.php
+includes/specials/SpecialMovepage.php
+includes/specials/SpecialPreferences.php
+includes/specials/SpecialRandompage.php
+includes/specials/SpecialRandomredirect.php
+includes/specials/SpecialRecentchanges.php
+includes/specials/SpecialRecentchangeslinked.php
+includes/specials/SpecialResetpass.php
+includes/specials/SpecialSearch.php
+includes/specials/SpecialSpecialpages.php
+includes/specials/SpecialUpload.php
+includes/specials/SpecialUploadStash.php
+includes/specials/SpecialUserlogin.php
+includes/specials/SpecialUserlogout.php
+includes/specials/SpecialWatchlist.php
+includes/specials/SpecialWhatlinkshere.php
+includes/templates/NoLocalSettings.php
+includes/templates/PHP4.php
+includes/templates/Userlogin.php
+includes/upload/UploadBase.php
+includes/upload/UploadFromFile.php
+includes/upload/UploadFromStash.php
+includes/upload/UploadFromUrl.php
+includes/upload/UploadStash.php
+index.php
+languages/Language.php
+languages/LanguageConverter.php
+languages/Names.php
+load.php
+maintenance/commandLine.inc
+maintenance/doMaintenance.php
+maintenance/eval.php
+maintenance/install-utils.inc
+maintenance/Maintenance.php
+opensearch_desc.php
+profileinfo.php
+redirect.php
+resources/Resources.php
+serialized/serialize.php
+skins/Vector.deps.php
+skins/Vector.php
+thumb.php
+trackback.php

diff --git a/maintenance/hiphop/make b/maintenance/hiphop/make
new file mode 100755 (executable)
index 0000000..88b2913
--- /dev/null
+++ b/maintenance/hiphop/make
@@ -0,0 +1,162 @@
+#!/usr/bin/hphpi -f 
+<?php
+
+require( dirname( __FILE__ ) . '/../Maintenance.php' );
+
+class MakeHipHop extends Maintenance {
+
+       function execute() {
+               $startTime = time();
+
+               $sourceDir = realpath( dirname( __FILE__ ) );
+               $IP = realpath( "$sourceDir/../.." );
+               $buildDir = "$sourceDir/build";
+               $outDir = "$buildDir/hiphop-output";
+               $persistentDir = "$buildDir/persistent" ;
+
+               if ( !is_dir( $buildDir ) ) {
+                       mkdir( $buildDir, 0777, true );
+               }
+               if ( !is_dir( $persistentDir ) ) {
+                       mkdir( $persistentDir, 0777, true );
+               }
+
+               # With the CentOS RPMs, you just get g++44, no g++, so we have to 
+               # use the environment
+               if ( isset( $_ENV['CXX'] ) ) {
+                       $cxx = $_ENV['CXX'];
+               } else {
+                       $cxx = 'g++';
+               }
+
+               # Create a function that provides the HipHop compiler version, and 
+               # doesn't exist when MediaWiki is invoked in interpreter mode.
+               $version = trim( `hphp --version` );
+               file_put_contents(
+                       "$buildDir/HipHopCompilerVersion.php",
+                       "<" . "?php\n" .
+                       "function wfHipHopCompilerVersion() {\n" .
+                       "return " . var_export( $version, true ) . ";\n" .
+                       "}\n"
+               );
+
+               # Generate the C++
+               passthru(
+                       'hphp' .
+                       ' --target=cpp' .
+                       ' --format=file' .
+                       ' --input-dir=' . wfEscapeShellArg( $IP ) .
+                       ' --input-list=' . wfEscapeShellArg( "$sourceDir/file-list.small" ) .
+                       ' --inputs=' . wfEscapeShellArg( "$buildDir/HipHopCompilerVersion.php" ) .
+                       ' --config=' . wfEscapeShellArg( "$sourceDir/compiler.conf" ) .
+                       ' --parse-on-demand=false' .
+                       ' --program=mediawiki-hphp' .
+                       ' --output-dir=' . wfEscapeShellArg( $outDir ) .
+                       ' --log=3' );
+
+               # Copy the generated C++ files into the source directory for cmake
+               $iter = new RecursiveIteratorIterator( 
+                       new RecursiveDirectoryIterator( $outDir ),
+                       RecursiveIteratorIterator::SELF_FIRST );
+               $sourceFiles = array();
+               $regenerateMakefile = false;
+               $numFiles = 0;
+               $numFilesChanged = 0;
+               foreach ( $iter as $sourcePath => $file ) {
+                       $name = substr( $sourcePath, strlen( $outDir ) + 1 );
+                       $sourceFiles[$name] = true;
+                       $destPath = "$persistentDir/$name";
+                       if ( $file->isDir() ) {
+                               if ( !is_dir( $destPath ) ) {
+                                       mkdir( $destPath );
+                               }
+                               continue;
+                       }
+
+                       $numFiles++;
+                       # Remove any files that weren't touched, these may have been removed
+                       # from file-list, we should not compile them
+                       if ( $file->getMTime() < $startTime ) {
+                               if ( file_exists( $destPath ) ) {
+                                       unlink( $destPath );
+                                       # Files removed, regenerate the makefile
+                                       $regenerateMakefile = true;
+                               }
+                               unlink( $sourcePath );
+                               $numFilesChanged++;
+                               continue;
+                       }
+
+                       if ( file_exists( $destPath ) ) {
+                               $sourceHash = md5( file_get_contents( $sourcePath ) );
+                               $destHash = md5( file_get_contents( $destPath ) );
+                               if ( $sourceHash == $destHash ) {
+                                       continue;
+                               }
+                       } else {
+                               # New files added, regenerate the makefile
+                               $regenerateMakefile = true;
+                       }
+                       $numFilesChanged++;
+                       copy( $sourcePath, $destPath );
+               }
+
+               echo "MediaWiki: $numFilesChanged files changed out of $numFiles\n";
+
+               if ( !file_exists( "$persistentDir/CMakeLists.txt" ) ) {
+                       # Run cmake for the first time
+                       $regenerateMakefile = true;
+               }
+
+               # Do our own version of $HPHP_HOME/bin/run.sh, which isn't so broken.
+               # HipHop's RELEASE mode seems to be stuck always on, so symbols get 
+               # stripped. Also we will try keeping the generated .o files instead of 
+               # throwing away hours of CPU time every time you make a typo.
+
+               chdir( $persistentDir );
+
+               if ( $regenerateMakefile ) {
+                       copy( $_ENV['HPHP_HOME'] . '/bin/CMakeLists.base.txt', 
+                               "$persistentDir/CMakeLists.txt" );
+
+                       if ( file_exists( "$persistentDir/CMakeCache.txt" ) ) {
+                               unlink( "$persistentDir/CMakeCache.txt" );
+                       }
+
+                       $cmd = 'cmake' .
+                               ' -D CMAKE_BUILD_TYPE:string=Debug' .
+                               ' -D PROGRAM_NAME:string=mediawiki-hphp';
+                       
+                       if ( file_exists( '/usr/bin/ccache' ) ) {
+                               $cmd .= ' -D CMAKE_CXX_COMPILER:string=ccache' .
+                                       ' -D CMAKE_CXX_COMPILER_ARG1:string=' . wfEscapeShellArg( $cxx );
+                       }
+
+                       $cmd .= ' .';
+                       echo "$cmd\n";
+                       passthru( $cmd );
+               }
+
+               # Run make. This is the slow step.
+               passthru( 'make' );
+
+               $elapsed = time() - $startTime;
+
+               echo "Completed in ";
+               if ( $elapsed >= 3600 ) {
+                       $hours = floor( $elapsed / 3600 );
+                       echo $hours . 'h ';
+                       $elapsed -= $hours * 3600;
+               }
+               if ( $elapsed >= 60 ) {
+                       $minutes = floor( $elapsed / 60 );
+                       echo $minutes . 'm ';
+                       $elapsed -= $minutes * 60;
+               }
+               echo $elapsed . "s\n";
+               echo "The MediaWiki executable is at build/persistent/mediawiki-hphp\n";
+       }
+}
+
+$maintClass = 'MakeHipHop';
+require_once( RUN_MAINTENANCE_IF_MAIN );

diff --git a/maintenance/hiphop/run-server b/maintenance/hiphop/run-server
new file mode 100755 (executable)
index 0000000..87695dd
--- /dev/null
+++ b/maintenance/hiphop/run-server
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+sourceDir=`dirname "$0"`
+
+"$sourceDir"/build/persistent/mediawiki-hphp \
+       --config="$sourceDir/server.conf" \
+       --mode=server \
+       --port=8080
+
+

diff --git a/maintenance/hiphop/server.conf b/maintenance/hiphop/server.conf
new file mode 100644 (file)
index 0000000..a8cf0b1
--- /dev/null
+++ b/maintenance/hiphop/server.conf
@@ -0,0 +1,13 @@
+Log {
+       Level = Verbose
+       UseLogFile = true
+       File = /dev/stdout
+       NativeStackTrace = true
+       InjectedStackTrace = true
+}
+Debug {
+       FullBacktrace = true
+       ServerStackTrace = true
+       ServerErrorMessage = true
+       TranslateSource = true
+}