From: Tim Starling <tstarling@users.mediawiki.org>
Date: Mon, 4 Apr 2011 06:17:46 +0000 (+0000)
Subject: HipHop build support files.
X-Git-Tag: 1.31.0-rc.0~31032
X-Git-Url: https://git.cyclocoop.org/%7B%24admin_url%7Dmembres/cotisations/voir.php?a=commitdiff_plain;h=1ab6b47e5ab4fc0fc5729bcdc1ff6f293bbe85b1;p=lhc%2Fweb%2Fwiklou.git

HipHop build support files.
* Added a script called "make" which works around some of the inadequacies in the hphp build system:
** Allow incremental rebuilds. CMake is very smart and can do this with very little prompting, including header dependencies. This cuts down the build time for certain types of small change in the PHP source.
** Use ccache if it is present.
** Use debug mode, don't strip symbols.

* Added a script which runs the compiled program in server mode
* Added a PHP file list which is greatly cut down to reduce build time. Suitable only for initial testing. Current MediaWiki build time is about 36 minutes on my laptop.
---

diff --git a/maintenance/hiphop/compiler.conf b/maintenance/hiphop/compiler.conf
new file mode 100644
index 0000000000..122b7b5f52
--- /dev/null
+++ b/maintenance/hiphop/compiler.conf
@@ -0,0 +1,3 @@
+GenerateSourceInfo = true
+EnableEval = 2
+AllDynamic = true
diff --git a/maintenance/hiphop/file-list.small b/maintenance/hiphop/file-list.small
new file mode 100644
index 0000000000..84acdda147
--- /dev/null
+++ b/maintenance/hiphop/file-list.small
@@ -0,0 +1,286 @@
+img_auth.php
+includes/AjaxDispatcher.php
+includes/AjaxFunctions.php
+includes/AjaxResponse.php
+includes/Article.php
+includes/AuthPlugin.php
+includes/AutoLoader.php
+includes/Autopromote.php
+includes/BacklinkCache.php
+includes/Block.php
+includes/CacheDependency.php
+includes/Category.php
+includes/CategoryPage.php
+includes/Categoryfinder.php
+includes/Cdb.php
+includes/Cdb_PHP.php
+includes/ChangeTags.php
+includes/ChangesFeed.php
+includes/ChangesList.php
+includes/Collation.php
+includes/ConfEditor.php
+includes/Cookie.php
+includes/Credits.php
+includes/DefaultSettings.php
+includes/Defines.php
+includes/DjVuImage.php
+includes/EditPage.php
+includes/Exception.php
+includes/Exif.php
+includes/Export.php
+includes/ExternalEdit.php
+includes/ExternalStore.php
+includes/ExternalStoreDB.php
+includes/ExternalStoreHttp.php
+includes/ExternalUser.php
+includes/FakeTitle.php
+includes/Fallback.php
+includes/Feed.php
+includes/FeedUtils.php
+includes/FileDeleteForm.php
+includes/FileRevertForm.php
+includes/ForkController.php
+includes/FormOptions.php
+includes/GenderCache.php
+includes/GlobalFunctions.php
+includes/HTMLCacheUpdate.php
+includes/HTMLFileCache.php
+includes/HTMLForm.php
+includes/HistoryBlob.php
+includes/HistoryPage.php
+includes/Hooks.php
+includes/Html.php
+includes/HttpFunctions.old.php
+includes/HttpFunctions.php
+includes/IP.php
+includes/ImageFunctions.php
+includes/ImageGallery.php
+includes/ImagePage.php
+includes/ImageQueryPage.php
+includes/Import.php
+includes/Init.php
+includes/Interwiki.php
+includes/Licenses.php
+includes/LinkBatch.php
+includes/LinkCache.php
+includes/LinkFilter.php
+includes/Linker.php
+includes/LinksUpdate.php
+includes/LocalisationCache.php
+includes/LogEventsList.php
+includes/LogPage.php
+includes/MWFunction.php
+includes/MacBinary.php
+includes/MagicWord.php
+includes/Math.php
+includes/MemcachedSessions.php
+includes/Message.php
+includes/MessageBlobStore.php
+includes/MessageCache.php
+includes/Metadata.php
+includes/MimeMagic.php
+includes/Namespace.php
+includes/OutputHandler.php
+includes/OutputPage.php
+includes/PageQueryPage.php
+includes/Pager.php
+includes/PatrolLog.php
+includes/PoolCounter.php
+includes/Preferences.php
+includes/PrefixSearch.php
+includes/ProfilerStub.php
+includes/ProtectionForm.php
+includes/ProxyTools.php
+includes/QueryPage.php
+includes/RawPage.php
+includes/RecentChange.php
+includes/RequestContext.php
+includes/Revision.php
+includes/Sanitizer.php
+includes/SeleniumWebSettings.php
+includes/Setup.php
+includes/SiteConfiguration.php
+includes/SiteStats.php
+includes/Skin.php
+includes/SkinLegacy.php
+includes/SkinTemplate.php
+includes/SpecialPage.php
+includes/SquidPurgeClient.php
+includes/SquidUpdate.php
+includes/Status.php
+includes/StreamFile.php
+includes/StringUtils.php
+includes/StubObject.php
+includes/Title.php
+includes/TitleArray.php
+includes/User.php
+includes/UserArray.php
+includes/UserMailer.php
+includes/UserRightsProxy.php
+includes/ViewCountUpdate.php
+includes/WatchedItem.php
+includes/WebRequest.php
+includes/WebResponse.php
+includes/WebStart.php
+includes/Wiki.php
+includes/WikiError.php
+includes/WikiMap.php
+includes/Xml.php
+includes/XmlTypeCheck.php
+includes/ZipDirectoryReader.php
+includes/db/CloneDatabase.php
+includes/db/Database.php
+includes/db/DatabaseMysql.php
+includes/db/LBFactory.php
+includes/db/LBFactory_Multi.php
+includes/db/LBFactory_Single.php
+includes/db/LoadBalancer.php
+includes/db/LoadMonitor.php
+includes/diff/DairikiDiff.php
+includes/diff/DifferenceEngine.php
+includes/diff/WikiDiff3.php
+includes/extauth/Hardcoded.php
+includes/extauth/MediaWiki.php
+includes/extauth/vB.php
+includes/filerepo/ArchivedFile.php
+includes/filerepo/FSRepo.php
+includes/filerepo/File.php
+includes/filerepo/FileRepo.php
+includes/filerepo/FileRepoStatus.php
+includes/filerepo/ForeignAPIFile.php
+includes/filerepo/ForeignAPIRepo.php
+includes/filerepo/ForeignDBFile.php
+includes/filerepo/ForeignDBRepo.php
+includes/filerepo/ForeignDBViaLBRepo.php
+includes/filerepo/LocalFile.php
+includes/filerepo/LocalRepo.php
+includes/filerepo/NullRepo.php
+includes/filerepo/OldLocalFile.php
+includes/filerepo/RepoGroup.php
+includes/filerepo/UnregisteredLocalFile.php
+includes/job/DoubleRedirectJob.php
+includes/job/EmaillingJob.php
+includes/job/EnotifNotifyJob.php
+includes/job/JobQueue.php
+includes/job/RefreshLinksJob.php
+includes/job/UploadFromUrlJob.php
+includes/json/FormatJson.php
+includes/json/Services_JSON.php
+includes/libs/CSSJanus.php
+includes/libs/CSSMin.php
+includes/libs/IEContentAnalyzer.php
+includes/libs/JavaScriptDistiller.php
+includes/libs/JavaScriptMinifier.php
+includes/libs/spyc.php
+includes/media/BMP.php
+includes/media/Bitmap.php
+includes/media/Bitmap_ClientOnly.php
+includes/media/DjVu.php
+includes/media/GIF.php
+includes/media/GIFMetadataExtractor.php
+includes/media/Generic.php
+includes/media/MediaTransformOutput.php
+includes/media/PNG.php
+includes/media/PNGMetadataExtractor.php
+includes/media/SVG.php
+includes/media/SVGMetadataExtractor.php
+includes/media/Tiff.php
+includes/normal/UtfNormal.php
+includes/normal/UtfNormalDefines.php
+includes/normal/UtfNormalUtil.php
+includes/objectcache/APCBagOStuff.php
+includes/objectcache/BagOStuff.php
+includes/objectcache/DBABagOStuff.php
+includes/objectcache/EhcacheBagOStuff.php
+includes/objectcache/EmptyBagOStuff.php
+includes/objectcache/HashBagOStuff.php
+includes/objectcache/MemcachedClient.php
+includes/objectcache/MemcachedPhpBagOStuff.php
+includes/objectcache/MultiWriteBagOStuff.php
+includes/objectcache/ObjectCache.php
+includes/objectcache/SqlBagOStuff.php
+includes/parser/CoreLinkFunctions.php
+includes/parser/CoreParserFunctions.php
+includes/parser/CoreTagHooks.php
+includes/parser/DateFormatter.php
+includes/parser/LinkHolderArray.php
+includes/parser/Parser.php
+includes/parser/ParserCache.php
+includes/parser/ParserOptions.php
+includes/parser/ParserOutput.php
+includes/parser/Parser_DiffTest.php
+includes/parser/Parser_LinkHooks.php
+includes/parser/Preprocessor.php
+includes/parser/Preprocessor_DOM.php
+includes/parser/Preprocessor_Hash.php
+includes/parser/StripState.php
+includes/parser/Tidy.php
+includes/proxy_check.php
+includes/resourceloader/ResourceLoader.php
+includes/resourceloader/ResourceLoaderContext.php
+includes/resourceloader/ResourceLoaderFileModule.php
+includes/resourceloader/ResourceLoaderModule.php
+includes/resourceloader/ResourceLoaderNoscriptModule.php
+includes/resourceloader/ResourceLoaderSiteModule.php
+includes/resourceloader/ResourceLoaderStartUpModule.php
+includes/resourceloader/ResourceLoaderUserGroupsModule.php
+includes/resourceloader/ResourceLoaderUserModule.php
+includes/resourceloader/ResourceLoaderUserOptionsModule.php
+includes/resourceloader/ResourceLoaderWikiModule.php
+includes/revisiondelete/RevisionDelete.php
+includes/revisiondelete/RevisionDeleteAbstracts.php
+includes/revisiondelete/RevisionDeleteUser.php
+includes/revisiondelete/RevisionDeleter.php
+includes/search/SearchEngine.php
+includes/search/SearchMySQL.php
+includes/search/SearchUpdate.php
+includes/specials/SpecialBlankpage.php
+includes/specials/SpecialCategories.php
+includes/specials/SpecialContributions.php
+includes/specials/SpecialEditWatchlist.php
+includes/specials/SpecialEmailuser.php
+includes/specials/SpecialExport.php
+includes/specials/SpecialImport.php
+includes/specials/SpecialLog.php
+includes/specials/SpecialMovepage.php
+includes/specials/SpecialPreferences.php
+includes/specials/SpecialRandompage.php
+includes/specials/SpecialRandomredirect.php
+includes/specials/SpecialRecentchanges.php
+includes/specials/SpecialRecentchangeslinked.php
+includes/specials/SpecialResetpass.php
+includes/specials/SpecialSearch.php
+includes/specials/SpecialSpecialpages.php
+includes/specials/SpecialUpload.php
+includes/specials/SpecialUploadStash.php
+includes/specials/SpecialUserlogin.php
+includes/specials/SpecialUserlogout.php
+includes/specials/SpecialWatchlist.php
+includes/specials/SpecialWhatlinkshere.php
+includes/templates/NoLocalSettings.php
+includes/templates/PHP4.php
+includes/templates/Userlogin.php
+includes/upload/UploadBase.php
+includes/upload/UploadFromFile.php
+includes/upload/UploadFromStash.php
+includes/upload/UploadFromUrl.php
+includes/upload/UploadStash.php
+index.php
+languages/Language.php
+languages/LanguageConverter.php
+languages/Names.php
+load.php
+maintenance/commandLine.inc
+maintenance/doMaintenance.php
+maintenance/eval.php
+maintenance/install-utils.inc
+maintenance/Maintenance.php
+opensearch_desc.php
+profileinfo.php
+redirect.php
+resources/Resources.php
+serialized/serialize.php
+skins/Vector.deps.php
+skins/Vector.php
+thumb.php
+trackback.php
diff --git a/maintenance/hiphop/make b/maintenance/hiphop/make
new file mode 100755
index 0000000000..88b2913fb7
--- /dev/null
+++ b/maintenance/hiphop/make
@@ -0,0 +1,162 @@
+#!/usr/bin/hphpi -f 
+<?php
+
+require( dirname( __FILE__ ) . '/../Maintenance.php' );
+
+class MakeHipHop extends Maintenance {
+
+	function execute() {
+		$startTime = time();
+
+		$sourceDir = realpath( dirname( __FILE__ ) );
+		$IP = realpath( "$sourceDir/../.." );
+		$buildDir = "$sourceDir/build";
+		$outDir = "$buildDir/hiphop-output";
+		$persistentDir = "$buildDir/persistent" ;
+
+		if ( !is_dir( $buildDir ) ) {
+			mkdir( $buildDir, 0777, true );
+		}
+		if ( !is_dir( $persistentDir ) ) {
+			mkdir( $persistentDir, 0777, true );
+		}
+
+		# With the CentOS RPMs, you just get g++44, no g++, so we have to 
+		# use the environment
+		if ( isset( $_ENV['CXX'] ) ) {
+			$cxx = $_ENV['CXX'];
+		} else {
+			$cxx = 'g++';
+		}
+
+		# Create a function that provides the HipHop compiler version, and 
+		# doesn't exist when MediaWiki is invoked in interpreter mode.
+		$version = trim( `hphp --version` );
+		file_put_contents(
+			"$buildDir/HipHopCompilerVersion.php",
+			"<" . "?php\n" .
+			"function wfHipHopCompilerVersion() {\n" .
+			"return " . var_export( $version, true ) . ";\n" .
+			"}\n"
+		);
+
+		# Generate the C++
+		passthru(
+			'hphp' .
+			' --target=cpp' .
+			' --format=file' .
+			' --input-dir=' . wfEscapeShellArg( $IP ) .
+			' --input-list=' . wfEscapeShellArg( "$sourceDir/file-list.small" ) .
+			' --inputs=' . wfEscapeShellArg( "$buildDir/HipHopCompilerVersion.php" ) .
+			' --config=' . wfEscapeShellArg( "$sourceDir/compiler.conf" ) .
+			' --parse-on-demand=false' .
+			' --program=mediawiki-hphp' .
+			' --output-dir=' . wfEscapeShellArg( $outDir ) .
+			' --log=3' );
+
+		# Copy the generated C++ files into the source directory for cmake
+		$iter = new RecursiveIteratorIterator( 
+			new RecursiveDirectoryIterator( $outDir ),
+			RecursiveIteratorIterator::SELF_FIRST );
+		$sourceFiles = array();
+		$regenerateMakefile = false;
+		$numFiles = 0;
+		$numFilesChanged = 0;
+		foreach ( $iter as $sourcePath => $file ) {
+			$name = substr( $sourcePath, strlen( $outDir ) + 1 );
+			$sourceFiles[$name] = true;
+			$destPath = "$persistentDir/$name";
+			if ( $file->isDir() ) {
+				if ( !is_dir( $destPath ) ) {
+					mkdir( $destPath );
+				}
+				continue;
+			}
+
+			$numFiles++;
+			# Remove any files that weren't touched, these may have been removed
+			# from file-list, we should not compile them
+			if ( $file->getMTime() < $startTime ) {
+				if ( file_exists( $destPath ) ) {
+					unlink( $destPath );
+					# Files removed, regenerate the makefile
+					$regenerateMakefile = true;
+				}
+				unlink( $sourcePath );
+				$numFilesChanged++;
+				continue;
+			}
+
+			if ( file_exists( $destPath ) ) {
+				$sourceHash = md5( file_get_contents( $sourcePath ) );
+				$destHash = md5( file_get_contents( $destPath ) );
+				if ( $sourceHash == $destHash ) {
+					continue;
+				}
+			} else {
+				# New files added, regenerate the makefile
+				$regenerateMakefile = true;
+			}
+			$numFilesChanged++;
+			copy( $sourcePath, $destPath );
+		}
+
+		echo "MediaWiki: $numFilesChanged files changed out of $numFiles\n";
+
+		if ( !file_exists( "$persistentDir/CMakeLists.txt" ) ) {
+			# Run cmake for the first time
+			$regenerateMakefile = true;
+		}
+
+		# Do our own version of $HPHP_HOME/bin/run.sh, which isn't so broken.
+		# HipHop's RELEASE mode seems to be stuck always on, so symbols get 
+		# stripped. Also we will try keeping the generated .o files instead of 
+		# throwing away hours of CPU time every time you make a typo.
+
+		chdir( $persistentDir );
+
+		if ( $regenerateMakefile ) {
+			copy( $_ENV['HPHP_HOME'] . '/bin/CMakeLists.base.txt', 
+				"$persistentDir/CMakeLists.txt" );
+
+			if ( file_exists( "$persistentDir/CMakeCache.txt" ) ) {
+				unlink( "$persistentDir/CMakeCache.txt" );
+			}
+
+			$cmd = 'cmake' .
+				' -D CMAKE_BUILD_TYPE:string=Debug' .
+				' -D PROGRAM_NAME:string=mediawiki-hphp';
+			
+			if ( file_exists( '/usr/bin/ccache' ) ) {
+				$cmd .= ' -D CMAKE_CXX_COMPILER:string=ccache' .
+					' -D CMAKE_CXX_COMPILER_ARG1:string=' . wfEscapeShellArg( $cxx );
+			}
+
+			$cmd .= ' .';
+			echo "$cmd\n";
+			passthru( $cmd );
+		}
+
+		# Run make. This is the slow step.
+		passthru( 'make' );
+
+		$elapsed = time() - $startTime;
+
+		echo "Completed in ";
+		if ( $elapsed >= 3600 ) {
+			$hours = floor( $elapsed / 3600 );
+			echo $hours . 'h ';
+			$elapsed -= $hours * 3600;
+		}
+		if ( $elapsed >= 60 ) {
+			$minutes = floor( $elapsed / 60 );
+			echo $minutes . 'm ';
+			$elapsed -= $minutes * 60;
+		}
+		echo $elapsed . "s\n";
+		echo "The MediaWiki executable is at build/persistent/mediawiki-hphp\n";
+	}
+}
+
+$maintClass = 'MakeHipHop';
+require_once( RUN_MAINTENANCE_IF_MAIN );
diff --git a/maintenance/hiphop/run-server b/maintenance/hiphop/run-server
new file mode 100755
index 0000000000..87695dd02a
--- /dev/null
+++ b/maintenance/hiphop/run-server
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+sourceDir=`dirname "$0"`
+
+"$sourceDir"/build/persistent/mediawiki-hphp \
+	--config="$sourceDir/server.conf" \
+	--mode=server \
+	--port=8080
+
+
diff --git a/maintenance/hiphop/server.conf b/maintenance/hiphop/server.conf
new file mode 100644
index 0000000000..a8cf0b1195
--- /dev/null
+++ b/maintenance/hiphop/server.conf
@@ -0,0 +1,13 @@
+Log {
+	Level = Verbose
+	UseLogFile = true
+	File = /dev/stdout
+	NativeStackTrace = true
+	InjectedStackTrace = true
+}
+Debug {
+	FullBacktrace = true
+	ServerStackTrace = true
+	ServerErrorMessage = true
+	TranslateSource = true
+}