From bf63ffaa2c7e472181d74829a8bb380ad2ee9735 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 2 Oct 2006 17:04:13 +0000 Subject: [PATCH] Committed a bunch of live hacks from Wikimedia servers A few left, but these don't look too evil --- RELEASE-NOTES | 1 + includes/Database.php | 25 +++++++++--- includes/FileStore.php | 2 +- includes/LoadBalancer.php | 4 +- includes/MagicWord.php | 5 +++ includes/Parser.php | 8 +++- includes/ProfilerSimple.php | 6 +++ includes/ProfilerSimpleUDP.php | 6 +++ includes/Setup.php | 3 ++ includes/SiteStatsUpdate.php | 10 +++++ includes/SpecialExport.php | 10 ++--- includes/SpecialShortpages.php | 3 ++ includes/StreamFile.php | 6 ++- includes/Wiki.php | 6 +++ maintenance/addwiki.php | 56 +++++++++++++++++++-------- maintenance/deleteBatch.php | 8 +++- maintenance/dumpInterwiki.inc | 1 + maintenance/mctest.php | 13 ++++--- maintenance/ourusers.php | 6 +-- maintenance/runJobs.php | 12 ++++++ maintenance/storage/compressOld.inc | 11 ++++++ serialized/serialize-localisation.php | 1 + serialized/serialize.php | 1 + thumb.php | 7 +++- 24 files changed, 166 insertions(+), 45 deletions(-) diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 13096c98b8..22babed168 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -251,6 +251,7 @@ it from source control: http://www.mediawiki.org/wiki/Download_from_SVN "I can't import dumps from Wikipedia" complaints * (bug 7460) Revert broken patch for bug 7226 which slows down Special:Allmessages by a factor of 16 +* Committed a bunch of live hacks from Wikimedia servers == Languages updated == diff --git a/includes/Database.php b/includes/Database.php index eaecc1a0ef..f620b28ea3 100644 --- a/includes/Database.php +++ b/includes/Database.php @@ -490,12 +490,26 @@ class Database { $success = false; - if ( $this->mFlags & DBO_PERSISTENT ) { - @/**/$this->mConn = mysql_pconnect( $server, $user, $password ); - } else { - # Create a new connection... - @/**/$this->mConn = mysql_connect( $server, $user, $password, true ); + wfProfileIn("dbconnect-$server"); + + # LIVE PATCH by Tim, ask Domas for why: retry loop + $this->mConn = false; + for ( $i = 0; $i < 3 && !$this->mConn; $i++ ) { + if ( $i > 1 ) { + usleep( 1000 ); + } + if ( $this->mFlags & DBO_PERSISTENT ) { + @/**/$this->mConn = mysql_pconnect( $server, $user, $password ); + } else { + # Create a new connection... + @/**/$this->mConn = mysql_connect( $server, $user, $password, true ); + } + if ($this->mConn === false) { + wfLogDBError("Connect loop error ($server): " . mysql_errno() . " - " . mysql_error()."\n"); + } } + + wfProfileOut("dbconnect-$server"); if ( $dbName != '' ) { if ( $this->mConn !== false ) { @@ -503,6 +517,7 @@ class Database { if ( !$success ) { $error = "Error selecting database $dbName on server {$this->mServer} " . "from client host {$wguname['nodename']}\n"; + wfLogDBError(" Error selecting database $dbname on server {$this->mServer} \n"); wfDebug( $error ); } } else { diff --git a/includes/FileStore.php b/includes/FileStore.php index 85aaedfe15..c81504a7fa 100644 --- a/includes/FileStore.php +++ b/includes/FileStore.php @@ -374,4 +374,4 @@ class FSTransaction { class FSException extends MWException { } -?> \ No newline at end of file +?> diff --git a/includes/LoadBalancer.php b/includes/LoadBalancer.php index cbee03ea06..33c23bb100 100644 --- a/includes/LoadBalancer.php +++ b/includes/LoadBalancer.php @@ -623,6 +623,9 @@ class LoadBalancer { wfProfileOut( __METHOD__ ); return $times; } + wfIncrStats( 'lag_cache_miss_expired' ); + } else { + wfIncrStats( 'lag_cache_miss_absent' ); } # Cache key missing or expired @@ -642,7 +645,6 @@ class LoadBalancer { # But don't give the timestamp to the caller unset($times['timestamp']); - wfIncrStats( 'lag_cache_miss' ); wfProfileOut( __METHOD__ ); return $times; } diff --git a/includes/MagicWord.php b/includes/MagicWord.php index 826a7e9d04..511efab4cf 100644 --- a/includes/MagicWord.php +++ b/includes/MagicWord.php @@ -155,6 +155,11 @@ class MagicWord { global $wgContLang; $this->mId = $id; $wgContLang->getMagic( $this ); + if ( !$this->mSynonyms ) { + $this->mSynonyms = array( 'dkjsagfjsgashfajsh' ); + #throw new MWException( "Error: invalid magic word '$id'" ); + wfDebugLog( 'exception', "Error: invalid magic word '$id'\n" ); + } } /** diff --git a/includes/Parser.php b/includes/Parser.php index 733218028f..31ec709931 100644 --- a/includes/Parser.php +++ b/includes/Parser.php @@ -272,7 +272,7 @@ class Parser */ global $wgUseTidy, $wgAlwaysUseTidy, $wgContLang; - $fname = 'Parser::parse'; + $fname = 'Parser::parse-' . wfGetCaller(); wfProfileIn( $fname ); if ( $clearState ) { @@ -3787,7 +3787,11 @@ class Parser wfProfileIn($fname); - $this->mTitle = $wgTitle; + if ( $wgTitle ) { + $this->mTitle = $wgTitle; + } else { + $this->mTitle = Title::newFromText('msg'); + } $this->mOptions = $options; $this->setOutputType( OT_MSG ); $this->clearState(); diff --git a/includes/ProfilerSimple.php b/includes/ProfilerSimple.php index 0f2b59a87b..d5bdaf94dd 100644 --- a/includes/ProfilerSimple.php +++ b/includes/ProfilerSimple.php @@ -11,6 +11,8 @@ require_once(dirname(__FILE__).'/Profiler.php'); class ProfilerSimple extends Profiler { + var $mMinimumTime = 0; + function ProfilerSimple() { global $wgRequestTime,$wgRUstart; if (!empty($wgRequestTime) && !empty($wgRUstart)) { @@ -33,6 +35,10 @@ class ProfilerSimple extends Profiler { } } + function setMinimum( $min ) { + $this->mMinimumTime = $min; + } + function profileIn($functionname) { global $wgDebugFunctionEntry; if ($wgDebugFunctionEntry) { diff --git a/includes/ProfilerSimpleUDP.php b/includes/ProfilerSimpleUDP.php index 5c8da1362b..1eb2c5f01f 100644 --- a/includes/ProfilerSimpleUDP.php +++ b/includes/ProfilerSimpleUDP.php @@ -12,6 +12,12 @@ class ProfilerSimpleUDP extends ProfilerSimple { global $wgUDPProfilerPort; global $wgDBname; + if ( $this->mCollated['-total']['real'] < $this->mMinimumTime ) { + # Less than minimum, ignore + return; + } + + $sock = socket_create(AF_INET, SOCK_DGRAM, SOL_UDP); $plength=0; $packet=""; diff --git a/includes/Setup.php b/includes/Setup.php index 2a8e6e5b92..d529fea995 100644 --- a/includes/Setup.php +++ b/includes/Setup.php @@ -184,7 +184,10 @@ wfProfileIn( $fname.'-extensions' ); # of the extension file. This allows the extension to perform # any necessary initialisation in the fully initialised environment foreach ( $wgExtensionFunctions as $func ) { + $profName = $fname.'-extensions-'.strval( $func ); + wfProfileIn( $profName ); call_user_func( $func ); + wfProfileOut( $profName ); } // For compatibility diff --git a/includes/SiteStatsUpdate.php b/includes/SiteStatsUpdate.php index 1b6d3804e3..b91dcfeb54 100644 --- a/includes/SiteStatsUpdate.php +++ b/includes/SiteStatsUpdate.php @@ -75,8 +75,18 @@ class SiteStatsUpdate { if ( $updates ) { $site_stats = $dbw->tableName( 'site_stats' ); $sql = $dbw->limitResultForUpdate("UPDATE $site_stats SET $updates", 1); + $dbw->begin(); $dbw->query( $sql, $fname ); + $dbw->commit(); } + + /* + global $wgDBname, $wgTitle; + if ( $this->mGood && $wgDBname == 'enwiki' ) { + $good = $dbw->selectField( 'site_stats', 'ss_good_articles', '', $fname ); + error_log( $good . ' ' . $wgTitle->getPrefixedDBkey() . "\n", 3, '/home/wikipedia/logs/million.log' ); + } + */ } } ?> diff --git a/includes/SpecialExport.php b/includes/SpecialExport.php index a402b659ab..3e56f2fc0f 100644 --- a/includes/SpecialExport.php +++ b/includes/SpecialExport.php @@ -31,13 +31,12 @@ require_once( 'Export.php' ); function wfSpecialExport( $page = '' ) { global $wgOut, $wgRequest, $wgExportAllowListContributors; global $wgExportAllowHistory, $wgExportMaxHistory; - $maxLimit = 200; $curonly = true; $fullHistory = array( 'dir' => 'asc', 'offset' => false, - 'limit' => $maxLimit, + 'limit' => $wgExportMaxHistory, ); if( $wgRequest->wasPosted() ) { $page = $wgRequest->getText( 'pages' ); @@ -53,13 +52,13 @@ function wfSpecialExport( $page = '' ) { $history = array( 'dir' => 'asc', 'offset' => false, - 'limit' => $maxLimit, + 'limit' => $wgExportMaxHistory, ); $historyCheck = $wgRequest->getCheck( 'history' ); if ( $curonly ) { $history = MW_EXPORT_CURRENT; } elseif ( !$historyCheck ) { - if ( $limit > 0 && $limit < $maxLimit ) { + if ( $limit > 0 && $limit < $wgExportMaxHistory ) { $history['limit'] = $limit; } if ( !is_null( $offset ) ) { @@ -107,6 +106,7 @@ function wfSpecialExport( $page = '' ) { $exporter->openStream(); foreach( $pages as $page ) { + /* if( $wgExportMaxHistory && !$curonly ) { $title = Title::newFromText( $page ); if( $title ) { @@ -117,7 +117,7 @@ function wfSpecialExport( $page = '' ) { continue; } } - } + }*/ $exporter->pageByName( $page ); } diff --git a/includes/SpecialShortpages.php b/includes/SpecialShortpages.php index d8e13c7b35..34b3505bbd 100644 --- a/includes/SpecialShortpages.php +++ b/includes/SpecialShortpages.php @@ -65,6 +65,9 @@ class ShortPagesPage extends QueryPage { $dm = $wgContLang->getDirMark(); $title = Title::makeTitleSafe( $result->namespace, $result->title ); + if ( !$title ) { + return ''; + } $hlink = $skin->makeKnownLinkObj( $title, wfMsgHtml( 'hist' ), 'action=history' ); $plink = $this->isCached() ? $skin->makeLinkObj( $title ) diff --git a/includes/StreamFile.php b/includes/StreamFile.php index 83417185fd..99cb4d7cc8 100644 --- a/includes/StreamFile.php +++ b/includes/StreamFile.php @@ -6,10 +6,12 @@ function wfStreamFile( $fname ) { $stat = @stat( $fname ); if ( !$stat ) { header( 'HTTP/1.0 404 Not Found' ); + $encFile = htmlspecialchars( $fname ); + $encScript = htmlspecialchars( $_SERVER['SCRIPT_NAME'] ); echo "

File not found

-

Although this PHP script ({$_SERVER['SCRIPT_NAME']}) exists, the file requested for output -does not.

+

Although this PHP script ($encScript) exists, the file requested for output +($encFile) does not.

"; return; } diff --git a/includes/Wiki.php b/includes/Wiki.php index f6136389da..401756bec5 100644 --- a/includes/Wiki.php +++ b/includes/Wiki.php @@ -264,8 +264,14 @@ class MediaWiki { */ function doUpdates ( &$updates ) { wfProfileIn( 'MediaWiki::doUpdates' ); + $dbw =& wfGetDB( DB_MASTER ); foreach( $updates as $up ) { $up->doUpdate(); + + # Commit after every update to prevent lock contention + if ( $dbw->trxLevel() ) { + $dbw->commit(); + } } wfProfileOut( 'MediaWiki::doUpdates' ); } diff --git a/maintenance/addwiki.php b/maintenance/addwiki.php index 253033a3da..b7843632b2 100644 --- a/maintenance/addwiki.php +++ b/maintenance/addwiki.php @@ -33,26 +33,40 @@ function addWiki( $lang, $site, $dbName ) print "Initialising tables\n"; dbsource( "$maintenance/tables.sql", $dbw ); dbsource( "$IP/extensions/OAI/update_table.sql", $dbw ); + dbsource( "$IP/extensions/AntiSpoof/mysql/patch-antispoof.sql", $dbw ); $dbw->query( "INSERT INTO site_stats(ss_row_id) VALUES (1)" ); # Initialise external storage - if ( $wgDefaultExternalStore && preg_match( '!^DB://(.*)$!', $wgDefaultExternalStore, $m ) ) { - print "Initialising external storage...\n"; + if ( is_array( $wgDefaultExternalStore ) ) { + $stores = $wgDefaultExternalStore; + } elseif ( $stores ) { + $stores = array( $wgDefaultExternalStore ); + } else { + $stores = array(); + } + if ( count( $stores ) ) { require_once( 'ExternalStoreDB.php' ); + print "Initialising external storage $store...\n"; global $wgDBuser, $wgDBpassword, $wgExternalServers; - $cluster = $m[1]; - - # Hack - $wgExternalServers[$cluster][0]['user'] = $wgDBuser; - $wgExternalServers[$cluster][0]['password'] = $wgDBpassword; - - $store = new ExternalStoreDB; - $extdb =& $store->getMaster( $cluster ); - $extdb->query( "SET table_type=InnoDB" ); - $extdb->query( "CREATE DATABASE $dbName" ); - $extdb->selectDB( $dbName ); - dbsource( "$maintenance/storage/blobs.sql", $extdb ); - $extdb->immediateCommit(); + foreach ( $stores as $storeURL ) { + if ( !preg_match( '!^DB://(.*)$!', $storeURL, $m ) ) { + continue; + } + + $cluster = $m[1]; + + # Hack + $wgExternalServers[$cluster][0]['user'] = $wgDBuser; + $wgExternalServers[$cluster][0]['password'] = $wgDBpassword; + + $store = new ExternalStoreDB; + $extdb =& $store->getMaster( $cluster ); + $extdb->query( "SET table_type=InnoDB" ); + $extdb->query( "CREATE DATABASE $dbName" ); + $extdb->selectDB( $dbName ); + dbsource( "$maintenance/storage/blobs.sql", $extdb ); + $extdb->immediateCommit(); + } } $wgTitle = Title::newMainPage(); @@ -203,7 +217,17 @@ See the [http://www.wikipedia.org Wikipedia portal] for other language Wikipedia fclose( $file ); print "Sourcing interwiki SQL\n"; dbsource( $tempname, $dbw ); - unlink( $tempname ); + #unlink( $tempname ); + + # Create the upload dir + global $wgUploadDirectory; + if( file_exists( $wgUploadDirectory ) ) { + echo "$wgUploadDirectory already exists.\n"; + } else { + echo "Creating $wgUploadDirectory...\n"; + mkdir( $wgUploadDirectory, 0777 ); + chmod( $wgUploadDirectory, 0777 ); + } print "Script ended. You now want to run sync-common-all to publish *dblist files (check them for duplicates first)\n"; } diff --git a/maintenance/deleteBatch.php b/maintenance/deleteBatch.php index 697dffd7a4..234744c3d8 100644 --- a/maintenance/deleteBatch.php +++ b/maintenance/deleteBatch.php @@ -71,9 +71,13 @@ for ( $linenum = 1; !feof( $file ); $linenum++ ) { } else { $art = new Article( $page ); } - $art->doDelete( $reason ); + $success = $art->doDeleteArticle( $reason ); $dbw->immediateCommit(); - print "\n"; + if ( $success ) { + print "\n"; + } else { + print " FAILED\n"; + } if ( $interval ) { sleep( $interval ); diff --git a/maintenance/dumpInterwiki.inc b/maintenance/dumpInterwiki.inc index 3cca1e02d9..f7a56ee7af 100644 --- a/maintenance/dumpInterwiki.inc +++ b/maintenance/dumpInterwiki.inc @@ -43,6 +43,7 @@ function getRebuildInterwikiDump() { 'wikinews' => new Site( 'wikinews', 'n', 'wikinews.org' ), 'wikisource' => new Site( 'wikisource', 's', 'wikisource.org' ), 'wikimedia' => new Site( 'wikimedia', 'chapter', 'wikimedia.org' ), + 'wikiversity' => new Site( 'wikiversity', 'v', 'wikiversity.org' ), ); # List of language prefixes likely to be found in multi-language sites diff --git a/maintenance/mctest.php b/maintenance/mctest.php index 7deaa07706..316620d78e 100644 --- a/maintenance/mctest.php +++ b/maintenance/mctest.php @@ -5,6 +5,13 @@ $optionsWithArgs = array( 'i' ); require_once('commandLine.inc'); +function microtime_float() +{ + list($usec, $sec) = explode(" ", microtime()); + return ((float)$usec + (float)$sec); +} + + #$wgDebugLogFile = '/dev/stdout'; if ( isset( $args[0] ) ) { @@ -49,11 +56,5 @@ foreach ( $wgMemCachedServers as $server ) { print "set: $set incr: $incr get: $get time: $exectime\n"; } -function microtime_float() -{ - list($usec, $sec) = explode(" ", microtime()); - return ((float)$usec + (float)$sec); -} - ?> diff --git a/maintenance/ourusers.php b/maintenance/ourusers.php index e244578674..b50519d273 100644 --- a/maintenance/ourusers.php +++ b/maintenance/ourusers.php @@ -54,8 +54,7 @@ print "/*!40100 set global old_passwords=1 */;"; foreach( $hosts as $host ) { print "--\n-- $host\n--\n\n-- wikiuser\n\n"; print "GRANT REPLICATION CLIENT,PROCESS ON *.* TO 'wikiuser'@'$host' IDENTIFIED BY '$wikiuser_pass';\n"; - print "GRANT ALL PRIVILEGES ON `boardvote`.* TO 'wikiuser'@'$host' IDENTIFIED BY '$wikiuser_pass';\n"; - print "GRANT ALL PRIVILEGES ON `boardvote2005`.* TO 'wikiuser'@'$host' IDENTIFIED BY '$wikiuser_pass';\n"; + print "GRANT ALL PRIVILEGES ON `boardvote%`.* TO 'wikiuser'@'$host' IDENTIFIED BY '$wikiuser_pass';\n"; foreach( $databases as $db ) { print "GRANT SELECT, INSERT, UPDATE, DELETE ON `$db`.* TO 'wikiuser'@'$host' IDENTIFIED BY '$wikiuser_pass';\n"; } @@ -84,8 +83,7 @@ EOS; print "\n-- wikiadmin\n\n"; print "GRANT PROCESS, REPLICATION CLIENT ON *.* TO 'wikiadmin'@'$host' IDENTIFIED BY '$wikiadmin_pass';\n"; - print "GRANT ALL PRIVILEGES ON `boardvote`.* TO wikiadmin@'$host' IDENTIFIED BY '$wikiadmin_pass';\n"; - print "GRANT ALL PRIVILEGES ON `boardvote2005`.* TO wikiadmin@'$host' IDENTIFIED BY '$wikiadmin_pass';\n"; + print "GRANT ALL PRIVILEGES ON `boardvote%`.* TO wikiadmin@'$host' IDENTIFIED BY '$wikiadmin_pass';\n"; foreach ( $databases as $db ) { print "GRANT ALL PRIVILEGES ON `$db`.* TO wikiadmin@'$host' IDENTIFIED BY '$wikiadmin_pass';\n"; } diff --git a/maintenance/runJobs.php b/maintenance/runJobs.php index d72addc769..343cda8a3a 100644 --- a/maintenance/runJobs.php +++ b/maintenance/runJobs.php @@ -1,13 +1,22 @@ selectField( 'job', 'count(*)', '', 'runJobs.php' ) ) { while ( false != ($job = Job::pop()) ) { wfWaitForSlaves( 5 ); @@ -15,6 +24,9 @@ while ( $dbw->selectField( 'job', 'count(*)', '', 'runJobs.php' ) ) { if ( !$job->run() ) { print "Error: {$job->error}\n"; } + if ( $maxJobs && ++$n > $maxJobs ) { + break 2; + } } } ?> diff --git a/maintenance/storage/compressOld.inc b/maintenance/storage/compressOld.inc index b7d7094f4a..3c42684104 100644 --- a/maintenance/storage/compressOld.inc +++ b/maintenance/storage/compressOld.inc @@ -155,6 +155,17 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh $titleObj = Title::makeTitle( $pageRow->page_namespace, $pageRow->page_title ); print "$pageId\t" . $titleObj->getPrefixedDBkey() . " "; + print_r( + array( + 'rev_page' => $pageRow->page_id, + # Don't operate on the current revision + # Use < instead of <> in case the current revision has changed + # since the page select, which wasn't locking + 'rev_id < ' . $pageRow->page_latest + ) + $conds + ); + exit; + # Load revisions $revRes = $dbw->select( $tables, $fields, array( diff --git a/serialized/serialize-localisation.php b/serialized/serialize-localisation.php index 10624ba541..925806b0b5 100644 --- a/serialized/serialize-localisation.php +++ b/serialized/serialize-localisation.php @@ -1,5 +1,6 @@ path ) { +if ( $img->lastError && $img->lastError !== true ) { + header( 'HTTP/1.0 500 Internal Server Error' ); + echo "

Thumbnail generation error

" . + htmlspecialchars( $img->lastError ) . "
" . wfHostname() . + "

"; +} elseif ( $thumb && $thumb->path ) { wfStreamFile( $thumb->path ); } else { $badtitle = wfMsg( 'badtitle' ); -- 2.20.1