From: Brion Vibber Date: Sun, 2 Apr 2006 04:19:27 +0000 (+0000) Subject: Live fixes: lots of scary magic runes; php 5 fixes; reporting; etc X-Git-Tag: 1.6.0~66 X-Git-Url: https://git.cyclocoop.org/admin/?a=commitdiff_plain;h=c4409658c6c5b468d9df0842ac7ff441a26c1f45;p=lhc%2Fweb%2Fwiklou.git Live fixes: lots of scary magic runes; php 5 fixes; reporting; etc --- diff --git a/maintenance/storage/compressOld.inc b/maintenance/storage/compressOld.inc index efd45b3567..c1fd7049fe 100644 --- a/maintenance/storage/compressOld.inc +++ b/maintenance/storage/compressOld.inc @@ -103,10 +103,13 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh */ # For each article, get a list of revisions which fit the criteria + # No recompression, use a condition on old_flags + # Don't compress object type entities, because that might produce data loss when + # overwriting bulk storage concat rows. Don't compress external references, because + # the script doesn't yet delete rows from external storage. $conds = array( - "old_flags NOT LIKE '%object%' " . - " AND (old_flags NOT LIKE '%external%' OR old_text NOT LIKE 'DB://%/%/%')"); + "old_flags NOT LIKE '%object%' AND old_flags NOT LIKE '%external%'"); if ( $beginDate ) { $conds[] = "rev_timestamp>'" . $beginDate . "'"; @@ -133,8 +136,12 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh $totalMatchingRevisions = 0; $masterPos = false; for ( $pageId = $startId; $pageId <= $maxPageId; $pageId++ ) { - wfWaitForSlaves( 10 ); + wfWaitForSlaves( 5 ); + # Wake up + $dbr->ping(); + + # Get the page row $pageRes = $dbr->select( 'page', array('page_id', 'page_namespace', 'page_title'), $pageConds + array('page_id' => $pageId), $fname ); if ( $dbr->numRows( $pageRes ) == 0 ) { @@ -213,7 +220,6 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh $stub = $chunk->addItem( $text ); $stub->setLocation( $primaryOldid ); $stub->setReferrer( $oldid ); - $hash = $stub->getHash(); print '.'; $usedChunk = true; } @@ -277,6 +283,7 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh print "/"; $dbw->commit(); $i += $thisChunkSize; + wfWaitForSlaves( 5 ); } print "\n"; } diff --git a/maintenance/storage/compressOld.php b/maintenance/storage/compressOld.php index 9cee05546c..d597f1dfc5 100644 --- a/maintenance/storage/compressOld.php +++ b/maintenance/storage/compressOld.php @@ -45,7 +45,7 @@ $defaults = array( 't' => 'concat', 'c' => 20, 's' => 0, - 'f' => 3, + 'f' => 5, 'h' => 100, 'b' => '', 'e' => '', diff --git a/maintenance/storage/moveToExternal.php b/maintenance/storage/moveToExternal.php index aabc5e6c36..0b46f70bdf 100644 --- a/maintenance/storage/moveToExternal.php +++ b/maintenance/storage/moveToExternal.php @@ -1,7 +1,6 @@ old_flags ); - if ( in_array( 'object', $flags ) - && substr( $row->old_text, 0, strlen( STUB_HEADER ) ) === STUB_HEADER ) - { - resolveStub( $id, $row->old_text, $row->old_flags ); + $text = $row->old_text; + if ( $row->old_flags === '' ) { + $flags = 'external'; + } else { + $flags = "{$row->old_flags},external"; + } + + if ( strpos( $flags, 'object' ) !== false ) { + $obj = unserialize( $text ); + $className = strtolower( get_class( $obj ) ); + if ( $className == 'historyblobstub' ) { + resolveStub( $id, $row->old_text, $row->old_flags ); + continue; + } elseif ( $className == 'historyblobcurstub' ) { + $text = gzdeflate( $obj->getText() ); + $flags = 'utf-8,gzip,external'; + } elseif ( $className == 'concatenatedgziphistoryblob' ) { + // Do nothing + } else { + print "Warning: unrecognised object class \"$className\"\n"; + continue; + } + } + + if ( strlen( $text ) < 100 ) { + // Don't move tiny revisions continue; } - $url = $ext->store( $cluster, $row->old_text ); + #print "Storing " . strlen( $text ) . " bytes to $url\n"; + + $url = $ext->store( $cluster, $text ); if ( !$url ) { print "Error writing to external storage\n"; exit; } - if ( $row->old_flags === '' ) { - $flags = 'external'; - } else { - $flags = "{$row->old_flags},external"; - } $dbw->update( 'text', array( 'old_flags' => $flags, 'old_text' => $url ), array( 'old_id' => $id ), $fname ); diff --git a/maintenance/storage/resolveStubs.php b/maintenance/storage/resolveStubs.php index 583341848f..e93d5c972f 100644 --- a/maintenance/storage/resolveStubs.php +++ b/maintenance/storage/resolveStubs.php @@ -18,45 +18,37 @@ if ( !defined( 'MEDIAWIKI' ) ) { function resolveStubs() { $fname = 'resolveStubs'; - print "Retrieving stub rows...\n"; $dbr =& wfGetDB( DB_SLAVE ); + $dbw =& wfGetDB( DB_MASTER ); $maxID = $dbr->selectField( 'text', 'MAX(old_id)', false, $fname ); - $stubs = array(); - $flagsArray = array(); - - # Do it in 100 blocks - for ( $b = 0; $b < 100; $b++ ) { - print "$b%\r"; - $start = intval($maxID / 100) * $b + 1; - $end = intval($maxID / 100) * ($b + 1); - + $blockSize = 10000; + $numBlocks = intval( $maxID / $blockSize ) + 1; + + for ( $b = 0; $b < $numBlocks; $b++ ) { + wfWaitForSlaves( 5 ); + + printf( "%5.2f%%\n", $b / $numBlocks * 100 ); + $start = intval($maxID / $numBlocks) * $b + 1; + $end = intval($maxID / $numBlocks) * ($b + 1); + $stubs = array(); + $flagsArray = array(); + + $res = $dbr->select( 'text', array( 'old_id', 'old_text', 'old_flags' ), - "old_id>=$start AND old_id<=$end AND old_flags like '%object%' ". + "old_id>=$start AND old_id<=$end " . + # Using a more restrictive flag set for now, until I do some more analysis -- TS + #"AND old_flags LIKE '%object%' AND old_flags NOT LIKE '%external%' ". + + "AND old_flags='object' " . "AND old_text LIKE 'O:15:\"historyblobstub\"%'", $fname ); while ( $row = $dbr->fetchObject( $res ) ) { - $stubs[$row->old_id] = $row->old_text; - $flagsArray[$row->old_id] = $row->old_flags; + resolveStub( $row->old_id, $row->old_text, $row->old_flags ); } $dbr->freeResult( $res ); - } - print "100%\n"; - - print "\nConverting " . count( $stubs ) . " rows ...\n"; - # Get master database, no transactions - $dbw =& wfGetDB( DB_MASTER ); - $dbw->clearFlag( DBO_TRX ); - $dbw->immediateCommit(); - - $i = 0; - foreach( $stubs as $id => $stub ) { - if ( !(++$i % REPORTING_INTERVAL) ) { - print "$i\n"; - wfWaitForSlaves( 5 ); - } - - resolveStub( $id, $stub, $flagsArray[$id] ); + } + print "100%\n"; } /** @@ -71,8 +63,8 @@ function resolveStub( $id, $stubText, $flags ) { $dbr =& wfGetDB( DB_SLAVE ); $dbw =& wfGetDB( DB_MASTER ); - if ( get_class( $stub ) !== 'historyblobstub' ) { - print "Error, invalid stub object\n"; + if ( strtolower( get_class( $stub ) ) !== 'historyblobstub' ) { + print "Error found object of class " . get_class( $stub ) . ", expecting historyblobstub\n"; return; } @@ -84,7 +76,7 @@ function resolveStub( $id, $stubText, $flags ) { if ( !$externalRow ) { # Object wasn't external - continue; + return; } # Preserve the legacy encoding flag, but switch from object to external