3 require_once( dirname( __FILE__
) . '/../Maintenance.php' );
5 class FixBug20757
extends Maintenance
{
6 var $batchSize = 10000;
7 var $mapCache = array();
9 var $maxMapCacheSize = 1000000;
11 function __construct() {
12 parent
::__construct();
13 $this->mDescription
= 'Script to fix bug 20757 assuming that blob_tracking is intact';
14 $this->addOption( 'dry-run', 'Report only' );
15 $this->addOption( 'start', 'old_id to start at', false, true );
19 $dbr = wfGetDB( DB_SLAVE
);
20 $dbw = wfGetDB( DB_MASTER
);
22 $dryRun = $this->getOption( 'dry-run' );
24 print "Dry run only.\n";
27 $startId = $this->getOption( 'start', 0 );
32 $totalRevs = $dbr->selectField( 'text', 'MAX(old_id)', false, __METHOD__
);
34 if ( $dbr->getType() == 'mysql'
35 && version_compare( $dbr->getServerVersion(), '4.1.0', '>=' ) )
37 // In MySQL 4.1+, the binary field old_text has a non-working LOWER() function
38 $lowerLeft = 'LOWER(CONVERT(LEFT(old_text,22) USING latin1))';
40 // No CONVERT() in MySQL 4.0
41 $lowerLeft = 'LOWER(LEFT(old_text,22))';
45 print "ID: $startId / $totalRevs\r";
49 array( 'old_id', 'old_flags', 'old_text' ),
51 'old_id > ' . intval( $startId ),
52 'old_flags LIKE \'%object%\' AND old_flags NOT LIKE \'%external%\'',
53 "$lowerLeft = 'o:15:\"historyblobstub\"'",
57 'ORDER BY' => 'old_id',
58 'LIMIT' => $this->batchSize
,
62 if ( !$res->numRows() ) {
66 $secondaryIds = array();
69 foreach ( $res as $row ) {
70 $startId = $row->old_id
;
72 // Basic sanity checks
73 $obj = unserialize( $row->old_text
);
74 if ( $obj === false ) {
75 print "{$row->old_id}: unrecoverable: cannot unserialize\n";
80 if ( !is_object( $obj ) ) {
81 print "{$row->old_id}: unrecoverable: unserialized to type " .
82 gettype( $obj ) . ", possible double-serialization\n";
87 if ( strtolower( get_class( $obj ) ) !== 'historyblobstub' ) {
88 print "{$row->old_id}: unrecoverable: unexpected object class " .
89 get_class( $obj ) . "\n";
95 $flags = explode( ',', $row->old_flags
);
96 if ( in_array( 'utf-8', $flags ) ||
in_array( 'utf8', $flags ) ) {
97 $legacyEncoding = false;
99 $legacyEncoding = true;
102 // Queue the stub for future batch processing
103 $id = intval( $obj->mOldId
);
104 $secondaryIds[] = $id;
105 $stubs[$row->old_id
] = array(
106 'legacyEncoding' => $legacyEncoding,
107 'secondaryId' => $id,
108 'hash' => $obj->mHash
,
112 $secondaryIds = array_unique( $secondaryIds );
114 if ( !count( $secondaryIds ) ) {
118 // Run the batch query on blob_tracking
123 'bt_text_id' => $secondaryIds,
127 $trackedBlobs = array();
128 foreach ( $res as $row ) {
129 $trackedBlobs[$row->bt_text_id
] = $row;
133 foreach ( $stubs as $primaryId => $stub ) {
134 $secondaryId = $stub['secondaryId'];
135 if ( !isset( $trackedBlobs[$secondaryId] ) ) {
136 // No tracked blob. Work out what went wrong
137 $secondaryRow = $dbr->selectRow(
139 array( 'old_flags', 'old_text' ),
140 array( 'old_id' => $secondaryId ),
143 if ( !$secondaryRow ) {
144 print "$primaryId: unrecoverable: secondary row is missing\n";
146 } elseif ( $this->isUnbrokenStub( $stub, $secondaryRow ) ) {
147 // Not broken yet, and not in the tracked clusters so it won't get
148 // broken by the current RCT run.
150 } elseif ( strpos( $secondaryRow->old_flags
, 'external' ) !== false ) {
151 print "$primaryId: unrecoverable: secondary gone to {$secondaryRow->old_text}\n";
154 print "$primaryId: unrecoverable: miscellaneous corruption of secondary row\n";
157 unset( $stubs[$primaryId] );
160 $trackRow = $trackedBlobs[$secondaryId];
162 // Check that the specified text really is available in the tracked source row
163 $url = "DB://{$trackRow->bt_cluster}/{$trackRow->bt_blob_id}/{$stub['hash']}";
164 $text = ExternalStore
::fetchFromURL( $url );
165 if ( $text === false ) {
166 print "$primaryId: unrecoverable: source text missing\n";
168 unset( $stubs[$primaryId] );
171 if ( md5( $text ) !== $stub['hash'] ) {
172 print "$primaryId: unrecoverable: content hashes do not match\n";
174 unset( $stubs[$primaryId] );
178 // Find the page_id and rev_id
179 // The page is probably the same as the page of the secondary row
180 $pageId = intval( $trackRow->bt_page
);
182 $revId = $pageId = 0;
184 $revId = $this->findTextIdInPage( $pageId, $primaryId );
186 // Actually an orphan
187 $pageId = $revId = 0;
191 $newFlags = $stub['legacyEncoding'] ?
'external' : 'external,utf-8';
194 // Reset the text row to point to the original copy
200 'old_flags' => $newFlags,
204 array( 'old_id' => $primaryId ),
208 // Add a blob_tracking row so that the new reference can be recompressed
209 // without needing to run trackBlobs.php again
210 $dbw->insert( 'blob_tracking',
212 'bt_page' => $pageId,
213 'bt_rev_id' => $revId,
214 'bt_text_id' => $primaryId,
215 'bt_cluster' => $trackRow->bt_cluster
,
216 'bt_blob_id' => $trackRow->bt_blob_id
,
217 'bt_cgz_hash' => $stub['hash'],
218 'bt_new_url' => null,
224 $this->waitForSlaves();
227 print "$primaryId: resolved to $url\n";
233 print "Fixed: $numFixed\n";
234 print "Unrecoverable: $numBad\n";
235 print "Good stubs: $numGood\n";
238 function waitForSlaves() {
239 static $iteration = 0;
241 if ( ++
$iteration > 50 == 0 ) {
242 wfWaitForSlaves( 5 );
247 function findTextIdInPage( $pageId, $textId ) {
248 $ids = $this->getRevTextMap( $pageId );
249 if ( !isset( $ids[$textId] ) ) {
252 return $ids[$textId];
256 function getRevTextMap( $pageId ) {
257 if ( !isset( $this->mapCache
[$pageId] ) ) {
259 while ( $this->mapCacheSize
> $this->maxMapCacheSize
) {
260 $key = key( $this->mapCache
);
261 $this->mapCacheSize
-= count( $this->mapCache
[$key] );
262 unset( $this->mapCache
[$key] );
265 $dbr = wfGetDB( DB_SLAVE
);
267 $res = $dbr->select( 'revision',
268 array( 'rev_id', 'rev_text_id' ),
269 array( 'rev_page' => $pageId ),
272 foreach ( $res as $row ) {
273 $map[$row->rev_text_id
] = $row->rev_id
;
275 $this->mapCache
[$pageId] = $map;
276 $this->mapCacheSize +
= count( $map );
278 return $this->mapCache
[$pageId];
282 * This is based on part of HistoryBlobStub::getText().
283 * Determine if the text can be retrieved from the row in the normal way.
285 function isUnbrokenStub( $stub, $secondaryRow ) {
286 $flags = explode( ',', $secondaryRow->old_flags
);
287 $text = $secondaryRow->old_text
;
288 if ( in_array( 'external', $flags ) ) {
290 @list
( /* $proto */ , $path ) = explode( '://', $url, 2 );
294 $text = ExternalStore
::fetchFromUrl( $url );
296 if ( !in_array( 'object', $flags ) ) {
300 if ( in_array( 'gzip', $flags ) ) {
301 $obj = unserialize( gzinflate( $text ) );
303 $obj = unserialize( $text );
306 if ( !is_object( $obj ) ) {
307 // Correct for old double-serialization bug.
308 $obj = unserialize( $obj );
311 if ( !is_object( $obj ) ) {
316 $text = $obj->getItem( $stub['hash'] );
317 return $text !== false;
321 $maintClass = 'FixBug20757';
322 require_once( DO_MAINTENANCE
);