maintenance/storage/fixBug20757.php

   1 <?php
   2
   3 require_once( dirname( __FILE__ ) . '/../Maintenance.php' );
   4
   5 class FixBug20757 extends Maintenance {
   6         var $batchSize = 10000;
   7         var $mapCache = array();
   8         var $mapCacheSize = 0;
   9         var $maxMapCacheSize = 1000000;
  10
  11         function __construct() {
  12                 parent::__construct();
  13                 $this->mDescription = 'Script to fix bug 20757 assuming that blob_tracking is intact';
  14                 $this->addOption( 'dry-run', 'Report only' );
  15         }
  16
  17         function execute() {
  18                 $dbr = wfGetDB( DB_SLAVE );
  19                 $dbw = wfGetDB( DB_MASTER );
  20
  21                 $dryRun = $this->getOption( 'dry-run' );
  22                 if ( $dryRun ) {
  23                         print "Dry run only.\n";
  24                 }
  25
  26                 $startId = 0;
  27                 $numGood = 0;
  28                 $numFixed = 0;
  29                 $numBad = 0;
  30
  31                 $totalRevs = $dbr->selectField( 'text', 'MAX(old_id)', false, __METHOD__ );
  32
  33                 while ( true ) {
  34                         print "ID: $startId / $totalRevs\r";
  35
  36                         $res = $dbr->select(
  37                                 'text',
  38                                 array( 'old_id', 'old_flags', 'old_text' ),
  39                                 array(
  40                                         'old_id > ' . intval( $startId ),
  41                                         'old_flags' => 'object'
  42                                 ),
  43                                 __METHOD__,
  44                                 array(
  45                                         'ORDER BY' => 'old_id',
  46                                         'LIMIT' => $this->batchSize,
  47                                 )
  48                         );
  49
  50                         if ( !$res->numRows() ) {
  51                                 break;
  52                         }
  53
  54                         $secondaryIds = array();
  55                         $stubs = array();
  56
  57                         foreach ( $res as $row ) {
  58                                 $startId = $row->old_id;
  59
  60                                 // Basic sanity checks
  61                                 $obj = unserialize( $row->old_text );
  62                                 if ( $obj === false ) {
  63                                         print "{$row->old_id}: unrecoverable: cannot unserialize\n";
  64                                         ++$numBad;
  65                                         continue;
  66                                 }
  67
  68                                 if ( !is_object( $obj ) ) {
  69                                         print "{$row->old_id}: unrecoverable: unserialized to type " .
  70                                                 gettype( $obj ) . ", possible double-serialization\n";
  71                                         ++$numBad;
  72                                         continue;
  73                                 }
  74
  75                                 // Check if it really is broken
  76                                 $text = Revision::getRevisionText( $row );
  77                                 if ( $text !== false ) {
  78                                         // Not broken yet
  79                                         ++$numGood;
  80                                         continue;
  81                                 }
  82
  83                                 if ( strtolower( get_class( $obj ) ) !== 'historyblobstub' ) {
  84                                         print "{$row->old_id}: unrecoverable: unexpected object class " .
  85                                                 get_class( $obj ) . "\n";
  86                                         ++$numBad;
  87                                         continue;
  88                                 }
  89
  90                                 // Queue the stub for future batch processing
  91                                 $id = intval( $obj->mOldId );
  92                                 $secondaryIds[] = $id;
  93                                 $stubs[$row->old_id] = array(
  94                                         'secondaryId' => $id,
  95                                         'hash' => $obj->mHash,
  96                                 );
  97                         }
  98
  99                         $secondaryIds = array_unique( $secondaryIds );
 100
 101                         if ( !count( $secondaryIds ) ) {
 102                                 continue;
 103                         }
 104
 105                         // Run the batch query on blob_tracking
 106                         $res = $dbr->select(
 107                                 'blob_tracking',
 108                                 '*',
 109                                 array(
 110                                         'bt_text_id' => $secondaryIds,
 111                                         'bt_moved' => 1,
 112                                 ),
 113                                 __METHOD__
 114                         );
 115                         $trackedBlobs = array();
 116                         foreach ( $res as $row ) {
 117                                 $trackedBlobs[$row->bt_text_id] = $row;
 118                         }
 119
 120                         // Process the stubs
 121                         $stubsToFix = array();
 122                         foreach ( $stubs as $primaryId => $stub ) {
 123                                 $secondaryId = $stub['secondaryId'];
 124                                 if ( !isset( $trackedBlobs[$secondaryId] ) ) {
 125                                         $secondaryRow = $dbr->selectRow(
 126                                                 'text',
 127                                                 array( 'old_flags', 'old_text' ),
 128                                                 array( 'old_id' => $secondaryId ),
 129                                                 __METHOD__
 130                                         );
 131                                         if ( !$secondaryRow ) {
 132                                                 print "$primaryId: unrecoverable: secondary row is missing\n";
 133                                         } elseif ( strpos( $secondaryRow->old_flags, 'external' ) !== false ) {
 134                                                 print "$primaryId: unrecoverable: secondary gone to {$secondaryRow->old_text}\n";
 135                                         } else {
 136                                                 print "$primaryId: unrecoverable: miscellaneous corruption of secondary row\n";
 137                                         }
 138                                         ++$numBad;
 139                                         unset( $stubs[$primaryId] );
 140                                         continue;
 141                                 }
 142                                 $trackRow = $trackedBlobs[$secondaryId];
 143
 144                                 // Check that the specified text really is available in the tracked source row
 145                                 $url = "DB://{$trackRow->bt_cluster}/{$trackRow->bt_blob_id}/{$stub['hash']}";
 146                                 $text = ExternalStore::fetchFromURL( $url );
 147                                 if ( $text === false ) {
 148                                         print "$primaryId: unrecoverable: source text missing\n";
 149                                         ++$numBad;
 150                                         unset( $stubs[$primaryId] );
 151                                         continue;
 152                                 }
 153                                 if ( md5( $text ) !== $stub['hash'] ) {
 154                                         print "$primaryId: unrecoverable: content hashes do not match\n";
 155                                         ++$numBad;
 156                                         unset( $stubs[$primaryId] );
 157                                         continue;
 158                                 }
 159
 160                                 // Find the page_id and rev_id
 161                                 // The page is probably the same as the page of the secondary row
 162                                 $pageId = $trackRow->bt_page;
 163                                 if ( $pageId === null ) {
 164                                         $revId = null;
 165                                 } else {
 166                                         $revId = $this->findTextIdInPage( $pageId, $primaryId );
 167                                         if ( $revId === null ) {
 168                                                 // Actually an orphan
 169                                                 $pageId = null;
 170                                         }
 171                                 }
 172
 173                                 if ( !$dryRun ) {
 174                                         // Reset the text row to point to the original copy
 175                                         $dbw->begin();
 176                                         $dbw->update(
 177                                                 'text',
 178                                                 // SET
 179                                                 array(
 180                                                         'old_flags' => 'external', // use legacy encoding
 181                                                         'old_text' => $url
 182                                                 ),
 183                                                 // WHERE
 184                                                 array( 'old_id' => $primaryId ),
 185                                                 __METHOD__
 186                                         );
 187
 188                                         // Add a blob_tracking row so that the new reference can be recompressed
 189                                         // without needing to run trackBlobs.php again
 190                                         $dbw->insert( 'blob_tracking',
 191                                                 array(
 192                                                         'bt_page' => $trackRow->bt_page,
 193                                                         'bt_rev_id' => $revId,
 194                                                         'bt_text_id' => $primaryId,
 195                                                         'bt_cluster' => $trackRow->bt_cluster,
 196                                                         'bt_blob_id' => $trackRow->bt_blob_id,
 197                                                         'bt_cgz_hash' => $stub['hash'],
 198                                                         'bt_new_url' => null,
 199                                                         'bt_moved' => 0,
 200                                                 ),
 201                                                 __METHOD__
 202                                         );
 203                                         $dbw->commit();
 204                                         $this->waitForSlaves();
 205                                 }
 206
 207                                 print "$primaryId: resolved to $url\n";
 208                                 ++$numFixed;
 209                         }
 210                 }
 211
 212                 print "\n";
 213                 print "Fixed: $numFixed\n";
 214                 print "Unrecoverable: $numBad\n";
 215                 print "Not yet broken: $numGood\n";
 216         }
 217
 218         function waitForSlaves() {
 219                 static $iteration = 0;
 220                 ++$iteration;
 221                 if ( ++$iteration > 50 == 0 ) {
 222                         wfWaitForSlaves( 5 );
 223                         $iteration = 0;
 224                 }
 225         }
 226
 227         function findTextIdInPage( $pageId, $textId ) {
 228                 $ids = $this->getRevTextMap( $pageId );
 229                 if ( !isset( $ids[$textId] ) ) {
 230                         return null;
 231                 } else {
 232                         return $ids[$textId];
 233                 }
 234         }
 235
 236         function getRevTextMap( $pageId ) {
 237                 if ( !isset( $this->mapCache[$pageId] ) ) {
 238                         // Limit cache size
 239                         while ( $this->mapCacheSize > $this->maxMapCacheSize ) {
 240                                 $key = key( $this->mapCache );
 241                                 $this->mapCacheSize -= count( $this->mapCache[$key] );
 242                                 unset( $this->mapCache[$key] );
 243                         }
 244
 245                         $dbr = wfGetDB( DB_SLAVE );
 246                         $map = array();
 247                         $res = $dbr->select( 'revision',
 248                                 array( 'rev_id', 'rev_text_id' ),
 249                                 array( 'rev_page' => $pageId ),
 250                                 __METHOD__
 251                         );
 252                         foreach ( $res as $row ) {
 253                                 $map[$row->rev_text_id] = $row->rev_id;
 254                         }
 255                         $this->mapCache[$pageId] = $map;
 256                         $this->mapCacheSize += count( $map );
 257                 }
 258                 return $this->mapCache[$pageId];
 259         }
 260
 261 }
 262
 263 $maintClass = 'FixBug20757';
 264 require_once( DO_MAINTENANCE );
 265