maintenance/rebuildImages.php

   1 <?php
   2 /*
   3  * Script to update image metadata records
   4  *
   5  * Usage: php rebuildImages.php [--missing] [--dry-run]
   6  * Options:
   7  *   --missing  Crawl the uploads dir for images without records, and
   8  *              add them only.
   9  *
  10  * Copyright (C) 2005 Brion Vibber <brion@pobox.com>
  11  * http://www.mediawiki.org/
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License along
  24  * with this program; if not, write to the Free Software Foundation, Inc.,
  25  * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  26  * http://www.gnu.org/copyleft/gpl.html
  27  *
  28  * @author Brion Vibber <brion at pobox.com>
  29  * @package MediaWiki
  30  * @subpackage maintenance
  31  */
  32
  33 $options = array( 'missing', 'dry-run' );
  34
  35 require_once( 'commandLine.inc' );
  36 require_once( 'FiveUpgrade.inc' );
  37
  38 class ImageBuilder extends FiveUpgrade {
  39         function ImageBuilder( $dryrun = false ) {
  40                 parent::FiveUpgrade();
  41
  42                 $this->maxLag = 10; # if slaves are lagged more than 10 secs, wait
  43                 $this->dryrun = $dryrun;
  44         }
  45
  46         function build() {
  47                 $this->buildImage();
  48                 $this->buildOldImage();
  49         }
  50
  51         function init( $count, $table ) {
  52                 $this->processed = 0;
  53                 $this->updated = 0;
  54                 $this->count = $count;
  55                 $this->startTime = wfTime();
  56                 $this->table = $table;
  57         }
  58
  59         function progress( $updated ) {
  60                 $this->updated += $updated;
  61                 $this->processed++;
  62                 if( $this->processed % 100 != 0 ) {
  63                         return;
  64                 }
  65                 $portion = $this->processed / $this->count;
  66                 $updateRate = $this->updated / $this->processed;
  67
  68                 $now = wfTime();
  69                 $delta = $now - $this->startTime;
  70                 $estimatedTotalTime = $delta / $portion;
  71                 $eta = $this->startTime + $estimatedTotalTime;
  72
  73                 printf( "%s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
  74                         wfTimestamp( TS_DB, intval( $now ) ),
  75                         $portion * 100.0,
  76                         $this->table,
  77                         wfTimestamp( TS_DB, intval( $eta ) ),
  78                         $completed,
  79                         $this->count,
  80                         $rate,
  81                         $updateRate * 100.0 );
  82                 flush();
  83         }
  84
  85         function buildTable( $table, $key, $callback ) {
  86                 $fname = 'ImageBuilder::buildTable';
  87
  88                 $count = $this->dbw->selectField( $table, 'count(*)', '', $fname );
  89                 $this->init( $count, $table );
  90                 $this->log( "Processing $table..." );
  91
  92                 $tableName = $this->dbr->tableName( $table );
  93                 $sql = "SELECT * FROM $tableName";
  94                 $result = $this->dbr->query( $sql, $fname );
  95
  96                 while( $row = $this->dbr->fetchObject( $result ) ) {
  97                         $update = call_user_func( $callback, $row );
  98                         if( is_array( $update ) ) {
  99                                 if( !$this->dryrun ) {
 100                                         $this->dbw->update( $table,
 101                                                 $update,
 102                                                 array( $key => $row->$key ),
 103                                                 $fname );
 104                                 }
 105                                 $this->progress( 1 );
 106                         } else {
 107                                 $this->progress( 0 );
 108                         }
 109                 }
 110                 $this->log( "Finished $table... $this->updated of $this->processed rows updated" );
 111                 $this->dbr->freeResult( $result );
 112         }
 113
 114         function buildImage() {
 115                 $callback = array( &$this, 'imageCallback' );
 116                 $this->buildTable( 'image', 'img_name', $callback );
 117         }
 118
 119         function imageCallback( $row ) {
 120                 if( $row->img_width ) {
 121                         // Already processed
 122                         return null;
 123                 }
 124
 125                 // Fill in the new image info fields
 126                 $info = $this->imageInfo( $row->img_name );
 127                 return array(
 128                         'img_width'      => $info['width'],
 129                         'img_height'     => $info['height'],
 130                         'img_bits'       => $info['bits'],
 131                         'img_media_type' => $info['media'],
 132                         'img_major_mime' => $info['major'],
 133                         'img_minor_mime' => $info['minor'] );
 134         }
 135
 136
 137         function buildOldImage() {
 138                 $this->buildTable( 'oldimage', 'oi_archive_name',
 139                         array( &$this, 'oldimageCallback' ) );
 140         }
 141
 142         function oldimageCallback( $row ) {
 143                 if( $row->oi_width ) {
 144                         return null;
 145                 }
 146
 147                 // Fill in the new image info fields
 148                 $info = $this->imageInfo( $row->oi_archive_name, 'wfImageArchiveDir', $row->oi_name );
 149                 return array(
 150                         'oi_width'  => $info['width' ],
 151                         'oi_height' => $info['height'],
 152                         'oi_bits'   => $info['bits'  ] );
 153         }
 154
 155         function crawlMissing() {
 156                 global $wgUploadDirectory, $wgHashedUploadDirectory;
 157                 if( $wgHashedUploadDirectory ) {
 158                         for( $i = 0; $i < 16; $i++ ) {
 159                                 for( $j = 0; $j < 16; $j++ ) {
 160                                         $dir = sprintf( '%s%s%01x%s%02x',
 161                                                 $wgUploadDirectory,
 162                                                 DIRECTORY_SEPARATOR,
 163                                                 $i,
 164                                                 DIRECTORY_SEPARATOR,
 165                                                 $i * 16 + $j );
 166                                         $this->crawlDirectory( $dir );
 167                                 }
 168                         }
 169                 } else {
 170                         $this->crawlDirectory( $wgUploadDirectory );
 171                 }
 172         }
 173
 174         function crawlDirectory( $dir ) {
 175                 if( !file_exists( $dir ) ) {
 176                         return $this->log( "no directory, skipping $dir" );
 177                 }
 178                 if( !is_dir( $dir ) ) {
 179                         return $this->log( "not a directory?! skipping $dir" );
 180                 }
 181                 if( !is_readable( $dir ) ) {
 182                         return $this->log( "dir not readable, skipping $dir" );
 183                 }
 184                 $source = opendir( $dir );
 185                 if( $source === false ) {
 186                         return $this->log( "couldn't open dir, skipping $dir" );
 187                 }
 188
 189                 $this->log( "crawling $dir" );
 190                 while( false !== ( $filename = readdir( $source ) ) ) {
 191                         $fullpath = $dir . DIRECTORY_SEPARATOR . $filename;
 192                         if( is_dir( $fullpath ) ) {
 193                                 continue;
 194                         }
 195                         if( is_link( $fullpath ) ) {
 196                                 $this->log( "skipping symlink at $fullpath" );
 197                                 continue;
 198                         }
 199                         $this->checkMissingImage( $filename, $fullpath );
 200                 }
 201                 closedir( $source );
 202         }
 203
 204         function checkMissingImage( $filename, $fullpath ) {
 205                 $fname = 'ImageBuilder::checkMissingImage';
 206                 $row = $this->dbw->selectRow( 'image',
 207                         array( 'img_name' ),
 208                         array( 'img_name' => $filename ),
 209                         $fname );
 210
 211                 if( $row ) {
 212                         // already known, move on
 213                         return;
 214                 } else {
 215                         $this->addMissingImage( $filename, $fullpath );
 216                 }
 217         }
 218
 219         function addMissingImage( $filename, $fullpath ) {
 220                 $fname = 'ImageBuilder::addMissingImage';
 221
 222                 $size = filesize( $fullpath );
 223                 $info = $this->imageInfo( $filename );
 224                 $timestamp = $this->dbw->timestamp( filemtime( $fullpath ) );
 225
 226                 global $wgContLang;
 227                 $altname = $wgContLang->checkTitleEncoding( $filename );
 228                 if( $altname != $filename ) {
 229                         if( $this->dryrun ) {
 230                                 $filename = $altname;
 231                                 $this->log( "Estimating transcoding... $altname" );
 232                         } else {
 233                                 $filename = $this->renameFile( $filename );
 234                         }
 235                 }
 236
 237                 $fields = array(
 238                         'img_name'       => $filename,
 239                         'img_size'       => $size,
 240                         'img_width'      => $info['width'],
 241                         'img_height'     => $info['height'],
 242                         'img_metadata'   => '', // filled in on-demand
 243                         'img_bits'       => $info['bits'],
 244                         'img_media_type' => $info['media'],
 245                         'img_major_mime' => $info['major'],
 246                         'img_minor_mime' => $info['minor'],
 247                         'img_description' => '(recovered file, missing upload log entry)',
 248                         'img_user'        => 0,
 249                         'img_user_text'   => 'Conversion script',
 250                         'img_timestamp'   => $timestamp );
 251                 if( !$this->dryrun ) {
 252                         $this->dbw->insert( 'image', $fields, $fname );
 253                 }
 254                 $this->log( $fullpath );
 255         }
 256 }
 257
 258 $builder = new ImageBuilder( isset( $options['dry-run'] ) );
 259 if( isset( $options['missing'] ) ) {
 260         $builder->crawlMissing();
 261 } else {
 262         $builder->build();
 263 }
 264
 265 ?>