maintenance/rebuildImages.php

   1 <?php
   2 /*
   3  * Script to update image metadata records
   4  *
   5  * Usage: php rebuildImages.php [--missing] [--dry-run]
   6  * Options:
   7  *   --missing  Crawl the uploads dir for images without records, and
   8  *              add them only.
   9  *
  10  * Copyright (C) 2005 Brion Vibber <brion@pobox.com>
  11  * http://www.mediawiki.org/
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License along
  24  * with this program; if not, write to the Free Software Foundation, Inc.,
  25  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  26  * http://www.gnu.org/copyleft/gpl.html
  27  *
  28  * @author Brion Vibber <brion at pobox.com>
  29  * @addtogroup maintenance
  30  */
  31
  32 $options = array( 'missing', 'dry-run' );
  33
  34 require_once( 'commandLine.inc' );
  35 require_once( 'FiveUpgrade.inc' );
  36
  37 class ImageBuilder extends FiveUpgrade {
  38         function ImageBuilder( $dryrun = false ) {
  39                 parent::FiveUpgrade();
  40
  41                 $this->maxLag = 10; # if slaves are lagged more than 10 secs, wait
  42                 $this->dryrun = $dryrun;
  43         }
  44
  45         function build() {
  46                 $this->buildImage();
  47                 $this->buildOldImage();
  48         }
  49
  50         function init( $count, $table ) {
  51                 $this->processed = 0;
  52                 $this->updated = 0;
  53                 $this->count = $count;
  54                 $this->startTime = wfTime();
  55                 $this->table = $table;
  56         }
  57
  58         function progress( $updated ) {
  59                 $this->updated += $updated;
  60                 $this->processed++;
  61                 if( $this->processed % 100 != 0 ) {
  62                         return;
  63                 }
  64                 $portion = $this->processed / $this->count;
  65                 $updateRate = $this->updated / $this->processed;
  66
  67                 $now = wfTime();
  68                 $delta = $now - $this->startTime;
  69                 $estimatedTotalTime = $delta / $portion;
  70                 $eta = $this->startTime + $estimatedTotalTime;
  71
  72                 printf( "%s: %6.2f%% done on %s; ETA %s [%d/%d] %.2f/sec <%.2f%% updated>\n",
  73                         wfTimestamp( TS_DB, intval( $now ) ),
  74                         $portion * 100.0,
  75                         $this->table,
  76                         wfTimestamp( TS_DB, intval( $eta ) ),
  77                         $completed,   // $completed does not appear to be defined.
  78                         $this->count,
  79                         $rate,        // $rate does not appear to be defined.
  80                         $updateRate * 100.0 );
  81                 flush();
  82         }
  83
  84         function buildTable( $table, $key, $callback ) {
  85                 $fname = 'ImageBuilder::buildTable';
  86
  87                 $count = $this->dbw->selectField( $table, 'count(*)', '', $fname );
  88                 $this->init( $count, $table );
  89                 $this->log( "Processing $table..." );
  90
  91                 $tableName = $this->dbr->tableName( $table );
  92                 $sql = "SELECT * FROM $tableName";
  93                 $result = $this->dbr->query( $sql, $fname );
  94
  95                 while( $row = $this->dbr->fetchObject( $result ) ) {
  96                         $update = call_user_func( $callback, $row );
  97                         if( is_array( $update ) ) {
  98                                 if( !$this->dryrun ) {
  99                                         $this->dbw->update( $table,
 100                                                 $update,
 101                                                 array( $key => $row->$key ),
 102                                                 $fname );
 103                                 }
 104                                 $this->progress( 1 );
 105                         } else {
 106                                 $this->progress( 0 );
 107                         }
 108                 }
 109                 $this->log( "Finished $table... $this->updated of $this->processed rows updated" );
 110                 $this->dbr->freeResult( $result );
 111         }
 112
 113         function buildImage() {
 114                 $callback = array( &$this, 'imageCallback' );
 115                 $this->buildTable( 'image', 'img_name', $callback );
 116         }
 117
 118         function imageCallback( $row ) {
 119                 if( $row->img_width ) {
 120                         // Already processed
 121                         return null;
 122                 }
 123
 124                 // Fill in the new image info fields
 125                 $info = $this->imageInfo( $row->img_name );
 126
 127                 global $wgMemc;
 128                 $key = wfMemcKey( "Image", md5( $row->img_name ) );
 129                 $wgMemc->delete( $key );
 130
 131                 return array(
 132                         'img_width'      => $info['width'],
 133                         'img_height'     => $info['height'],
 134                         'img_bits'       => $info['bits'],
 135                         'img_media_type' => $info['media'],
 136                         'img_major_mime' => $info['major'],
 137                         'img_minor_mime' => $info['minor'] );
 138         }
 139
 140
 141         function buildOldImage() {
 142                 $this->buildTable( 'oldimage', 'oi_archive_name',
 143                         array( &$this, 'oldimageCallback' ) );
 144         }
 145
 146         function oldimageCallback( $row ) {
 147                 if( $row->oi_width && $row->oi_media_type ) {
 148                         return null;
 149                 }
 150
 151                 // Fill in the new image info fields
 152                 $info = $this->imageInfo( $row->oi_archive_name, 'wfImageArchiveDir', $row->oi_name );
 153                 return array(
 154                         'oi_width'  => $info['width' ],
 155                         'oi_height' => $info['height'],
 156                         'oi_bits'   => $info['bits'  ],
 157                         'oi_metadata' => '', // filled in on-demand
 158                         'oi_media_type' => $info['media'],
 159                         'oi_major_mime' => $info['major'],
 160                         'oi_minor_mime' => $info['minor'] );
 161         }
 162
 163         function crawlMissing() {
 164                 global $wgUploadDirectory, $wgHashedUploadDirectory;
 165                 if( $wgHashedUploadDirectory ) {
 166                         for( $i = 0; $i < 16; $i++ ) {
 167                                 for( $j = 0; $j < 16; $j++ ) {
 168                                         $dir = sprintf( '%s%s%01x%s%02x',
 169                                                 $wgUploadDirectory,
 170                                                 DIRECTORY_SEPARATOR,
 171                                                 $i,
 172                                                 DIRECTORY_SEPARATOR,
 173                                                 $i * 16 + $j );
 174                                         $this->crawlDirectory( $dir );
 175                                 }
 176                         }
 177                 } else {
 178                         $this->crawlDirectory( $wgUploadDirectory );
 179                 }
 180         }
 181
 182         function crawlDirectory( $dir ) {
 183                 if( !file_exists( $dir ) ) {
 184                         return $this->log( "no directory, skipping $dir" );
 185                 }
 186                 if( !is_dir( $dir ) ) {
 187                         return $this->log( "not a directory?! skipping $dir" );
 188                 }
 189                 if( !is_readable( $dir ) ) {
 190                         return $this->log( "dir not readable, skipping $dir" );
 191                 }
 192                 $source = opendir( $dir );
 193                 if( $source === false ) {
 194                         return $this->log( "couldn't open dir, skipping $dir" );
 195                 }
 196
 197                 $this->log( "crawling $dir" );
 198                 while( false !== ( $filename = readdir( $source ) ) ) {
 199                         $fullpath = $dir . DIRECTORY_SEPARATOR . $filename;
 200                         if( is_dir( $fullpath ) ) {
 201                                 continue;
 202                         }
 203                         if( is_link( $fullpath ) ) {
 204                                 $this->log( "skipping symlink at $fullpath" );
 205                                 continue;
 206                         }
 207                         $this->checkMissingImage( $filename, $fullpath );
 208                 }
 209                 closedir( $source );
 210         }
 211
 212         function checkMissingImage( $filename, $fullpath ) {
 213                 $fname = 'ImageBuilder::checkMissingImage';
 214                 $row = $this->dbw->selectRow( 'image',
 215                         array( 'img_name' ),
 216                         array( 'img_name' => $filename ),
 217                         $fname );
 218
 219                 if( $row ) {
 220                         // already known, move on
 221                         return;
 222                 } else {
 223                         $this->addMissingImage( $filename, $fullpath );
 224                 }
 225         }
 226
 227         function addMissingImage( $filename, $fullpath ) {
 228                 $fname = 'ImageBuilder::addMissingImage';
 229
 230                 $size = filesize( $fullpath );
 231                 $info = $this->imageInfo( $filename );
 232                 $timestamp = $this->dbw->timestamp( filemtime( $fullpath ) );
 233
 234                 global $wgContLang;
 235                 $altname = $wgContLang->checkTitleEncoding( $filename );
 236                 if( $altname != $filename ) {
 237                         if( $this->dryrun ) {
 238                                 $filename = $altname;
 239                                 $this->log( "Estimating transcoding... $altname" );
 240                         } else {
 241                                 $filename = $this->renameFile( $filename );
 242                         }
 243                 }
 244
 245                 if( $filename == '' ) {
 246                         $this->log( "Empty filename for $fullpath" );
 247                         return;
 248                 }
 249
 250                 $fields = array(
 251                         'img_name'       => $filename,
 252                         'img_size'       => $size,
 253                         'img_width'      => $info['width'],
 254                         'img_height'     => $info['height'],
 255                         'img_metadata'   => '', // filled in on-demand
 256                         'img_bits'       => $info['bits'],
 257                         'img_media_type' => $info['media'],
 258                         'img_major_mime' => $info['major'],
 259                         'img_minor_mime' => $info['minor'],
 260                         'img_description' => '(recovered file, missing upload log entry)',
 261                         'img_user'        => 0,
 262                         'img_user_text'   => 'Conversion script',
 263                         'img_timestamp'   => $timestamp );
 264                 if( !$this->dryrun ) {
 265                         $this->dbw->insert( 'image', $fields, $fname );
 266                 }
 267                 $this->log( $fullpath );
 268         }
 269 }
 270
 271 $builder = new ImageBuilder( isset( $options['dry-run'] ) );
 272 if( isset( $options['missing'] ) ) {
 273         $builder->crawlMissing();
 274 } else {
 275         $builder->build();
 276 }
 277
 278 ?>