From ddc1d231497f15271170a23c278989ffe7e3014e Mon Sep 17 00:00:00 2001 From: Brian Wolff Date: Thu, 25 Aug 2011 05:33:32 +0000 Subject: [PATCH] New maintenance script for refreshing image metadata (refreshImageMetadata.php) This is very similar to rebuildImages.php, except more specific to img_metadata field, and does the images in batches instead of all at once. Also, while I'm here, I added a line to Maintenance.php to make sure it casted $this->mBatchSize to an integer when gotten from command line (thought it was weird that it didn't do that) (I'm going to tag this revision 1.18 because I think it'd be nice to have this script in 1.18 given new image metadata stuff added in 1.18, but not super-important because rebuildImages.php does already work to refresh image metadata) --- RELEASE-NOTES-1.18 | 1 + maintenance/Maintenance.php | 2 +- maintenance/refreshImageMetadata.php | 199 +++++++++++++++++++++++++++ 3 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 maintenance/refreshImageMetadata.php diff --git a/RELEASE-NOTES-1.18 b/RELEASE-NOTES-1.18 index 04fa698a54..c53d9b8d86 100644 --- a/RELEASE-NOTES-1.18 +++ b/RELEASE-NOTES-1.18 @@ -205,6 +205,7 @@ production. targets and $wgServer. * Introduced $wgVaryOnXFPForAPI which will cause the API to send Vary: X-Forwarded-Proto headers. +* New maintenance script to refresh image metadata (maintenance/refreshImageMetadata.php) === Bug fixes in 1.18 === * mw.util.getScript has been implemented (like wfScript in GlobalFunctions.php) diff --git a/maintenance/Maintenance.php b/maintenance/Maintenance.php index 36e9fa4bcc..a1496034df 100644 --- a/maintenance/Maintenance.php +++ b/maintenance/Maintenance.php @@ -699,7 +699,7 @@ abstract class Maintenance { $this->mQuiet = true; } if ( $this->hasOption( 'batch-size' ) ) { - $this->mBatchSize = $this->getOption( 'batch-size' ); + $this->mBatchSize = intval( $this->getOption( 'batch-size' ) ); } } diff --git a/maintenance/refreshImageMetadata.php b/maintenance/refreshImageMetadata.php new file mode 100644 index 0000000000..ec612183cb --- /dev/null +++ b/maintenance/refreshImageMetadata.php @@ -0,0 +1,199 @@ +mDescription = 'Script to update image metadata records'; + $this->setBatchSize( 200 ); + + $this->addOption( 'force', 'Reload metadata from file even if the metadata looks ok', false, false, 'f' ); + $this->addOption( 'broken-only', 'Only fix really broken records, leave old but still compatible records alone.' ); + $this->addOption( 'verbose', 'Output extra information about each upgraded/non-upgraded file.', false, false, 'v' ); + $this->addOption( 'start', 'Name of file to start with', false, true ); + $this->addOption( 'end', 'Name of file to end with', false, true ); + + $this->addOption( 'mime', '(Inefficient!) Only refresh files with this mime type. Can accept wild-card image/*' , false, true ); + $this->addOption( 'metadata-contains', '(Inefficient!) Only refresh files where the img_metadata field contains this string. Can be used if its known a specific property was being extracted incorrectly.', false, true ); + + } + + public function execute() { + $force = $this->hasOption( 'force' ); + $brokenOnly = $this->hasOption( 'broken-only' ); + $verbose = $this->hasOption( 'verbose' ); + $start = $this->getOption( 'start', false ); + $this->setupParameters( $force, $brokenOnly ); + + $upgraded = 0; + $leftAlone = 0; + $error = 0; + + $dbw = wfGetDB( DB_MASTER ); + if ( $this->mBatchSize <= 0 ) { + $this->error( "Batch size is too low...", 12 ); + } + + $repo = RepoGroup::singleton()->getLocalRepo(); + $conds = $this->getConditions( $dbw ); + + // For the WHERE img_name > 'foo' condition that comes after doing a batch + $conds2 = array(); + if ( $start !== false ) { + $conds2[] = 'img_name >= ' . $dbw->addQuotes( $start ); + } + + $options = array( + 'LIMIT' => $this->mBatchSize, + 'ORDER BY' => 'img_name ASC', + ); + + do { + $res = $dbw->select( + 'image', + '*', + array_merge( $conds, $conds2 ), + __METHOD__, + $options + ); + + if ( $res->numRows() > 0 ) { + $row1 = $res->current(); + $this->output( "Processing next {$this->mBatchSize} rows starting with {$row1->img_name}.\n"); + $res->rewind(); + } else { + $this->error( "No images to process.", 4 ); + } + + foreach ( $res as $row ) { + $file = $repo->newFileFromRow( $row ); + if ( $file->getUpgraded() ) { + // File was upgraded. + $upgraded++; + $newLength = strlen( $file->getMetadata() ); + $oldLength = strlen( $row->img_metadata ); + if ( $newLength < $oldLength - 5 ) { + // If after updating, the metadata is smaller then + // what it was before, that's probably not a good thing + // because we extract more data with time, not less. + // Thus this probably indicates an error of some sort, + // or at the very least is suspicious. Have the - 5 just + // to weed out any inconsequential changes. + $error++; + $this->output( "Warning: File:{$row->img_name} used to have " . + "$oldLength bytes of metadata but now has $newLength bytes.\n" ); + } elseif ( $verbose ) { + $this->output("Refreshed File:{$row->img_name}.\n" ); + } + } else { + $leftAlone++; + if ( $force ) { + $file->upgradeRow(); + $newLength = strlen( $file->getMetadata() ); + $oldLength = strlen( $row->img_metadata ); + if ( $newLength < $oldLength - 5 ) { + $error++; + $this->output( "Warning: File:{$row->img_name} used to have " . + "$oldLength bytes of metadata but now has $newLength bytes. (forced)\n" ); + + } + if ( $verbose ) { + $this->output("Forcibly refreshed File:{$row->img_name}.\n" ); + } + } + else { + if ( $verbose ) { + $this->output( "Skipping File:{$row->img_name}.\n" ); + } + } + } + + } + $conds2 = array( 'img_name > ' . $dbw->addQuotes( $row->img_name ) ); + wfWaitForSlaves(); + } while( $res->numRows() === $this->mBatchSize ); + + $total = $upgraded + $leftAlone; + if ( $force ) { + $this->output( "\nFinished refreshing file metadata for $total files. $upgraded needed to be refreshed, $leftAlone did not need to be but were refreshed anyways, and $error refreshes were suspicious.\n" ); + } else { + $this->output( "\nFinished refreshing file metadata for $total files. $upgraded were refreshed, $leftAlone were already up to date, and $error refreshes were suspicious.\n" ); + } + } + + function getConditions( $dbw ) { + $conds = array(); + + $end = $this->getOption( 'end', false ); + $mime = $this->getOption( 'mime', false ); + $like = $this->getOption( 'metadata-contains', false ); + + if ( $end !== false ) { + $conds[] = 'img_name <= ' . $dbw->addQuotes( $end ) ; + } + if ( $mime !== false ) { + list( $major, $minor ) = File::splitMime( $mime ); + $conds['img_major_mime'] = $major; + if ( $minor !== '*' ) { + $conds['img_minor_mime'] = $minor; + } + } + if ( $like ) { + $conds[] = 'img_metadata ' . $dbw->buildLike( $dbw->anyString(), $like, $dbw->anyString() ); + } + return $conds; + } + + function setupParameters( $force, $brokenOnly ) { + global $wgUpdateCompatibleMetadata, $wgReadOnly; + + if ( $brokenOnly ) { + $wgUpdateCompatibleMetadata = false; + } else { + $wgUpdateCompatibleMetadata = true; + } + + if ( $brokenOnly && $force ) { + $this->error( 'Cannot use --broken-only and --force together. ', 2 ); + } + } +} + + +$maintClass = 'RefreshImageMetadata'; +require_once( RUN_MAINTENANCE_IF_MAIN ); -- 2.20.1