From ca76169bbef2f9c430a664c710a07c77672357e1 Mon Sep 17 00:00:00 2001 From: Tim Starling Date: Sun, 22 Jul 2007 14:45:12 +0000 Subject: [PATCH] * Introduced FileRepoStatus -- result class for file repo operations. * Ported file delete/restore to the filerepo framework. Some user-visible changes in error reporting. * $wgSaveDeletedFiles has been removed, the feature is now enabled unconditionally. Added a "deleted" directory for the default location, protected by a .htaccess file and the practical obscurity of content hashes. * Fixed bug 2735: "Preview" shown in title bar for action=submit on special pages * Removed "restore" links from the deletion log embedded in Special:Undelete * Added img_sha1/oi_sha1 fields, preserved through upload, delete and restore * Referenced the new oi_metadata etc. fields to preserve metadata across upload and delete/restore. --- RELEASE-NOTES | 7 +- StartProfiler.php | 10 +- images/deleted/.htaccess | 1 + includes/Article.php | 3 +- includes/AutoLoader.php | 3 + includes/DefaultSettings.php | 16 +- includes/EditPage.php | 4 + includes/FileStore.php | 6 +- includes/GlobalFunctions.php | 11 +- includes/ImagePage.php | 79 +- includes/OutputPage.php | 26 +- includes/PageHistory.php | 1 + includes/Setup.php | 9 +- includes/SpecialLog.php | 78 +- includes/SpecialUndelete.php | 38 +- includes/SpecialUpload.php | 18 +- includes/filerepo/FSRepo.php | 380 ++++++-- includes/filerepo/File.php | 65 +- includes/filerepo/FileRepo.php | 132 ++- includes/filerepo/FileRepoStatus.php | 170 ++++ includes/filerepo/ForeignDBRepo.php | 6 + includes/filerepo/LocalFile.php | 1054 +++++++++++++---------- includes/filerepo/LocalRepo.php | 29 + includes/filerepo/OldLocalFile.php | 21 +- languages/messages/MessagesEn.php | 16 + maintenance/archives/patch-img_sha1.sql | 8 + maintenance/rebuildImages.php | 2 - maintenance/tables.sql | 13 +- maintenance/updaters.inc | 1 + 29 files changed, 1509 insertions(+), 698 deletions(-) create mode 100644 images/deleted/.htaccess create mode 100644 includes/filerepo/FileRepoStatus.php create mode 100644 maintenance/archives/patch-img_sha1.sql diff --git a/RELEASE-NOTES b/RELEASE-NOTES index a5fea50f38..9c96b52588 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -26,6 +26,7 @@ it from source control: http://www.mediawiki.org/wiki/Download_from_SVN usergroups * $wgEnotifImpersonal, $wgEnotifUseJobQ - Bulk mail options for large sites * $wgShowHostnames - Expose server host names through the API and HTML comments +* $wgSaveDeletedFiles has been removed, the feature is now enabled unconditionally == New features since 1.10 == @@ -318,7 +319,11 @@ it from source control: http://www.mediawiki.org/wiki/Download_from_SVN * (bug 10642) Fix shift-click checkbox behavior for Opera 9.0+ and 6.0 * Work around Safari bug with pages ending in ".gz" or ".tgz" * Removed obsolete maintenance/changeuser.sql script; use RenameUser extension - +* (bug 2735) "Preview" shown in title bar for action=submit on special pages +* Removed "restore" links from the deletion log embedded in Special:Undelete +* Improved error reporting and robustness for file delete/undelete. +* Improved speed of file delete by storing the SHA-1 hash in image/oldimage +* Fixed leading zero in base 36 SHA-1 hash == API changes since 1.10 == diff --git a/StartProfiler.php b/StartProfiler.php index 3fcf69e6e5..15c39da4f9 100644 --- a/StartProfiler.php +++ b/StartProfiler.php @@ -1,22 +1,24 @@ mTitle->getSubjectPage(); $wgOut->setPagetitle( $page->getPrefixedText() ); + $wgOut->setPageTitleActionText( wfMsg( 'info_short' ) ); $wgOut->setSubtitle( wfMsg( 'infosubtitle' )); # first, see if the page exists at all. @@ -3063,4 +3064,4 @@ class Article { $wgOut->addParserOutput( $parserOutput ); } -} \ No newline at end of file +} diff --git a/includes/AutoLoader.php b/includes/AutoLoader.php index 29ffb052d4..222222b4bb 100644 --- a/includes/AutoLoader.php +++ b/includes/AutoLoader.php @@ -258,11 +258,14 @@ function __autoload($className) { 'ArchivedFile' => 'includes/filerepo/ArchivedFile.php', 'File' => 'includes/filerepo/File.php', 'FileRepo' => 'includes/filerepo/FileRepo.php', + 'FileRepoStatus' => 'includes/filerepo/FileRepoStatus.php', 'ForeignDBFile' => 'includes/filerepo/ForeignDBFile.php', 'ForeignDBRepo' => 'includes/filerepo/ForeignDBRepo.php', 'FSRepo' => 'includes/filerepo/FSRepo.php', 'Image' => 'includes/filerepo/LocalFile.php', 'LocalFile' => 'includes/filerepo/LocalFile.php', + 'LocalFileDeleteBatch' => 'includes/filerepo/LocalFile.php', + 'LocalFileRestoreBatch' => 'includes/filerepo/LocalFile.php', 'LocalRepo' => 'includes/filerepo/LocalRepo.php', 'OldLocalFile' => 'includes/filerepo/OldLocalFile.php', 'RepoGroup' => 'includes/filerepo/RepoGroup.php', diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index bfec7cd5f9..cef2715915 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -163,16 +163,6 @@ $wgTmpDirectory = false; /// defaults to "{$wgUploadDirectory}/tmp" $wgUploadBaseUrl = ""; /**#@-*/ -/** - * By default deleted files are simply discarded; to save them and - * make it possible to undelete images, create a directory which - * is writable to the web server but is not exposed to the internet. - * - * Set $wgSaveDeletedFiles to true and set up the save path in - * $wgFileStore['deleted']['directory']. - */ -$wgSaveDeletedFiles = false; - /** * New file storage paths; currently used only for deleted files. * Set it like this: @@ -181,7 +171,7 @@ $wgSaveDeletedFiles = false; * */ $wgFileStore = array(); -$wgFileStore['deleted']['directory'] = null; // Don't forget to set this. +$wgFileStore['deleted']['directory'] = false;// Defaults to $wgUploadDirectory/deleted $wgFileStore['deleted']['url'] = null; // Private $wgFileStore['deleted']['hash'] = 3; // 3-level subdirectory split @@ -208,6 +198,10 @@ $wgFileStore['deleted']['hash'] = 3; // 3-level subdirectory split * start with a capital letter. The current implementation may give incorrect * description page links when the local $wgCapitalLinks and initialCapital * are mismatched. + * pathDisclosureProtection + * May be 'paranoid' to remove all parameters from error messages, 'none' to + * leave the paths in unchanged, or 'simple' to replace paths with + * placeholders. Default for LocalRepo is 'simple'. * * These settings describe a foreign MediaWiki installation. They are optional, and will be ignored * for local repositories: diff --git a/includes/EditPage.php b/includes/EditPage.php index 867431b14c..c6807e3cff 100644 --- a/includes/EditPage.php +++ b/includes/EditPage.php @@ -939,6 +939,10 @@ class EditPage { # Enabled article-related sidebar, toplinks, etc. $wgOut->setArticleRelated( true ); + if ( $this->formtype == 'preview' ) { + $wgOut->setPageTitleActionText( wfMsg( 'preview' ) ); + } + if ( $this->isConflict ) { $s = wfMsg( 'editconflict', $this->mTitle->getPrefixedText() ); $wgOut->setPageTitle( $s ); diff --git a/includes/FileStore.php b/includes/FileStore.php index 108b416d75..1554d66eb8 100644 --- a/includes/FileStore.php +++ b/includes/FileStore.php @@ -219,7 +219,7 @@ class FileStore { * Confirm that the given file key is valid. * Note that a valid key may refer to a file that does not exist. * - * Key should consist of a 32-digit base-36 SHA-1 hash and + * Key should consist of a 31-digit base-36 SHA-1 hash and * an optional alphanumeric extension, all lowercase. * The whole must not exceed 64 characters. * @@ -227,7 +227,7 @@ class FileStore { * @return boolean */ static function validKey( $key ) { - return preg_match( '/^[0-9a-z]{32}(\.[0-9a-z]{1,31})?$/', $key ); + return preg_match( '/^[0-9a-z]{31,32}(\.[0-9a-z]{1,31})?$/', $key ); } @@ -249,7 +249,7 @@ class FileStore { return false; } - $base36 = wfBaseConvert( $hash, 16, 36, 32 ); + $base36 = wfBaseConvert( $hash, 16, 36, 31 ); if( $extension == '' ) { $key = $base36; } else { diff --git a/includes/GlobalFunctions.php b/includes/GlobalFunctions.php index abf101f3d0..f8d0c767de 100644 --- a/includes/GlobalFunctions.php +++ b/includes/GlobalFunctions.php @@ -237,6 +237,13 @@ function wfLogDBError( $text ) { * Log to a file without getting "file size exceeded" signals */ function wfErrorLog( $text, $file ) { + # Temp: add unique request prefix + static $prefix; + if ( !isset( $prefix ) ) { + $prefix = chr( mt_rand( 33, 126 ) ) . chr( mt_rand( 33, 126 ) ) . chr( mt_rand( 33, 126 ) ) . '| '; + } + $text = $prefix . $text; + wfSuppressWarnings(); $exists = file_exists( $file ); $size = $exists ? filesize( $file ) : false; @@ -2139,7 +2146,9 @@ function wfSetupSession() { } session_set_cookie_params( 0, $wgCookiePath, $wgCookieDomain, $wgCookieSecure); session_cache_limiter( 'private, must-revalidate' ); + wfDebug( "Starting session..." ); @session_start(); + wfDebug( "ok\n" ); } /** @@ -2296,4 +2305,4 @@ function wfScript( $script = 'index' ) { */ function wfBoolToStr( $value ) { return $value ? 'true' : 'false'; -} \ No newline at end of file +} diff --git a/includes/ImagePage.php b/includes/ImagePage.php index 3265b7f386..d95e19f9de 100644 --- a/includes/ImagePage.php +++ b/includes/ImagePage.php @@ -579,56 +579,56 @@ EOT $wgOut->showUnexpectedValueError( 'oldimage', htmlspecialchars($oldimage) ); return; } - if ( !$this->doDeleteOldImage( $oldimage ) ) { - return; - } + $status = $this->doDeleteOldImage( $oldimage ); $deleted = $oldimage; } else { - $ok = $this->img->delete( $reason ); - if( !$ok ) { - # If the deletion operation actually failed, bug out: - $wgOut->showFileDeleteError( $this->img->getName() ); - return; + $status = $this->img->delete( $reason ); + if ( !$status->isGood() ) { + // Warning or error + $wgOut->addWikiText( $status->getWikiText( 'filedeleteerror-short', 'filedeleteerror-long' ) ); + } + if ( $status->ok ) { + # Image itself is now gone, and database is cleaned. + # Now we remove the image description page. + $article = new Article( $this->mTitle ); + $article->doDeleteArticle( $reason ); # ignore errors + $deleted = $this->img->getName(); } - - # Image itself is now gone, and database is cleaned. - # Now we remove the image description page. - - $article = new Article( $this->mTitle ); - $article->doDeleteArticle( $reason ); # ignore errors - - $deleted = $this->img->getName(); } - $wgOut->setPagetitle( wfMsg( 'actioncomplete' ) ); $wgOut->setRobotpolicy( 'noindex,nofollow' ); - $loglink = '[[Special:Log/delete|' . wfMsg( 'deletionlog' ) . ']]'; - $text = wfMsg( 'deletedtext', $deleted, $loglink ); - - $wgOut->addWikiText( $text ); - - $wgOut->returnToMain( false, $this->mTitle->getPrefixedText() ); + if ( !$status->ok ) { + // Fatal error flagged + $wgOut->setPagetitle( wfMsg( 'errorpagetitle' ) ); + $wgOut->returnToMain( false, $this->mTitle->getPrefixedText() ); + } else { + // Operation completed + $wgOut->setPagetitle( wfMsg( 'actioncomplete' ) ); + $loglink = '[[Special:Log/delete|' . wfMsg( 'deletionlog' ) . ']]'; + $text = wfMsg( 'deletedtext', $deleted, $loglink ); + $wgOut->addWikiText( $text ); + $wgOut->returnToMain( false, $this->mTitle->getPrefixedText() ); + } } /** - * @return success + * Delete an old revision of an image, + * @return FileRepoStatus */ - function doDeleteOldImage( $oldimage ) - { + function doDeleteOldImage( $oldimage ) { global $wgOut; - $ok = $this->img->deleteOld( $oldimage, '' ); - if( !$ok ) { - # If we actually have a file and can't delete it, throw an error. - # Something went awry... - $wgOut->showFileDeleteError( "$oldimage" ); - } else { + $status = $this->img->deleteOld( $oldimage, '' ); + if( !$status->isGood() ) { + $wgOut->addWikiText( $status->getWikiText( 'filedeleteerror-short', 'filedeleteerror-long' ) ); + } + if ( $status->ok ) { # Log the deletion $log = new LogPage( 'delete' ); $log->addEntry( 'delete', $this->mTitle, wfMsg('deletedrevision',$oldimage) ); } - return $ok; + return $status; } function revert() { @@ -667,10 +667,11 @@ EOT $sourcePath = $this->img->getArchiveVirtualUrl( $oldimage ); $comment = wfMsg( "reverted" ); - $result = $this->img->upload( $sourcePath, $comment, $comment ); + // TODO: preserve file properties from DB instead of reloading from file + $status = $this->img->upload( $sourcePath, $comment, $comment ); - if ( WikiError::isError( $result ) ) { - $this->showError( $result ); + if ( !$status->isGood() ) { + $this->showError( $status->getWikiText() ); return; } @@ -699,15 +700,15 @@ EOT } /** - * Display an error from a wikitext-formatted WikiError object + * Display an error with a wikitext description */ - function showError( WikiError $error ) { + function showError( $description ) { global $wgOut; $wgOut->setPageTitle( wfMsg( "internalerror" ) ); $wgOut->setRobotpolicy( "noindex,nofollow" ); $wgOut->setArticleRelated( false ); $wgOut->enableClientCache( false ); - $wgOut->addWikiText( $error->getMessage() ); + $wgOut->addWikiText( $description ); } } diff --git a/includes/OutputPage.php b/includes/OutputPage.php index fc7a621e37..cecff52f30 100644 --- a/includes/OutputPage.php +++ b/includes/OutputPage.php @@ -28,6 +28,7 @@ class OutputPage { var $mNewSectionLink = false; var $mNoGallery = false; + var $mPageTitleActionText = ''; /** * Constructor @@ -189,26 +190,13 @@ class OutputPage { } } + function setPageTitleActionText( $text ) { + $this->mPageTitleActionText = $text; + } + function getPageTitleActionText () { - global $action; - switch($action) { - case 'edit': - case 'delete': - case 'protect': - case 'unprotect': - case 'watch': - case 'unwatch': - // Display title is already customized - return ''; - case 'history': - return wfMsg('history_short'); - case 'submit': - // FIXME: bug 2735; not correct for special pages etc - return wfMsg('preview'); - case 'info': - return wfMsg('info_short'); - default: - return ''; + if ( isset( $this->mPageTitleActionText ) ) { + return $this->mPageTitleActionText; } } diff --git a/includes/PageHistory.php b/includes/PageHistory.php index cfc447a32a..c92b3ddb3c 100644 --- a/includes/PageHistory.php +++ b/includes/PageHistory.php @@ -62,6 +62,7 @@ class PageHistory { * Setup page variables. */ $wgOut->setPageTitle( $this->mTitle->getPrefixedText() ); + $wgOut->setPageTitleActionText( wfMsg( 'history_short' ) ); $wgOut->setArticleFlag( false ); $wgOut->setArticleRelated( true ); $wgOut->setRobotpolicy( 'noindex,nofollow' ); diff --git a/includes/Setup.php b/includes/Setup.php index b3749afee7..66bae0a8ca 100644 --- a/includes/Setup.php +++ b/includes/Setup.php @@ -54,6 +54,11 @@ if( $wgTmpDirectory === false ) $wgTmpDirectory = "{$wgUploadDirectory}/tmp"; if( $wgReadOnlyFile === false ) $wgReadOnlyFile = "{$wgUploadDirectory}/lock_yBgMBwiR"; if( $wgFileCacheDirectory === false ) $wgFileCacheDirectory = "{$wgUploadDirectory}/cache"; +if ( empty( $wgFileStore['deleted']['directory'] ) ) { + $wgFileStore['deleted']['directory'] = "{$wgUploadDirectory}/deleted"; +} + + /** * Initialise $wgLocalFileRepo from backwards-compatible settings */ @@ -67,6 +72,8 @@ if ( !$wgLocalFileRepo ) { 'thumbScriptUrl' => $wgThumbnailScriptPath, 'transformVia404' => !$wgGenerateThumbnailOnParse, 'initialCapital' => $wgCapitalLinks, + 'deletedDir' => $wgFileStore['deleted']['directory'], + 'deletedHashLevels' => $wgFileStore['deleted']['hash'] ); } /** @@ -87,7 +94,7 @@ if ( $wgUseSharedUploads ) { 'dbUser' => $wgDBuser, 'dbPassword' => $wgDBpassword, 'dbName' => $wgSharedUploadDBname, - 'dbFlags' => DBO_DEFAULT, + 'dbFlags' => ($wgDebugDumpSql ? DBO_DEBUG : 0) | DBO_DEFAULT, 'tablePrefix' => $wgSharedUploadDBprefix, 'hasSharedCache' => $wgCacheSharedUploads, 'descBaseUrl' => $wgRepositoryBaseUrl, diff --git a/includes/SpecialLog.php b/includes/SpecialLog.php index 0ed417ce42..6d251b3ce7 100644 --- a/includes/SpecialLog.php +++ b/includes/SpecialLog.php @@ -241,19 +241,25 @@ class LogReader { * @addtogroup SpecialPage */ class LogViewer { + const NO_ACTION_LINK = 1; + /** * @var LogReader $reader */ var $reader; var $numResults = 0; + var $flags = 0; /** * @param LogReader &$reader where to get our data from + * @param integer $flags Bitwise combination of flags: + * self::NO_ACTION_LINK Don't show restore/unblock/block links */ - function LogViewer( &$reader ) { + function LogViewer( &$reader, $flags = 0 ) { global $wgUser; $this->skin = $wgUser->getSkin(); $this->reader =& $reader; + $this->flags = $flags; } /** @@ -363,41 +369,43 @@ class LogViewer { $paramArray = LogPage::extractParams( $s->log_params ); $revert = ''; // show revertmove link - if ( $s->log_type == 'move' && isset( $paramArray[0] ) ) { - $destTitle = Title::newFromText( $paramArray[0] ); - if ( $destTitle ) { - $revert = '(' . $this->skin->makeKnownLinkObj( SpecialPage::getTitleFor( 'Movepage' ), - wfMsg( 'revertmove' ), - 'wpOldTitle=' . urlencode( $destTitle->getPrefixedDBkey() ) . - '&wpNewTitle=' . urlencode( $title->getPrefixedDBkey() ) . - '&wpReason=' . urlencode( wfMsgForContent( 'revertmove' ) ) . - '&wpMovetalk=0' ) . ')'; - } - // show undelete link - } elseif ( $s->log_action == 'delete' && $wgUser->isAllowed( 'delete' ) ) { - $revert = '(' . $this->skin->makeKnownLinkObj( SpecialPage::getTitleFor( 'Undelete' ), - wfMsg( 'undeletebtn' ) , - 'target='. urlencode( $title->getPrefixedDBkey() ) ) . ')'; - - // show unblock link - } elseif ( $s->log_action == 'block' && $wgUser->isAllowed( 'block' ) ) { - $revert = '(' . $skin->makeKnownLinkObj( SpecialPage::getTitleFor( 'Ipblocklist' ), - wfMsg( 'unblocklink' ), - 'action=unblock&ip=' . urlencode( $s->log_title ) ) . ')'; - // show change protection link - } elseif ( ( $s->log_action == 'protect' || $s->log_action == 'modify' ) && $wgUser->isAllowed( 'protect' ) ) { - $revert = '(' . $skin->makeKnownLinkObj( $title, wfMsg( 'protect_change' ), 'action=unprotect' ) . ')'; - // show user tool links for self created users - // TODO: The extension should be handling this, get it out of core! - } elseif ( $s->log_action == 'create2' ) { - if( isset( $paramArray[0] ) ) { - $revert = $this->skin->userToolLinks( $paramArray[0], $s->log_title, true ); - } else { - # Fall back to a blue contributions link - $revert = $this->skin->userToolLinks( 1, $s->log_title ); + if ( !( $this->flags & self::NO_ACTION_LINK ) ) { + if ( $s->log_type == 'move' && isset( $paramArray[0] ) ) { + $destTitle = Title::newFromText( $paramArray[0] ); + if ( $destTitle ) { + $revert = '(' . $this->skin->makeKnownLinkObj( SpecialPage::getTitleFor( 'Movepage' ), + wfMsg( 'revertmove' ), + 'wpOldTitle=' . urlencode( $destTitle->getPrefixedDBkey() ) . + '&wpNewTitle=' . urlencode( $title->getPrefixedDBkey() ) . + '&wpReason=' . urlencode( wfMsgForContent( 'revertmove' ) ) . + '&wpMovetalk=0' ) . ')'; + } + // show undelete link + } elseif ( $s->log_action == 'delete' && $wgUser->isAllowed( 'delete' ) ) { + $revert = '(' . $this->skin->makeKnownLinkObj( SpecialPage::getTitleFor( 'Undelete' ), + wfMsg( 'undeletebtn' ) , + 'target='. urlencode( $title->getPrefixedDBkey() ) ) . ')'; + + // show unblock link + } elseif ( $s->log_action == 'block' && $wgUser->isAllowed( 'block' ) ) { + $revert = '(' . $skin->makeKnownLinkObj( SpecialPage::getTitleFor( 'Ipblocklist' ), + wfMsg( 'unblocklink' ), + 'action=unblock&ip=' . urlencode( $s->log_title ) ) . ')'; + // show change protection link + } elseif ( ( $s->log_action == 'protect' || $s->log_action == 'modify' ) && $wgUser->isAllowed( 'protect' ) ) { + $revert = '(' . $skin->makeKnownLinkObj( $title, wfMsg( 'protect_change' ), 'action=unprotect' ) . ')'; + // show user tool links for self created users + // TODO: The extension should be handling this, get it out of core! + } elseif ( $s->log_action == 'create2' ) { + if( isset( $paramArray[0] ) ) { + $revert = $this->skin->userToolLinks( $paramArray[0], $s->log_title, true ); + } else { + # Fall back to a blue contributions link + $revert = $this->skin->userToolLinks( 1, $s->log_title ); + } + # Suppress $comment from old entries, not needed and can contain incorrect links + $comment = ''; } - # Suppress $comment from old entries, not needed and can contain incorrect links - $comment = ''; } $action = LogPage::actionText( $s->log_type, $s->log_action, $title, $this->skin, $paramArray, true, true ); diff --git a/includes/SpecialUndelete.php b/includes/SpecialUndelete.php index 8f49a29317..52f1ecc04a 100644 --- a/includes/SpecialUndelete.php +++ b/includes/SpecialUndelete.php @@ -23,6 +23,7 @@ function wfSpecialUndelete( $par ) { */ class PageArchive { protected $title; + var $fileStatus; function __construct( $title ) { if( is_null( $title ) ) { @@ -270,7 +271,8 @@ class PageArchive { if( $restoreFiles && $this->title->getNamespace() == NS_IMAGE ) { $img = wfLocalFile( $this->title ); - $filesRestored = $img->restore( $fileVersions ); + $this->fileStatus = $img->restore( $fileVersions ); + $filesRestored = $this->fileStatus->successCount; } else { $filesRestored = 0; } @@ -280,7 +282,7 @@ class PageArchive { } else { $textRestored = 0; } - + // Touch the log! global $wgContLang; $log = new LogPage( 'delete' ); @@ -303,8 +305,12 @@ class PageArchive { if( trim( $comment ) != '' ) $reason .= ": {$comment}"; $log->addEntry( 'restore', $this->title, $reason ); - - return true; + + if ( $this->fileStatus && !$this->fileStatus->ok ) { + return false; + } else { + return true; + } } /** @@ -448,6 +454,7 @@ class PageArchive { return $restored; } + function getFileStatus() { return $this->fileStatus; } } /** @@ -731,8 +738,13 @@ class UndeleteForm { $logViewer = new LogViewer( new LogReader( new FauxRequest( - array( 'page' => $this->mTargetObj->getPrefixedText(), - 'type' => 'delete' ) ) ) ); + array( + 'page' => $this->mTargetObj->getPrefixedText(), + 'type' => 'delete' + ) + ) + ), LogViewer::NO_ACTION_LINK + ); $logViewer->showList( $wgOut ); if( $this->mAllowed && ( $haveRevisions || $haveFiles ) ) { @@ -836,15 +848,23 @@ class UndeleteForm { $this->mTargetTimestamp, $this->mComment, $this->mFileVersions ); - + if( $ok ) { $skin = $wgUser->getSkin(); $link = $skin->makeKnownLinkObj( $this->mTargetObj ); $wgOut->addHtml( wfMsgWikiHtml( 'undeletedpage', $link ) ); - return true; + } else { + $wgOut->showFatalError( wfMsg( "cannotundelete" ) ); } + + // Show file deletion warnings and errors + $status = $archive->getFileStatus(); + if ( $status && !$status->isGood() ) { + $wgOut->addWikiText( $status->getWikiText( 'undelete-error-short', 'undelete-error-long' ) ); + } + } else { + $wgOut->showFatalError( wfMsg( "cannotundelete" ) ); } - $wgOut->showFatalError( wfMsg( "cannotundelete" ) ); return false; } } diff --git a/includes/SpecialUpload.php b/includes/SpecialUpload.php index 18ebf0a532..0c66e93482 100644 --- a/includes/SpecialUpload.php +++ b/includes/SpecialUpload.php @@ -433,8 +433,8 @@ class UploadForm { $status = $this->mLocalFile->upload( $this->mTempPath, $this->mComment, $pageText, File::DELETE_SOURCE, $this->mFileProps ); - if ( WikiError::isError( $status ) ) { - $this->showError( $status ); + if ( !$status->isGood() ) { + $this->showError( $status->getWikiText() ); } else { if ( $this->mWatchthis ) { global $wgUser; @@ -592,12 +592,12 @@ class UploadForm { function saveTempUploadedFile( $saveName, $tempName ) { global $wgOut; $repo = RepoGroup::singleton()->getLocalRepo(); - $result = $repo->storeTemp( $saveName, $tempName ); - if ( WikiError::isError( $result ) ) { - $this->showError( $result ); + $status = $repo->storeTemp( $saveName, $tempName ); + if ( !$status->isGood() ) { + $this->showError( $status->getWikiText() ); return false; } else { - return $result; + return $status->value; } } @@ -1354,15 +1354,15 @@ EOT } /** - * Display an error from a wikitext-formatted WikiError object + * Display an error with a wikitext description */ - function showError( WikiError $error ) { + function showError( $description ) { global $wgOut; $wgOut->setPageTitle( wfMsg( "internalerror" ) ); $wgOut->setRobotpolicy( "noindex,nofollow" ); $wgOut->setArticleRelated( false ); $wgOut->enableClientCache( false ); - $wgOut->addWikiText( $error->getMessage() ); + $wgOut->addWikiText( $description ); } /** diff --git a/includes/filerepo/FSRepo.php b/includes/filerepo/FSRepo.php index dc404447f1..d5a6783e05 100644 --- a/includes/filerepo/FSRepo.php +++ b/includes/filerepo/FSRepo.php @@ -6,9 +6,10 @@ */ class FSRepo extends FileRepo { - var $directory, $url, $hashLevels; + var $directory, $deletedDir, $url, $hashLevels, $deletedHashLevels; var $fileFactory = array( 'UnregisteredLocalFile', 'newFromTitle' ); var $oldFileFactory = false; + var $pathDisclosureProtection = 'simple'; function __construct( $info ) { parent::__construct( $info ); @@ -16,7 +17,12 @@ class FSRepo extends FileRepo { // Required settings $this->directory = $info['directory']; $this->url = $info['url']; - $this->hashLevels = $info['hashLevels']; + + // Optional settings + $this->hashLevels = isset( $info['hashLevels'] ) ? $info['hashLevels'] : 2; + $this->deletedHashLevels = isset( $info['deletedHashLevels'] ) ? + $info['deletedHashLevels'] : $this->hashLevels; + $this->deletedDir = isset( $info['deletedDir'] ) ? $info['deletedDir'] : false; } /** @@ -50,7 +56,7 @@ class FSRepo extends FileRepo { case 'temp': return "{$this->directory}/temp"; case 'deleted': - return $GLOBALS['wgFileStore']['deleted']['directory']; + return $this->deletedDir; default: return false; } @@ -66,7 +72,7 @@ class FSRepo extends FileRepo { case 'temp': return "{$this->url}/temp"; case 'deleted': - return $GLOBALS['wgFileStore']['deleted']['url']; + return false; // no public URL default: return false; } @@ -109,47 +115,101 @@ class FSRepo extends FileRepo { } /** - * Store a file to a given destination. + * Store a batch of files + * + * @param array $triplets (src,zone,dest) triplets as per store() + * @param integer $flags Bitwise combination of the following flags: + * self::DELETE_SOURCE Delete the source file after upload + * self::OVERWRITE Overwrite an existing destination file instead of failing + * self::OVERWRITE_SAME Overwrite the file if the destination exists and has the + * same contents as the source */ - function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) { + function storeBatch( $triplets, $flags = 0 ) { if ( !is_writable( $this->directory ) ) { - return new WikiErrorMsg( 'upload_directory_read_only', wfEscapeWikiText( $this->directory ) ); - } - $root = $this->getZonePath( $dstZone ); - if ( !$root ) { - throw new MWException( "Invalid zone: $dstZone" ); + return $this->newFatal( 'upload_directory_read_only', $this->directory ); } - $dstPath = "$root/$dstRel"; + $status = $this->newGood(); + foreach ( $triplets as $i => $triplet ) { + list( $srcPath, $dstZone, $dstRel ) = $triplet; - if ( !is_dir( dirname( $dstPath ) ) ) { - wfMkdirParents( dirname( $dstPath ) ); + $root = $this->getZonePath( $dstZone ); + if ( !$root ) { + throw new MWException( "Invalid zone: $dstZone" ); + } + if ( !$this->validateFilename( $dstRel ) ) { + throw new MWException( 'Validation error in $dstRel' ); + } + $dstPath = "$root/$dstRel"; + $dstDir = dirname( $dstPath ); + + if ( !is_dir( $dstDir ) && !wfMkdirParents( $dstDir ) ) { + return $this->newFatal( 'directorycreateerror', $dstDir ); + } + + if ( self::isVirtualUrl( $srcPath ) ) { + $srcPath = $triplets[$i][0] = $this->resolveVirtualUrl( $srcPath ); + } + if ( !is_file( $srcPath ) ) { + // Make a list of files that don't exist for return to the caller + $status->fatal( 'filenotfound', $srcPath ); + continue; + } + if ( !( $flags & self::OVERWRITE ) && file_exists( $dstPath ) ) { + if ( $flags & self::OVERWRITE_SAME ) { + $hashSource = sha1_file( $srcPath ); + $hashDest = sha1_file( $dstPath ); + if ( $hashSource != $hashDest ) { + $status->fatal( 'fileexists', $dstPath ); + } + } else { + $status->fatal( 'fileexists', $dstPath ); + } + } } - - if ( self::isVirtualUrl( $srcPath ) ) { - $srcPath = $this->resolveVirtualUrl( $srcPath ); + + $deleteDest = wfIsWindows() && ( $flags & self::OVERWRITE ); + + // Abort now on failure + if ( !$status->ok ) { + return $status; } - if ( $flags & self::DELETE_SOURCE ) { - if ( !rename( $srcPath, $dstPath ) ) { - return new WikiErrorMsg( 'filerenameerror', wfEscapeWikiText( $srcPath ), - wfEscapeWikiText( $dstPath ) ); + foreach ( $triplets as $triplet ) { + list( $srcPath, $dstZone, $dstRel ) = $triplet; + $root = $this->getZonePath( $dstZone ); + $dstPath = "$root/$dstRel"; + $good = true; + + if ( $flags & self::DELETE_SOURCE ) { + if ( $deleteDest ) { + unlink( $dstPath ); + } + if ( !rename( $srcPath, $dstPath ) ) { + $status->error( 'filerenameerror', $srcPath, $dstPath ); + $good = false; + } + } else { + if ( !copy( $srcPath, $dstPath ) ) { + $status->error( 'filecopyerror', $srcPath, $dstPath ); + $good = false; + } } - } else { - if ( !copy( $srcPath, $dstPath ) ) { - return new WikiErrorMsg( 'filecopyerror', wfEscapeWikiText( $srcPath ), - wfEscapeWikiText( $dstPath ) ); + if ( $good ) { + chmod( $dstPath, 0644 ); + $status->successCount++; + } else { + $status->failCount++; } } - chmod( $dstPath, 0644 ); - return true; + return $status; } /** * Pick a random name in the temp zone and store a file to it. - * Returns the URL, or a WikiError on failure. * @param string $originalName The base name of the file as specified * by the user. The file extension will be maintained. * @param string $srcPath The current location of the file. + * @return FileRepoStatus object with the URL in the value. */ function storeTemp( $originalName, $srcPath ) { $date = gmdate( "YmdHis" ); @@ -158,11 +218,8 @@ class FSRepo extends FileRepo { $dstUrlRel = $hashPath . $date . '!' . rawurlencode( $originalName ); $result = $this->store( $srcPath, 'temp', $dstRel ); - if ( WikiError::isError( $result ) ) { - return $result; - } else { - return $this->getVirtualUrl( 'temp' ) . '/' . $dstUrlRel; - } + $result->value = $this->getVirtualUrl( 'temp' ) . '/' . $dstUrlRel; + return $result; } /** @@ -183,82 +240,186 @@ class FSRepo extends FileRepo { return $success; } - /** - * Copy or move a file either from the local filesystem or from an mwrepo:// - * virtual URL, into this repository at the specified destination location. - * - * @param string $srcPath The source path or URL - * @param string $dstRel The destination relative path - * @param string $archiveRel The relative path where the existing file is to - * be archived, if there is one. Relative to the public zone root. + * Publish a batch of files + * @param array $triplets (source,dest,archive) triplets as per publish() * @param integer $flags Bitfield, may be FileRepo::DELETE_SOURCE to indicate - * that the source file should be deleted if possible + * that the source files should be deleted if possible */ - function publish( $srcPath, $dstRel, $archiveRel, $flags = 0 ) { + function publishBatch( $triplets, $flags = 0 ) { + // Perform initial checks if ( !is_writable( $this->directory ) ) { - return new WikiErrorMsg( 'upload_directory_read_only', wfEscapeWikiText( $this->directory ) ); - } - if ( substr( $srcPath, 0, 9 ) == 'mwrepo://' ) { - $srcPath = $this->resolveVirtualUrl( $srcPath ); + return $this->newFatal( 'upload_directory_read_only', $this->directory ); } - if ( !$this->validateFilename( $dstRel ) ) { - throw new MWException( 'Validation error in $dstRel' ); + $status = $this->newGood( array() ); + foreach ( $triplets as $i => $triplet ) { + list( $srcPath, $dstRel, $archiveRel ) = $triplet; + + if ( substr( $srcPath, 0, 9 ) == 'mwrepo://' ) { + $triplets[$i][0] = $srcPath = $this->resolveVirtualUrl( $srcPath ); + } + if ( !$this->validateFilename( $dstRel ) ) { + throw new MWException( 'Validation error in $dstRel' ); + } + if ( !$this->validateFilename( $archiveRel ) ) { + throw new MWException( 'Validation error in $archiveRel' ); + } + $dstPath = "{$this->directory}/$dstRel"; + $archivePath = "{$this->directory}/$archiveRel"; + + $dstDir = dirname( $dstPath ); + $archiveDir = dirname( $archivePath ); + // Abort immediately on directory creation errors since they're likely to be repetitive + if ( !is_dir( $dstDir ) && !wfMkdirParents( $dstDir ) ) { + return $this->newFatal( 'directorycreateerror', $dstDir ); + } + if ( !is_dir( $archiveDir ) && !wfMkdirParents( $archiveDir ) ) { + return $this->newFatal( 'directorycreateerror', $archiveDir ); + } + if ( !is_file( $srcPath ) ) { + // Make a list of files that don't exist for return to the caller + $status->fatal( 'filenotfound', $srcPath ); + } } - if ( !$this->validateFilename( $archiveRel ) ) { - throw new MWException( 'Validation error in $archiveRel' ); + + if ( !$status->ok ) { + return $status; } - $dstPath = "{$this->directory}/$dstRel"; - $archivePath = "{$this->directory}/$archiveRel"; - $dstDir = dirname( $dstPath ); - if ( !is_dir( $dstDir ) ) wfMkdirParents( $dstDir ); + foreach ( $triplets as $i => $triplet ) { + list( $srcPath, $dstRel, $archiveRel ) = $triplet; + $dstPath = "{$this->directory}/$dstRel"; + $archivePath = "{$this->directory}/$archiveRel"; - // Check if the source is missing before we attempt to move the dest to archive - if ( !is_file( $srcPath ) ) { - return new WikiErrorMsg( 'filenotfound', wfEscapeWikiText( $srcPath ) ); - } + // Archive destination file if it exists + if( is_file( $dstPath ) ) { + // Check if the archive file exists + // This is a sanity check to avoid data loss. In UNIX, the rename primitive + // unlinks the destination file if it exists. DB-based synchronisation in + // publishBatch's caller should prevent races. In Windows there's no + // problem because the rename primitive fails if the destination exists. + if ( is_file( $archivePath ) ) { + $success = false; + } else { + wfSuppressWarnings(); + $success = rename( $dstPath, $archivePath ); + wfRestoreWarnings(); + } - if( is_file( $dstPath ) ) { - $archiveDir = dirname( $archivePath ); - if ( !is_dir( $archiveDir ) ) wfMkdirParents( $archiveDir ); + if( !$success ) { + $status->error( 'filerenameerror',$dstPath, $archivePath ); + $status->failCount++; + continue; + } else { + wfDebug(__METHOD__.": moved file $dstPath to $archivePath\n"); + } + $status->value[$i] = 'archived'; + } else { + $status->value[$i] = 'new'; + } + + $good = true; wfSuppressWarnings(); - $success = rename( $dstPath, $archivePath ); + if ( $flags & self::DELETE_SOURCE ) { + if ( !rename( $srcPath, $dstPath ) ) { + $status->error( 'filerenameerror', $srcPath, $dstPath ); + $good = false; + } + } else { + if ( !copy( $srcPath, $dstPath ) ) { + $status->error( 'filecopyerror', $srcPath, $dstPath ); + $good = false; + } + } wfRestoreWarnings(); - if( ! $success ) { - return new WikiErrorMsg( 'filerenameerror', wfEscapeWikiText( $dstPath ), - wfEscapeWikiText( $archivePath ) ); + if ( $good ) { + $status->successCount++; + wfDebug(__METHOD__.": wrote tempfile $srcPath to $dstPath\n"); + // Thread-safe override for umask + chmod( $dstPath, 0644 ); + } else { + $status->failCount++; } - else wfDebug(__METHOD__.": moved file $dstPath to $archivePath\n"); - $status = 'archived'; } - else { - $status = 'new'; + return $status; + } + + /** + * Move a group of files to the deletion archive. + * If no valid deletion archive is configured, this may either delete the + * file or throw an exception, depending on the preference of the repository. + * + * @param array $sourceDestPairs Array of source/destination pairs. Each element + * is a two-element array containing the source file path relative to the + * public root in the first element, and the archive file path relative + * to the deleted zone root in the second element. + * @return FileRepoStatus + */ + function deleteBatch( $sourceDestPairs ) { + $status = $this->newGood(); + if ( !$this->deletedDir ) { + throw new MWException( __METHOD__.': no valid deletion archive directory' ); } - $error = false; - wfSuppressWarnings(); - if ( $flags & self::DELETE_SOURCE ) { - if ( !rename( $srcPath, $dstPath ) ) { - $error = new WikiErrorMsg( 'filerenameerror', wfEscapeWikiText( $srcPath ), - wfEscapeWikiText( $dstPath ) ); + /** + * Validate filenames and create archive directories + */ + foreach ( $sourceDestPairs as $pair ) { + list( $srcRel, $archiveRel ) = $pair; + if ( !$this->validateFilename( $srcRel ) ) { + throw new MWException( __METHOD__.':Validation error in $srcRel' ); } - } else { - if ( !copy( $srcPath, $dstPath ) ) { - $error = new WikiErrorMsg( 'filerenameerror', wfEscapeWikiText( $srcPath ), - wfEscapeWikiText( $dstPath ) ); + if ( !$this->validateFilename( $archiveRel ) ) { + throw new MWException( __METHOD__.':Validation error in $archiveRel' ); + } + $archivePath = "{$this->deletedDir}/$archiveRel"; + $archiveDir = dirname( $archivePath ); + if ( !wfMkdirParents( $archiveDir ) ) { + $status->fatal( 'directorycreateerror', $archiveDir ); + continue; + } + // Check if the archive directory is writable + // This doesn't appear to work on NTFS + if ( !is_writable( $archiveDir ) ) { + $status->fatal( 'filedelete-archive-read-only', $archiveDir ); } } - wfRestoreWarnings(); - - if( $error ) { - return $error; - } else { - wfDebug(__METHOD__.": wrote tempfile $srcPath to $dstPath\n"); + if ( !$status->ok ) { + // Abort early + return $status; } - chmod( $dstPath, 0644 ); + /** + * Move the files + * We're now committed to returning an OK result, which will lead to + * the files being moved in the DB also. + */ + foreach ( $sourceDestPairs as $pair ) { + list( $srcRel, $archiveRel ) = $pair; + $srcPath = "{$this->directory}/$srcRel"; + $archivePath = "{$this->deletedDir}/$archiveRel"; + $good = true; + if ( file_exists( $archivePath ) ) { + # A file with this content hash is already archived + if ( !@unlink( $srcPath ) ) { + $status->error( 'filedeleteerror', $srcPath ); + $good = false; + } + } else{ + if ( !@rename( $srcPath, $archivePath ) ) { + $status->error( 'filerenameerror', $srcPath, $archivePath ); + $good = false; + } else { + chmod( $archivePath, 0644 ); + } + } + if ( $good ) { + $status->successCount++; + } else { + $status->failCount++; + } + } return $status; } @@ -270,6 +431,18 @@ class FSRepo extends FileRepo { return FileRepo::getHashPathForLevel( $name, $this->hashLevels ); } + /** + * Get a relative path for a deletion archive key, + * e.g. s/z/a/ for sza251lrxrc1jad41h5mgilp8nysje52.jpg + */ + function getDeletedHashPath( $key ) { + $path = ''; + for ( $i = 0; $i < $this->deletedHashLevels; $i++ ) { + $path .= $key[$i] . '/'; + } + return $path; + } + /** * Call a callback function for every file in the repository. * Uses the filesystem even in child classes. @@ -308,6 +481,39 @@ class FSRepo extends FileRepo { $path = $this->resolveVirtualUrl( $virtualUrl ); return File::getPropsFromPath( $path ); } + + /** + * Path disclosure protection functions + * + * Get a callback function to use for cleaning error message parameters + */ + function getErrorCleanupFunction() { + switch ( $this->pathDisclosureProtection ) { + case 'simple': + $callback = array( $this, 'simpleClean' ); + break; + default: + $callback = parent::getErrorCleanupFunction(); + } + return $callback; + } + + function simpleClean( $param ) { + if ( !isset( $this->simpleCleanPairs ) ) { + global $IP; + $this->simpleCleanPairs = array( + $this->directory => 'public', + "{$this->directory}/temp" => 'temp', + $IP => '$IP', + dirname( __FILE__ ) => '$IP/extensions/WebStore', + ); + if ( $this->deletedDir ) { + $this->simpleCleanPairs[$this->deletedDir] = 'deleted'; + } + } + return strtr( $param, $this->simpleCleanPairs ); + } + } diff --git a/includes/filerepo/File.php b/includes/filerepo/File.php index 90937d2997..3b586da064 100644 --- a/includes/filerepo/File.php +++ b/includes/filerepo/File.php @@ -593,11 +593,9 @@ abstract class File { /** * Purge metadata and all affected pages when the file is created, - * deleted, or majorly updated. A set of additional URLs may be - * passed to purge, such as specific file files which have changed. - * @param $urlArray array + * deleted, or majorly updated. */ - function purgeEverything( $urlArr=array() ) { + function purgeEverything() { // Delete thumbnails and refresh file metadata cache $this->purgeCache(); $this->purgeDescription(); @@ -656,9 +654,9 @@ abstract class File { return $this->getHashPath() . rawurlencode( $this->getName() ); } - /** Get the path of the archive directory, or a particular file if $suffix is specified */ - function getArchivePath( $suffix = false ) { - $path = $this->repo->getZonePath('public') . '/archive/' . $this->getHashPath(); + /** Get the relative path for an archive file */ + function getArchiveRel( $suffix = false ) { + $path = 'archive/' . $this->getHashPath(); if ( $suffix === false ) { $path = substr( $path, 0, -1 ); } else { @@ -667,15 +665,25 @@ abstract class File { return $path; } - /** Get the path of the thumbnail directory, or a particular file if $suffix is specified */ - function getThumbPath( $suffix = false ) { - $path = $this->repo->getZonePath('public') . '/thumb/' . $this->getRel(); + /** Get relative path for a thumbnail file */ + function getThumbRel( $suffix = false ) { + $path = 'thumb/' . $this->getRel(); if ( $suffix !== false ) { $path .= '/' . $suffix; } return $path; } + /** Get the path of the archive directory, or a particular file if $suffix is specified */ + function getArchivePath( $suffix = false ) { + return $this->repo->getZonePath('public') . '/' . $this->getArchiveRel(); + } + + /** Get the path of the thumbnail directory, or a particular file if $suffix is specified */ + function getThumbPath( $suffix = false ) { + return $this->repo->getZonePath('public') . '/' . $this->getThumbRel( $suffix ); + } + /** Get the URL of the archive directory, or a particular file if $suffix is specified */ function getArchiveUrl( $suffix = false ) { $path = $this->repo->getZoneUrl('public') . '/archive/' . $this->getHashPath(); @@ -980,13 +988,20 @@ abstract class File { /** * Get the 14-character timestamp of the file upload, or false if */ - function getTimestmap() { + function getTimestamp() { $path = $this->getPath(); if ( !file_exists( $path ) ) { return false; } return wfTimestamp( filemtime( $path ) ); } + + /** + * Get the SHA-1 base 36 hash of the file + */ + function getSha1() { + return self::sha1Base36( $this->getPath() ); + } /** * Determine if the current user is allowed to view a particular @@ -1031,12 +1046,14 @@ abstract class File { $gis = false; $info['metadata'] = ''; } + $info['sha1'] = self::sha1Base36( $path ); wfDebug(__METHOD__.": $path loaded, {$info['size']} bytes, {$info['mime']}.\n"); } else { $info['mime'] = NULL; $info['media_type'] = MEDIATYPE_UNKNOWN; $info['metadata'] = ''; + $info['sha1'] = ''; wfDebug(__METHOD__.": $path NOT FOUND!\n"); } if( $gis ) { @@ -1056,6 +1073,30 @@ abstract class File { wfProfileOut( __METHOD__ ); return $info; } -} + /** + * Get a SHA-1 hash of a file in the local filesystem, in base-36 lower case + * encoding, zero padded to 31 digits. + * + * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36 + * fairly neatly. + * + * Returns false on failure + */ + static function sha1Base36( $path ) { + $hash = sha1_file( $path ); + if ( $hash === false ) { + return false; + } else { + return wfBaseConvert( $hash, 16, 36, 31 ); + } + } +} +/** + * Aliases for backwards compatibility with 1.6 + */ +define( 'MW_IMG_DELETED_FILE', File::DELETED_FILE ); +define( 'MW_IMG_DELETED_COMMENT', File::DELETED_COMMENT ); +define( 'MW_IMG_DELETED_USER', File::DELETED_USER ); +define( 'MW_IMG_DELETED_RESTRICTED', File::DELETED_RESTRICTED ); diff --git a/includes/filerepo/FileRepo.php b/includes/filerepo/FileRepo.php index b5e3787a85..cf6d65c2ce 100644 --- a/includes/filerepo/FileRepo.php +++ b/includes/filerepo/FileRepo.php @@ -6,9 +6,12 @@ */ abstract class FileRepo { const DELETE_SOURCE = 1; + const OVERWRITE = 2; + const OVERWRITE_SAME = 4; var $thumbScriptUrl, $transformVia404; var $descBaseUrl, $scriptDirUrl, $articleUrl, $fetchDescription, $initialCapital; + var $pathDisclosureProtection = 'paranoid'; /** * Factory functions for creating new files @@ -23,7 +26,7 @@ abstract class FileRepo { // Optional settings $this->initialCapital = true; // by default foreach ( array( 'descBaseUrl', 'scriptDirUrl', 'articleUrl', 'fetchDescription', - 'thumbScriptUrl', 'initialCapital' ) as $var ) + 'thumbScriptUrl', 'initialCapital', 'pathDisclosureProtection' ) as $var ) { if ( isset( $info[$var] ) ) { $this->$var = $info[$var]; @@ -200,12 +203,37 @@ abstract class FileRepo { /** * Store a file to a given destination. + * + * @param string $srcPath Source path or virtual URL + * @param string $dstZone Destination zone + * @param string $dstRel Destination relative path + * @param integer $flags Bitwise combination of the following flags: + * self::DELETE_SOURCE Delete the source file after upload + * self::OVERWRITE Overwrite an existing destination file instead of failing + * self::OVERWRITE_SAME Overwrite the file if the destination exists and has the + * same contents as the source + * @return FileRepoStatus + */ + function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) { + $status = $this->storeBatch( array( array( $srcPath, $dstZone, $dstRel ) ), $flags ); + if ( $status->successCount == 0 ) { + $status->ok = false; + } + return $status; + } + + /** + * Store a batch of files + * + * @param array $triplets (src,zone,dest) triplets as per store() + * @param integer $flags Flags as per store */ - abstract function store( $srcPath, $dstZone, $dstRel, $flags = 0 ); + abstract function storeBatch( $triplets, $flags = 0 ); /** * Pick a random name in the temp zone and store a file to it. - * Returns the URL, or a WikiError on failure. + * Returns a FileRepoStatus object with the URL in the value. + * * @param string $originalName The base name of the file as specified * by the user. The file extension will be maintained. * @param string $srcPath The current location of the file. @@ -226,6 +254,9 @@ abstract class FileRepo { * Copy or move a file either from the local filesystem or from an mwrepo:// * virtual URL, into this repository at the specified destination location. * + * Returns a FileRepoStatus object. On success, the value contains "new" or + * "archived", to indicate whether the file was new with that name. + * * @param string $srcPath The source path or URL * @param string $dstRel The destination relative path * @param string $archiveRel The relative path where the existing file is to @@ -233,7 +264,57 @@ abstract class FileRepo { * @param integer $flags Bitfield, may be FileRepo::DELETE_SOURCE to indicate * that the source file should be deleted if possible */ - abstract function publish( $srcPath, $dstRel, $archiveRel, $flags = 0 ); + function publish( $srcPath, $dstRel, $archiveRel, $flags = 0 ) { + $status = $this->publishBatch( array( array( $srcPath, $dstRel, $archiveRel ) ), $flags ); + if ( $status->successCount == 0 ) { + $status->ok = false; + } + if ( isset( $status->value[0] ) ) { + $status->value = $status->value[0]; + } else { + $status->value = false; + } + return $status; + } + + /** + * Publish a batch of files + * @param array $triplets (source,dest,archive) triplets as per publish() + * @param integer $flags Bitfield, may be FileRepo::DELETE_SOURCE to indicate + * that the source files should be deleted if possible + */ + abstract function publishBatch( $triplets, $flags = 0 ); + + /** + * Move a group of files to the deletion archive. + * + * If no valid deletion archive is configured, this may either delete the + * file or throw an exception, depending on the preference of the repository. + * + * The overwrite policy is determined by the repository -- currently FSRepo + * assumes a naming scheme in the deleted zone based on content hash, as + * opposed to the public zone which is assumed to be unique. + * + * @param array $sourceDestPairs Array of source/destination pairs. Each element + * is a two-element array containing the source file path relative to the + * public root in the first element, and the archive file path relative + * to the deleted zone root in the second element. + * @return FileRepoStatus + */ + abstract function deleteBatch( $sourceDestPairs ); + + /** + * Move a file to the deletion archive. + * If no valid deletion archive exists, this may either delete the file + * or throw an exception, depending on the preference of the repository + * @param mixed $srcRel Relative path for the file to be deleted + * @param mixed $archiveRel Relative path for the archive location. + * Relative to a private archive directory. + * @return WikiError object (wikitext-formatted), or true for success + */ + function delete( $srcRel, $archiveRel ) { + return $this->deleteBatch( array( array( $srcRel, $archiveRel ) ) ); + } /** * Get properties of a file with a given virtual URL @@ -276,5 +357,48 @@ abstract class FileRepo { return true; } } + + /**#@+ + * Path disclosure protection functions + */ + function paranoidClean( $param ) { return '[hidden]'; } + function passThrough( $param ) { return $param; } + + /** + * Get a callback function to use for cleaning error message parameters + */ + function getErrorCleanupFunction() { + switch ( $this->pathDisclosureProtection ) { + case 'none': + $callback = array( $this, 'passThrough' ); + break; + default: // 'paranoid' + $callback = array( $this, 'paranoidClean' ); + } + return $callback; + } + /**#@-*/ + + /** + * Create a new fatal error + */ + function newFatal( $message /*, parameters...*/ ) { + $params = func_get_args(); + array_unshift( $params, $this ); + return call_user_func_array( array( 'FileRepoStatus', 'newFatal' ), $params ); + } + + /** + * Create a new good result + */ + function newGood( $value = null ) { + return FileRepoStatus::newGood( $this, $value ); + } + + /** + * Delete files in the deleted directory if they are not referenced in the filearchive table + * STUB + */ + function cleanupDeletedBatch( $storageKeys ) {} } diff --git a/includes/filerepo/FileRepoStatus.php b/includes/filerepo/FileRepoStatus.php new file mode 100644 index 0000000000..9e8e079efc --- /dev/null +++ b/includes/filerepo/FileRepoStatus.php @@ -0,0 +1,170 @@ +ok = false; + } + + static function newGood( $repo = false, $value = null ) { + $result = new self( $repo ); + $result->value = $value; + return $result; + } + + function __construct( $repo = false ) { + if ( $repo ) { + $this->cleanCallback = $repo->getErrorCleanupFunction(); + } + } + + function setResult( $ok, $value = null ) { + $this->ok = $ok; + $this->value = $value; + } + + function isGood() { + return $this->ok && !$this->errors; + } + + function isOK() { + return $this->ok; + } + + function warning( $message /*, parameters... */ ) { + $params = array_slice( func_get_args(), 1 ); + $this->errors[] = array( + 'type' => 'warning', + 'message' => $message, + 'params' => $params ); + } + + /** + * Add an error, do not set fatal flag + * This can be used for non-fatal errors + */ + function error( $message /*, parameters... */ ) { + $params = array_slice( func_get_args(), 1 ); + $this->errors[] = array( + 'type' => 'error', + 'message' => $message, + 'params' => $params ); + } + + /** + * Add an error and set OK to false, indicating that the operation as a whole was fatal + */ + function fatal( $message /*, parameters... */ ) { + $params = array_slice( func_get_args(), 1 ); + $this->errors[] = array( + 'type' => 'error', + 'message' => $message, + 'params' => $params ); + $this->ok = false; + } + + protected function cleanParams( $params ) { + if ( !$this->cleanCallback ) { + return $params; + } + $cleanParams = array(); + foreach ( $params as $i => $param ) { + $cleanParams[$i] = call_user_func( $this->cleanCallback, $param ); + } + return $cleanParams; + } + + protected function getItemXML( $item ) { + $params = $this->cleanParams( $item['params'] ); + $xml = "<{$item['type']}>\n" . + Xml::element( 'message', null, $item['message'] ) . "\n" . + Xml::element( 'text', null, wfMsgReal( $item['message'], $params ) ) ."\n"; + foreach ( $params as $param ) { + $xml .= Xml::element( 'param', null, $param ); + } + $xml .= "type}>\n"; + return $xml; + } + + /** + * Get the error list as XML + */ + function getXML() { + $xml = "\n"; + foreach ( $this->errors as $error ) { + $xml .= $this->getItemXML( $error ); + } + $xml .= "\n"; + return $xml; + } + + /** + * Get the error list as a wikitext formatted list + * @param string $shortContext A short enclosing context message name, to be used + * when there is a single error + * @param string $longContext A long enclosing context message name, for a list + */ + function getWikiText( $shortContext = false, $longContext = false ) { + if ( count( $this->errors ) == 0 ) { + if ( $this->ok ) { + $this->fatal( 'internalerror_info', + __METHOD__." called for a good result, this is incorrect\n" ); + } else { + $this->fatal( 'internalerror_info', + __METHOD__.": Invalid result object: no error text but not OK\n" ); + } + } + if ( count( $this->errors ) == 1 ) { + $params = array_map( 'wfEscapeWikiText', $this->cleanParams( $this->errors[0]['params'] ) ); + $s = wfMsgReal( $this->errors[0]['message'], $params ); + if ( $shortContext ) { + $s = wfMsg( $shortContext, $s ); + } elseif ( $longContext ) { + $s = wfMsg( $longContext, "* $s\n" ); + } + } else { + $s = ''; + foreach ( $this->errors as $error ) { + $params = array_map( 'wfEscapeWikiText', $this->cleanParams( $error['params'] ) ); + $s .= '* ' . wfMsgReal( $error['message'], $params ) . "\n"; + } + if ( $longContext ) { + $s = wfMsg( $longContext, $s ); + } elseif ( $shortContext ) { + $s = wfMsg( $shortContext, "\n* $s\n" ); + } + } + return $s; + } + + /** + * Merge another status object into this one + */ + function merge( $other, $overwriteValue = false ) { + $this->errors = array_merge( $this->errors, $other->errors ); + $this->ok = $this->ok && $other->ok; + if ( $overwriteValue ) { + $this->value = $other->value; + } + $this->successCount += $other->successCount; + $this->failCount += $other->failCount; + } +} diff --git a/includes/filerepo/ForeignDBRepo.php b/includes/filerepo/ForeignDBRepo.php index 96baff40a3..13dcd029c1 100644 --- a/includes/filerepo/ForeignDBRepo.php +++ b/includes/filerepo/ForeignDBRepo.php @@ -46,6 +46,12 @@ class ForeignDBRepo extends LocalRepo { function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) { throw new MWException( get_class($this) . ': write operations are not supported' ); } + function publish( $srcPath, $dstRel, $archiveRel, $flags = 0 ) { + throw new MWException( get_class($this) . ': write operations are not supported' ); + } + function deleteBatch( $fileMap ) { + throw new MWException( get_class($this) . ': write operations are not supported' ); + } } diff --git a/includes/filerepo/LocalFile.php b/includes/filerepo/LocalFile.php index 3f25d91fd7..8a6b3de165 100644 --- a/includes/filerepo/LocalFile.php +++ b/includes/filerepo/LocalFile.php @@ -41,10 +41,12 @@ class LocalFile extends File $major_mime, # Major mime type $minor_mine, # Minor mime type $size, # Size in bytes (loadFromXxx) - $metadata, # Metadata + $metadata, # Handler-specific metadata $timestamp, # Upload timestamp + $sha1, # SHA-1 base 36 content hash $dataLoaded, # Whether or not all this has been loaded from the database (loadFromXxx) - $upgraded; # Whether the row was upgraded on load + $upgraded, # Whether the row was upgraded on load + $locked; # True if the image row is locked /**#@-*/ @@ -156,7 +158,7 @@ class LocalFile extends File function getCacheFields( $prefix = 'img_' ) { static $fields = array( 'size', 'width', 'height', 'bits', 'media_type', - 'major_mime', 'minor_mime', 'metadata', 'timestamp' ); + 'major_mime', 'minor_mime', 'metadata', 'timestamp', 'sha1' ); static $results = array(); if ( $prefix == '' ) { return $fields; @@ -175,7 +177,9 @@ class LocalFile extends File * Load file metadata from the DB */ function loadFromDB() { - wfProfileIn( __METHOD__ ); + # Polymorphic function name to distinguish foreign and local fetches + $fname = get_class( $this ) . '::' . __FUNCTION__; + wfProfileIn( $fname ); # Unconditionally set loaded=true, we don't want the accessors constantly rechecking $this->dataLoaded = true; @@ -183,14 +187,14 @@ class LocalFile extends File $dbr = $this->repo->getSlaveDB(); $row = $dbr->selectRow( 'image', $this->getCacheFields( 'img_' ), - array( 'img_name' => $this->getName() ), __METHOD__ ); + array( 'img_name' => $this->getName() ), $fname ); if ( $row ) { $this->loadFromRow( $row ); } else { $this->fileExists = false; } - wfProfileOut( __METHOD__ ); + wfProfileOut( $fname ); } /** @@ -218,6 +222,8 @@ class LocalFile extends File } $decoded['mime'] = $decoded['major_mime'].'/'.$decoded['minor_mime']; } + # Trim zero padding from char/binary field + $decoded['sha1'] = rtrim( $decoded['sha1'], "\0" ); return $decoded; } @@ -254,7 +260,10 @@ class LocalFile extends File if ( wfReadOnly() ) { return; } - if ( is_null($this->media_type) || $this->mime == 'image/svg' ) { + if ( is_null($this->media_type) || + $this->mime == 'image/svg' || + $this->sha1 == '' + ) { $this->upgradeRow(); $this->upgraded = true; } else { @@ -292,6 +301,7 @@ class LocalFile extends File 'img_major_mime' => $major, 'img_minor_mime' => $minor, 'img_metadata' => $this->metadata, + 'img_sha1' => $this->sha1, ), array( 'img_name' => $this->getName() ), __METHOD__ ); @@ -480,12 +490,23 @@ class LocalFile extends File function purgeMetadataCache() { $this->loadFromDB(); $this->saveToCache(); + $this->purgeHistory(); + } + + /** + * Purge the shared history (OldLocalFile) cache + */ + function purgeHistory() { + global $wgMemc; + $hashedName = md5($this->getName()); + $oldKey = wfMemcKey( 'oldfile', $hashedName ); + $wgMemc->delete( $oldKey ); } /** * Delete all previously generated thumbnails, refresh metadata in memcached and purge the squid */ - function purgeCache( $archiveFiles = array() ) { + function purgeCache() { // Refresh metadata cache $this->purgeMetadataCache(); @@ -596,6 +617,8 @@ class LocalFile extends File /** getHashPath inherited */ /** getRel inherited */ /** getUrlRel inherited */ + /** getArchiveRel inherited */ + /** getThumbRel inherited */ /** getArchivePath inherited */ /** getThumbPath inherited */ /** getArchiveUrl inherited */ @@ -615,18 +638,19 @@ class LocalFile extends File * is already known * @param string $timestamp Timestamp for img_timestamp, or false to use the current time * - * @return Returns the archive name on success or an empty string if it was a new upload. - * Returns a wikitext-formatted WikiError on failure. + * @return FileRepoStatus object. On success, the value member contains the + * archive name, or an empty string if it was a new file. */ function upload( $srcPath, $comment, $pageText, $flags = 0, $props = false, $timestamp = false ) { - $archive = $this->publish( $srcPath, $flags ); - if ( WikiError::isError( $archive ) ){ - return $archive; - } - if ( !$this->recordUpload2( $archive, $comment, $pageText, $props, $timestamp ) ) { - return new WikiErrorMsg( 'filenotfound', wfEscapeWikiText( $srcPath ) ); + $this->lock(); + $status = $this->publish( $srcPath, $flags ); + if ( $status->ok ) { + if ( !$this->recordUpload2( $status->value, $comment, $pageText, $props, $timestamp ) ) { + $status->fatal( 'filenotfound', $srcPath ); + } } - return $archive; + $this->unlock(); + return $status; } /** @@ -695,6 +719,7 @@ class LocalFile extends File 'img_user' => $wgUser->getID(), 'img_user_text' => $wgUser->getName(), 'img_metadata' => $this->metadata, + 'img_sha1' => $this->sha1 ), __METHOD__, 'IGNORE' @@ -715,6 +740,11 @@ class LocalFile extends File 'oi_description' => 'img_description', 'oi_user' => 'img_user', 'oi_user_text' => 'img_user_text', + 'oi_metadata' => 'img_metadata', + 'oi_media_type' => 'img_media_type', + 'oi_major_mime' => 'img_major_mime', + 'oi_minor_mime' => 'img_minor_mime', + 'oi_sha1' => 'img_sha1', ), array( 'img_name' => $this->getName() ), __METHOD__ ); @@ -733,6 +763,7 @@ class LocalFile extends File 'img_user' => $wgUser->getID(), 'img_user_text' => $wgUser->getName(), 'img_metadata' => $this->metadata, + 'img_sha1' => $this->sha1 ), array( /* WHERE */ 'img_name' => $this->getName() ), __METHOD__ @@ -792,22 +823,23 @@ class LocalFile extends File * @param integer $flags A bitwise combination of: * File::DELETE_SOURCE Delete the source file, i.e. move * rather than copy - * @return The archive name on success or an empty string if it was a new - * file, and a wikitext-formatted WikiError object on failure. + * @return FileRepoStatus object. On success, the value member contains the + * archive name, or an empty string if it was a new file. */ function publish( $srcPath, $flags = 0 ) { + $this->lock(); $dstRel = $this->getRel(); $archiveName = gmdate( 'YmdHis' ) . '!'. $this->getName(); $archiveRel = 'archive/' . $this->getHashPath() . $archiveName; $flags = $flags & File::DELETE_SOURCE ? LocalRepo::DELETE_SOURCE : 0; $status = $this->repo->publish( $srcPath, $dstRel, $archiveRel, $flags ); - if ( WikiError::isError( $status ) ) { - return $status; - } elseif ( $status == 'new' ) { - return ''; + if ( $status->value == 'new' ) { + $status->value = ''; } else { - return $archiveName; + $status->value = $archiveName; } + $this->unlock(); + return $status; } /** getLinksTo inherited */ @@ -824,62 +856,34 @@ class LocalFile extends File * Cache purging is done; logging is caller's responsibility. * * @param $reason - * @return true on success, false on some kind of failure + * @return FileRepoStatus object. */ - function delete( $reason, $suppress=false ) { - $transaction = new FSTransaction(); - $urlArr = array( $this->getURL() ); + function delete( $reason ) { + $this->lock(); + $batch = new LocalFileDeleteBatch( $this, $reason ); + $batch->addCurrent(); - if( !FileStore::lock() ) { - wfDebug( __METHOD__.": failed to acquire file store lock, aborting\n" ); - return false; + # Get old version relative paths + $dbw = $this->repo->getMasterDB(); + $result = $dbw->select( 'oldimage', + array( 'oi_archive_name' ), + array( 'oi_name' => $this->getName() ) ); + while ( $row = $dbw->fetchObject( $result ) ) { + $batch->addOld( $row->oi_archive_name ); } + $status = $batch->execute(); - try { - $dbw = $this->repo->getMasterDB(); - $dbw->begin(); - - // Delete old versions - $result = $dbw->select( 'oldimage', - array( 'oi_archive_name' ), - array( 'oi_name' => $this->getName() ) ); - - while( $row = $dbw->fetchObject( $result ) ) { - $oldName = $row->oi_archive_name; - - $transaction->add( $this->prepareDeleteOld( $oldName, $reason, $suppress ) ); - - // We'll need to purge this URL from caches... - $urlArr[] = $this->getArchiveUrl( $oldName ); - } - $dbw->freeResult( $result ); - - // And the current version... - $transaction->add( $this->prepareDeleteCurrent( $reason, $suppress ) ); - - $dbw->immediateCommit(); - } catch( MWException $e ) { - wfDebug( __METHOD__.": db error, rolling back file transactions\n" ); - $transaction->rollback(); - FileStore::unlock(); - throw $e; + if ( $status->ok ) { + // Update site_stats + $site_stats = $dbw->tableName( 'site_stats' ); + $dbw->query( "UPDATE $site_stats SET ss_images=ss_images-1", __METHOD__ ); + $this->purgeEverything(); } - wfDebug( __METHOD__.": deleted db items, applying file transactions\n" ); - $transaction->commit(); - FileStore::unlock(); - - - // Update site_stats - $site_stats = $dbw->tableName( 'site_stats' ); - $dbw->query( "UPDATE $site_stats SET ss_images=ss_images-1", __METHOD__ ); - - $this->purgeEverything( $urlArr ); - - return true; + $this->unlock(); + return $status; } - /** * Delete an old version of the file. * @@ -890,187 +894,19 @@ class LocalFile extends File * * @param $reason * @throws MWException or FSException on database or filestore failure - * @return true on success, false on some kind of failure + * @return FileRepoStatus object. */ - function deleteOld( $archiveName, $reason, $suppress=false ) { - $transaction = new FSTransaction(); - $urlArr = array(); - - if( !FileStore::lock() ) { - wfDebug( __METHOD__.": failed to acquire file store lock, aborting\n" ); - return false; - } - - $transaction = new FSTransaction(); - try { - $dbw = $this->repo->getMasterDB(); - $dbw->begin(); - $transaction->add( $this->prepareDeleteOld( $archiveName, $reason, $suppress ) ); - $dbw->immediateCommit(); - } catch( MWException $e ) { - wfDebug( __METHOD__.": db error, rolling back file transaction\n" ); - $transaction->rollback(); - FileStore::unlock(); - throw $e; - } - - wfDebug( __METHOD__.": deleted db items, applying file transaction\n" ); - $transaction->commit(); - FileStore::unlock(); - - $this->purgeDescription(); - - // Squid purging - global $wgUseSquid; - if ( $wgUseSquid ) { - $urlArr = array( - $this->getArchiveUrl( $archiveName ), - ); - wfPurgeSquidServers( $urlArr ); + function deleteOld( $archiveName, $reason ) { + $this->lock(); + $batch = new LocalFileDeleteBatch( $this, $reason ); + $batch->addOld( $archiveName ); + $status = $batch->execute(); + $this->unlock(); + if ( $status->ok ) { + $this->purgeDescription(); + $this->purgeHistory(); } - return true; - } - - /** - * Delete the current version of a file. - * May throw a database error. - * @return true on success, false on failure - */ - private function prepareDeleteCurrent( $reason, $suppress=false ) { - return $this->prepareDeleteVersion( - $this->getFullPath(), - $reason, - 'image', - array( - 'fa_name' => 'img_name', - 'fa_archive_name' => 'NULL', - 'fa_size' => 'img_size', - 'fa_width' => 'img_width', - 'fa_height' => 'img_height', - 'fa_metadata' => 'img_metadata', - 'fa_bits' => 'img_bits', - 'fa_media_type' => 'img_media_type', - 'fa_major_mime' => 'img_major_mime', - 'fa_minor_mime' => 'img_minor_mime', - 'fa_description' => 'img_description', - 'fa_user' => 'img_user', - 'fa_user_text' => 'img_user_text', - 'fa_timestamp' => 'img_timestamp' ), - array( 'img_name' => $this->getName() ), - $suppress, - __METHOD__ ); - } - - /** - * Delete a given older version of a file. - * May throw a database error. - * @return true on success, false on failure - */ - private function prepareDeleteOld( $archiveName, $reason, $suppress=false ) { - $oldpath = $this->getArchivePath() . - DIRECTORY_SEPARATOR . $archiveName; - return $this->prepareDeleteVersion( - $oldpath, - $reason, - 'oldimage', - array( - 'fa_name' => 'oi_name', - 'fa_archive_name' => 'oi_archive_name', - 'fa_size' => 'oi_size', - 'fa_width' => 'oi_width', - 'fa_height' => 'oi_height', - 'fa_metadata' => 'NULL', - 'fa_bits' => 'oi_bits', - 'fa_media_type' => 'NULL', - 'fa_major_mime' => 'NULL', - 'fa_minor_mime' => 'NULL', - 'fa_description' => 'oi_description', - 'fa_user' => 'oi_user', - 'fa_user_text' => 'oi_user_text', - 'fa_timestamp' => 'oi_timestamp' ), - array( - 'oi_name' => $this->getName(), - 'oi_archive_name' => $archiveName ), - $suppress, - __METHOD__ ); - } - - /** - * Do the dirty work of backing up an image row and its file - * (if $wgSaveDeletedFiles is on) and removing the originals. - * - * Must be run while the file store is locked and a database - * transaction is open to avoid race conditions. - * - * @return FSTransaction - */ - private function prepareDeleteVersion( $path, $reason, $table, $fieldMap, $where, $suppress=false, $fname ) { - global $wgUser, $wgSaveDeletedFiles; - - // Dupe the file into the file store - if( file_exists( $path ) ) { - if( $wgSaveDeletedFiles ) { - $group = 'deleted'; - - $store = FileStore::get( $group ); - $key = FileStore::calculateKey( $path, $this->getExtension() ); - $transaction = $store->insert( $key, $path, - FileStore::DELETE_ORIGINAL ); - } else { - $group = null; - $key = null; - $transaction = FileStore::deleteFile( $path ); - } - } else { - wfDebug( __METHOD__." deleting already-missing '$path'; moving on to database\n" ); - $group = null; - $key = null; - $transaction = new FSTransaction(); // empty - } - - if( $transaction === false ) { - // Fail to restore? - wfDebug( __METHOD__.": import to file store failed, aborting\n" ); - throw new MWException( "Could not archive and delete file $path" ); - return false; - } - - // Bitfields to further supress the file content - // Note that currently, live files are stored elsewhere - // and cannot be partially deleted - $bitfield = 0; - if ( $suppress ) { - $bitfield |= self::DELETED_FILE; - $bitfield |= self::DELETED_COMMENT; - $bitfield |= self::DELETED_USER; - $bitfield |= self::DELETED_RESTRICTED; - } - - $dbw = $this->repo->getMasterDB(); - $storageMap = array( - 'fa_storage_group' => $dbw->addQuotes( $group ), - 'fa_storage_key' => $dbw->addQuotes( $key ), - - 'fa_deleted_user' => $dbw->addQuotes( $wgUser->getId() ), - 'fa_deleted_timestamp' => $dbw->timestamp(), - 'fa_deleted_reason' => $dbw->addQuotes( $reason ), - 'fa_deleted' => $bitfield); - $allFields = array_merge( $storageMap, $fieldMap ); - - try { - if( $wgSaveDeletedFiles ) { - $dbw->insertSelect( 'filearchive', $table, $allFields, $where, $fname ); - } - $dbw->delete( $table, $where, $fname ); - } catch( DBQueryError $e ) { - // Something went horribly wrong! - // Leave the file as it was... - wfDebug( __METHOD__.": database error, rolling back file transaction\n" ); - $transaction->rollback(); - throw $e; - } - - return $transaction; + return $status; } /** @@ -1081,202 +917,25 @@ class LocalFile extends File * * @param $versions set of record ids of deleted items to restore, * or empty to restore all revisions. - * @return the number of file revisions restored if successful, - * or false on failure + * @return FileRepoStatus */ - function restore( $versions=array(), $Unsuppress=false ) { - global $wgUser; - - if( !FileStore::lock() ) { - wfDebug( __METHOD__." could not acquire filestore lock\n" ); - return false; - } - - $transaction = new FSTransaction(); - try { - $dbw = $this->repo->getMasterDB(); - $dbw->begin(); - - // Re-confirm whether this file presently exists; - // if no we'll need to create an file record for the - // first item we restore. - $exists = $dbw->selectField( 'image', '1', - array( 'img_name' => $this->getName() ), - __METHOD__ ); - - // Fetch all or selected archived revisions for the file, - // sorted from the most recent to the oldest. - $conditions = array( 'fa_name' => $this->getName() ); - if( $versions ) { - $conditions['fa_id'] = $versions; - } - - $result = $dbw->select( 'filearchive', '*', - $conditions, - __METHOD__, - array( 'ORDER BY' => 'fa_timestamp DESC' ) ); - - if( $dbw->numRows( $result ) < count( $versions ) ) { - // There's some kind of conflict or confusion; - // we can't restore everything we were asked to. - wfDebug( __METHOD__.": couldn't find requested items\n" ); - $dbw->rollback(); - FileStore::unlock(); - return false; - } - - if( $dbw->numRows( $result ) == 0 ) { - // Nothing to do. - wfDebug( __METHOD__.": nothing to do\n" ); - $dbw->rollback(); - FileStore::unlock(); - return true; - } - - $revisions = 0; - while( $row = $dbw->fetchObject( $result ) ) { - if ( $Unsuppress ) { - // Currently, fa_deleted flags fall off upon restore, lets be careful about this - } else if ( ($row->fa_deleted & Revision::DELETED_RESTRICTED) && !$wgUser->isAllowed('hiderevision') ) { - // Skip restoring file revisions that the user cannot restore - continue; - } - $revisions++; - $store = FileStore::get( $row->fa_storage_group ); - if( !$store ) { - wfDebug( __METHOD__.": skipping row with no file.\n" ); - continue; - } - - $restoredImage = new self( Title::makeTitle( NS_IMAGE, $row->fa_name ), $this->repo ); - - if( $revisions == 1 && !$exists ) { - $destPath = $restoredImage->getFullPath(); - $destDir = dirname( $destPath ); - if ( !is_dir( $destDir ) ) { - wfMkdirParents( $destDir ); - } - - // We may have to fill in data if this was originally - // an archived file revision. - if( is_null( $row->fa_metadata ) ) { - $tempFile = $store->filePath( $row->fa_storage_key ); - - $magic = MimeMagic::singleton(); - $mime = $magic->guessMimeType( $tempFile, true ); - $media_type = $magic->getMediaType( $tempFile, $mime ); - list( $major_mime, $minor_mime ) = self::splitMime( $mime ); - $handler = MediaHandler::getHandler( $mime ); - if ( $handler ) { - $metadata = $handler->getMetadata( false, $tempFile ); - } else { - $metadata = ''; - } - } else { - $metadata = $row->fa_metadata; - $major_mime = $row->fa_major_mime; - $minor_mime = $row->fa_minor_mime; - $media_type = $row->fa_media_type; - } - - $table = 'image'; - $fields = array( - 'img_name' => $row->fa_name, - 'img_size' => $row->fa_size, - 'img_width' => $row->fa_width, - 'img_height' => $row->fa_height, - 'img_metadata' => $metadata, - 'img_bits' => $row->fa_bits, - 'img_media_type' => $media_type, - 'img_major_mime' => $major_mime, - 'img_minor_mime' => $minor_mime, - 'img_description' => $row->fa_description, - 'img_user' => $row->fa_user, - 'img_user_text' => $row->fa_user_text, - 'img_timestamp' => $row->fa_timestamp ); - } else { - $archiveName = $row->fa_archive_name; - if( $archiveName == '' ) { - // This was originally a current version; we - // have to devise a new archive name for it. - // Format is ! - $archiveName = - wfTimestamp( TS_MW, $row->fa_deleted_timestamp ) . - '!' . $row->fa_name; - } - $destDir = $restoredImage->getArchivePath(); - if ( !is_dir( $destDir ) ) { - wfMkdirParents( $destDir ); - } - $destPath = $destDir . DIRECTORY_SEPARATOR . $archiveName; - - $table = 'oldimage'; - $fields = array( - 'oi_name' => $row->fa_name, - 'oi_archive_name' => $archiveName, - 'oi_size' => $row->fa_size, - 'oi_width' => $row->fa_width, - 'oi_height' => $row->fa_height, - 'oi_bits' => $row->fa_bits, - 'oi_description' => $row->fa_description, - 'oi_user' => $row->fa_user, - 'oi_user_text' => $row->fa_user_text, - 'oi_timestamp' => $row->fa_timestamp ); - } - - $dbw->insert( $table, $fields, __METHOD__ ); - // @todo this delete is not totally safe, potentially - $dbw->delete( 'filearchive', - array( 'fa_id' => $row->fa_id ), - __METHOD__ ); - - // Check if any other stored revisions use this file; - // if so, we shouldn't remove the file from the deletion - // archives so they will still work. - $useCount = $dbw->selectField( 'filearchive', - 'COUNT(*)', - array( - 'fa_storage_group' => $row->fa_storage_group, - 'fa_storage_key' => $row->fa_storage_key ), - __METHOD__ ); - if( $useCount == 0 ) { - wfDebug( __METHOD__.": nothing else using {$row->fa_storage_key}, will deleting after\n" ); - $flags = FileStore::DELETE_ORIGINAL; - } else { - $flags = 0; - } - - $transaction->add( $store->export( $row->fa_storage_key, - $destPath, $flags ) ); - } - - $dbw->immediateCommit(); - } catch( MWException $e ) { - wfDebug( __METHOD__." caught error, aborting\n" ); - $transaction->rollback(); - $dbw->rollback(); - throw $e; + function restore( $versions = array(), $unsuppress = false ) { + $batch = new LocalFileRestoreBatch( $this ); + if ( !$versions ) { + $batch->addAll(); + } else { + $batch->addIds( $versions ); } - - $transaction->commit(); - FileStore::unlock(); - - if( $revisions > 0 ) { - if( !$exists ) { - wfDebug( __METHOD__." restored $revisions items, creating a new current\n" ); - - // Update site_stats - $site_stats = $dbw->tableName( 'site_stats' ); - $dbw->query( "UPDATE $site_stats SET ss_images=ss_images+1", __METHOD__ ); - - $this->purgeEverything(); - } else { - wfDebug( __METHOD__." restored $revisions as archived versions\n" ); - $this->purgeDescription(); - } + $status = $batch->execute(); + if ( !$status->ok ) { + return $status; } - return $revisions; + $cleanupStatus = $batch->cleanup(); + $cleanupStatus->successCount = 0; + $cleanupStatus->failCount = 0; + $status->merge( $cleanupStatus ); + return $status; } /** isMultipage inherited */ @@ -1310,8 +969,52 @@ class LocalFile extends File $this->load(); return $this->timestamp; } + + function getSha1() { + $this->load(); + return $this->sha1; + } + + /** + * Start a transaction and lock the image for update + * Increments a reference counter if the lock is already held + * @return boolean True if the image exists, false otherwise + */ + function lock() { + $dbw = $this->repo->getMasterDB(); + if ( !$this->locked ) { + $dbw->begin(); + $this->locked++; + } + return $dbw->selectField( 'image', '1', array( 'img_name' => $this->getName() ), __METHOD__ ); + } + + /** + * Decrement the lock reference count. If the reference count is reduced to zero, commits + * the transaction and thereby releases the image lock. + */ + function unlock() { + if ( $this->locked ) { + --$this->locked; + if ( !$this->locked ) { + $dbw = $this->repo->getMasterDB(); + $dbw->commit(); + } + } + } + + /** + * Roll back the DB transaction and mark the image unlocked + */ + function unlockAndRollback() { + $this->locked = false; + $dbw = $this->repo->getMasterDB(); + $dbw->rollback(); + } } // LocalFile class +#------------------------------------------------------------------------------ + /** * Backwards compatibility class */ @@ -1379,12 +1082,467 @@ class Image extends LocalFile { } } +#------------------------------------------------------------------------------ + /** - * Aliases for backwards compatibility with 1.6 + * Helper class for file deletion */ -define( 'MW_IMG_DELETED_FILE', File::DELETED_FILE ); -define( 'MW_IMG_DELETED_COMMENT', File::DELETED_COMMENT ); -define( 'MW_IMG_DELETED_USER', File::DELETED_USER ); -define( 'MW_IMG_DELETED_RESTRICTED', File::DELETED_RESTRICTED ); +class LocalFileDeleteBatch { + var $file, $reason, $srcRels = array(), $archiveUrls = array(), $deletionBatch; + var $status; + + function __construct( File $file, $reason = '' ) { + $this->file = $file; + $this->reason = $reason; + $this->status = $file->repo->newGood(); + } + + function addCurrent() { + $this->srcRels['.'] = $this->file->getRel(); + } + + function addOld( $oldName ) { + $this->srcRels[$oldName] = $this->file->getArchiveRel( $oldName ); + $this->archiveUrls[] = $this->file->getArchiveUrl( $oldName ); + } + + function getOldRels() { + if ( !isset( $this->srcRels['.'] ) ) { + $oldRels =& $this->srcRels; + $deleteCurrent = false; + } else { + $oldRels = $this->srcRels; + unset( $oldRels['.'] ); + $deleteCurrent = true; + } + return array( $oldRels, $deleteCurrent ); + } + + /*protected*/ function getHashes() { + $hashes = array(); + list( $oldRels, $deleteCurrent ) = $this->getOldRels(); + if ( $deleteCurrent ) { + $hashes['.'] = $this->file->getSha1(); + } + if ( count( $oldRels ) ) { + $dbw = $this->file->repo->getMasterDB(); + $res = $dbw->select( 'oldimage', array( 'oi_archive_name', 'oi_sha1' ), + 'oi_archive_name IN(' . $dbw->makeList( array_keys( $oldRels ) ) . ')', + __METHOD__ ); + while ( $row = $dbw->fetchObject( $res ) ) { + if ( rtrim( $row->oi_sha1, "\0" ) === '' ) { + // Get the hash from the file + $oldUrl = $this->file->getArchiveVirtualUrl( $row->oi_archive_name ); + $props = $this->file->repo->getFileProps( $oldUrl ); + if ( $props['fileExists'] ) { + // Upgrade the oldimage row + $dbw->update( 'oldimage', + array( 'oi_sha1' => $props['sha1'] ), + array( 'oi_name' => $this->file->getName(), 'oi_archive_name' => $row->oi_archive_name ), + __METHOD__ ); + $hashes[$row->oi_archive_name] = $props['sha1']; + } else { + $hashes[$row->oi_archive_name] = false; + } + } else { + $hashes[$row->oi_archive_name] = $row->oi_sha1; + } + } + } + $missing = array_diff_key( $this->srcRels, $hashes ); + foreach ( $missing as $name => $rel ) { + $this->status->error( 'filedelete-old-unregistered', $name ); + } + foreach ( $hashes as $name => $hash ) { + if ( !$hash ) { + $this->status->error( 'filedelete-missing', $this->srcRels[$name] ); + unset( $hashes[$name] ); + } + } + + return $hashes; + } + + function doDBInserts() { + global $wgUser; + $dbw = $this->file->repo->getMasterDB(); + $encTimestamp = $dbw->addQuotes( $dbw->timestamp() ); + $encUserId = $dbw->addQuotes( $wgUser->getId() ); + $encReason = $dbw->addQuotes( $this->reason ); + $encGroup = $dbw->addQuotes( 'deleted' ); + $ext = $this->file->getExtension(); + $dotExt = $ext === '' ? '' : ".$ext"; + $encExt = $dbw->addQuotes( $dotExt ); + list( $oldRels, $deleteCurrent ) = $this->getOldRels(); + + if ( $deleteCurrent ) { + $where = array( 'img_name' => $this->file->getName() ); + $dbw->insertSelect( 'filearchive', 'image', + array( + 'fa_storage_group' => $encGroup, + 'fa_storage_key' => "IF(img_sha1='', '', CONCAT(img_sha1,$encExt))", + + 'fa_deleted_user' => $encUserId, + 'fa_deleted_timestamp' => $encTimestamp, + 'fa_deleted_reason' => $encReason, + 'fa_deleted' => 0, + + 'fa_name' => 'img_name', + 'fa_archive_name' => 'NULL', + 'fa_size' => 'img_size', + 'fa_width' => 'img_width', + 'fa_height' => 'img_height', + 'fa_metadata' => 'img_metadata', + 'fa_bits' => 'img_bits', + 'fa_media_type' => 'img_media_type', + 'fa_major_mime' => 'img_major_mime', + 'fa_minor_mime' => 'img_minor_mime', + 'fa_description' => 'img_description', + 'fa_user' => 'img_user', + 'fa_user_text' => 'img_user_text', + 'fa_timestamp' => 'img_timestamp' + ), $where, __METHOD__ ); + } + + if ( count( $oldRels ) ) { + $where = array( + 'oi_name' => $this->file->getName(), + 'oi_archive_name IN (' . $dbw->makeList( array_keys( $oldRels ) ) . ')' ); + + $dbw->insertSelect( 'filearchive', 'oldimage', + array( + 'fa_storage_group' => $encGroup, + 'fa_storage_key' => "IF(oi_sha1='', '', CONCAT(oi_sha1,$encExt))", + + 'fa_deleted_user' => $encUserId, + 'fa_deleted_timestamp' => $encTimestamp, + 'fa_deleted_reason' => $encReason, + 'fa_deleted' => 0, + + 'fa_name' => 'oi_name', + 'fa_archive_name' => 'oi_archive_name', + 'fa_size' => 'oi_size', + 'fa_width' => 'oi_width', + 'fa_height' => 'oi_height', + 'fa_metadata' => 'oi_metadata', + 'fa_bits' => 'oi_bits', + 'fa_media_type' => 'oi_media_type', + 'fa_major_mime' => 'oi_major_mime', + 'fa_minor_mime' => 'oi_minor_mime', + 'fa_description' => 'oi_description', + 'fa_user' => 'oi_user', + 'fa_user_text' => 'oi_user_text', + 'fa_timestamp' => 'oi_timestamp' + ), $where, __METHOD__ ); + } + } + + function doDBDeletes() { + $dbw = $this->file->repo->getMasterDB(); + list( $oldRels, $deleteCurrent ) = $this->getOldRels(); + if ( $deleteCurrent ) { + $where = array( 'img_name' => $this->file->getName() ); + $dbw->delete( 'image', array( 'img_name' => $this->file->getName() ), __METHOD__ ); + } + if ( count( $oldRels ) ) { + $dbw->delete( 'oldimage', + array( + 'oi_name' => $this->file->getName(), + 'oi_archive_name IN (' . $dbw->makeList( array_keys( $oldRels ) ) . ')' + ), __METHOD__ ); + } + } + + /** + * Run the transaction + */ + function execute() { + global $wgUser, $wgUseSquid; + wfProfileIn( __METHOD__ ); + + $this->file->lock(); + + // Prepare deletion batch + $hashes = $this->getHashes(); + $this->deletionBatch = array(); + $ext = $this->file->getExtension(); + $dotExt = $ext === '' ? '' : ".$ext"; + foreach ( $this->srcRels as $name => $srcRel ) { + // Skip files that have no hash (missing source) + if ( isset( $hashes[$name] ) ) { + $hash = $hashes[$name]; + $key = $hash . $dotExt; + $dstRel = $this->file->repo->getDeletedHashPath( $key ) . $key; + $this->deletionBatch[$name] = array( $srcRel, $dstRel ); + } + } + + // Lock the filearchive rows so that the files don't get deleted by a cleanup operation + // We acquire this lock by running the inserts now, before the file operations. + // + // This potentially has poor lock contention characteristics -- an alternative + // scheme would be to insert stub filearchive entries with no fa_name and commit + // them in a separate transaction, then run the file ops, then update the fa_name fields. + $this->doDBInserts(); + + // Execute the file deletion batch + $status = $this->file->repo->deleteBatch( $this->deletionBatch ); + if ( !$status->isGood() ) { + $this->status->merge( $status ); + } + + if ( !$this->status->ok ) { + // Critical file deletion error + // Roll back inserts, release lock and abort + // TODO: delete the defunct filearchive rows if we are using a non-transactional DB + $this->file->unlockAndRollback(); + return $this->status; + } + + // Purge squid + if ( $wgUseSquid ) { + $urls = array(); + foreach ( $this->srcRels as $srcRel ) { + $urlRel = str_replace( '%2F', '/', rawurlencode( $srcRel ) ); + $urls[] = $this->repo->getZoneUrl( 'public' ) . '/' . $urlRel; + } + SquidUpdate::purge( $urls ); + } + + // Delete image/oldimage rows + $this->doDBDeletes(); + + // Commit and return + $this->file->unlock(); + wfProfileOut( __METHOD__ ); + return $this->status; + } +} +#------------------------------------------------------------------------------ +/** + * Helper class for file undeletion + */ +class LocalFileRestoreBatch { + var $file, $cleanupBatch, $ids, $all, $unsuppress = false; + + function __construct( File $file ) { + $this->file = $file; + $this->cleanupBatch = $this->ids = array(); + $this->ids = array(); + } + + /** + * Add a file by ID + */ + function addId( $fa_id ) { + $this->ids[] = $fa_id; + } + + /** + * Add a whole lot of files by ID + */ + function addIds( $ids ) { + $this->ids = array_merge( $this->ids, $ids ); + } + + /** + * Add all revisions of the file + */ + function addAll() { + $this->all = true; + } + + /** + * Run the transaction, except the cleanup batch. + * The cleanup batch should be run in a separate transaction, because it locks different + * rows and there's no need to keep the image row locked while it's acquiring those locks + * The caller may have its own transaction open. + * So we save the batch and let the caller call cleanup() + */ + function execute() { + global $wgUser, $wgLang; + if ( !$this->all && !$this->ids ) { + // Do nothing + return $this->file->repo->newGood(); + } + + $exists = $this->file->lock(); + $dbw = $this->file->repo->getMasterDB(); + $status = $this->file->repo->newGood(); + + // Fetch all or selected archived revisions for the file, + // sorted from the most recent to the oldest. + $conditions = array( 'fa_name' => $this->file->getName() ); + if( !$this->all ) { + $conditions[] = 'fa_id IN (' . $dbw->makeList( $this->ids ) . ')'; + } + + $result = $dbw->select( 'filearchive', '*', + $conditions, + __METHOD__, + array( 'ORDER BY' => 'fa_timestamp DESC' ) ); + + $idsPresent = array(); + $storeBatch = array(); + $insertBatch = array(); + $insertCurrent = false; + $deleteIds = array(); + $first = true; + $archiveNames = array(); + while( $row = $dbw->fetchObject( $result ) ) { + $idsPresent[] = $row->fa_id; + if ( $this->unsuppress ) { + // Currently, fa_deleted flags fall off upon restore, lets be careful about this + } else if ( ($row->fa_deleted & Revision::DELETED_RESTRICTED) && !$wgUser->isAllowed('hiderevision') ) { + // Skip restoring file revisions that the user cannot restore + continue; + } + if ( $row->fa_name != $this->file->getName() ) { + $status->error( 'undelete-filename-mismatch', $wgLang->timeanddate( $row->fa_timestamp ) ); + $status->failCount++; + continue; + } + if ( $row->fa_storage_key == '' ) { + // Revision was missing pre-deletion + $status->error( 'undelete-bad-store-key', $wgLang->timeanddate( $row->fa_timestamp ) ); + $status->failCount++; + continue; + } + + $deletedRel = $this->file->repo->getDeletedHashPath( $row->fa_storage_key ) . $row->fa_storage_key; + $deletedUrl = $this->file->repo->getVirtualUrl() . '/deleted/' . $deletedRel; + + $sha1 = substr( $row->fa_storage_key, 0, strcspn( $row->fa_storage_key, '.' ) ); + # Fix leading zero + if ( strlen( $sha1 ) == 32 && $sha1[0] == '0' ) { + $sha1 = substr( $sha1, 1 ); + } + + if ( $first && !$exists ) { + // This revision will be published as the new current version + $destRel = $this->file->getRel(); + $info = $this->file->repo->getFileProps( $deletedUrl ); + $insertCurrent = array( + 'img_name' => $row->fa_name, + 'img_size' => $row->fa_size, + 'img_width' => $row->fa_width, + 'img_height' => $row->fa_height, + 'img_metadata' => $row->fa_metadata, + 'img_bits' => $row->fa_bits, + 'img_media_type' => $row->fa_media_type, + 'img_major_mime' => $row->fa_major_mime, + 'img_minor_mime' => $row->fa_minor_mime, + 'img_description' => $row->fa_description, + 'img_user' => $row->fa_user, + 'img_user_text' => $row->fa_user_text, + 'img_timestamp' => $row->fa_timestamp, + 'img_sha1' => $sha1); + } else { + $archiveName = $row->fa_archive_name; + if( $archiveName == '' ) { + // This was originally a current version; we + // have to devise a new archive name for it. + // Format is ! + $timestamp = wfTimestamp( TS_UNIX, $row->fa_deleted_timestamp ); + do { + $archiveName = wfTimestamp( TS_MW, $timestamp ) . '!' . $row->fa_name; + $timestamp++; + } while ( isset( $archiveNames[$archiveName] ) ); + } + $archiveNames[$archiveName] = true; + $destRel = $this->file->getArchiveRel( $archiveName ); + $insertBatch[] = array( + 'oi_name' => $row->fa_name, + 'oi_archive_name' => $archiveName, + 'oi_size' => $row->fa_size, + 'oi_width' => $row->fa_width, + 'oi_height' => $row->fa_height, + 'oi_bits' => $row->fa_bits, + 'oi_description' => $row->fa_description, + 'oi_user' => $row->fa_user, + 'oi_user_text' => $row->fa_user_text, + 'oi_timestamp' => $row->fa_timestamp, + 'oi_metadata' => $row->fa_metadata, + 'oi_media_type' => $row->fa_media_type, + 'oi_major_mime' => $row->fa_major_mime, + 'oi_minor_mime' => $row->fa_minor_mime, + 'oi_deleted' => $row->fa_deleted, + 'oi_sha1' => $sha1 ); + } + + $deleteIds[] = $row->fa_id; + $storeBatch[] = array( $deletedUrl, 'public', $destRel ); + $this->cleanupBatch[] = $row->fa_storage_key; + $first = false; + } + unset( $result ); + + // Add a warning to the status object for missing IDs + $missingIds = array_diff( $this->ids, $idsPresent ); + foreach ( $missingIds as $id ) { + $status->error( 'undelete-missing-filearchive', $id ); + } + + // Run the store batch + // Use the OVERWRITE_SAME flag to smooth over a common error + $storeStatus = $this->file->repo->storeBatch( $storeBatch, FileRepo::OVERWRITE_SAME ); + $status->merge( $storeStatus ); + + if ( !$status->ok ) { + // Store batch returned a critical error -- this usually means nothing was stored + // Stop now and return an error + $this->file->unlock(); + return $status; + } + + // Run the DB updates + // Because we have locked the image row, key conflicts should be rare. + // If they do occur, we can roll back the transaction at this time with + // no data loss, but leaving unregistered files scattered throughout the + // public zone. + // This is not ideal, which is why it's important to lock the image row. + if ( $insertCurrent ) { + $dbw->insert( 'image', $insertCurrent, __METHOD__ ); + } + if ( $insertBatch ) { + $dbw->insert( 'oldimage', $insertBatch, __METHOD__ ); + } + if ( $deleteIds ) { + $dbw->delete( 'filearchive', + array( 'fa_id IN (' . $dbw->makeList( $deleteIds ) . ')' ), + __METHOD__ ); + } + + if( $status->successCount > 0 ) { + if( !$exists ) { + wfDebug( __METHOD__." restored {$status->successCount} items, creating a new current\n" ); + + // Update site_stats + $site_stats = $dbw->tableName( 'site_stats' ); + $dbw->query( "UPDATE $site_stats SET ss_images=ss_images+1", __METHOD__ ); + + $this->file->purgeEverything(); + } else { + wfDebug( __METHOD__." restored {$status->successCount} as archived versions\n" ); + $this->file->purgeDescription(); + $this->file->purgeHistory(); + } + } + $this->file->unlock(); + return $status; + } + + /** + * Delete unused files in the deleted zone. + * This should be called from outside the transaction in which execute() was called. + */ + function cleanup() { + if ( !$this->cleanupBatch ) { + return $this->file->repo->newGood(); + } + $status = $this->file->repo->cleanupDeletedBatch( $this->cleanupBatch ); + return $status; + } +} diff --git a/includes/filerepo/LocalRepo.php b/includes/filerepo/LocalRepo.php index be32c02ac3..57b1872d47 100644 --- a/includes/filerepo/LocalRepo.php +++ b/includes/filerepo/LocalRepo.php @@ -28,4 +28,33 @@ class LocalRepo extends FSRepo { function newFromArchiveName( $title, $archiveName ) { return OldLocalFile::newFromArchiveName( $title, $this, $archiveName ); } + + /** + * Delete files in the deleted directory if they are not referenced in the + * filearchive table. This needs to be done in the repo because it needs to + * interleave database locks with file operations, which is potentially a + * remote operation. + * @return FileRepoStatus + */ + function cleanupDeletedBatch( $storageKeys ) { + $root = $this->getZonePath( 'deleted' ); + $dbw = $this->getMasterDB(); + $status = $this->newGood(); + foreach ( $storageKeys as $key ) { + $hashPath = $this->getDeletedHashPath( $key ); + $path = "$root/$hashPath$key"; + $dbw->begin(); + $inuse = $dbw->select( 'filearchive', '1', + array( 'fa_storage_group' => 'deleted', 'fa_storage_key' => $key ), + __METHOD__, array( 'FOR UPDATE' ) ); + if ( !$inuse && !unlink( $path ) ) { + $status->error( 'undelete-cleanup-error', $path ); + $status->failCount++; + } else { + $status->successCount++; + } + $dbw->commit(); + } + return $status; + } } diff --git a/includes/filerepo/OldLocalFile.php b/includes/filerepo/OldLocalFile.php index fef97fe721..4d9dffbdb7 100644 --- a/includes/filerepo/OldLocalFile.php +++ b/includes/filerepo/OldLocalFile.php @@ -70,7 +70,7 @@ class OldLocalFile extends LocalFile { } $oldImages = $wgMemc->get( $key ); - if ( isset( $oldImages['version'] ) && $oldImages['version'] == MW_OLDFILE_VERSION ) { + if ( isset( $oldImages['version'] ) && $oldImages['version'] == self::CACHE_VERSION ) { unset( $oldImages['version'] ); $more = isset( $oldImages['more'] ); unset( $oldImages['more'] ); @@ -94,7 +94,8 @@ class OldLocalFile extends LocalFile { if ( $found ) { wfDebug( "Pulling file metadata from cache key {$key}[{$timestamp}]\n" ); $this->dataLoaded = true; - foreach ( $cachedValues as $name => $value ) { + $this->fileExists = true; + foreach ( $info as $name => $value ) { $this->$name = $value; } } elseif ( $more ) { @@ -130,7 +131,7 @@ class OldLocalFile extends LocalFile { wfProfileIn( __METHOD__ ); $dbr = $this->repo->getSlaveDB(); - $res = $dbr->select( 'oldimage', $this->getCacheFields(), + $res = $dbr->select( 'oldimage', $this->getCacheFields( 'oi_' ), array( 'oi_name' => $this->getName() ), __METHOD__, array( 'LIMIT' => self::MAX_CACHE_ROWS + 1, @@ -144,8 +145,8 @@ class OldLocalFile extends LocalFile { } for ( $i = 0; $i < $numRows; $i++ ) { $row = $dbr->fetchObject( $res ); - $this->decodeRow( $row, 'oi_' ); - $cache[$row->oi_timestamp] = $row; + $decoded = $this->decodeRow( $row, 'oi_' ); + $cache[$row->oi_timestamp] = $decoded; } $dbr->freeResult( $res ); $wgMemc->set( $key, $cache, 7*86400 /* 1 week */ ); @@ -169,6 +170,7 @@ class OldLocalFile extends LocalFile { $this->fileExists = false; } $this->dataLoaded = true; + wfProfileOut( __METHOD__ ); } function getCacheFields( $prefix = 'img_' ) { @@ -207,15 +209,14 @@ class OldLocalFile extends LocalFile { #'oi_major_mime' => $major, #'oi_minor_mime' => $minor, #'oi_metadata' => $this->metadata, - ), array( 'oi_name' => $this->getName(), 'oi_timestamp' => $this->requestedTime ), + 'oi_sha1' => $this->sha1, + ), array( + 'oi_name' => $this->getName(), + 'oi_archive_name' => $this->archive_name ), __METHOD__ ); wfProfileOut( __METHOD__ ); } - - // XXX: Temporary hack before schema update - function maybeUpgradeRow() {} - } diff --git a/languages/messages/MessagesEn.php b/languages/messages/MessagesEn.php index a741eca4a1..c88aa36fdd 100644 --- a/languages/messages/MessagesEn.php +++ b/languages/messages/MessagesEn.php @@ -761,10 +761,13 @@ If this is not the case, you may have found a bug in the software. Please report this to an administrator, making note of the URL.', 'readonly_lag' => 'The database has been automatically locked while the slave database servers catch up to the master', 'internalerror' => 'Internal error', +'internalerror_info' => 'Internal error: $1', 'filecopyerror' => 'Could not copy file "$1" to "$2".', 'filerenameerror' => 'Could not rename file "$1" to "$2".', 'filedeleteerror' => 'Could not delete file "$1".', +'directorycreateerror' => 'Could not create directory "$1".', 'filenotfound' => 'Could not find file "$1".', +'fileexists' => 'Unable to write to file "$1": file exists', 'unexpected' => 'Unexpected value: "$1"="$2".', 'formerror' => 'Error: could not submit form', 'badarticleerror' => 'This action cannot be performed on this page.', @@ -1885,6 +1888,13 @@ Consult the [[Special:Log/delete|deletion log]] for a record of recent deletions 'undelete-search-prefix' => 'Show pages starting with:', 'undelete-search-submit' => 'Search', 'undelete-no-results' => 'No matching pages found in the deletion archive.', +'undelete-filename-mismatch' => 'Cannot undelete file revision with timestamp $1: filename mismatch', +'undelete-bad-store-key' => 'Cannot undelete file revision with timestamp $1: file was missing before deletion.', +'undelete-cleanup-error' => 'Error deleting unused archive file "$1".', +'undelete-missing-filearchive' => 'Unable to restore file archive ID $1 because it isn\'t in the database. ' . + 'It may have already been undeleted.', +'undelete-error-short' => 'Error undeleting file: $1', +'undelete-error-long' => "Errors were encountered while undeleting the file:\n\n$1\n", # Namespace form on various pages 'namespace' => 'Namespace:', @@ -2362,6 +2372,12 @@ All transwiki import actions are logged at the [[Special:Log/import|import log]] # Image deletion 'deletedrevision' => 'Deleted old revision $1.', +'filedeleteerror-short' => "Error deleting file: $1", +'filedeleteerror-long' => "Errors were encountered while deleting the file:\n\n$1\n", +'filedelete-missing' => 'The file "$1" cannot be deleted, because it doesn\'t exist.', +'filedelete-old-unregistered' => 'The specified file revision "$1" is not in the database.', +'filedelete-current-unregistered' => 'The specified file "$1" is not in the database.', +'filedelete-archive-read-only' => 'The archive directory "$1" is not writable by the webserver.', # Browsing diffs 'previousdiff' => '← Previous diff', diff --git a/maintenance/archives/patch-img_sha1.sql b/maintenance/archives/patch-img_sha1.sql new file mode 100644 index 0000000000..b882fbfb71 --- /dev/null +++ b/maintenance/archives/patch-img_sha1.sql @@ -0,0 +1,8 @@ +-- Add img_sha1, oi_sha1 and related indexes +ALTER TABLE image + ADD COLUMN img_sha1 varbinary(31) NOT NULL default '', + ADD INDEX img_sha1 (img_sha1); + +ALTER TABLE oldimage + ADD COLUMN oi_sha1 varbinary(31) NOT NULL default '', + ADD INDEX oi_sha1 (oi_sha1); diff --git a/maintenance/rebuildImages.php b/maintenance/rebuildImages.php index 06f84c9a55..dfdb3b20ee 100644 --- a/maintenance/rebuildImages.php +++ b/maintenance/rebuildImages.php @@ -173,8 +173,6 @@ class ImageBuilder extends FiveUpgrade { function addMissingImage( $filename, $fullpath ) { $fname = 'ImageBuilder::addMissingImage'; - $size = filesize( $fullpath ); - $info = $this->imageInfo( $fullpath ); $timestamp = $this->dbw->timestamp( filemtime( $fullpath ) ); global $wgContLang; diff --git a/maintenance/tables.sql b/maintenance/tables.sql index 39d632e7de..8523e0d31b 100644 --- a/maintenance/tables.sql +++ b/maintenance/tables.sql @@ -686,13 +686,20 @@ CREATE TABLE /*$wgDBprefix*/image ( -- Time of the upload. img_timestamp varbinary(14) NOT NULL default '', + -- SHA-1 content hash in base-36 + img_sha1 varbinary(32) NOT NULL default '', + PRIMARY KEY img_name (img_name), INDEX img_usertext_timestamp (img_user_text,img_timestamp), -- Used by Special:Imagelist for sort-by-size INDEX img_size (img_size), -- Used by Special:Newimages and Special:Imagelist - INDEX img_timestamp (img_timestamp) + INDEX img_timestamp (img_timestamp), + + -- For future use + INDEX img_sha1 (img_sha1), + ) /*$wgDBTableOptions*/; @@ -724,11 +731,13 @@ CREATE TABLE /*$wgDBprefix*/oldimage ( oi_major_mime ENUM("unknown", "application", "audio", "image", "text", "video", "message", "model", "multipart") NOT NULL default "unknown", oi_minor_mime varbinary(32) NOT NULL default "unknown", oi_deleted tinyint unsigned NOT NULL default '0', + oi_sha1 varbinary(32) NOT NULL default '', INDEX oi_usertext_timestamp (oi_user_text,oi_timestamp), INDEX oi_name_timestamp (oi_name,oi_timestamp), -- oi_archive_name truncated to 14 to avoid key length overflow - INDEX oi_name_archive_name (oi_name,oi_archive_name(14)) + INDEX oi_name_archive_name (oi_name,oi_archive_name(14)), + INDEX oi_sha1 (oi_sha1) ) /*$wgDBTableOptions*/; diff --git a/maintenance/updaters.inc b/maintenance/updaters.inc index 5dffc0c128..8cba69ab6f 100644 --- a/maintenance/updaters.inc +++ b/maintenance/updaters.inc @@ -81,6 +81,7 @@ $wgNewFields = array( array( 'ipblocks', 'ipb_block_email', 'patch-ipb_emailban.sql' ), array( 'oldimage', 'oi_metadata', 'patch-oi_metadata.sql'), array( 'archive', 'ar_page', 'patch-archive-ar_page.sql'), + array( 'image', 'img_sha1', 'patch-img_sha1.sql' ), ); # For extensions only, should be populated via hooks -- 2.20.1