From 5275f9b097cdae5dbab0845e3ea127086e3a6570 Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Tue, 20 Dec 2011 03:52:06 +0000 Subject: [PATCH] Merged FileBackend branch. Manually avoiding merging the many prop-only changes SVN likes to sprinkle in (easy to spot from the change list). Did not add SwiftFileBackend.php as it still is in development. --- img_auth.php | 38 +- includes/AutoLoader.php | 26 + includes/DefaultSettings.php | 21 + includes/GlobalFunctions.php | 4 + includes/Setup.php | 72 ++ includes/filerepo/FSRepo.php | 768 +------------ includes/filerepo/FileRepo.php | 1002 ++++++++++++++--- includes/filerepo/ForeignAPIRepo.php | 46 +- includes/filerepo/ForeignDBViaLBRepo.php | 1 + includes/filerepo/LocalRepo.php | 26 +- includes/filerepo/RepoGroup.php | 2 +- includes/filerepo/backend/FSFileBackend.php | 608 ++++++++++ includes/filerepo/backend/FileBackend.php | 835 ++++++++++++++ .../filerepo/backend/FileBackendGroup.php | 96 ++ .../backend/FileBackendMultiWrite.php | 267 +++++ includes/filerepo/backend/FileOp.php | 922 +++++++++++++++ .../backend/lockmanager/DBLockManager.php | 454 ++++++++ .../backend/lockmanager/FSLockManager.php | 199 ++++ .../backend/lockmanager/LSLockManager.php | 287 +++++ .../backend/lockmanager/LockManager.php | 172 +++ .../backend/lockmanager/LockManagerGroup.php | 68 ++ includes/filerepo/file/FSFile.php | 211 ++++ includes/filerepo/file/File.php | 242 ++-- includes/filerepo/file/ForeignAPIFile.php | 42 +- includes/filerepo/file/LocalFile.php | 57 +- includes/filerepo/file/OldLocalFile.php | 2 +- includes/filerepo/file/TempFSFile.php | 96 ++ .../filerepo/file/UnregisteredLocalFile.php | 20 +- includes/media/Bitmap.php | 2 +- includes/media/Bitmap_ClientOnly.php | 2 +- includes/media/DjVu.php | 2 +- includes/media/ExifBitmap.php | 2 +- includes/media/Generic.php | 13 +- includes/media/MediaTransformOutput.php | 61 +- includes/media/SVG.php | 2 +- includes/specials/SpecialRevisiondelete.php | 2 +- includes/specials/SpecialUndelete.php | 2 +- includes/upload/UploadBase.php | 47 +- includes/upload/UploadFromChunks.php | 19 +- includes/upload/UploadStash.php | 13 +- languages/messages/MessagesEn.php | 32 + maintenance/language/messages.inc | 32 + maintenance/locking/LockServerDaemon.php | 427 +++++++ maintenance/locking/file_locks.sql | 11 + tests/parser/parserTest.inc | 21 +- tests/phpunit/includes/LocalFileTest.php | 34 +- .../includes/api/ApiTestCaseUpload.php | 2 +- tests/phpunit/includes/api/ApiUploadTest.php | 2 +- .../includes/filerepo/FileBackendTest.php | 592 ++++++++++ .../includes/media/ExifRotationTest.php | 41 +- .../includes/media/FormatMetadataTest.php | 19 +- tests/phpunit/includes/media/GIFTest.php | 31 +- tests/phpunit/includes/media/PNGTest.php | 30 +- .../phpunit/includes/parser/NewParserTest.php | 53 +- .../phpunit/suites/UploadFromUrlTestSuite.php | 21 +- thumb.php | 24 +- 56 files changed, 6795 insertions(+), 1328 deletions(-) create mode 100644 includes/filerepo/backend/FSFileBackend.php create mode 100644 includes/filerepo/backend/FileBackend.php create mode 100644 includes/filerepo/backend/FileBackendGroup.php create mode 100644 includes/filerepo/backend/FileBackendMultiWrite.php create mode 100644 includes/filerepo/backend/FileOp.php create mode 100644 includes/filerepo/backend/lockmanager/DBLockManager.php create mode 100644 includes/filerepo/backend/lockmanager/FSLockManager.php create mode 100644 includes/filerepo/backend/lockmanager/LSLockManager.php create mode 100644 includes/filerepo/backend/lockmanager/LockManager.php create mode 100644 includes/filerepo/backend/lockmanager/LockManagerGroup.php create mode 100644 includes/filerepo/file/FSFile.php create mode 100644 includes/filerepo/file/TempFSFile.php create mode 100644 maintenance/locking/LockServerDaemon.php create mode 100644 maintenance/locking/file_locks.sql create mode 100644 tests/phpunit/includes/filerepo/FileBackendTest.php diff --git a/img_auth.php b/img_auth.php index 968a60e144..7f922c5223 100644 --- a/img_auth.php +++ b/img_auth.php @@ -72,36 +72,26 @@ function wfImageAuthMain() { return; } - // Get the full file path - $filename = realpath( $wgUploadDirectory . $path ); - $realUpload = realpath( $wgUploadDirectory ); + // Get the local file repository + $repo = RepoGroup::singleton()->getRepo( 'local' ); - // Basic directory traversal check - if ( substr( $filename, 0, strlen( $realUpload ) ) != $realUpload ) { - wfForbidden( 'img-auth-accessdenied', 'img-auth-notindir' ); - return; + // Get the full file storage path and extract the source file name. + // (e.g. 120px-Foo.png => Foo.png or page2-120px-Foo.png => Foo.png). + // This only applies to thumbnails, and all thumbnails should + // be under a folder that has the source file name. + if ( strpos( $path, '/thumb/' ) === 0 ) { + $name = wfBaseName( dirname( $path ) ); // file is a thumbnail + $filename = $repo->getZonePath( 'thumb' ) . substr( $path, 6 ); // strip "/thumb" + } else { + $name = wfBaseName( $path ); // file is a source file + $filename = $repo->getZonePath( 'public' ) . $path; } // Check to see if the file exists - if ( !file_exists( $filename ) ) { + if ( !$repo->fileExists( $filename, FileRepo::FILES_ONLY ) ) { wfForbidden( 'img-auth-accessdenied','img-auth-nofile', $filename ); return; } - - // Check to see if tried to access a directory - if ( is_dir( $filename ) ) { - wfForbidden( 'img-auth-accessdenied','img-auth-isdir', $filename ); - return; - } - - // Extract the file name and chop off the size specifier. - // (e.g. 120px-Foo.png => Foo.png or page2-120px-Foo.png => Foo.png). - // This only applies to thumbnails, and all thumbnails should - // be under a folder that has the source file name. - $name = wfBaseName( $path ); - if ( strpos( $path, '/thumb/' ) === 0 ) { - $name = wfBaseName( dirname( $path ) ); // this file is a thumbnail - } $title = Title::makeTitleSafe( NS_FILE, $name ); if ( !$title instanceof Title ) { // files have valid titles @@ -124,7 +114,7 @@ function wfImageAuthMain() { // Stream the requested file wfDebugLog( 'img_auth', "Streaming `".$filename."`." ); - StreamFile::stream( $filename, array( 'Cache-Control: private', 'Vary: Cookie' ) ); + $repo->streamFile( $filename, array( 'Cache-Control: private', 'Vary: Cookie' ) ); } /** diff --git a/includes/AutoLoader.php b/includes/AutoLoader.php index 710ea61ef6..f03c859144 100644 --- a/includes/AutoLoader.php +++ b/includes/AutoLoader.php @@ -481,6 +481,32 @@ $wgAutoloadLocalClasses = array( 'LocalFileRestoreBatch' => 'includes/filerepo/file/LocalFile.php', 'OldLocalFile' => 'includes/filerepo/file/OldLocalFile.php', 'UnregisteredLocalFile' => 'includes/filerepo/file/UnregisteredLocalFile.php', + 'FSFile' => 'includes/filerepo/file/FSFile.php', + 'TempFSFile' => 'includes/filerepo/file/TempFSFile.php', + + # includes/filerepo/backend + 'FileBackendGroup' => 'includes/filerepo/backend/FileBackendGroup.php', + 'FileBackendBase' => 'includes/filerepo/backend/FileBackend.php', + 'FileBackend' => 'includes/filerepo/backend/FileBackend.php', + 'FileBackendMultiWrite' => 'includes/filerepo/backend/FileBackendMultiWrite.php', + 'FSFileBackend' => 'includes/filerepo/backend/FSFileBackend.php', + 'FSFileIterator' => 'includes/filerepo/backend/FSFileBackend.php', + 'LockManagerGroup' => 'includes/filerepo/backend/lockmanager/LockManagerGroup.php', + 'LockManager' => 'includes/filerepo/backend/lockmanager/LockManager.php', + 'ScopedLock' => 'includes/filerepo/backend/lockmanager/LockManager.php', + 'FSLockManager' => 'includes/filerepo/backend/lockmanager/FSLockManager.php', + 'DBLockManager' => 'includes/filerepo/backend/lockmanager/DBLockManager.php', + 'LSLockManager' => 'includes/filerepo/backend/lockmanager/LSLockManager.php', + 'MySqlLockManager'=> 'includes/filerepo/backend/lockmanager/DBLockManager.php', + 'NullLockManager' => 'includes/filerepo/backend/lockmanager/LockManager.php', + 'FileOp' => 'includes/filerepo/backend/FileOp.php', + 'StoreFileOp' => 'includes/filerepo/backend/FileOp.php', + 'CopyFileOp' => 'includes/filerepo/backend/FileOp.php', + 'MoveFileOp' => 'includes/filerepo/backend/FileOp.php', + 'DeleteFileOp' => 'includes/filerepo/backend/FileOp.php', + 'ConcatenateFileOp' => 'includes/filerepo/backend/FileOp.php', + 'CreateFileOp' => 'includes/filerepo/backend/FileOp.php', + 'NullFileOp' => 'includes/filerepo/backend/FileOp.php', # includes/installer 'CliInstaller' => 'includes/installer/CliInstaller.php', diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index 1fe7870ad3..c7c6d52e0d 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -366,6 +366,27 @@ $wgForeignFileRepos = array(); */ $wgUseInstantCommons = false; +/** + * File backend structure configuration. + * This is an array of file backend configuration arrays. + * Each backend configuration has the following parameters: + * 'name' : A unique name for the backend + * 'class' : The file backend class to use + * 'wikiId' : A unique string that identifies the wiki (container prefix) + * 'lockManager' : The name of a lock manager (see $wgFileLockManagers) + * Additional parameters are specific to the class used. + */ +$wgFileBackends = array(); + +/** + * Array of configuration arrays for each lock manager. + * Each backend configuration has the following parameters: + * 'name' : A unique name for the lock manger + * 'class' : The lock manger class to use + * Additional parameters are specific to the class used. + */ +$wgFileLockManagers = array(); + /** * Show EXIF data, on by default if available. * Requires PHP's EXIF extension: http://www.php.net/manual/en/ref.exif.php diff --git a/includes/GlobalFunctions.php b/includes/GlobalFunctions.php index 331922788d..de0221c97c 100644 --- a/includes/GlobalFunctions.php +++ b/includes/GlobalFunctions.php @@ -2391,6 +2391,10 @@ function wfTempDir() { function wfMkdirParents( $dir, $mode = null, $caller = null ) { global $wgDirectoryMode; + if ( FileBackend::isStoragePath( $dir ) ) { // sanity + throw new MWException( __FUNCTION__ . " given storage path `$dir`."); + } + if ( !is_null( $caller ) ) { wfDebug( "$caller: called wfMkdirParents($dir)\n" ); } diff --git a/includes/Setup.php b/includes/Setup.php index 5c08261abf..6182086171 100644 --- a/includes/Setup.php +++ b/includes/Setup.php @@ -114,6 +114,19 @@ $wgNamespaceProtection[NS_MEDIAWIKI] = 'editinterface'; $wgNamespaceAliases['Image'] = NS_FILE; $wgNamespaceAliases['Image_talk'] = NS_FILE_TALK; +/** + * Initialise $wgLockManagers to include basic FS version + */ +$wgLockManagers[] = array( + 'name' => 'fsLockManager', + 'class' => 'FSLockManager', + 'lockDirectory' => "{$wgUploadDirectory}/lockdir", +); +$wgLockManagers[] = array( + 'name' => 'nullLockManager', + 'class' => 'NullLockManager', +); + /** * Initialise $wgLocalFileRepo from backwards-compatible settings */ @@ -179,6 +192,7 @@ if ( $wgUseInstantCommons ) { $wgForeignFileRepos[] = array( 'class' => 'ForeignAPIRepo', 'name' => 'wikimediacommons', + 'directory' => $wgUploadDirectory, 'apibase' => 'http://commons.wikimedia.org/w/api.php', 'hashLevels' => 2, 'fetchDescription' => true, @@ -186,6 +200,59 @@ if ( $wgUseInstantCommons ) { 'apiThumbCacheExpiry' => 86400, ); } +/* + * Add on default file backend config for repos to $wgFileBackends + */ +if ( !isset( $wgLocalFileRepo['backend'] ) ) { + $wgFileBackends[] = wfBackendForLegacyRepoConf( $wgLocalFileRepo ); +} +foreach ( $wgForeignFileRepos as &$repo ) { + if ( !isset( $repo['backend'] ) ) { + $wgFileBackends[] = wfBackendForLegacyRepoConf( $repo ); + } +} +unset( $repo ); // no global pollution; destroy reference +/* + * Get file backend configuration for a given repo + * configuration that lacks a backend parameter. + * Also updates the repo config to use the backend. + */ +function wfBackendForLegacyRepoConf( &$info ) { + // Local vars that used to be FSRepo members... + $directory = $info['directory']; + $deletedDir = isset( $info['deletedDir'] ) + ? $info['deletedDir'] + : false; + $thumbDir = isset( $info['thumbDir'] ) + ? $info['thumbDir'] + : "{$directory}/thumb"; + $fileMode = isset( $info['fileMode'] ) + ? $info['fileMode'] + : 0644; + + // Make a backend name (based on repo name) + $backendName = $info['name'] . '-backend'; + // Update repo config to use this backend + $info['backend'] = $backendName; + // Disable "deleted" zone in repo config if deleted dir not set + if ( $deletedDir !== false ) { + $info['zones']['deleted'] = array( + 'container' => 'images-deleted', 'directory' => '' ); + } + // Get the FS backend configuration + return array( + 'name' => $backendName, + 'class' => 'FSFileBackend', + 'lockManager' => 'fsLockManager', + 'containerPaths' => array( + "images-public" => "{$directory}", + "images-temp" => "{$directory}/temp", + "images-thumb" => $thumbDir, + "images-deleted" => $deletedDir + ), + 'fileMode' => $fileMode, + ); +} if ( is_null( $wgEnableAutoRotation ) ) { // Only enable auto-rotation when the bitmap handler can rotate @@ -462,6 +529,11 @@ if ( !is_object( $wgAuth ) ) { wfRunHooks( 'AuthPluginSetup', array( &$wgAuth ) ); } +# Register file lock managers +LockManagerGroup::singleton()->register( $wgLockManagers ); +# Register file backends +FileBackendGroup::singleton()->register( $wgFileBackends ); + # Placeholders in case of DB error $wgTitle = null; diff --git a/includes/filerepo/FSRepo.php b/includes/filerepo/FSRepo.php index cec4c23f7e..8dd83335f4 100644 --- a/includes/filerepo/FSRepo.php +++ b/includes/filerepo/FSRepo.php @@ -10,775 +10,13 @@ * A repository for files accessible via the local filesystem. Does not support * database access or registration. * @ingroup FileRepo + * @deprecated since 1.19 */ class FSRepo extends FileRepo { - var $directory, $deletedDir, $deletedHashLevels, $fileMode; - var $fileFactory = array( 'UnregisteredLocalFile', 'newFromTitle' ); - var $oldFileFactory = false; - var $pathDisclosureProtection = 'simple'; - function __construct( $info ) { parent::__construct( $info ); - - // Required settings - $this->directory = $info['directory']; - $this->url = $info['url']; - - // Optional settings - $this->hashLevels = isset( $info['hashLevels'] ) ? $info['hashLevels'] : 2; - $this->deletedHashLevels = isset( $info['deletedHashLevels'] ) ? - $info['deletedHashLevels'] : $this->hashLevels; - $this->deletedDir = isset( $info['deletedDir'] ) ? $info['deletedDir'] : false; - $this->fileMode = isset( $info['fileMode'] ) ? $info['fileMode'] : 0644; - if ( isset( $info['thumbDir'] ) ) { - $this->thumbDir = $info['thumbDir']; - } else { - $this->thumbDir = "{$this->directory}/thumb"; - } - if ( isset( $info['thumbUrl'] ) ) { - $this->thumbUrl = $info['thumbUrl']; - } else { - $this->thumbUrl = "{$this->url}/thumb"; - } - } - - /** - * Get the public root directory of the repository. - * @return string - */ - function getRootDirectory() { - return $this->directory; - } - - /** - * Get the public root URL of the repository - * @return string - */ - function getRootUrl() { - return $this->url; - } - - /** - * Returns true if the repository uses a multi-level directory structure - * @return string - */ - function isHashed() { - return (bool)$this->hashLevels; - } - - /** - * Get the local directory corresponding to one of the three basic zones - * - * @param $zone string - * - * @return string - */ - function getZonePath( $zone ) { - switch ( $zone ) { - case 'public': - return $this->directory; - case 'temp': - return "{$this->directory}/temp"; - case 'deleted': - return $this->deletedDir; - case 'thumb': - return $this->thumbDir; - default: - return false; - } - } - - /** - * @see FileRepo::getZoneUrl() - * - * @param $zone string - * - * @return string url - */ - function getZoneUrl( $zone ) { - switch ( $zone ) { - case 'public': - return $this->url; - case 'temp': - return "{$this->url}/temp"; - case 'deleted': - return parent::getZoneUrl( $zone ); // no public URL - case 'thumb': - return $this->thumbUrl; - default: - return parent::getZoneUrl( $zone ); + if ( !( $this->backend instanceof FSFileBackend ) ) { + throw new MWException( "FSRepo only supports FSFileBackend." ); } } - - /** - * Get a URL referring to this repository, with the private mwrepo protocol. - * The suffix, if supplied, is considered to be unencoded, and will be - * URL-encoded before being returned. - * - * @param $suffix string - * - * @return string - */ - function getVirtualUrl( $suffix = false ) { - $path = 'mwrepo://' . $this->name; - if ( $suffix !== false ) { - $path .= '/' . rawurlencode( $suffix ); - } - return $path; - } - - /** - * Get the local path corresponding to a virtual URL - * - * @param $url string - * - * @return string - */ - function resolveVirtualUrl( $url ) { - if ( substr( $url, 0, 9 ) != 'mwrepo://' ) { - throw new MWException( __METHOD__.': unknown protocol' ); - } - - $bits = explode( '/', substr( $url, 9 ), 3 ); - if ( count( $bits ) != 3 ) { - throw new MWException( __METHOD__.": invalid mwrepo URL: $url" ); - } - list( $repo, $zone, $rel ) = $bits; - if ( $repo !== $this->name ) { - throw new MWException( __METHOD__.": fetching from a foreign repo is not supported" ); - } - $base = $this->getZonePath( $zone ); - if ( !$base ) { - throw new MWException( __METHOD__.": invalid zone: $zone" ); - } - return $base . '/' . rawurldecode( $rel ); - } - - /** - * Store a batch of files - * - * @param $triplets Array: (src,zone,dest) triplets as per store() - * @param $flags Integer: bitwise combination of the following flags: - * self::DELETE_SOURCE Delete the source file after upload - * self::OVERWRITE Overwrite an existing destination file instead of failing - * self::OVERWRITE_SAME Overwrite the file if the destination exists and has the - * same contents as the source - * @return Status - */ - function storeBatch( $triplets, $flags = 0 ) { - wfDebug( __METHOD__ . ': Storing ' . count( $triplets ) . - " triplets; flags: {$flags}\n" ); - - // Try creating directories - if ( !wfMkdirParents( $this->directory, null, __METHOD__ ) ) { - return $this->newFatal( 'upload_directory_missing', $this->directory ); - } - if ( !is_writable( $this->directory ) ) { - return $this->newFatal( 'upload_directory_read_only', $this->directory ); - } - - // Validate each triplet - $status = $this->newGood(); - foreach ( $triplets as $i => $triplet ) { - list( $srcPath, $dstZone, $dstRel ) = $triplet; - - // Resolve destination path - $root = $this->getZonePath( $dstZone ); - if ( !$root ) { - throw new MWException( "Invalid zone: $dstZone" ); - } - if ( !$this->validateFilename( $dstRel ) ) { - throw new MWException( 'Validation error in $dstRel' ); - } - $dstPath = "$root/$dstRel"; - $dstDir = dirname( $dstPath ); - - // Create destination directories for this triplet - if ( !is_dir( $dstDir ) ) { - if ( !wfMkdirParents( $dstDir, null, __METHOD__ ) ) { - return $this->newFatal( 'directorycreateerror', $dstDir ); - } - if ( $dstZone == 'deleted' ) { - $this->initDeletedDir( $dstDir ); - } - } - - // Resolve source - if ( self::isVirtualUrl( $srcPath ) ) { - $srcPath = $triplets[$i][0] = $this->resolveVirtualUrl( $srcPath ); - } - if ( !is_file( $srcPath ) ) { - // Make a list of files that don't exist for return to the caller - $status->fatal( 'filenotfound', $srcPath ); - continue; - } - - // Check overwriting - if ( !( $flags & self::OVERWRITE ) && file_exists( $dstPath ) ) { - if ( $flags & self::OVERWRITE_SAME ) { - $hashSource = sha1_file( $srcPath ); - $hashDest = sha1_file( $dstPath ); - if ( $hashSource != $hashDest ) { - $status->fatal( 'fileexistserror', $dstPath ); - $status->failCount++; - } - } else { - $status->fatal( 'fileexistserror', $dstPath ); - $status->failCount++; - } - } - } - - // Windows does not support moving over existing files, so explicitly delete them - $deleteDest = wfIsWindows() && ( $flags & self::OVERWRITE ); - - // Abort now on failure - if ( !$status->ok ) { - return $status; - } - - // Execute the store operation for each triplet - foreach ( $triplets as $i => $triplet ) { - list( $srcPath, $dstZone, $dstRel ) = $triplet; - $root = $this->getZonePath( $dstZone ); - $dstPath = "$root/$dstRel"; - $good = true; - - if ( $flags & self::DELETE_SOURCE ) { - if ( $deleteDest ) { - unlink( $dstPath ); - } - if ( !rename( $srcPath, $dstPath ) ) { - $status->error( 'filerenameerror', $srcPath, $dstPath ); - $good = false; - } - } else { - if ( !copy( $srcPath, $dstPath ) ) { - $status->error( 'filecopyerror', $srcPath, $dstPath ); - $good = false; - } - if ( !( $flags & self::SKIP_VALIDATION ) ) { - wfSuppressWarnings(); - $hashSource = sha1_file( $srcPath ); - $hashDest = sha1_file( $dstPath ); - wfRestoreWarnings(); - - if ( $hashDest === false || $hashSource !== $hashDest ) { - wfDebug( __METHOD__ . ': File copy validation failed: ' . - "$srcPath ($hashSource) to $dstPath ($hashDest)\n" ); - - $status->error( 'filecopyerror', $srcPath, $dstPath ); - $good = false; - } - } - } - if ( $good ) { - $this->chmod( $dstPath ); - $status->successCount++; - } else { - $status->failCount++; - } - $status->success[$i] = $good; - } - return $status; - } - - /** - * Deletes a batch of files. Each file can be a (zone, rel) pairs, a - * virtual url or a real path. It will try to delete each file, but - * ignores any errors that may occur - * - * @param $pairs array List of files to delete - * @return void - */ - function cleanupBatch( $files ) { - foreach ( $files as $file ) { - if ( is_array( $file ) ) { - // This is a pair, extract it - list( $zone, $rel ) = $file; - $root = $this->getZonePath( $zone ); - $path = "$root/$rel"; - } else { - if ( self::isVirtualUrl( $file ) ) { - // This is a virtual url, resolve it - $path = $this->resolveVirtualUrl( $file ); - } else { - // This is a full file name - $path = $file; - } - } - - wfSuppressWarnings(); - unlink( $path ); - wfRestoreWarnings(); - } - } - /** - * Concatenate a list of files into a target file location. - * - * @param $fileList array of files - * @param $targetFile String target path - * @param $flags Integer: bitwise combination of the following flags: - * self::FILES_ONLY Mark file as existing only if it is a file (not directory) - */ - function concatenate( $fileList, $targetPath, $flags = 0 ){ - $status = $this->newGood(); - // Resolve the virtual URL for taget: - if ( self::isVirtualUrl( $targetPath ) ) { - $targetPath = $this->resolveVirtualUrl( $targetPath ); - // empty out the target file: - if ( is_file( $targetPath ) ){ - unlink( $targetPath ); - } - } - foreach( $fileList as $sourcePath ){ - // Resolve the virtual URL for source: - if ( self::isVirtualUrl( $sourcePath ) ) { - $sourcePath = $this->resolveVirtualUrl( $sourcePath ); - } - if ( !is_file( $sourcePath ) ) - $status->fatal( 'filenotfound', $sourcePath ); - - if ( !$status->isOk() ){ - return $status; - } - - // Do the append - $chunk = file_get_contents( $sourcePath ); - if( $chunk === false ) { - $status->fatal( 'fileconcatenateerrorread', $sourcePath ); - return $status; - } - if( $status->isOk() ) { - if ( file_put_contents( $targetPath, $chunk, FILE_APPEND ) ) { - $status->value = $targetPath; - } else { - $status->fatal( 'fileconcatenateerror', $sourcePath, $targetPath); - } - } - if ( $flags & self::DELETE_SOURCE ) { - unlink( $sourcePath ); - } - } - return $status; - } - /** - * @deprecated 1.19 - * - * @return Status - */ - function append( $srcPath, $toAppendPath, $flags = 0 ) { - wfDeprecated( __METHOD__, '1.19' ); - - $status = $this->newGood(); - - // Resolve the virtual URL - if ( self::isVirtualUrl( $toAppendPath ) ) { - $toAppendPath = $this->resolveVirtualUrl( $toAppendPath ); - } - // Make sure the files are there - if ( !is_file( $toAppendPath ) ) - $status->fatal( 'filenotfound', $toAppendPath ); - - if ( !is_file( $srcPath ) ) - $status->fatal( 'filenotfound', $srcPath ); - - if ( !$status->isOk() ) return $status; - - // Do the append - $chunk = file_get_contents( $srcPath ); - if( $chunk === false ) { - $status->fatal( 'fileappenderrorread', $srcPath ); - } - - if( $status->isOk() ) { - if ( file_put_contents( $toAppendPath, $chunk, FILE_APPEND ) ) { - $status->value = $toAppendPath; - } else { - $status->fatal( 'fileappenderror', $srcPath, $toAppendPath); - } - } - - if ( $flags & self::DELETE_SOURCE ) { - unlink( $srcPath ); - } - - return $status; - } - - /* We can actually append to the files, so no-op needed here. */ - function appendFinish( $toAppendPath ) {} - - /** - * Checks existence of specified array of files. - * - * @param $files Array: URLs of files to check - * @param $flags Integer: bitwise combination of the following flags: - * self::FILES_ONLY Mark file as existing only if it is a file (not directory) - * @return Either array of files and existence flags, or false - */ - function fileExistsBatch( $files, $flags = 0 ) { - if ( !file_exists( $this->directory ) || !is_readable( $this->directory ) ) { - return false; - } - $result = array(); - foreach ( $files as $key => $file ) { - if ( self::isVirtualUrl( $file ) ) { - $file = $this->resolveVirtualUrl( $file ); - } - if( $flags & self::FILES_ONLY ) { - $result[$key] = is_file( $file ); - } else { - $result[$key] = file_exists( $file ); - } - } - - return $result; - } - - /** - * Take all available measures to prevent web accessibility of new deleted - * directories, in case the user has not configured offline storage - * @return void - */ - protected function initDeletedDir( $dir ) { - // Add a .htaccess file to the root of the deleted zone - $root = $this->getZonePath( 'deleted' ); - if ( !file_exists( "$root/.htaccess" ) ) { - file_put_contents( "$root/.htaccess", "Deny from all\n" ); - } - // Seed new directories with a blank index.html, to prevent crawling - file_put_contents( "$dir/index.html", '' ); - } - - /** - * Pick a random name in the temp zone and store a file to it. - * @param $originalName String: the base name of the file as specified - * by the user. The file extension will be maintained. - * @param $srcPath String: the current location of the file. - * @return FileRepoStatus object with the URL in the value. - */ - function storeTemp( $originalName, $srcPath ) { - $date = gmdate( "YmdHis" ); - $hashPath = $this->getHashPath( $originalName ); - $dstRel = "$hashPath$date!$originalName"; - $dstUrlRel = $hashPath . $date . '!' . rawurlencode( $originalName ); - - $result = $this->store( $srcPath, 'temp', $dstRel ); - $result->value = $this->getVirtualUrl( 'temp' ) . '/' . $dstUrlRel; - return $result; - } - - /** - * Remove a temporary file or mark it for garbage collection - * @param $virtualUrl String: the virtual URL returned by storeTemp - * @return Boolean: true on success, false on failure - */ - function freeTemp( $virtualUrl ) { - $temp = "mwrepo://{$this->name}/temp"; - if ( substr( $virtualUrl, 0, strlen( $temp ) ) != $temp ) { - wfDebug( __METHOD__.": Invalid virtual URL\n" ); - return false; - } - $path = $this->resolveVirtualUrl( $virtualUrl ); - wfSuppressWarnings(); - $success = unlink( $path ); - wfRestoreWarnings(); - return $success; - } - - /** - * Publish a batch of files - * @param $triplets Array: (source,dest,archive) triplets as per publish() - * @param $flags Integer: bitfield, may be FileRepo::DELETE_SOURCE to indicate - * that the source files should be deleted if possible - * @return Status - */ - function publishBatch( $triplets, $flags = 0 ) { - // Perform initial checks - if ( !wfMkdirParents( $this->directory, null, __METHOD__ ) ) { - return $this->newFatal( 'upload_directory_missing', $this->directory ); - } - if ( !is_writable( $this->directory ) ) { - return $this->newFatal( 'upload_directory_read_only', $this->directory ); - } - $status = $this->newGood( array() ); - foreach ( $triplets as $i => $triplet ) { - list( $srcPath, $dstRel, $archiveRel ) = $triplet; - - if ( substr( $srcPath, 0, 9 ) == 'mwrepo://' ) { - $triplets[$i][0] = $srcPath = $this->resolveVirtualUrl( $srcPath ); - } - if ( !$this->validateFilename( $dstRel ) ) { - throw new MWException( 'Validation error in $dstRel' ); - } - if ( !$this->validateFilename( $archiveRel ) ) { - throw new MWException( 'Validation error in $archiveRel' ); - } - $dstPath = "{$this->directory}/$dstRel"; - $archivePath = "{$this->directory}/$archiveRel"; - - $dstDir = dirname( $dstPath ); - $archiveDir = dirname( $archivePath ); - // Abort immediately on directory creation errors since they're likely to be repetitive - if ( !is_dir( $dstDir ) && !wfMkdirParents( $dstDir, null, __METHOD__ ) ) { - return $this->newFatal( 'directorycreateerror', $dstDir ); - } - if ( !is_dir( $archiveDir ) && !wfMkdirParents( $archiveDir, null, __METHOD__ ) ) { - return $this->newFatal( 'directorycreateerror', $archiveDir ); - } - if ( !is_file( $srcPath ) ) { - // Make a list of files that don't exist for return to the caller - $status->fatal( 'filenotfound', $srcPath ); - } - } - - if ( !$status->ok ) { - return $status; - } - - foreach ( $triplets as $i => $triplet ) { - list( $srcPath, $dstRel, $archiveRel ) = $triplet; - $dstPath = "{$this->directory}/$dstRel"; - $archivePath = "{$this->directory}/$archiveRel"; - - // Archive destination file if it exists - if( is_file( $dstPath ) ) { - // Check if the archive file exists - // This is a sanity check to avoid data loss. In UNIX, the rename primitive - // unlinks the destination file if it exists. DB-based synchronisation in - // publishBatch's caller should prevent races. In Windows there's no - // problem because the rename primitive fails if the destination exists. - if ( is_file( $archivePath ) ) { - $success = false; - } else { - wfSuppressWarnings(); - $success = rename( $dstPath, $archivePath ); - wfRestoreWarnings(); - } - - if( !$success ) { - $status->error( 'filerenameerror',$dstPath, $archivePath ); - $status->failCount++; - continue; - } else { - wfDebug(__METHOD__.": moved file $dstPath to $archivePath\n"); - } - $status->value[$i] = 'archived'; - } else { - $status->value[$i] = 'new'; - } - - $good = true; - wfSuppressWarnings(); - if ( $flags & self::DELETE_SOURCE ) { - if ( !rename( $srcPath, $dstPath ) ) { - $status->error( 'filerenameerror', $srcPath, $dstPath ); - $good = false; - } - } else { - if ( !copy( $srcPath, $dstPath ) ) { - $status->error( 'filecopyerror', $srcPath, $dstPath ); - $good = false; - } - } - wfRestoreWarnings(); - - if ( $good ) { - $status->successCount++; - wfDebug(__METHOD__.": wrote tempfile $srcPath to $dstPath\n"); - // Thread-safe override for umask - $this->chmod( $dstPath ); - } else { - $status->failCount++; - } - } - return $status; - } - - /** - * Move a group of files to the deletion archive. - * If no valid deletion archive is configured, this may either delete the - * file or throw an exception, depending on the preference of the repository. - * - * @param $sourceDestPairs Array of source/destination pairs. Each element - * is a two-element array containing the source file path relative to the - * public root in the first element, and the archive file path relative - * to the deleted zone root in the second element. - * @return FileRepoStatus - */ - function deleteBatch( $sourceDestPairs ) { - $status = $this->newGood(); - if ( !$this->deletedDir ) { - throw new MWException( __METHOD__.': no valid deletion archive directory' ); - } - - /** - * Validate filenames and create archive directories - */ - foreach ( $sourceDestPairs as $pair ) { - list( $srcRel, $archiveRel ) = $pair; - if ( !$this->validateFilename( $srcRel ) ) { - throw new MWException( __METHOD__.':Validation error in $srcRel' ); - } - if ( !$this->validateFilename( $archiveRel ) ) { - throw new MWException( __METHOD__.':Validation error in $archiveRel' ); - } - $archivePath = "{$this->deletedDir}/$archiveRel"; - $archiveDir = dirname( $archivePath ); - if ( !is_dir( $archiveDir ) ) { - if ( !wfMkdirParents( $archiveDir, null, __METHOD__ ) ) { - $status->fatal( 'directorycreateerror', $archiveDir ); - continue; - } - $this->initDeletedDir( $archiveDir ); - } - // Check if the archive directory is writable - // This doesn't appear to work on NTFS - if ( !is_writable( $archiveDir ) ) { - $status->fatal( 'filedelete-archive-read-only', $archiveDir ); - } - } - if ( !$status->ok ) { - // Abort early - return $status; - } - - /** - * Move the files - * We're now committed to returning an OK result, which will lead to - * the files being moved in the DB also. - */ - foreach ( $sourceDestPairs as $pair ) { - list( $srcRel, $archiveRel ) = $pair; - $srcPath = "{$this->directory}/$srcRel"; - $archivePath = "{$this->deletedDir}/$archiveRel"; - if ( file_exists( $archivePath ) ) { - # A file with this content hash is already archived - wfSuppressWarnings(); - $good = unlink( $srcPath ); - wfRestoreWarnings(); - if ( !$good ) { - $status->error( 'filedeleteerror', $srcPath ); - } - } else{ - wfSuppressWarnings(); - $good = rename( $srcPath, $archivePath ); - wfRestoreWarnings(); - if ( !$good ) { - $status->error( 'filerenameerror', $srcPath, $archivePath ); - } else { - $this->chmod( $archivePath ); - } - } - if ( $good ) { - $status->successCount++; - } else { - $status->failCount++; - } - } - return $status; - } - - /** - * Get a relative path for a deletion archive key, - * e.g. s/z/a/ for sza251lrxrc1jad41h5mgilp8nysje52.jpg - * @return string - */ - function getDeletedHashPath( $key ) { - $path = ''; - for ( $i = 0; $i < $this->deletedHashLevels; $i++ ) { - $path .= $key[$i] . '/'; - } - return $path; - } - - /** - * Call a callback function for every file in the repository. - * Uses the filesystem even in child classes. - * @return void - */ - function enumFilesInFS( $callback ) { - $numDirs = 1 << ( $this->hashLevels * 4 ); - for ( $flatIndex = 0; $flatIndex < $numDirs; $flatIndex++ ) { - $hexString = sprintf( "%0{$this->hashLevels}x", $flatIndex ); - $path = $this->directory; - for ( $hexPos = 0; $hexPos < $this->hashLevels; $hexPos++ ) { - $path .= '/' . substr( $hexString, 0, $hexPos + 1 ); - } - if ( !file_exists( $path ) || !is_dir( $path ) ) { - continue; - } - $dir = opendir( $path ); - if ($dir) { - while ( false !== ( $name = readdir( $dir ) ) ) { - call_user_func( $callback, $path . '/' . $name ); - } - closedir( $dir ); - } - } - } - - /** - * Call a callback function for every file in the repository - * May use either the database or the filesystem - * @return void - */ - function enumFiles( $callback ) { - $this->enumFilesInFS( $callback ); - } - - /** - * Get properties of a file with a given virtual URL - * The virtual URL must refer to this repo - * @return array - */ - function getFileProps( $virtualUrl ) { - $path = $this->resolveVirtualUrl( $virtualUrl ); - return File::getPropsFromPath( $path ); - } - - /** - * Path disclosure protection functions - * - * Get a callback function to use for cleaning error message parameters - */ - function getErrorCleanupFunction() { - switch ( $this->pathDisclosureProtection ) { - case 'simple': - $callback = array( $this, 'simpleClean' ); - break; - default: - $callback = parent::getErrorCleanupFunction(); - } - return $callback; - } - - function simpleClean( $param ) { - if ( !isset( $this->simpleCleanPairs ) ) { - global $IP; - $this->simpleCleanPairs = array( - $this->directory => 'public', - "{$this->directory}/temp" => 'temp', - $IP => '$IP', - dirname( __FILE__ ) => '$IP/extensions/WebStore', - ); - if ( $this->deletedDir ) { - $this->simpleCleanPairs[$this->deletedDir] = 'deleted'; - } - } - return strtr( $param, $this->simpleCleanPairs ); - } - - /** - * Chmod a file, supressing the warnings. - * @param $path String: the path to change - * @return void - */ - protected function chmod( $path ) { - wfSuppressWarnings(); - chmod( $path, $this->fileMode ); - wfRestoreWarnings(); - } - } diff --git a/includes/filerepo/FileRepo.php b/includes/filerepo/FileRepo.php index 58b64e4f51..9142065235 100644 --- a/includes/filerepo/FileRepo.php +++ b/includes/filerepo/FileRepo.php @@ -7,60 +7,245 @@ */ /** - * Base class for file repositories. - * Do not instantiate, use a derived class. + * Base class for file repositories * * @ingroup FileRepo */ -abstract class FileRepo { +class FileRepo { const FILES_ONLY = 1; const DELETE_SOURCE = 1; const OVERWRITE = 2; const OVERWRITE_SAME = 4; const SKIP_VALIDATION = 8; + /** @var FileBackendBase */ + protected $backend; + /** @var Array Map of zones to config */ + protected $zones = array(); + var $thumbScriptUrl, $transformVia404; var $descBaseUrl, $scriptDirUrl, $scriptExtension, $articleUrl; var $fetchDescription, $initialCapital; - var $pathDisclosureProtection = 'paranoid'; - var $descriptionCacheExpiry, $hashLevels, $url, $thumbUrl; + var $pathDisclosureProtection = 'simple'; // 'paranoid' + var $descriptionCacheExpiry, $url, $thumbUrl; + var $hashLevels, $deletedHashLevels; /** * Factory functions for creating new files * Override these in the base class */ - var $fileFactory = false, $oldFileFactory = false; + var $fileFactory = array( 'UnregisteredLocalFile', 'newFromTitle' ); + var $oldFileFactory = false; var $fileFactoryKey = false, $oldFileFactoryKey = false; function __construct( $info ) { // Required settings $this->name = $info['name']; + $this->url = isset( $info['url'] ) + ? $info['url'] + : false; // a subclass will need to set the URL (e.g. ForeignAPIRepo) + if ( $info['backend'] instanceof FileBackendBase ) { + $this->backend = $info['backend']; // useful for testing + } else { + $this->backend = FileBackendGroup::singleton()->get( $info['backend'] ); + } - // Optional settings - $this->initialCapital = MWNamespace::isCapitalized( NS_FILE ); - foreach ( array( 'descBaseUrl', 'scriptDirUrl', 'articleUrl', 'fetchDescription', - 'thumbScriptUrl', 'initialCapital', 'pathDisclosureProtection', - 'descriptionCacheExpiry', 'hashLevels', 'url', 'thumbUrl', 'scriptExtension' ) - as $var ) - { + // Optional settings that can have no value + $optionalSettings = array( + 'descBaseUrl', 'scriptDirUrl', 'articleUrl', 'fetchDescription', + 'thumbScriptUrl', 'pathDisclosureProtection', 'descriptionCacheExpiry', + 'scriptExtension' + ); + foreach ( $optionalSettings as $var ) { if ( isset( $info[$var] ) ) { $this->$var = $info[$var]; } } + + // Optional settings that have a default + $this->initialCapital = isset( $info['initialCapital'] ) + ? $info['initialCapital'] + : MWNamespace::isCapitalized( NS_FILE ); + $this->thumbUrl = isset( $info['thumbUrl'] ) + ? $info['thumbUrl'] + : "{$this->url}/thumb"; + $this->hashLevels = isset( $info['hashLevels'] ) + ? $info['hashLevels'] + : 2; + $this->deletedHashLevels = isset( $info['deletedHashLevels'] ) + ? $info['deletedHashLevels'] + : $this->hashLevels; $this->transformVia404 = !empty( $info['transformVia404'] ); + $this->zones = isset( $info['zones'] ) + ? $info['zones'] + : array(); + // Give defaults for the basic zones... + foreach ( array( 'public', 'thumb', 'temp', 'deleted' ) as $zone ) { + if ( !isset( $this->zones[$zone] ) ) { + if ( $zone === 'deleted' ) { + $this->zones[$zone] = array( + 'container' => null, // user must set this up + 'directory' => '' // container root + ); + } else { + $this->zones[$zone] = array( + 'container' => "images-$zone", + 'directory' => '' // container root + ); + } + } + } + } + + /** + * Get the file backend instance + * + * @return FileBackendBase + */ + public function getBackend() { + return $this->backend; + } + + /** + * Prepare all the zones for basic usage. + * See initDeletedDir() for additional setup needed for the 'deleted' zone. + * + * @param $doZones Array Only do a particular zones + * @return Status + */ + protected function initZones( $doZones = array() ) { + $status = $this->newGood(); + $doZones = (array)$doZones; // string => array + foreach ( $this->zones as $zone => $info ) { + if ( $doZones && !in_array( $zone, $doZones ) ) { + continue; + } + $root = $this->getZonePath( $zone ); + if ( $root !== null ) { + $params = array( 'dir' => $this->getZonePath( $zone ) ); + $status->merge( $this->backend->prepare( $params ) ); + } + } + return $status; + } + + /** + * Take all available measures to prevent web accessibility of new deleted + * directories, in case the user has not configured offline storage + * + * @return void + */ + protected function initDeletedDir( $dir ) { + // Add a .htaccess file to the root of the deleted zone + $root = $this->getZonePath( 'deleted' ); + $this->backend->secure( array( 'dir' => $root, 'noAccess' => true ) ); + // Seed new directories with a blank index.html, to prevent crawling + $this->backend->secure( array( 'dir' => $dir, 'noListing' => true ) ); } /** * Determine if a string is an mwrepo:// URL * * @param $url string - * * @return bool */ - static function isVirtualUrl( $url ) { + public static function isVirtualUrl( $url ) { return substr( $url, 0, 9 ) == 'mwrepo://'; } + /** + * Get a URL referring to this repository, with the private mwrepo protocol. + * The suffix, if supplied, is considered to be unencoded, and will be + * URL-encoded before being returned. + * + * @param $suffix string + * @return string + */ + public function getVirtualUrl( $suffix = false ) { + $path = 'mwrepo://' . $this->name; + if ( $suffix !== false ) { + $path .= '/' . rawurlencode( $suffix ); + } + return $path; + } + + /** + * Get the URL corresponding to one of the four basic zones + * + * @param $zone String: one of: public, deleted, temp, thumb + * @return String or false + */ + public function getZoneUrl( $zone ) { + switch ( $zone ) { + case 'public': + return $this->url; + case 'temp': + return "{$this->url}/temp"; + case 'deleted': + return false; // no public URL + case 'thumb': + return $this->thumbUrl; + default: + return false; + } + } + + /** + * Get the backend storage path corresponding to a virtual URL + * + * @param $url string + * @return string + */ + function resolveVirtualUrl( $url ) { + if ( substr( $url, 0, 9 ) != 'mwrepo://' ) { + throw new MWException( __METHOD__.': unknown protocol' ); + } + $bits = explode( '/', substr( $url, 9 ), 3 ); + if ( count( $bits ) != 3 ) { + throw new MWException( __METHOD__.": invalid mwrepo URL: $url" ); + } + list( $repo, $zone, $rel ) = $bits; + if ( $repo !== $this->name ) { + throw new MWException( __METHOD__.": fetching from a foreign repo is not supported" ); + } + $base = $this->getZonePath( $zone ); + if ( !$base ) { + throw new MWException( __METHOD__.": invalid zone: $zone" ); + } + return $base . '/' . rawurldecode( $rel ); + } + + /** + * The the storage container and base path of a zone + * + * @param $zone string + * @return Array (container, base path) or (null, null) + */ + protected function getZoneLocation( $zone ) { + if ( !isset( $this->zones[$zone] ) ) { + return array( null, null ); // bogus + } + return array( $this->zones[$zone]['container'], $this->zones[$zone]['directory'] ); + } + + /** + * Get the storage path corresponding to one of the zones + * + * @param $zone string + * @return string|null + */ + public function getZonePath( $zone ) { + list( $container, $base ) = $this->getZoneLocation( $zone ); + if ( $container === null || $base === null ) { + return null; + } + $backendName = $this->backend->getName(); + if ( $base != '' ) { // may not be set + $base = "/{$base}"; + } + return "mwstore://$backendName/{$container}{$base}"; + } + /** * Create a new File object from the local repository * @@ -70,10 +255,9 @@ abstract class FileRepo { * instance of the repository's old file class instead of a * current file. Repositories not supporting version control * should return false if this parameter is set. - * * @return File|null A File, or null if passed an invalid Title */ - function newFile( $title, $time = false ) { + public function newFile( $title, $time = false ) { $title = File::normalizeTitle( $title ); if ( !$title ) { return null; @@ -105,10 +289,9 @@ abstract class FileRepo { * private: If true, return restricted (deleted) files if the current * user is allowed to view them. Otherwise, such files will not * be found. - * * @return File|false */ - function findFile( $title, $options = array() ) { + public function findFile( $title, $options = array() ) { $title = File::normalizeTitle( $title ); if ( !$title ) { return false; @@ -139,12 +322,12 @@ abstract class FileRepo { return false; } $redir = $this->checkRedirect( $title ); - if( $redir && $title->getNamespace() == NS_FILE) { + if ( $redir && $title->getNamespace() == NS_FILE) { $img = $this->newFile( $redir ); - if( !$img ) { + if ( !$img ) { return false; } - if( $img->exists() ) { + if ( $img->exists() ) { $img->redirectedFrom( $title->getDBkey() ); return $img; } @@ -154,16 +337,16 @@ abstract class FileRepo { /** * Find many files at once. + * * @param $items An array of titles, or an array of findFile() options with * the "title" option giving the title. Example: * * $findItem = array( 'title' => $title, 'private' => true ); * $findBatch = array( $findItem ); * $repo->findFiles( $findBatch ); - * * @return array */ - function findFiles( $items ) { + public function findFiles( $items ) { $result = array(); foreach ( $items as $item ) { if ( is_array( $item ) ) { @@ -189,8 +372,9 @@ abstract class FileRepo { * * @param $sha1 String base 36 SHA-1 hash * @param $options Option array, same as findFile(). + * @return File|false */ - function findFileFromKey( $sha1, $options = array() ) { + public function findFileFromKey( $sha1, $options = array() ) { $time = isset( $options['time'] ) ? $options['time'] : false; # First try to find a matching current version of a file... @@ -217,19 +401,40 @@ abstract class FileRepo { } /** - * Get the URL of thumb.php + * Get an array or iterator of file objects for files that have a given + * SHA-1 content hash. + * + * STUB */ - function getThumbScriptUrl() { - return $this->thumbScriptUrl; + public function findBySha1( $hash ) { + return array(); } /** - * Get the URL corresponding to one of the four basic zones - * @param $zone String: one of: public, deleted, temp, thumb - * @return String or false + * Get the public root URL of the repository + * + * @return string */ - function getZoneUrl( $zone ) { - return false; + public function getRootUrl() { + return $this->url; + } + + /** + * Returns true if the repository uses a multi-level directory structure + * + * @return string + */ + public function isHashed() { + return (bool)$this->hashLevels; + } + + /** + * Get the URL of thumb.php + * + * @return string + */ + public function getThumbScriptUrl() { + return $this->thumbScriptUrl; } /** @@ -237,17 +442,18 @@ abstract class FileRepo { * * @return bool */ - function canTransformVia404() { + public function canTransformVia404() { return $this->transformVia404; } /** * Get the name of an image from its title object + * * @param $title Title */ - function getNameFromTitle( Title $title ) { + public function getNameFromTitle( Title $title ) { + global $wgContLang; if ( $this->initialCapital != MWNamespace::isCapitalized( NS_FILE ) ) { - global $wgContLang; $name = $title->getUserCaseDBKey(); if ( $this->initialCapital ) { $name = $wgContLang->ucfirst( $name ); @@ -258,6 +464,26 @@ abstract class FileRepo { return $name; } + /** + * Get the public zone root storage directory of the repository + * + * @return string + */ + public function getRootDirectory() { + return $this->getZonePath( 'public' ); + } + + /** + * Get a relative path including trailing slash, e.g. f/fa/ + * If the repo is not hashed, returns an empty string + * + * @param $name string + * @return string + */ + public function getHashPath( $name ) { + return self::getHashPathForLevel( $name, $this->hashLevels ); + } + /** * @param $name * @param $levels @@ -281,26 +507,16 @@ abstract class FileRepo { * * @return integer */ - function getHashLevels() { + public function getHashLevels() { return $this->hashLevels; } /** - * Get a relative path including trailing slash, e.g. f/fa/ - * If the repo is not hashed, returns an empty string - * - * @param $name string + * Get the name of this repository, as specified by $info['name]' to the constructor * * @return string */ - function getHashPath( $name ) { - return self::getHashPathForLevel( $name, $this->hashLevels ); - } - - /** - * Get the name of this repository, as specified by $info['name]' to the constructor - */ - function getName() { + public function getName() { return $this->name; } @@ -311,7 +527,7 @@ abstract class FileRepo { * @param $entry string Entry point; defaults to index * @return string */ - function makeUrl( $query = '', $entry = 'index' ) { + public function makeUrl( $query = '', $entry = 'index' ) { $ext = isset( $this->scriptExtension ) ? $this->scriptExtension : '.php'; return wfAppendQuery( "{$this->scriptDirUrl}/{$entry}{$ext}", $query ); } @@ -324,8 +540,11 @@ abstract class FileRepo { * * In particular, it uses the article paths as specified to the repository * constructor, whereas local repositories use the local Title functions. + * + * @param $name string + * @return string */ - function getDescriptionUrl( $name ) { + public function getDescriptionUrl( $name ) { $encName = wfUrlencode( $name ); if ( !is_null( $this->descBaseUrl ) ) { # "http://example.com/wiki/Image:" @@ -355,10 +574,12 @@ abstract class FileRepo { * MediaWiki this means action=render. This should only be called by the * repository's file class, since it may return invalid results. User code * should use File::getDescriptionText(). + * * @param $name String: name of image to fetch * @param $lang String: language to fetch it in, if any. + * @return string */ - function getDescriptionRenderUrl( $name, $lang = null ) { + public function getDescriptionRenderUrl( $name, $lang = null ) { $query = 'action=render'; if ( !is_null( $lang ) ) { $query .= '&uselang=' . $lang; @@ -380,9 +601,10 @@ abstract class FileRepo { /** * Get the URL of the stylesheet to apply to description pages + * * @return string */ - function getDescriptionStylesheetUrl() { + public function getDescriptionStylesheetUrl() { if ( $this->scriptDirUrl ) { return $this->makeUrl( 'title=MediaWiki:Filepage.css&' . wfArrayToCGI( Skin::getDynamicStylesheetQuery() ) ); @@ -392,7 +614,7 @@ abstract class FileRepo { /** * Store a file to a given destination. * - * @param $srcPath String: source path or virtual URL + * @param $srcPath String: source FS path, storage path, or virtual URL * @param $dstZone String: destination zone * @param $dstRel String: destination relative path * @param $flags Integer: bitwise combination of the following flags: @@ -402,7 +624,7 @@ abstract class FileRepo { * same contents as the source * @return FileRepoStatus */ - function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) { + public function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) { $status = $this->storeBatch( array( array( $srcPath, $dstZone, $dstRel ) ), $flags ); if ( $status->successCount == 0 ) { $status->ok = false; @@ -413,10 +635,133 @@ abstract class FileRepo { /** * Store a batch of files * - * @param $triplets Array: (src,zone,dest) triplets as per store() - * @param $flags Integer: flags as per store + * @param $triplets Array: (src, dest zone, dest rel) triplets as per store() + * @param $flags Integer: bitwise combination of the following flags: + * self::DELETE_SOURCE Delete the source file after upload + * self::OVERWRITE Overwrite an existing destination file instead of failing + * self::OVERWRITE_SAME Overwrite the file if the destination exists and has the + * same contents as the source + * @return FileRepoStatus */ - abstract function storeBatch( $triplets, $flags = 0 ); + public function storeBatch( $triplets, $flags = 0 ) { + $backend = $this->backend; // convenience + + // Try creating directories + $status = $this->initZones(); + if ( !$status->isOK() ) { + return $status; + } + + $status = $this->newGood(); + + $operations = array(); + $sourceFSFilesToDelete = array(); // cleanup for disk source files + // Validate each triplet and get the store operation... + foreach ( $triplets as $i => $triplet ) { + list( $srcPath, $dstZone, $dstRel ) = $triplet; + + // Resolve destination path + $root = $this->getZonePath( $dstZone ); + if ( !$root ) { + throw new MWException( "Invalid zone: $dstZone" ); + } + if ( !$this->validateFilename( $dstRel ) ) { + throw new MWException( 'Validation error in $dstRel' ); + } + $dstPath = "$root/$dstRel"; + $dstDir = dirname( $dstPath ); + + // Create destination directories for this triplet + if ( !$backend->prepare( array( 'dir' => $dstDir ) )->isOK() ) { + return $this->newFatal( 'directorycreateerror', $dstDir ); + } + + if ( $dstZone == 'deleted' ) { + $this->initDeletedDir( $dstDir ); + } + + // Resolve source to a storage path if virtual + if ( self::isVirtualUrl( $srcPath ) ) { + $srcPath = $this->resolveVirtualUrl( $srcPath ); + } + + // Get the appropriate file operation + if ( FileBackend::isStoragePath( $srcPath ) ) { + $opName = ( $flags & self::DELETE_SOURCE ) ? 'move' : 'copy'; + } else { + $opName = 'store'; + if ( $flags & self::DELETE_SOURCE ) { + $sourceFSFilesToDelete[] = $srcPath; + } + } + $operations[] = array( + 'op' => $opName, + 'src' => $srcPath, + 'dst' => $dstPath, + 'overwriteDest' => $flags & self::OVERWRITE, + 'overwriteSame' => $flags & self::OVERWRITE_SAME, + ); + } + + // Execute the store operation for each triplet + $opts = array( 'ignoreErrors' => true ); + $status->merge( $backend->doOperations( $operations, $opts ) ); + // Cleanup for disk source files... + foreach ( $sourceFSFilesToDelete as $file ) { + wfSuppressWarnings(); + unlink( $file ); // FS cleanup + wfRestoreWarnings(); + } + + return $status; + } + + /** + * Deletes a batch of files. + * Each file can be a (zone, rel) pair, virtual url, storage path, or FS path. + * It will try to delete each file, but ignores any errors that may occur. + * + * @param $pairs array List of files to delete + * @return void + */ + public function cleanupBatch( $files ) { + $operations = array(); + $sourceFSFilesToDelete = array(); // cleanup for disk source files + foreach ( $files as $file ) { + if ( is_array( $file ) ) { + // This is a pair, extract it + list( $zone, $rel ) = $file; + $root = $this->getZonePath( $zone ); + $path = "$root/$rel"; + } else { + if ( self::isVirtualUrl( $file ) ) { + // This is a virtual url, resolve it + $path = $this->resolveVirtualUrl( $file ); + } else { + // This is a full file name + $path = $file; + } + } + // Get a file operation if needed + if ( FileBackend::isStoragePath( $path ) ) { + $operations[] = array( + 'op' => 'delete', + 'src' => $path, + ); + } else { + $sourceFSFilesToDelete[] = $path; + } + } + // Actually delete files from storage... + $opts = array( 'ignoreErrors' => true ); + $this->backend->doOperations( $operations, $opts ); + // Cleanup for disk source files... + foreach ( $sourceFSFilesToDelete as $file ) { + wfSuppressWarnings(); + unlink( $path ); // FS cleanup + wfRestoreWarnings(); + } + } /** * Pick a random name in the temp zone and store a file to it. @@ -425,61 +770,97 @@ abstract class FileRepo { * @param $originalName String: the base name of the file as specified * by the user. The file extension will be maintained. * @param $srcPath String: the current location of the file. + * @return FileRepoStatus object with the URL in the value. */ - abstract function storeTemp( $originalName, $srcPath ); + public function storeTemp( $originalName, $srcPath ) { + $date = gmdate( "YmdHis" ); + $hashPath = $this->getHashPath( $originalName ); + $dstRel = "{$hashPath}{$date}!{$originalName}"; + $dstUrlRel = $hashPath . $date . '!' . rawurlencode( $originalName ); + $result = $this->store( $srcPath, 'temp', $dstRel ); + $result->value = $this->getVirtualUrl( 'temp' ) . '/' . $dstUrlRel; + return $result; + } /** - * Concatenate and array of file sources. - * @param $fileList Array of file sources - * @param $targetPath String target destination for file. - * @throws MWException - */ - abstract function concatenate( $fileList, $targetPath, $flags = 0 ); - - /** - * Append the contents of the source path to the given file, OR queue - * the appending operation in anticipation of a later appendFinish() call. - * @param $srcPath String: location of the source file - * @param $toAppendPath String: path to append to. - * @param $flags Integer: bitfield, may be FileRepo::DELETE_SOURCE to indicate - * that the source file should be deleted if possible - * @return mixed Status or false + * Concatenate a list of files into a target file location. + * + * @param $srcPaths Array Ordered list of source virtual URLs/storage paths + * @param $dstPath String Target virtual URL/storage path + * @param $flags Integer: bitwise combination of the following flags: + * self::DELETE_SOURCE Delete the source files + * @return FileRepoStatus */ - abstract function append( $srcPath, $toAppendPath, $flags = 0 ); + function concatenate( $srcPaths, $dstPath, $flags = 0 ) { + $status = $this->newGood(); + // Resolve target to a storage path if virtual + $dest = $this->resolveToStoragePath( $dstPath ); - /** - * Finish the append operation. - * @param $toAppendPath String: path to append to. - * @return mixed Status or false - */ - abstract function appendFinish( $toAppendPath ); + $sources = array(); + $deleteOperations = array(); // post-concatenate ops + foreach ( $srcPaths as $srcPath ) { + // Resolve source to a storage path if virtual + $source = $this->resolveToStoragePath( $srcPath ); + $sources[] = $source; // chunk to merge + if ( $flags & self::DELETE_SOURCE ) { + $deleteOperations[] = array( 'op' => 'delete', 'src' => $source ); + } + } + + // Concatenate the chunks into one file + $op = array( 'op' => 'concatenate', + 'srcs' => $sources, 'dst' => $dest, 'overwriteDest' => true ); + $status->merge( $this->backend->doOperation( $op ) ); + if ( !$status->isOK() ) { + return $status; + } + + // Delete the sources if required + if ( $deleteOperations ) { + $opts = array( 'ignoreErrors' => true ); + $status->merge( $this->backend->doOperations( $deleteOperations, $opts ) ); + } + + // Make sure status is OK, despite any $deleteOperations fatals + $status->setResult( true ); + + return $status; + } /** * Remove a temporary file or mark it for garbage collection + * * @param $virtualUrl String: the virtual URL returned by storeTemp * @return Boolean: true on success, false on failure - * STUB */ - function freeTemp( $virtualUrl ) { - return true; + public function freeTemp( $virtualUrl ) { + $temp = "mwrepo://{$this->name}/temp"; + if ( substr( $virtualUrl, 0, strlen( $temp ) ) != $temp ) { + wfDebug( __METHOD__.": Invalid temp virtual URL\n" ); + return false; + } + $path = $this->resolveVirtualUrl( $virtualUrl ); + $op = array( 'op' => 'delete', 'src' => $path ); + $status = $this->backend->doOperation( $op ); + return $status->isOK(); } /** - * Copy or move a file either from the local filesystem or from an mwrepo:// - * virtual URL, into this repository at the specified destination location. + * Copy or move a file either from a storage path, virtual URL, + * or FS path, into this repository at the specified destination location. * * Returns a FileRepoStatus object. On success, the value contains "new" or * "archived", to indicate whether the file was new with that name. * - * @param $srcPath String: the source path or URL + * @param $srcPath String: the source FS path, storage path, or URL * @param $dstRel String: the destination relative path * @param $archiveRel String: the relative path where the existing file is to * be archived, if there is one. Relative to the public zone root. * @param $flags Integer: bitfield, may be FileRepo::DELETE_SOURCE to indicate * that the source file should be deleted if possible */ - function publish( $srcPath, $dstRel, $archiveRel, $flags = 0 ) { + public function publish( $srcPath, $dstRel, $archiveRel, $flags = 0 ) { $status = $this->publishBatch( array( array( $srcPath, $dstRel, $archiveRel ) ), $flags ); if ( $status->successCount == 0 ) { $status->ok = false; @@ -494,18 +875,123 @@ abstract class FileRepo { /** * Publish a batch of files - * @param $triplets Array: (source,dest,archive) triplets as per publish() + * + * @param $triplets Array: (source, dest, archive) triplets as per publish() * @param $flags Integer: bitfield, may be FileRepo::DELETE_SOURCE to indicate * that the source files should be deleted if possible + * @return FileRepoStatus */ - abstract function publishBatch( $triplets, $flags = 0 ); + public function publishBatch( $triplets, $flags = 0 ) { + $backend = $this->backend; // convenience + + // Try creating directories + $status = $this->initZones( 'public' ); + if ( !$status->isOK() ) { + return $status; + } + + $status = $this->newGood( array() ); + + $operations = array(); + $sourceFSFilesToDelete = array(); // cleanup for disk source files + // Validate each triplet and get the store operation... + foreach ( $triplets as $i => $triplet ) { + list( $srcPath, $dstRel, $archiveRel ) = $triplet; + // Resolve source to a storage path if virtual + if ( substr( $srcPath, 0, 9 ) == 'mwrepo://' ) { + $srcPath = $this->resolveVirtualUrl( $srcPath ); + } + if ( !$this->validateFilename( $dstRel ) ) { + throw new MWException( 'Validation error in $dstRel' ); + } + if ( !$this->validateFilename( $archiveRel ) ) { + throw new MWException( 'Validation error in $archiveRel' ); + } + + $publicRoot = $this->getZonePath( 'public' ); + $dstPath = "$publicRoot/$dstRel"; + $archivePath = "$publicRoot/$archiveRel"; + + $dstDir = dirname( $dstPath ); + $archiveDir = dirname( $archivePath ); + // Abort immediately on directory creation errors since they're likely to be repetitive + if ( !$backend->prepare( array( 'dir' => $dstDir ) )->isOK() ) { + return $this->newFatal( 'directorycreateerror', $dstDir ); + } + if ( !$backend->prepare( array( 'dir' => $archiveDir ) )->isOK() ) { + return $this->newFatal( 'directorycreateerror', $archiveDir ); + } + + // Archive destination file if it exists + if ( $backend->fileExists( array( 'src' => $dstPath ) ) ) { + // Check if the archive file exists + // This is a sanity check to avoid data loss. In UNIX, the rename primitive + // unlinks the destination file if it exists. DB-based synchronisation in + // publishBatch's caller should prevent races. In Windows there's no + // problem because the rename primitive fails if the destination exists. + if ( $backend->fileExists( array( 'src' => $archivePath ) ) ) { + $operations[] = array( 'op' => 'null' ); + continue; + } else { + $operations[] = array( + 'op' => 'move', + 'src' => $dstPath, + 'dst' => $archivePath + ); + } + $status->value[$i] = 'archived'; + } else { + $status->value[$i] = 'new'; + } + // Copy (or move) the source file to the destination + if ( FileBackend::isStoragePath( $srcPath ) ) { + if ( $flags & self::DELETE_SOURCE ) { + $operations[] = array( + 'op' => 'move', + 'src' => $srcPath, + 'dst' => $dstPath + ); + } else { + $operations[] = array( + 'op' => 'copy', + 'src' => $srcPath, + 'dst' => $dstPath + ); + } + } else { // FS source path + $operations[] = array( + 'op' => 'store', + 'src' => $srcPath, + 'dst' => $dstPath + ); + if ( $flags & self::DELETE_SOURCE ) { + $sourceFSFilesToDelete[] = $srcPath; + } + } + } + + // Execute the operations for each triplet + $opts = array( 'ignoreErrors' => true ); + $status->merge( $backend->doOperations( $operations, $opts ) ); + // Cleanup for disk source files... + foreach ( $sourceFSFilesToDelete as $file ) { + wfSuppressWarnings(); + unlink( $file ); // FS cleanup + wfRestoreWarnings(); + } + + return $status; + } /** - * @param $file - * @param int $flags + * Checks existence of a a file + * + * @param $file Virtual URL (or storage path) of file to check + * @param $flags Integer: bitwise combination of the following flags: + * self::FILES_ONLY Mark file as existing only if it is a file (not directory) * @return bool */ - function fileExists( $file, $flags = 0 ) { + public function fileExists( $file, $flags = 0 ) { $result = $this->fileExistsBatch( array( $file ), $flags ); return $result[0]; } @@ -513,12 +999,47 @@ abstract class FileRepo { /** * Checks existence of an array of files. * - * @param $files Array: URLs (or paths) of files to check + * @param $files Array: Virtual URLs (or storage paths) of files to check * @param $flags Integer: bitwise combination of the following flags: * self::FILES_ONLY Mark file as existing only if it is a file (not directory) * @return Either array of files and existence flags, or false */ - abstract function fileExistsBatch( $files, $flags = 0 ); + public function fileExistsBatch( $files, $flags = 0 ) { + if ( !$this->initZones() ) { + return false; + } + $result = array(); + foreach ( $files as $key => $file ) { + if ( self::isVirtualUrl( $file ) ) { + $file = $this->resolveVirtualUrl( $file ); + } + if ( FileBackend::isStoragePath( $file ) ) { + $result[$key] = $this->backend->fileExists( array( 'src' => $file ) ); + } else { + if ( $flags & self::FILES_ONLY ) { + $result[$key] = is_file( $file ); // FS only + } else { + $result[$key] = file_exists( $file ); // FS only + } + } + } + + return $result; + } + + /** + * Move a file to the deletion archive. + * If no valid deletion archive exists, this may either delete the file + * or throw an exception, depending on the preference of the repository + * + * @param $srcRel Mixed: relative path for the file to be deleted + * @param $archiveRel Mixed: relative path for the archive location. + * Relative to a private archive directory. + * @return FileRepoStatus object + */ + public function delete( $srcRel, $archiveRel ) { + return $this->deleteBatch( array( array( $srcRel, $archiveRel ) ) ); + } /** * Move a group of files to the deletion archive. @@ -536,47 +1057,217 @@ abstract class FileRepo { * to the deleted zone root in the second element. * @return FileRepoStatus */ - abstract function deleteBatch( $sourceDestPairs ); + public function deleteBatch( $sourceDestPairs ) { + $backend = $this->backend; // convenience + + if ( !isset( $this->zones['deleted']['container'] ) ) { + throw new MWException( __METHOD__.': no valid deletion archive directory' ); + } + + // Try creating directories + $status = $this->initZones( array( 'public', 'deleted' ) ); + if ( !$status->isOK() ) { + return $status; + } + + $status = $this->newGood(); + + $operations = array(); + // Validate filenames and create archive directories + foreach ( $sourceDestPairs as $pair ) { + list( $srcRel, $archiveRel ) = $pair; + if ( !$this->validateFilename( $srcRel ) ) { + throw new MWException( __METHOD__.':Validation error in $srcRel' ); + } + if ( !$this->validateFilename( $archiveRel ) ) { + throw new MWException( __METHOD__.':Validation error in $archiveRel' ); + } + + $publicRoot = $this->getZonePath( 'public' ); + $srcPath = "{$publicRoot}/$srcRel"; + + $deletedRoot = $this->getZonePath( 'deleted' ); + $archivePath = "{$deletedRoot}/{$archiveRel}"; + $archiveDir = dirname( $archivePath ); // does not touch FS + + // Create destination directories + if ( !$backend->prepare( array( 'dir' => $archiveDir ) )->isOK() ) { + return $this->newFatal( 'directorycreateerror', $archiveDir ); + } + $this->initDeletedDir( $archiveDir ); + + if ( $backend->fileExists( array( 'src' => $archivePath ) ) ) { + $operations[] = array( + 'op' => 'delete', + 'src' => $srcPath + ); + } else { + $operations[] = array( + 'op' => 'move', + 'src' => $srcPath, + 'dst' => $archivePath + ); + } + } + + // Move the files by execute the operations for each pair. + // We're now committed to returning an OK result, which will + // lead to the files being moved in the DB also. + $opts = array( 'ignoreErrors' => true ); + $status->merge( $backend->doOperations( $operations, $opts ) ); + + return $status; + } /** - * Move a file to the deletion archive. - * If no valid deletion archive exists, this may either delete the file - * or throw an exception, depending on the preference of the repository - * @param $srcRel Mixed: relative path for the file to be deleted - * @param $archiveRel Mixed: relative path for the archive location. - * Relative to a private archive directory. - * @return FileRepoStatus object + * Get a relative path for a deletion archive key, + * e.g. s/z/a/ for sza251lrxrc1jad41h5mgilp8nysje52.jpg + * + * @return string */ - function delete( $srcRel, $archiveRel ) { - return $this->deleteBatch( array( array( $srcRel, $archiveRel ) ) ); + public function getDeletedHashPath( $key ) { + $path = ''; + for ( $i = 0; $i < $this->deletedHashLevels; $i++ ) { + $path .= $key[$i] . '/'; + } + return $path; } /** - * Get properties of a file with a given virtual URL - * The virtual URL must refer to this repo - * Properties should ultimately be obtained via File::getPropsFromPath() + * If a path is a virtual URL, resolve it to a storage path. + * Otherwise, just return the path as it is. * + * @param $path string + * @return string + * @throws MWException + */ + protected function resolveToStoragePath( $path ) { + if ( $this->isVirtualUrl( $path ) ) { + return $this->resolveVirtualUrl( $path ); + } + return $path; + } + + /** + * Get a local FS copy of a file with a given virtual URL/storage path. + * Temporary files may be purged when the file object falls out of scope. + * + * @param $virtualUrl string + * @return TempFSFile|null Returns null on failure + */ + public function getLocalCopy( $virtualUrl ) { + $path = $this->resolveToStoragePath( $virtualUrl ); + return $this->backend->getLocalCopy( array( 'src' => $path ) ); + } + + /** + * Get a local FS file with a given virtual URL/storage path. + * The file is either an original or a copy. It should not be changed. + * Temporary files may be purged when the file object falls out of scope. + * * @param $virtualUrl string + * @return FSFile|null Returns null on failure. */ - abstract function getFileProps( $virtualUrl ); + public function getLocalReference( $virtualUrl ) { + $path = $this->resolveToStoragePath( $virtualUrl ); + return $this->backend->getLocalReference( array( 'src' => $path ) ); + } /** - * Call a callback function for every file in the repository - * May use either the database or the filesystem - * STUB + * Get properties of a file with a given virtual URL/storage path. + * Properties should ultimately be obtained via FSFile::getProps(). + * + * @param $virtualUrl string + * @return Array */ - function enumFiles( $callback ) { - throw new MWException( 'enumFiles is not supported by ' . get_class( $this ) ); + public function getFileProps( $virtualUrl ) { + $path = $this->resolveToStoragePath( $virtualUrl ); + return $this->backend->getFileProps( array( 'src' => $path ) ); + } + + /** + * Get the timestamp of a file with a given virtual URL/storage path + * + * @param $virtualUrl string + * @return string|false + */ + public function getFileTimestamp( $virtualUrl ) { + $path = $this->resolveToStoragePath( $virtualUrl ); + return $this->backend->getFileTimestamp( array( 'src' => $path ) ); + } + + /** + * Get the sha1 of a file with a given virtual URL/storage path + * + * @param $virtualUrl string + * @return string|false + */ + public function getFileSha1( $virtualUrl ) { + $path = $this->resolveToStoragePath( $virtualUrl ); + $tmpFile = $this->backend->getLocalReference( array( 'src' => $path ) ); + if ( !$tmpFile ) { + return false; + } + return $tmpFile->getSha1Base36(); + } + + /** + * Attempt to stream a file with the given virtual URL/storage path + * + * @param $virtualUrl string + * @param $headers Array Additional HTTP headers to send on success + * @return bool Success + */ + public function streamFile( $virtualUrl, $headers = array() ) { + $path = $this->resolveToStoragePath( $virtualUrl ); + $params = array( 'src' => $path, 'headers' => $headers ); + return $this->backend->streamFile( $params )->isOK(); + } + + /** + * Call a callback function for every public file in the repository. + * May use either the database or the filesystem. + * + * @param $callback Array|string + * @return void + */ + public function enumFiles( $callback ) { + return $this->enumFilesInStorage( $callback ); + } + + /** + * Call a callback function for every public file in the repository. + * May use either the database or the filesystem. + * + * @param $callback Array|string + * @return void + */ + protected function enumFilesInStorage( $callback ) { + $publicRoot = $this->getZonePath( 'public' ); + $numDirs = 1 << ( $this->hashLevels * 4 ); + // Use a priori assumptions about directory structure + // to reduce the tree height of the scanning process. + for ( $flatIndex = 0; $flatIndex < $numDirs; $flatIndex++ ) { + $hexString = sprintf( "%0{$this->hashLevels}x", $flatIndex ); + $path = $publicRoot; + for ( $hexPos = 0; $hexPos < $this->hashLevels; $hexPos++ ) { + $path .= '/' . substr( $hexString, 0, $hexPos + 1 ); + } + $iterator = $this->backend->getFileList( array( 'dir' => $path ) ); + foreach ( $iterator as $name ) { + // Each item returned is a public file + call_user_func( $callback, "{$path}/{$name}" ); + } + } } /** * Determine if a relative path is valid, i.e. not blank or involving directory traveral * * @param $filename string - * * @return bool */ - function validateFilename( $filename ) { + public function validateFilename( $filename ) { if ( strval( $filename ) == '' ) { return false; } @@ -599,34 +1290,65 @@ abstract class FileRepo { } } - /**#@+ - * Path disclosure protection functions - */ - function paranoidClean( $param ) { return '[hidden]'; } - - /** - * @param $param - * @return - */ - function passThrough( $param ) { return $param; } - /** * Get a callback function to use for cleaning error message parameters + * + * @return Array */ function getErrorCleanupFunction() { switch ( $this->pathDisclosureProtection ) { case 'none': $callback = array( $this, 'passThrough' ); break; + case 'simple': + $callback = array( $this, 'simpleClean' ); + break; default: // 'paranoid' $callback = array( $this, 'paranoidClean' ); } return $callback; } - /**#@-*/ + + /** + * Path disclosure protection function + * + * @param $param string + * @return string + */ + function paranoidClean( $param ) { + return '[hidden]'; + } + + /** + * Path disclosure protection function + * + * @param $param string + * @return string + */ + function simpleClean( $param ) { + global $IP; + if ( !isset( $this->simpleCleanPairs ) ) { + $this->simpleCleanPairs = array( + $IP => '$IP', // sanity + ); + } + return strtr( $param, $this->simpleCleanPairs ); + } + + /** + * Path disclosure protection function + * + * @param $param string + * @return string + */ + function passThrough( $param ) { + return $param; + } /** * Create a new fatal error + * + * @return FileRepoStatus */ function newFatal( $message /*, parameters...*/ ) { $params = func_get_args(); @@ -645,9 +1367,10 @@ abstract class FileRepo { /** * Delete files in the deleted directory if they are not referenced in the filearchive table + * * STUB */ - function cleanupDeletedBatch( $storageKeys ) {} + public function cleanupDeletedBatch( $storageKeys ) {} /** * Checks if there is a redirect named as $title. If there is, return the @@ -657,7 +1380,7 @@ abstract class FileRepo { * @param $title Title of image * @return Bool */ - function checkRedirect( Title $title ) { + public function checkRedirect( Title $title ) { return false; } @@ -668,20 +1391,11 @@ abstract class FileRepo { * STUB * @param $title Title of image */ - function invalidateImageRedirect( Title $title ) {} + public function invalidateImageRedirect( Title $title ) {} /** - * Get an array or iterator of file objects for files that have a given - * SHA-1 content hash. + * Get the human-readable name of the repo * - * STUB - */ - function findBySha1( $hash ) { - return array(); - } - - /** - * Get the human-readable name of the repo. * @return string */ public function getDisplayName() { @@ -698,7 +1412,7 @@ abstract class FileRepo { * * @return bool */ - function isLocal() { + public function isLocal() { return $this->getName() == 'local'; } @@ -717,6 +1431,8 @@ abstract class FileRepo { * Get a key for this repo in the local cache domain. These cache keys are * not shared with remote instances of the repo. * The parameters are the parts of the key, as for wfMemcKey(). + * + * @return string */ function getLocalCacheKey( /*...*/ ) { $args = func_get_args(); @@ -729,7 +1445,7 @@ abstract class FileRepo { * * @return UploadStash */ - function getUploadStash() { + public function getUploadStash() { return new UploadStash( $this ); } } diff --git a/includes/filerepo/ForeignAPIRepo.php b/includes/filerepo/ForeignAPIRepo.php index 6a983ed570..91f3c18dcf 100644 --- a/includes/filerepo/ForeignAPIRepo.php +++ b/includes/filerepo/ForeignAPIRepo.php @@ -32,20 +32,19 @@ class ForeignAPIRepo extends FileRepo { var $apiThumbCacheExpiry = 86400; /* 24*60*60 */ /* Redownload thumbnail files after a month */ var $fileCacheExpiry = 2592000; /* 86400*30 */ - /* Local image directory */ - var $directory; - var $thumbDir; protected $mQueryCache = array(); protected $mFileExists = array(); function __construct( $info ) { + global $wgLocalFileRepo, $wgUploadDirectory; + if ( !isset( $info['directory'] ) ) { // b/c + $info['directory'] = $wgUploadDirectory; // Local image directory + } parent::__construct( $info ); - global $wgUploadDirectory; // http://commons.wikimedia.org/w/api.php $this->mApiBase = isset( $info['apibase'] ) ? $info['apibase'] : null; - $this->directory = isset( $info['directory'] ) ? $info['directory'] : $wgUploadDirectory; if( isset( $info['apiThumbCacheExpiry'] ) ) { $this->apiThumbCacheExpiry = $info['apiThumbCacheExpiry']; @@ -59,17 +58,11 @@ class ForeignAPIRepo extends FileRepo { } // If we can cache thumbs we can guess sane defaults for these if( $this->canCacheThumbs() && !$this->url ) { - global $wgLocalFileRepo; $this->url = $wgLocalFileRepo['url']; } if( $this->canCacheThumbs() && !$this->thumbUrl ) { $this->thumbUrl = $this->url . '/thumb'; } - if ( isset( $info['thumbDir'] ) ) { - $this->thumbDir = $info['thumbDir']; - } else { - $this->thumbDir = "{$this->directory}/thumb"; - } } /** @@ -284,9 +277,9 @@ class ForeignAPIRepo extends FileRepo { $localFilename = $localPath . "/" . $fileName; $localUrl = $this->getZoneUrl( 'thumb' ) . "/" . $this->getHashPath( $name ) . rawurlencode( $name ) . "/" . rawurlencode( $fileName ); - if( file_exists( $localFilename ) && isset( $metadata['timestamp'] ) ) { + if( $this->fileExists( $localFilename ) && isset( $metadata['timestamp'] ) ) { wfDebug( __METHOD__ . " Thumbnail was already downloaded before\n" ); - $modified = filemtime( $localFilename ); + $modified = $this->getFileTimestamp( $localFilename ); $remoteModified = strtotime( $metadata['timestamp'] ); $current = time(); $diff = abs( $modified - $current ); @@ -303,16 +296,12 @@ class ForeignAPIRepo extends FileRepo { wfDebug( __METHOD__ . " Could not download thumb\n" ); return false; } - if ( !is_dir($localPath) ) { - if( !wfMkdirParents( $localPath, null, __METHOD__ ) ) { - wfDebug( __METHOD__ . " could not create directory $localPath for thumb\n" ); - return $foreignUrl; - } - } # @todo FIXME: Delete old thumbs that aren't being used. Maintenance script? wfSuppressWarnings(); - if( !file_put_contents( $localFilename, $thumb ) ) { + $backend = $this->getBackend(); + $op = array( 'op' => 'create', 'dst' => $localFilename, 'content' => $thumb ); + if( !$backend->doOperation( $op )->isOK() ) { wfRestoreWarnings(); wfDebug( __METHOD__ . " could not write to thumb path\n" ); return $foreignUrl; @@ -339,17 +328,14 @@ class ForeignAPIRepo extends FileRepo { } /** - * Get the local directory corresponding to one of the three basic zones + * Get the local directory corresponding to one of the basic zones */ function getZonePath( $zone ) { - switch ( $zone ) { - case 'public': - return $this->directory; - case 'thumb': - return $this->thumbDir; - default: - return false; + $supported = array( 'public', 'thumb' ); + if ( in_array( $zone, $supported ) ) { + return parent::getZonePath( $zone ); } + return false; } /** @@ -392,4 +378,8 @@ class ForeignAPIRepo extends FileRepo { return false; } } + + function enumFiles( $callback ) { + throw new MWException( 'enumFiles is not supported by ' . get_class( $this ) ); + } } diff --git a/includes/filerepo/ForeignDBViaLBRepo.php b/includes/filerepo/ForeignDBViaLBRepo.php index 4c530b511d..28b48b5eb3 100644 --- a/includes/filerepo/ForeignDBViaLBRepo.php +++ b/includes/filerepo/ForeignDBViaLBRepo.php @@ -30,6 +30,7 @@ class ForeignDBViaLBRepo extends LocalRepo { function getSlaveDB() { return wfGetDB( DB_SLAVE, array(), $this->wiki ); } + function hasSharedCache() { return $this->hasSharedCache; } diff --git a/includes/filerepo/LocalRepo.php b/includes/filerepo/LocalRepo.php index 42aacdf870..8a58119c29 100644 --- a/includes/filerepo/LocalRepo.php +++ b/includes/filerepo/LocalRepo.php @@ -10,9 +10,10 @@ /** * A repository that stores files in the local filesystem and registers them * in the wiki's own database. This is the most commonly used repository class. + * * @ingroup FileRepo */ -class LocalRepo extends FSRepo { +class LocalRepo extends FileRepo { var $fileFactory = array( 'LocalFile', 'newFromTitle' ); var $fileFactoryKey = array( 'LocalFile', 'newFromKey' ); var $oldFileFactory = array( 'OldLocalFile', 'newFromTitle' ); @@ -22,7 +23,7 @@ class LocalRepo extends FSRepo { /** * @throws MWException - * @param $row + * @param $row * @return File */ function newFileFromRow( $row ) { @@ -55,6 +56,7 @@ class LocalRepo extends FSRepo { * @return FileRepoStatus */ function cleanupDeletedBatch( $storageKeys ) { + $backend = $this->backend; // convenience $root = $this->getZonePath( 'deleted' ); $dbw = $this->getMasterDB(); $status = $this->newGood(); @@ -69,10 +71,8 @@ class LocalRepo extends FSRepo { $hidden = $this->hiddenFileHasKey( $key, 'lock' ); if ( !$deleted && !$hidden ) { // not in use now wfDebug( __METHOD__ . ": deleting $key\n" ); - wfSuppressWarnings(); - $unlink = unlink( $path ); - wfRestoreWarnings(); - if ( !$unlink ) { + $op = array( 'op' => 'delete', 'src' => $path ); + if ( !$backend->doOperation( $op )->isOK() ) { $status->error( 'undelete-cleanup-error', $path ); $status->failCount++; } @@ -87,6 +87,7 @@ class LocalRepo extends FSRepo { /** * Check if a deleted (filearchive) file has this sha1 key + * * @param $key String File storage key (base-36 sha1 key with file extension) * @param $lock String|null Use "lock" to lock the row via FOR UPDATE * @return bool File with this key is in use @@ -103,6 +104,7 @@ class LocalRepo extends FSRepo { /** * Check if a hidden (revision delete) file has this sha1 key + * * @param $key String File storage key (base-36 sha1 key with file extension) * @param $lock String|null Use "lock" to lock the row via FOR UPDATE * @return bool File with this key is in use @@ -185,6 +187,7 @@ class LocalRepo extends FSRepo { /** * Function link Title::getArticleID(). * We can't say Title object, what database it should use, so we duplicate that function here. + * * @param $title Title */ protected function getArticleID( $title ) { @@ -193,13 +196,13 @@ class LocalRepo extends FSRepo { } $dbr = $this->getSlaveDB(); $id = $dbr->selectField( - 'page', // Table - 'page_id', //Field - array( //Conditions + 'page', // Table + 'page_id', //Field + array( //Conditions 'page_namespace' => $title->getNamespace(), 'page_title' => $title->getDBkey(), ), - __METHOD__ //Function name + __METHOD__ //Function name ); return $id; } @@ -207,6 +210,8 @@ class LocalRepo extends FSRepo { /** * Get an array or iterator of file objects for files that have a given * SHA-1 content hash. + * + * @param string * @return Array */ function findBySha1( $hash ) { @@ -244,6 +249,7 @@ class LocalRepo extends FSRepo { * Get a key on the primary cache for this repository. * Returns false if the repository's cache is not accessible at this site. * The parameters are the parts of the key, as for wfMemcKey(). + * * @return string */ function getSharedCacheKey( /*...*/ ) { diff --git a/includes/filerepo/RepoGroup.php b/includes/filerepo/RepoGroup.php index a2ddadc7b1..da6c7d9f30 100644 --- a/includes/filerepo/RepoGroup.php +++ b/includes/filerepo/RepoGroup.php @@ -366,7 +366,7 @@ class RepoGroup { $repo = $this->getRepo( $repoName ); return $repo->getFileProps( $fileName ); } else { - return File::getPropsFromPath( $fileName ); + return FSFile::getPropsFromPath( $fileName ); } } diff --git a/includes/filerepo/backend/FSFileBackend.php b/includes/filerepo/backend/FSFileBackend.php new file mode 100644 index 0000000000..e803467e59 --- /dev/null +++ b/includes/filerepo/backend/FSFileBackend.php @@ -0,0 +1,608 @@ +containerPaths = (array)$config['containerPaths']; + foreach ( $this->containerPaths as $container => &$path ) { + if ( substr( $path, -1 ) === '/' ) { + $path = substr( $path, 0, -1 ); // remove trailing slash + } + } + $this->fileMode = isset( $config['fileMode'] ) + ? $config['fileMode'] + : 0644; + } + + /** + * @see FileBackend::resolveContainerPath() + */ + protected function resolveContainerPath( $container, $relStoragePath ) { + // Get absolute path given the container base dir + if ( isset( $this->containerPaths[$container] ) ) { + return $this->containerPaths[$container] . "/{$relStoragePath}"; + } + return null; + } + + /** + * @see FileBackend::doStore() + */ + protected function doStore( array $params ) { + $status = Status::newGood(); + + list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] ); + if ( $dest === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; + } + if ( file_exists( $dest ) ) { + if ( !empty( $params['overwriteDest'] ) ) { + wfSuppressWarnings(); + $ok = unlink( $dest ); + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-delete', $params['dst'] ); + return $status; + } + } else { + $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); + return $status; + } + } else { + if ( !wfMkdirParents( dirname( $dest ) ) ) { + $status->fatal( 'directorycreateerror', $params['dst'] ); + return $status; + } + } + + wfSuppressWarnings(); + $ok = copy( $params['src'], $dest ); + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + return $status; + } + + $this->chmod( $dest ); + + return $status; + } + + /** + * @see FileBackend::doCopy() + */ + protected function doCopy( array $params ) { + $status = Status::newGood(); + + list( $c, $source ) = $this->resolveStoragePath( $params['src'] ); + if ( $source === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; + } + + list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] ); + if ( $dest === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; + } + + if ( file_exists( $dest ) ) { + if ( !empty( $params['overwriteDest'] ) ) { + wfSuppressWarnings(); + $ok = unlink( $dest ); + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-delete', $params['dst'] ); + return $status; + } + } else { + $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); + return $status; + } + } else { + if ( !wfMkdirParents( dirname( $dest ) ) ) { + $status->fatal( 'directorycreateerror', $params['dst'] ); + return $status; + } + } + + wfSuppressWarnings(); + $ok = copy( $source, $dest ); + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + return $status; + } + + $this->chmod( $dest ); + + return $status; + } + + /** + * @see FileBackend::doMove() + */ + protected function doMove( array $params ) { + $status = Status::newGood(); + + list( $c, $source ) = $this->resolveStoragePath( $params['src'] ); + if ( $source === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; + } + list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] ); + if ( $dest === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; + } + + if ( file_exists( $dest ) ) { + if ( !empty( $params['overwriteDest'] ) ) { + // Windows does not support moving over existing files + if ( wfIsWindows() ) { + wfSuppressWarnings(); + $ok = unlink( $dest ); + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-delete', $params['dst'] ); + return $status; + } + } + } else { + $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); + return $status; + } + } else { + if ( !wfMkdirParents( dirname( $dest ) ) ) { + $status->fatal( 'directorycreateerror', $params['dst'] ); + return $status; + } + } + + wfSuppressWarnings(); + $ok = rename( $source, $dest ); + clearstatcache(); // file no longer at source + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + return $status; + } + + return $status; + } + + /** + * @see FileBackend::doDelete() + */ + protected function doDelete( array $params ) { + $status = Status::newGood(); + + list( $c, $source ) = $this->resolveStoragePath( $params['src'] ); + if ( $source === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; + } + + if ( !is_file( $source ) ) { + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-delete', $params['src'] ); + } + return $status; // do nothing; either OK or bad status + } + + wfSuppressWarnings(); + $ok = unlink( $source ); + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-delete', $params['src'] ); + return $status; + } + + return $status; + } + + /** + * @see FileBackend::doConcatenate() + */ + protected function doConcatenate( array $params ) { + $status = Status::newGood(); + + list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] ); + if ( $dest === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; + } + + // Check if the destination file exists and we can't handle that + $destExists = file_exists( $dest ); + if ( $destExists && empty( $params['overwriteDest'] ) ) { + $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); + return $status; + } + + // Create a new temporary file... + wfSuppressWarnings(); + $tmpPath = tempnam( wfTempDir(), 'concatenate' ); + wfRestoreWarnings(); + if ( $tmpPath === false ) { + $status->fatal( 'backend-fail-createtemp' ); + return $status; + } + + // Build up that file using the source chunks (in order)... + wfSuppressWarnings(); + $tmpHandle = fopen( $tmpPath, 'a' ); + wfRestoreWarnings(); + if ( $tmpHandle === false ) { + $status->fatal( 'backend-fail-opentemp', $tmpPath ); + return $status; + } + foreach ( $params['srcs'] as $virtualSource ) { + list( $c, $source ) = $this->resolveStoragePath( $virtualSource ); + if ( $source === null ) { + fclose( $tmpHandle ); + $status->fatal( 'backend-fail-invalidpath', $virtualSource ); + return $status; + } + // Load chunk into memory (it should be a small file) + $sourceHandle = fopen( $source, 'r' ); + if ( $sourceHandle === false ) { + fclose( $tmpHandle ); + $status->fatal( 'backend-fail-read', $virtualSource ); + return $status; + } + // Append chunk to file (pass chunk size to avoid magic quotes) + if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) { + fclose( $sourceHandle ); + fclose( $tmpHandle ); + $status->fatal( 'backend-fail-writetemp', $tmpPath ); + return $status; + } + fclose( $sourceHandle ); + } + wfSuppressWarnings(); + if ( !fclose( $tmpHandle ) ) { + $status->fatal( 'backend-fail-closetemp', $tmpPath ); + return $status; + } + wfRestoreWarnings(); + + // Handle overwrite behavior of file destination if applicable. + // Note that we already checked if no overwrite params were set above. + if ( $destExists ) { + // Windows does not support moving over existing files + if ( wfIsWindows() ) { + wfSuppressWarnings(); + $ok = unlink( $dest ); + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-delete', $params['dst'] ); + return $status; + } + } + } else { + // Make sure destination directory exists + if ( !wfMkdirParents( dirname( $dest ) ) ) { + $status->fatal( 'directorycreateerror', $params['dst'] ); + return $status; + } + } + + // Rename the temporary file to the destination path + wfSuppressWarnings(); + $ok = rename( $tmpPath, $dest ); + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-move', $tmpPath, $params['dst'] ); + return $status; + } + + $this->chmod( $dest ); + + return $status; + } + + /** + * @see FileBackend::doCreate() + */ + protected function doCreate( array $params ) { + $status = Status::newGood(); + + list( $c, $dest ) = $this->resolveStoragePath( $params['dst'] ); + if ( $dest === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; + } + + if ( file_exists( $dest ) ) { + if ( !empty( $params['overwriteDest'] ) ) { + wfSuppressWarnings(); + $ok = unlink( $dest ); + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-delete', $params['dst'] ); + return $status; + } + } else { + $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); + return $status; + } + } else { + if ( !wfMkdirParents( dirname( $dest ) ) ) { + $status->fatal( 'directorycreateerror', $params['dst'] ); + return $status; + } + } + + wfSuppressWarnings(); + $ok = file_put_contents( $dest, $params['content'] ); + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-create', $params['dst'] ); + return $status; + } + + $this->chmod( $dest ); + + return $status; + } + + /** + * @see FileBackend::prepare() + */ + function prepare( array $params ) { + $status = Status::newGood(); + list( $c, $dir ) = $this->resolveStoragePath( $params['dir'] ); + if ( $dir === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); + return $status; // invalid storage path + } + if ( !wfMkdirParents( $dir ) ) { + $status->fatal( 'directorycreateerror', $params['dir'] ); + return $status; + } elseif ( !is_writable( $dir ) ) { + $status->fatal( 'directoryreadonlyerror', $params['dir'] ); + return $status; + } elseif ( !is_readable( $dir ) ) { + $status->fatal( 'directorynotreadableerror', $params['dir'] ); + return $status; + } + return $status; + } + + /** + * @see FileBackend::secure() + */ + function secure( array $params ) { + $status = Status::newGood(); + list( $c, $dir ) = $this->resolveStoragePath( $params['dir'] ); + if ( $dir === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); + return $status; // invalid storage path + } + if ( !wfMkdirParents( $dir ) ) { + $status->fatal( 'directorycreateerror', $params['dir'] ); + return $status; + } + // Add a .htaccess file to the root of the deleted zone + if ( !empty( $params['noAccess'] ) && !file_exists( "{$dir}/.htaccess" ) ) { + wfSuppressWarnings(); + $ok = file_put_contents( "{$dir}/.htaccess", "Deny from all\n" ); + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-create', $params['dir'] . '/.htaccess' ); + return $status; + } + } + // Seed new directories with a blank index.html, to prevent crawling + if ( !empty( $params['noListing'] ) && !file_exists( "{$dir}/index.html" ) ) { + wfSuppressWarnings(); + $ok = file_put_contents( "{$dir}/index.html", '' ); + wfRestoreWarnings(); + if ( !$ok ) { + $status->fatal( 'backend-fail-create', $params['dir'] . '/index.html' ); + return $status; + } + } + return $status; + } + + /** + * @see FileBackend::clean() + */ + function clean( array $params ) { + $status = Status::newGood(); + list( $c, $dir ) = $this->resolveStoragePath( $params['dir'] ); + if ( $dir === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); + return $status; // invalid storage path + } + wfSuppressWarnings(); + if ( is_dir( $dir ) ) { + rmdir( $dir ); // remove directory if empty + } + wfRestoreWarnings(); + return $status; + } + + /** + * @see FileBackend::fileExists() + */ + function fileExists( array $params ) { + list( $c, $source ) = $this->resolveStoragePath( $params['src'] ); + if ( $source === null ) { + return false; // invalid storage path + } + wfSuppressWarnings(); + $exists = is_file( $source ); + wfRestoreWarnings(); + return $exists; + } + + /** + * @see FileBackend::getFileTimestamp() + */ + function getFileTimestamp( array $params ) { + list( $c, $source ) = $this->resolveStoragePath( $params['src'] ); + if ( $source === null ) { + return false; // invalid storage path + } + $fsFile = new FSFile( $source ); + return $fsFile->getTimestamp(); + } + + /** + * @see FileBackend::getFileList() + */ + function getFileList( array $params ) { + list( $c, $dir ) = $this->resolveStoragePath( $params['dir'] ); + if ( $dir === null ) { // invalid storage path + return null; + } + wfSuppressWarnings(); + $exists = is_dir( $dir ); + wfRestoreWarnings(); + if ( !$exists ) { + return array(); // nothing under this dir + } + wfSuppressWarnings(); + $readable = is_readable( $dir ); + wfRestoreWarnings(); + if ( !$readable ) { + return null; // bad permissions? + } + return new FSFileIterator( $dir ); + } + + /** + * @see FileBackend::getLocalReference() + */ + function getLocalReference( array $params ) { + list( $c, $source ) = $this->resolveStoragePath( $params['src'] ); + if ( $source === null ) { + return null; + } + return new FSFile( $source ); + } + + /** + * @see FileBackend::getLocalCopy() + */ + function getLocalCopy( array $params ) { + list( $c, $source ) = $this->resolveStoragePath( $params['src'] ); + if ( $source === null ) { + return null; + } + + // Get source file extension + $i = strrpos( $source, '.' ); + $ext = strtolower( $i ? substr( $source, $i + 1 ) : '' ); + // Create a new temporary file... + $tmpFile = TempFSFile::factory( wfBaseName( $source ) . '_', $ext ); + if ( !$tmpFile ) { + return null; + } + $tmpPath = $tmpFile->getPath(); + + // Copy the source file over the temp file + wfSuppressWarnings(); + $ok = copy( $source, $tmpPath ); + wfRestoreWarnings(); + if ( !$ok ) { + return null; + } + + $this->chmod( $tmpPath ); + + return $tmpFile; + } + + /** + * Chmod a file, suppressing the warnings + * + * @param $path string Absolute file system path + * @return bool Success + */ + protected function chmod( $path ) { + wfSuppressWarnings(); + $ok = chmod( $path, $this->fileMode ); + wfRestoreWarnings(); + + return $ok; + } +} + +/** + * Wrapper around RecursiveDirectoryIterator that catches + * exception or does any custom behavoir that we may want. + * + * @ingroup FileBackend + */ +class FSFileIterator implements Iterator { + /** @var RecursiveIteratorIterator */ + protected $iter; + + /** + * Get an FSFileIterator from a file system directory + * + * @param $dir string + */ + public function __construct( $dir ) { + try { + $this->iter = new RecursiveIteratorIterator( new RecursiveDirectoryIterator( $dir ) ); + } catch ( UnexpectedValueException $e ) { + $this->iter = null; // bad permissions? deleted? + } + } + + public function current() { + return $this->iter->current(); + } + + public function key() { + return $this->iter->key(); + } + + public function next() { + try { + $this->iter->next(); + } catch ( UnexpectedValueException $e ) { + $this->iter = null; + } + } + + public function rewind() { + try { + $this->iter->rewind(); + } catch ( UnexpectedValueException $e ) { + $this->iter = null; + } + } + + public function valid() { + return $this->iter && $this->iter->valid(); + } +} diff --git a/includes/filerepo/backend/FileBackend.php b/includes/filerepo/backend/FileBackend.php new file mode 100644 index 0000000000..9b695b625e --- /dev/null +++ b/includes/filerepo/backend/FileBackend.php @@ -0,0 +1,835 @@ +name = $config['name']; + $this->wikiId = isset( $config['wikiId'] ) + ? $config['wikiId'] + : wfWikiID(); + $this->lockManager = LockManagerGroup::singleton()->get( $config['lockManager'] ); + } + + /** + * Get the unique backend name. + * + * We may have multiple different backends of the same type. + * For example, we can have two Swift backends using different proxies. + * + * @return string + */ + final public function getName() { + return $this->name; + } + + /** + * This is the main entry point into the backend for write operations. + * Callers supply an ordered list of operations to perform as a transaction. + * If any serious errors occur, all attempted operations will be rolled back. + * + * $ops is an array of arrays. The outer array holds a list of operations. + * Each inner array is a set of key value pairs that specify an operation. + * + * Supported operations and their parameters: + * a) Create a new file in storage with the contents of a string + * array( + * 'op' => 'create', + * 'dst' => , + * 'content' => , + * 'overwriteDest' => , + * 'overwriteSame' => + * ) + * b) Copy a file system file into storage + * array( + * 'op' => 'store', + * 'src' => , + * 'dst' => , + * 'overwriteDest' => , + * 'overwriteSame' => + * ) + * c) Copy a file within storage + * array( + * 'op' => 'copy', + * 'src' => , + * 'dst' => , + * 'overwriteDest' => , + * 'overwriteSame' => + * ) + * d) Move a file within storage + * array( + * 'op' => 'move', + * 'src' => , + * 'dst' => , + * 'overwriteDest' => , + * 'overwriteSame' => + * ) + * e) Delete a file within storage + * array( + * 'op' => 'delete', + * 'src' => , + * 'ignoreMissingSource' => + * ) + * f) Concatenate a list of files into a single file within storage + * array( + * 'op' => 'concatenate', + * 'srcs' => , + * 'dst' => , + * 'overwriteDest' => + * ) + * g) Do nothing (no-op) + * array( + * 'op' => 'null', + * ) + * + * Boolean flags for operations (operation-specific): + * 'ignoreMissingSource' : The operation will simply succeed and do + * nothing if the source file does not exist. + * 'overwriteDest' : Any destination file will be overwritten. + * 'overwriteSame' : An error will not be given if a file already + * exists at the destination that has the same + * contents as the new contents to be written there. + * + * $opts is an associative of options, including: + * 'nonLocking' : No locks are acquired for the operations. + * This can increase performance for non-critical writes. + * 'ignoreErrors' : Serious errors that would normally cause a rollback + * do not. The remaining operations are still attempted. + * + * Return value: + * This returns a Status, which contains all warnings and fatals that occured + * during the operation. The 'failCount', 'successCount', and 'success' members + * will reflect each operation attempted. The status will be "OK" unless any + * of the operations failed and the 'ignoreErrors' parameter was not set. + * + * @param $ops Array List of operations to execute in order + * @param $opts Array Batch operation options + * @return Status + */ + abstract public function doOperations( array $ops, array $opts = array() ); + + /** + * Same as doOperations() except it takes a single operation array + * + * @param $op Array + * @param $opts Array + * @return Status + */ + final public function doOperation( array $op, array $opts = array() ) { + return $this->doOperations( array( $op ), $opts ); + } + + /** + * Prepare a storage path for usage. This will create containers + * that don't yet exist or, on FS backends, create parent directories. + * + * $params include: + * dir : storage directory + * + * @param $params Array + * @return Status + */ + abstract public function prepare( array $params ); + + /** + * Take measures to block web access to a directory and + * the container it belongs to. FS backends might add .htaccess + * files wheras backends like Swift this might restrict container + * access to backend user that represents end-users in web request. + * This is not guaranteed to actually do anything. + * + * $params include: + * dir : storage directory + * noAccess : try to deny file access + * noListing : try to deny file listing + * + * @param $params Array + * @return Status + */ + abstract public function secure( array $params ); + + /** + * Clean up an empty storage directory. + * On FS backends, the directory will be deleted. Others may do nothing. + * + * $params include: + * dir : storage directory + * + * @param $params Array + * @return Status + */ + abstract public function clean( array $params ); + + /** + * Check if a file exists at a storage path in the backend. + * + * $params include: + * src : source storage path + * + * @param $params Array + * @return bool + */ + abstract public function fileExists( array $params ); + + /** + * Get a SHA-1 hash of the file at a storage path in the backend. + * + * $params include: + * src : source storage path + * + * @param $params Array + * @return string|false Hash string or false on failure + */ + abstract public function getFileSha1Base36( array $params ); + + /** + * Get the last-modified timestamp of the file at a storage path. + * + * $params include: + * src : source storage path + * + * @param $params Array + * @return string|false TS_MW timestamp or false on failure + */ + abstract public function getFileTimestamp( array $params ); + + /** + * Get the properties of the file at a storage path in the backend. + * Returns FSFile::placeholderProps() on failure. + * + * $params include: + * src : source storage path + * + * @param $params Array + * @return Array + */ + abstract public function getFileProps( array $params ); + + /** + * Stream the file at a storage path in the backend. + * Appropriate HTTP headers (Status, Content-Type, Content-Length) + * must be sent if streaming began, while none should be sent otherwise. + * Implementations should flush the output buffer before sending data. + * + * $params include: + * src : source storage path + * headers : additional HTTP headers to send on success + * + * @param $params Array + * @return Status + */ + abstract public function streamFile( array $params ); + + /** + * Get an iterator to list out all object files under a storage directory. + * If the directory is of the form "mwstore://container", then all items in + * the container should be listed. If of the form "mwstore://container/dir", + * then all items under that container directory should be listed. + * Results should be storage paths relative to the given directory. + * + * $params include: + * dir : storage path directory + * + * @return Traversable|Array|null Returns null on failure + */ + abstract public function getFileList( array $params ); + + /** + * Returns a file system file, identical to the file at a storage path. + * The file returned is either: + * a) A local copy of the file at a storage path in the backend. + * The temporary copy will have the same extension as the source. + * b) An original of the file at a storage path in the backend. + * Temporary files may be purged when the file object falls out of scope. + * + * Write operations should *never* be done on this file as some backends + * may do internal tracking or may be instances of FileBackendMultiWrite. + * In that later case, there are copies of the file that must stay in sync. + * + * $params include: + * src : source storage path + * + * @param $params Array + * @return FSFile|null Returns null on failure + */ + abstract public function getLocalReference( array $params ); + + /** + * Get a local copy on disk of the file at a storage path in the backend. + * The temporary copy will have the same file extension as the source. + * Temporary files may be purged when the file object falls out of scope. + * + * $params include: + * src : source storage path + * + * @param $params Array + * @return TempFSFile|null Returns null on failure + */ + abstract public function getLocalCopy( array $params ); + + /** + * Lock the files at the given storage paths in the backend. + * This will either lock all the files or none (on failure). + * + * Callers should consider using getScopedFileLocks() instead. + * + * @param $paths Array Storage paths + * @param $type integer LockManager::LOCK_EX, LockManager::LOCK_SH + * @return Status + */ + final public function lockFiles( array $paths, $type ) { + return $this->lockManager->lock( $paths, $type ); + } + + /** + * Unlock the files at the given storage paths in the backend. + * + * @param $paths Array Storage paths + * @param $type integer LockManager::LOCK_EX, LockManager::LOCK_SH + * @return Status + */ + final public function unlockFiles( array $paths, $type ) { + return $this->lockManager->unlock( $paths, $type ); + } + + /** + * Lock the files at the given storage paths in the backend. + * This will either lock all the files or none (on failure). + * On failure, the status object will be updated with errors. + * + * Once the return value goes out scope, the locks will be released and + * the status updated. Unlock fatals will not change the status "OK" value. + * + * @param $paths Array Storage paths + * @param $type integer LockManager::LOCK_EX, LockManager::LOCK_SH + * @param $status Status Status to update on lock/unlock + * @return ScopedLock|null Returns null on failure + */ + final public function getScopedFileLocks( array $paths, $type, Status $status ) { + return ScopedLock::factory( $this->lockManager, $paths, $type, $status ); + } +} + +/** + * Base class for all single-write backends. + * This class defines the methods as abstract that subclasses must implement. + * + * @ingroup FileBackend + * @since 1.19 + */ +abstract class FileBackend extends FileBackendBase { + /** @var Array */ + protected $cache = array(); // (storage path => key => value) + protected $maxCacheSize = 50; // integer; max paths with entries + + /** + * Store a file into the backend from a file on disk. + * Do not call this function from places outside FileBackend and FileOp. + * $params include: + * src : source path on disk + * dst : destination storage path + * overwriteDest : do nothing and pass if an identical file exists at destination + * + * @param $params Array + * @return Status + */ + final public function store( array $params ) { + $status = $this->doStore( $params ); + $this->clearCache( array( $params['dst'] ) ); + return $status; + } + + /** + * @see FileBackend::store() + */ + abstract protected function doStore( array $params ); + + /** + * Copy a file from one storage path to another in the backend. + * Do not call this function from places outside FileBackend and FileOp. + * $params include: + * src : source storage path + * dst : destination storage path + * overwriteDest : do nothing and pass if an identical file exists at destination + * + * @param $params Array + * @return Status + */ + final public function copy( array $params ) { + $status = $this->doCopy( $params ); + $this->clearCache( array( $params['dst'] ) ); + return $status; + } + + /** + * @see FileBackend::copy() + */ + abstract protected function doCopy( array $params ); + + /** + * Delete a file at the storage path. + * Do not call this function from places outside FileBackend and FileOp. + * $params include: + * src : source storage path + * + * @param $params Array + * @return Status + */ + final public function delete( array $params ) { + $status = $this->doDelete( $params ); + $this->clearCache( array( $params['src'] ) ); + return $status; + } + + /** + * @see FileBackend::delete() + */ + abstract protected function doDelete( array $params ); + + /** + * Move a file from one storage path to another in the backend. + * Do not call this function from places outside FileBackend and FileOp. + * $params include: + * src : source storage path + * dst : destination storage path + * overwriteDest : do nothing and pass if an identical file exists at destination + * + * @param $params Array + * @return Status + */ + final public function move( array $params ) { + $status = $this->doMove( $params ); + $this->clearCache( array( $params['src'], $params['dst'] ) ); + return $status; + } + + /** + * @see FileBackend::move() + */ + protected function doMove( array $params ) { + // Copy source to dest + $status = $this->backend->copy( $params ); + if ( !$status->isOK() ) { + return $status; + } + // Delete source (only fails due to races or medium going down) + $status->merge( $this->backend->delete( array( 'src' => $params['src'] ) ) ); + $status->setResult( true, $status->value ); // ignore delete() errors + return $status; + } + + /** + * Combines files from several storage paths into a new file in the backend. + * Do not call this function from places outside FileBackend and FileOp. + * $params include: + * srcs : ordered source storage paths (e.g. chunk1, chunk2, ...) + * dst : destination storage path + * overwriteDest : do nothing and pass if an identical file exists at destination + * + * @param $params Array + * @return Status + */ + final public function concatenate( array $params ) { + $status = $this->doConcatenate( $params ); + $this->clearCache( array( $params['dst'] ) ); + return $status; + } + + /** + * @see FileBackend::concatenate() + */ + abstract protected function doConcatenate( array $params ); + + /** + * Create a file in the backend with the given contents. + * Do not call this function from places outside FileBackend and FileOp. + * $params include: + * content : the raw file contents + * dst : destination storage path + * overwriteDest : do nothing and pass if an identical file exists at destination + * + * @param $params Array + * @return Status + */ + final public function create( array $params ) { + $status = $this->doCreate( $params ); + $this->clearCache( array( $params['dst'] ) ); + return $status; + } + + /** + * @see FileBackend::create() + */ + abstract protected function doCreate( array $params ); + + /** + * @see FileBackendBase::prepare() + */ + public function prepare( array $params ) { + return Status::newGood(); + } + + /** + * @see FileBackendBase::secure() + */ + public function secure( array $params ) { + return Status::newGood(); + } + + /** + * @see FileBackendBase::clean() + */ + public function clean( array $params ) { + return Status::newGood(); + } + + /** + * @see FileBackendBase::getFileSha1Base36() + */ + public function getFileSha1Base36( array $params ) { + $path = $params['src']; + if ( isset( $this->cache[$path]['sha1'] ) ) { + return $this->cache[$path]['sha1']; + } + $fsFile = $this->getLocalReference( array( 'src' => $path ) ); + if ( !$fsFile ) { + return false; + } else { + $sha1 = $fsFile->getSha1Base36(); + if ( $sha1 !== false ) { // don't cache negatives + $this->trimCache(); // limit memory + $this->cache[$path]['sha1'] = $sha1; + } + return $sha1; + } + } + + /** + * @see FileBackendBase::getFileProps() + */ + public function getFileProps( array $params ) { + $fsFile = $this->getLocalReference( array( 'src' => $params['src'] ) ); + if ( !$fsFile ) { + return FSFile::placeholderProps(); + } else { + return $fsFile->getProps(); + } + } + + /** + * @see FileBackendBase::getLocalReference() + */ + public function getLocalReference( array $params ) { + return $this->getLocalCopy( $params ); + } + + /** + * @see FileBackendBase::streamFile() + */ + function streamFile( array $params ) { + $status = Status::newGood(); + + $fsFile = $this->getLocalReference( array( 'src' => $params['src'] ) ); + if ( !$fsFile ) { + $status->fatal( 'backend-fail-stream', $params['src'] ); + return $status; + } + + $extraHeaders = isset( $params['headers'] ) + ? $params['headers'] + : array(); + + $ok = StreamFile::stream( $fsFile->getPath(), $extraHeaders, false ); + if ( !$ok ) { + $status->fatal( 'backend-fail-stream', $params['src'] ); + return $status; + } + + return $status; + } + + /** + * Get the list of supported operations and their corresponding FileOp classes. + * + * @return Array + */ + protected function supportedOperations() { + return array( + 'store' => 'StoreFileOp', + 'copy' => 'CopyFileOp', + 'move' => 'MoveFileOp', + 'delete' => 'DeleteFileOp', + 'concatenate' => 'ConcatenateFileOp', + 'create' => 'CreateFileOp', + 'null' => 'NullFileOp' + ); + } + + /** + * Return a list of FileOp objects from a list of operations. + * The result must have the same number of items as the input. + * An exception is thrown if an unsupported operation is requested. + * + * @param $ops Array Same format as doOperations() + * @return Array List of FileOp objects + * @throws MWException + */ + final public function getOperations( array $ops ) { + $supportedOps = $this->supportedOperations(); + + $performOps = array(); // array of FileOp objects + // Build up ordered array of FileOps... + foreach ( $ops as $operation ) { + $opName = $operation['op']; + if ( isset( $supportedOps[$opName] ) ) { + $class = $supportedOps[$opName]; + // Get params for this operation + $params = $operation; + // Append the FileOp class + $performOps[] = new $class( $this, $params ); + } else { + throw new MWException( "Operation `$opName` is not supported." ); + } + } + + return $performOps; + } + + /** + * @see FileBackendBase::doOperations() + */ + final public function doOperations( array $ops, array $opts = array() ) { + $status = Status::newGood(); + + // Build up a list of FileOps... + $performOps = $this->getOperations( $ops ); + + if ( empty( $opts['nonLocking'] ) ) { + // Build up a list of files to lock... + $filesLockEx = $filesLockSh = array(); + foreach ( $performOps as $index => $fileOp ) { + $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() ); + $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() ); + } + // Try to lock those files for the scope of this function... + $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status ); + $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status ); + if ( !$status->isOK() ) { + return $status; // abort + } + } + + // Clear any cache entries (after locks acquired) + $this->clearCache(); + // Actually attempt the operation batch... + $status->merge( FileOp::attemptBatch( $performOps, $opts ) ); + + return $status; + } + + /** + * Invalidate the file existence and property cache + * + * @param $paths Array Clear cache for specific files + * @return void + */ + final public function clearCache( array $paths = null ) { + if ( $paths === null ) { + $this->cache = array(); + } else { + foreach ( $paths as $path ) { + unset( $this->cache[$path] ); + } + } + } + + /** + * Prune the cache if it is too big to add an item + * + * @return void + */ + protected function trimCache() { + if ( count( $this->cache ) >= $this->maxCacheSize ) { + reset( $this->cache ); + $key = key( $this->cache ); + unset( $this->cache[$key] ); + } + } + + /** + * Check if a given path is a mwstore:// path. + * This does not do any actual validation or existence checks. + * + * @param $path string + * @return bool + */ + final public static function isStoragePath( $path ) { + return ( strpos( $path, 'mwstore://' ) === 0 ); + } + + /** + * Split a storage path (e.g. "mwstore://backend/container/path/to/object") + * into a backend name, a container name, and a relative object path. + * + * @param $storagePath string + * @return Array (backend, container, rel object) or (null, null, null) + */ + final public static function splitStoragePath( $storagePath ) { + if ( self::isStoragePath( $storagePath ) ) { + // Note: strlen( 'mwstore://' ) = 10 + $parts = explode( '/', substr( $storagePath, 10 ), 3 ); + if ( count( $parts ) == 3 ) { + return $parts; // e.g. "backend/container/path" + } elseif ( count( $parts ) == 2 ) { + return array( $parts[0], $parts[1], '' ); // e.g. "backend/container" + } + } + return array( null, null, null ); + } + + /** + * Validate a container name. + * Null is returned if the name has illegal characters. + * + * @param $container string + * @return bool + */ + final protected static function isValidContainerName( $container ) { + // This accounts for Swift and S3 restrictions. Also note + // that these urlencode to the same string, which is useful + // since the Swift size limit is *after* URL encoding. + return preg_match( '/^[a-zA-Z0-9._-]{1,256}$/u', $container ); + } + + /** + * Validate and normalize a relative storage path. + * Null is returned if the path involves directory traversal. + * Traversal is insecure for FS backends and broken for others. + * + * @param $path string + * @return string|null + */ + final protected static function normalizeStoragePath( $path ) { + // Normalize directory separators + $path = strtr( $path, '\\', '/' ); + // Use the same traversal protection as Title::secureAndSplit() + if ( strpos( $path, '.' ) !== false ) { + if ( + $path === '.' || + $path === '..' || + strpos( $path, './' ) === 0 || + strpos( $path, '../' ) === 0 || + strpos( $path, '/./' ) !== false || + strpos( $path, '/../' ) !== false + ) { + return null; + } + } + return $path; + } + + /** + * Split a storage path (e.g. "mwstore://backend/container/path/to/object") + * into an internal container name and an internal relative object name. + * This also checks that the storage path is valid and is within this backend. + * + * @param $storagePath string + * @return Array (container, object name) or (null, null) if path is invalid + */ + final protected function resolveStoragePath( $storagePath ) { + list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath ); + if ( $backend === $this->name ) { // must be for this backend + $relPath = self::normalizeStoragePath( $relPath ); + if ( $relPath !== null ) { + $relPath = $this->resolveContainerPath( $container, $relPath ); + if ( $relPath !== null ) { + $container = $this->fullContainerName( $container ); + if ( self::isValidContainerName( $container ) ) { + $container = $this->resolveContainerName( $container ); + if ( $container !== null ) { + return array( $container, $relPath ); + } + } + } + } + } + return array( null, null ); + } + + /** + * Get the full container name, including the wiki ID prefix + * + * @param $container string + * @return string + */ + final protected function fullContainerName( $container ) { + if ( $this->wikiId != '' ) { + return "{$this->wikiId}-$container"; + } else { + return $container; + } + } + + /** + * Resolve a container name, checking if it's allowed by the backend. + * This is intended for internal use, such as encoding illegal chars. + * Subclasses can override this to be more restrictive. + * + * @param $container string + * @return string|null + */ + protected function resolveContainerName( $container ) { + return $container; + } + + /** + * Resolve a relative storage path, checking if it's allowed by the backend. + * This is intended for internal use, such as encoding illegal chars + * or perhaps getting absolute paths (e.g. FS based backends). + * + * @param $container string Container the path is relative to + * @param $relStoragePath string Relative storage path + * @return string|null Path or null if not valid + */ + protected function resolveContainerPath( $container, $relStoragePath ) { + return $relStoragePath; + } +} diff --git a/includes/filerepo/backend/FileBackendGroup.php b/includes/filerepo/backend/FileBackendGroup.php new file mode 100644 index 0000000000..a0c1e85b7b --- /dev/null +++ b/includes/filerepo/backend/FileBackendGroup.php @@ -0,0 +1,96 @@ + ('class' =>, 'config' =>, 'instance' =>)) */ + protected $backends = array(); + + protected function __construct() {} + protected function __clone() {} + + public static function singleton() { + if ( self::$instance == null ) { + self::$instance = new self(); + } + return self::$instance; + } + + /** + * Destroy the singleton instance, so that a new one will be created next + * time singleton() is called. + */ + public static function destroySingleton() { + self::$instance = null; + } + + /** + * Register an array of file backend configurations + * + * @param $configs Array + * @return void + * @throws MWException + */ + public function register( array $configs ) { + foreach ( $configs as $config ) { + if ( !isset( $config['name'] ) ) { + throw new MWException( "Cannot register a backend with no name." ); + } + $name = $config['name']; + if ( !isset( $config['class'] ) ) { + throw new MWException( "Cannot register backend `{$name}` with no class." ); + } + $class = $config['class']; + + unset( $config['class'] ); // backend won't need this + $this->backends[$name] = array( + 'class' => $class, + 'config' => $config, + 'instance' => null + ); + } + } + + /** + * Get the backend object with a given name + * + * @param $name string + * @return FileBackendBase + * @throws MWException + */ + public function get( $name ) { + if ( !isset( $this->backends[$name] ) ) { + throw new MWException( "No backend defined with the name `$name`." ); + } + // Lazy-load the actual backend instance + if ( !isset( $this->backends[$name]['instance'] ) ) { + $class = $this->backends[$name]['class']; + $config = $this->backends[$name]['config']; + $this->backends[$name]['instance'] = new $class( $config ); + } + return $this->backends[$name]['instance']; + } + + /** + * Get an appropriate backend object from a storage path + * + * @param $storagePath string + * @return FileBackendBase|null Backend or null on failure + */ + public function backendFromPath( $storagePath ) { + list( $backend, $c, $p ) = FileBackend::splitStoragePath( $storagePath ); + if ( $backend !== null && isset( $this->backends[$backend] ) ) { + return $this->get( $backend ); + } + return null; + } +} diff --git a/includes/filerepo/backend/FileBackendMultiWrite.php b/includes/filerepo/backend/FileBackendMultiWrite.php new file mode 100644 index 0000000000..e361ed22c5 --- /dev/null +++ b/includes/filerepo/backend/FileBackendMultiWrite.php @@ -0,0 +1,267 @@ + backends) + protected $masterIndex = -1; // index of master backend + + /** + * Construct a proxy backend that consists of several internal backends. + * $config contains: + * 'name' : The name of the proxy backend + * 'lockManager' : Registered name of the file lock manager to use + * 'backends' : Array of backend config and multi-backend settings. + * Each value is the config used in the constructor of a + * FileBackend class, but with these additional settings: + * 'class' : The name of the backend class + * 'isMultiMaster' : This must be set for one backend. + * @param $config Array + */ + public function __construct( array $config ) { + parent::__construct( $config ); + // Construct backends here rather than via registration + // to keep these backends hidden from outside the proxy. + foreach ( $config['backends'] as $index => $config ) { + if ( !isset( $config['class'] ) ) { + throw new MWException( 'No class given for a backend config.' ); + } + $class = $config['class']; + $this->fileBackends[$index] = new $class( $config ); + if ( !empty( $config['isMultiMaster'] ) ) { + if ( $this->masterIndex >= 0 ) { + throw new MWException( 'More than one master backend defined.' ); + } + $this->masterIndex = $index; + } + } + if ( $this->masterIndex < 0 ) { // need backends and must have a master + throw new MWException( 'No master backend defined.' ); + } + } + + final public function doOperations( array $ops, array $opts = array() ) { + $status = Status::newGood(); + + $performOps = array(); // list of FileOp objects + $filesLockEx = $filesLockSh = array(); // storage paths to lock + // Build up a list of FileOps. The list will have all the ops + // for one backend, then all the ops for the next, and so on. + // These batches of ops are all part of a continuous array. + // Also build up a list of files to lock... + foreach ( $this->fileBackends as $index => $backend ) { + $backendOps = $this->substOpPaths( $ops, $backend ); + $performOps = array_merge( $performOps, $backend->getOperations( $backendOps ) ); + if ( $index == 0 && empty( $opts['nonLocking'] ) ) { + // Set "files to lock" from the first batch so we don't try to set all + // locks two or three times over (depending on the number of backends). + // A lock on one storage path is a lock on all the backends. + foreach ( $performOps as $index => $fileOp ) { + $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() ); + $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() ); + } + // Lock the paths under the proxy backend's name + $this->unsubstPaths( $filesLockSh ); + $this->unsubstPaths( $filesLockEx ); + } + } + + // Try to lock those files for the scope of this function... + $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status ); + $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status ); + if ( !$status->isOK() ) { + return $status; // abort + } + + // Clear any cache entries (after locks acquired) + foreach ( $this->fileBackends as $backend ) { + $backend->clearCache(); + } + // Actually attempt the operation batch... + $status->merge( FileOp::attemptBatch( $performOps, $opts ) ); + + return $status; + } + + /** + * Substitute the backend name in storage path parameters + * for a set of operations with a that of a given backend. + * + * @param $ops Array List of file operation arrays + * @param $backend FileBackend + * @return Array + */ + protected function substOpPaths( array $ops, FileBackend $backend ) { + $newOps = array(); // operations + foreach ( $ops as $op ) { + $newOp = $op; // operation + foreach ( array( 'src', 'srcs', 'dst' ) as $par ) { + if ( isset( $newOp[$par] ) ) { + $newOp[$par] = preg_replace( + '!^mwstore://' . preg_quote( $this->name ) . '/!', + 'mwstore://' . $backend->getName() . '/', + $newOp[$par] // string or array + ); + } + } + $newOps[] = $newOp; + } + return $newOps; + } + + /** + * Replace the backend part of storage paths with this backend's name + * + * @param &$paths Array + * @return void + */ + protected function unsubstPaths( array &$paths ) { + foreach ( $paths as &$path ) { + $path = preg_replace( '!^mwstore://([^/]+)!', "mwstore://{$this->name}", $path ); + } + } + + function prepare( array $params ) { + $status = Status::newGood(); + foreach ( $this->backends as $backend ) { + $realParams = $this->substOpPaths( $params, $backend ); + $status->merge( $backend->prepare( $realParams ) ); + } + return $status; + } + + function secure( array $params ) { + $status = Status::newGood(); + foreach ( $this->backends as $backend ) { + $realParams = $this->substOpPaths( $params, $backend ); + $status->merge( $backend->secure( $realParams ) ); + } + return $status; + } + + function clean( array $params ) { + $status = Status::newGood(); + foreach ( $this->backends as $backend ) { + $realParams = $this->substOpPaths( $params, $backend ); + $status->merge( $backend->clean( $realParams ) ); + } + return $status; + } + + function fileExists( array $params ) { + # Hit all backends in case of failed operations (out of sync) + foreach ( $this->backends as $backend ) { + $realParams = $this->substOpPaths( $params, $backend ); + if ( $backend->fileExists( $realParams ) ) { + return true; + } + } + return false; + } + + function getFileTimestamp( array $params ) { + // Skip non-master for consistent timestamps + $realParams = $this->substOpPaths( $params, $backend ); + return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams ); + } + + function getFileSha1Base36( array $params ) { + # Hit all backends in case of failed operations (out of sync) + foreach ( $this->backends as $backend ) { + $realParams = $this->substOpPaths( $params, $backend ); + $hash = $backend->getFileSha1Base36( $realParams ); + if ( $hash !== false ) { + return $hash; + } + } + return false; + } + + function getFileProps( array $params ) { + # Hit all backends in case of failed operations (out of sync) + foreach ( $this->backends as $backend ) { + $realParams = $this->substOpPaths( $params, $backend ); + $props = $backend->getFileProps( $realParams ); + if ( $props !== null ) { + return $props; + } + } + return null; + } + + function streamFile( array $params ) { + $status = Status::newGood(); + foreach ( $this->backends as $backend ) { + $realParams = $this->substOpPaths( $params, $backend ); + $subStatus = $backend->streamFile( $realParams ); + $status->merge( $subStatus ); + if ( $subStatus->isOK() ) { + // Pass isOK() despite fatals from other backends + $status->setResult( true ); + return $status; + } else { // failure + if ( headers_sent() ) { + return $status; // died mid-stream...so this is already fubar + } elseif ( strval( ob_get_contents() ) !== '' ) { + ob_clean(); // output was buffered but not sent; clear it + } + } + } + return $status; + } + + function getLocalReference( array $params ) { + # Hit all backends in case of failed operations (out of sync) + foreach ( $this->backends as $backend ) { + $realParams = $this->substOpPaths( $params, $backend ); + $fsFile = $backend->getLocalReference( $realParams ); + if ( $fsFile ) { + return $fsFile; + } + } + return null; + } + + function getLocalCopy( array $params ) { + # Hit all backends in case of failed operations (out of sync) + foreach ( $this->backends as $backend ) { + $realParams = $this->substOpPaths( $params, $backend ); + $tmpFile = $backend->getLocalCopy( $realParams ); + if ( $tmpFile ) { + return $tmpFile; + } + } + return null; + } + + function getFileList( array $params ) { + foreach ( $this->backends as $index => $backend ) { + # Get results from the first backend + $realParams = $this->substOpPaths( $params, $backend ); + return $backend->getFileList( $realParams ); + } + return array(); // sanity + } +} diff --git a/includes/filerepo/backend/FileOp.php b/includes/filerepo/backend/FileOp.php new file mode 100644 index 0000000000..d5059d6096 --- /dev/null +++ b/includes/filerepo/backend/FileOp.php @@ -0,0 +1,922 @@ +backend = $backend; + foreach ( $this->allowedParams() as $name ) { + if ( isset( $params[$name] ) ) { + $this->params[$name] = $params[$name]; + } + } + $this->params = $params; + } + + /** + * Disable file backups for this operation + * + * @return void + */ + final protected function disableBackups() { + $this->useBackups = false; + } + + /** + * Attempt a series of file operations. + * Callers are responsible for handling file locking. + * + * @param $performOps Array List of FileOp operations + * @param $opts Array Batch operation options + * @return Status + */ + final public static function attemptBatch( array $performOps, array $opts ) { + $status = Status::newGood(); + + $ignoreErrors = isset( $opts['ignoreErrors'] ) && $opts['ignoreErrors']; + $predicates = FileOp::newPredicates(); // account for previous op in prechecks + // Do pre-checks for each operation; abort on failure... + foreach ( $performOps as $index => $fileOp ) { + $status->merge( $fileOp->precheck( $predicates ) ); + if ( !$status->isOK() ) { // operation failed? + if ( $ignoreErrors ) { + ++$status->failCount; + $status->success[$index] = false; + } else { + return $status; + } + } + } + + // Attempt each operation; abort on failure... + foreach ( $performOps as $index => $fileOp ) { + if ( $fileOp->failed() ) { + continue; // nothing to do + } elseif ( $ignoreErrors ) { + $fileOp->disableBackups(); // no chance of revert() calls + } + $status->merge( $fileOp->attempt() ); + if ( !$status->isOK() ) { // operation failed? + if ( $ignoreErrors ) { + ++$status->failCount; + $status->success[$index] = false; + } else { + // Revert everything done so far and abort. + // Do this by reverting each previous operation in reverse order. + $pos = $index - 1; // last one failed; no need to revert() + while ( $pos >= 0 ) { + if ( !$performOps[$pos]->failed() ) { + $status->merge( $performOps[$pos]->revert() ); + } + $pos--; + } + return $status; + } + } + } + + // Finish each operation... + foreach ( $performOps as $index => $fileOp ) { + if ( $fileOp->failed() ) { + continue; // nothing to do + } + $subStatus = $fileOp->finish(); + if ( $subStatus->isOK() ) { + ++$status->successCount; + $status->success[$index] = true; + } else { + ++$status->failCount; + $status->success[$index] = false; + } + $status->merge( $subStatus ); + } + + // Make sure status is OK, despite any finish() fatals + $status->setResult( true, $status->value ); + + return $status; + } + + /** + * Get the value of the parameter with the given name. + * Returns null if the parameter is not set. + * + * @param $name string + * @return mixed + */ + final public function getParam( $name ) { + return isset( $this->params[$name] ) ? $this->params[$name] : null; + } + + /** + * Check if this operation failed precheck() or attempt() + * @return type + */ + final public function failed() { + return $this->failed; + } + + /** + * Get a new empty predicates array for precheck() + * + * @return Array + */ + final public static function newPredicates() { + return array( 'exists' => array() ); + } + + /** + * Check preconditions of the operation without writing anything + * + * @param $predicates Array + * @return Status + */ + final public function precheck( array &$predicates ) { + if ( $this->state !== self::STATE_NEW ) { + return Status::newFatal( 'fileop-fail-state', self::STATE_NEW, $this->state ); + } + $this->state = self::STATE_CHECKED; + $status = $this->doPrecheck( $predicates ); + if ( !$status->isOK() ) { + $this->failed = true; + } + return $status; + } + + /** + * Attempt the operation, backing up files as needed; this must be reversible + * + * @return Status + */ + final public function attempt() { + if ( $this->state !== self::STATE_CHECKED ) { + return Status::newFatal( 'fileop-fail-state', self::STATE_CHECKED, $this->state ); + } elseif ( $this->failed ) { // failed precheck + return Status::newFatal( 'fileop-fail-attempt-precheck' ); + } + $this->state = self::STATE_ATTEMPTED; + $status = $this->doAttempt(); + if ( !$status->isOK() ) { + $this->failed = true; + $this->logFailure( 'attempt' ); + } + return $status; + } + + /** + * Revert the operation; affected files are restored + * + * @return Status + */ + final public function revert() { + if ( $this->state !== self::STATE_ATTEMPTED ) { + return Status::newFatal( 'fileop-fail-state', self::STATE_ATTEMPTED, $this->state ); + } + $this->state = self::STATE_DONE; + if ( $this->failed ) { + $status = Status::newGood(); // nothing to revert + } else { + $status = $this->doRevert(); + if ( !$status->isOK() ) { + $this->logFailure( 'revert' ); + } + } + return $status; + } + + /** + * Finish the operation; this may be irreversible + * + * @return Status + */ + final public function finish() { + if ( $this->state !== self::STATE_ATTEMPTED ) { + return Status::newFatal( 'fileop-fail-state', self::STATE_ATTEMPTED, $this->state ); + } + $this->state = self::STATE_DONE; + if ( $this->failed ) { + $status = Status::newGood(); // nothing to finish + } else { + $status = $this->doFinish(); + } + return $status; + } + + /** + * Get a list of storage paths read from for this operation + * + * @return Array + */ + public function storagePathsRead() { + return array(); + } + + /** + * Get a list of storage paths written to for this operation + * + * @return Array + */ + public function storagePathsChanged() { + return array(); + } + + /** + * @return Array List of allowed parameters + */ + protected function allowedParams() { + return array(); + } + + /** + * @return Status + */ + protected function doPrecheck( array &$predicates ) { + return Status::newGood(); + } + + /** + * @return Status + */ + abstract protected function doAttempt(); + + /** + * @return Status + */ + abstract protected function doRevert(); + + /** + * @return Status + */ + protected function doFinish() { + return Status::newGood(); + } + + /** + * Check if the destination file exists and update the + * destAlreadyExists member variable. A bad status will + * be returned if there is no chance it can be overwritten. + * + * @param $predicates Array + * @return Status + */ + protected function precheckDestExistence( array $predicates ) { + $status = Status::newGood(); + if ( $this->fileExists( $this->params['dst'], $predicates ) ) { + $this->destAlreadyExists = true; + if ( !$this->getParam( 'overwriteDest' ) && !$this->getParam( 'overwriteSame' ) ) { + $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] ); + return $status; + } + } else { + $this->destAlreadyExists = false; + } + return $status; + } + + /** + * Backup any file at the source to a temporary file + * + * @return Status + */ + protected function backupSource() { + $status = Status::newGood(); + if ( $this->useBackups ) { + // Check if a file already exists at the source... + $params = array( 'src' => $this->params['src'] ); + if ( $this->backend->fileExists( $params ) ) { + // Create a temporary backup copy... + $this->tmpSourcePath = $this->backend->getLocalCopy( $params ); + if ( $this->tmpSourcePath === null ) { + $status->fatal( 'backend-fail-backup', $this->params['src'] ); + return $status; + } + } + } + return $status; + } + + /** + * Backup the file at the destination to a temporary file. + * Don't bother backing it up unless we might overwrite the file. + * This assumes that the destination is in the backend and that + * the source is either in the backend or on the file system. + * This also handles the 'overwriteSame' check logic and updates + * the destSameAsSource member variable. + * + * @return Status + */ + protected function checkAndBackupDest() { + $status = Status::newGood(); + $this->destSameAsSource = false; + + if ( $this->getParam( 'overwriteDest' ) ) { + if ( $this->useBackups ) { + // Create a temporary backup copy... + $params = array( 'src' => $this->params['dst'] ); + $this->tmpDestFile = $this->backend->getLocalCopy( $params ); + if ( !$this->tmpDestFile ) { + $status->fatal( 'backend-fail-backup', $this->params['dst'] ); + return $status; + } + } + } elseif ( $this->getParam( 'overwriteSame' ) ) { + $shash = $this->getSourceSha1Base36(); + // If there is a single source, then we can do some checks already. + // For things like concatenate(), we would need to build a temp file + // first and thus don't support 'overwriteSame' ($shash is null). + if ( $shash !== null ) { + $dhash = $this->getFileSha1Base36( $this->params['dst'] ); + if ( !strlen( $shash ) || !strlen( $dhash ) ) { + $status->fatal( 'backend-fail-hashes' ); + } elseif ( $shash !== $dhash ) { + // Give an error if the files are not identical + $status->fatal( 'backend-fail-notsame', $this->params['dst'] ); + } else { + $this->destSameAsSource = true; + } + return $status; // do nothing; either OK or bad status + } + } else { + $status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] ); + return $status; + } + + return $status; + } + + /** + * checkAndBackupDest() helper function to get the source file Sha1. + * Returns false on failure and null if there is no single source. + * + * @return string|false|null + */ + protected function getSourceSha1Base36() { + return null; // N/A + } + + /** + * checkAndBackupDest() helper function to get the Sha1 of a file. + * + * @return string|false False on failure + */ + protected function getFileSha1Base36( $path ) { + // Source file is in backend + if ( FileBackend::isStoragePath( $path ) ) { + // For some backends (e.g. Swift, Azure) we can get + // standard hashes to use for this types of comparisons. + $hash = $this->backend->getFileSha1Base36( array( 'src' => $path ) ); + // Source file is on file system + } else { + wfSuppressWarnings(); + $hash = sha1_file( $path ); + wfRestoreWarnings(); + if ( $hash !== false ) { + $hash = wfBaseConvert( $hash, 16, 36, 31 ); + } + } + return $hash; + } + + /** + * Restore any temporary source backup file + * + * @return Status + */ + protected function restoreSource() { + $status = Status::newGood(); + // Restore any file that was at the destination + if ( $this->tmpSourcePath !== null ) { + $params = array( + 'src' => $this->tmpSourcePath, + 'dst' => $this->params['src'], + 'overwriteDest' => true + ); + $status = $this->backend->store( $params ); + if ( !$status->isOK() ) { + return $status; + } + } + return $status; + } + + /** + * Restore any temporary destination backup file + * + * @return Status + */ + protected function restoreDest() { + $status = Status::newGood(); + // Restore any file that was at the destination + if ( $this->tmpDestFile ) { + $params = array( + 'src' => $this->tmpDestFile->getPath(), + 'dst' => $this->params['dst'], + 'overwriteDest' => true + ); + $status = $this->backend->store( $params ); + if ( !$status->isOK() ) { + return $status; + } + } + return $status; + } + + /** + * Check if a file will exist in storage when this operation is attempted + * + * @param $source string Storage path + * @param $predicates Array + * @return bool + */ + final protected function fileExists( $source, array $predicates ) { + if ( isset( $predicates['exists'][$source] ) ) { + return $predicates['exists'][$source]; // previous op assures this + } else { + return $this->backend->fileExists( array( 'src' => $source ) ); + } + } + + /** + * Log a file operation failure and preserve any temp files + * + * @param $fileOp FileOp + * @return void + */ + final protected function logFailure( $action ) { + $params = $this->params; + $params['failedAction'] = $action; + // Preserve backup files just in case (for recovery) + if ( $this->tmpSourceFile ) { + $this->tmpSourceFile->preserve(); // don't purge + $params['srcBackupPath'] = $this->tmpSourceFile->getPath(); + } + if ( $this->tmpDestFile ) { + $this->tmpDestFile->preserve(); // don't purge + $params['dstBackupPath'] = $this->tmpDestFile->getPath(); + } + try { + wfDebugLog( 'FileOperation', + get_class( $this ) . ' failed:' . serialize( $params ) ); + } catch ( Exception $e ) { + // bad config? debug log error? + } + } +} + +/** + * Store a file into the backend from a file on the file system. + * Parameters similar to FileBackend::store(), which include: + * src : source path on file system + * dst : destination storage path + * overwriteDest : do nothing and pass if an identical file exists at destination + * overwriteSame : override any existing file at destination + */ +class StoreFileOp extends FileOp { + protected function allowedParams() { + return array( 'src', 'dst', 'overwriteDest', 'overwriteSame' ); + } + + protected function doPrecheck( array &$predicates ) { + $status = Status::newGood(); + // Check if destination file exists + $status->merge( $this->precheckDestExistence( $predicates ) ); + if ( !$status->isOK() ) { + return $status; + } + // Check if the source file exists on the file system + if ( !is_file( $this->params['src'] ) ) { + $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; + } + // Update file existence predicates + $predicates['exists'][$this->params['dst']] = true; + return $status; + } + + protected function doAttempt() { + $status = Status::newGood(); + // Create a destination backup copy as needed + if ( $this->destAlreadyExists ) { + $status->merge( $this->checkAndBackupDest() ); + if ( !$status->isOK() ) { + return $status; + } + } + // Store the file at the destination + if ( !$this->destSameAsSource ) { + $status->merge( $this->backend->store( $this->params ) ); + } + return $status; + } + + protected function doRevert() { + $status = Status::newGood(); + if ( !$this->destSameAsSource ) { + // Restore any file that was at the destination, + // overwritting what was put there in attempt() + $status->merge( $this->restoreDest() ); + } + return $status; + } + + protected function getSourceSha1Base36() { + return $this->getFileSha1Base36( $this->params['src'] ); + } + + public function storagePathsChanged() { + return array( $this->params['dst'] ); + } +} + +/** + * Create a file in the backend with the given content. + * Parameters similar to FileBackend::create(), which include: + * content : a string of raw file contents + * dst : destination storage path + * overwriteDest : do nothing and pass if an identical file exists at destination + * overwriteSame : override any existing file at destination + */ +class CreateFileOp extends FileOp { + protected function allowedParams() { + return array( 'content', 'dst', 'overwriteDest', 'overwriteSame' ); + } + + protected function doPrecheck( array &$predicates ) { + $status = Status::newGood(); + // Check if destination file exists + $status->merge( $this->precheckDestExistence( $predicates ) ); + if ( !$status->isOK() ) { + return $status; + } + // Update file existence predicates + $predicates['exists'][$this->params['dst']] = true; + return $status; + } + + protected function doAttempt() { + $status = Status::newGood(); + // Create a destination backup copy as needed + if ( $this->destAlreadyExists ) { + $status->merge( $this->checkAndBackupDest() ); + if ( !$status->isOK() ) { + return $status; + } + } + // Create the file at the destination + if ( !$this->destSameAsSource ) { + $status->merge( $this->backend->create( $this->params ) ); + } + return $status; + } + + protected function doRevert() { + $status = Status::newGood(); + if ( !$this->destSameAsSource ) { + // Restore any file that was at the destination, + // overwritting what was put there in attempt() + $status->merge( $this->restoreDest() ); + } + return $status; + } + + protected function getSourceSha1Base36() { + return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 ); + } + + public function storagePathsChanged() { + return array( $this->params['dst'] ); + } +} + +/** + * Copy a file from one storage path to another in the backend. + * Parameters similar to FileBackend::copy(), which include: + * src : source storage path + * dst : destination storage path + * overwriteDest : do nothing and pass if an identical file exists at destination + * overwriteSame : override any existing file at destination + */ +class CopyFileOp extends FileOp { + protected function allowedParams() { + return array( 'src', 'dst', 'overwriteDest', 'overwriteSame' ); + } + + protected function doPrecheck( array &$predicates ) { + $status = Status::newGood(); + // Check if destination file exists + $status->merge( $this->precheckDestExistence( $predicates ) ); + if ( !$status->isOK() ) { + return $status; + } + // Check if the source file exists + if ( !$this->fileExists( $this->params['src'], $predicates ) ) { + $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; + } + // Update file existence predicates + $predicates['exists'][$this->params['dst']] = true; + return $status; + } + + protected function doAttempt() { + $status = Status::newGood(); + // Create a destination backup copy as needed + if ( $this->destAlreadyExists ) { + $status->merge( $this->checkAndBackupDest() ); + if ( !$status->isOK() ) { + return $status; + } + } + // Copy the file into the destination + if ( !$this->destSameAsSource ) { + $status->merge( $this->backend->copy( $this->params ) ); + } + return $status; + } + + protected function doRevert() { + $status = Status::newGood(); + if ( !$this->destSameAsSource ) { + // Restore any file that was at the destination, + // overwritting what was put there in attempt() + $status->merge( $this->restoreDest() ); + } + return $status; + } + + protected function getSourceSha1Base36() { + return $this->getFileSha1Base36( $this->params['src'] ); + } + + public function storagePathsRead() { + return array( $this->params['src'] ); + } + + public function storagePathsChanged() { + return array( $this->params['dst'] ); + } +} + +/** + * Move a file from one storage path to another in the backend. + * Parameters similar to FileBackend::move(), which include: + * src : source storage path + * dst : destination storage path + * overwriteDest : do nothing and pass if an identical file exists at destination + * overwriteSame : override any existing file at destination + */ +class MoveFileOp extends FileOp { + protected function allowedParams() { + return array( 'src', 'dst', 'overwriteDest', 'overwriteSame' ); + } + + protected function doPrecheck( array &$predicates ) { + $status = Status::newGood(); + // Check if destination file exists + $status->merge( $this->precheckDestExistence( $predicates ) ); + if ( !$status->isOK() ) { + return $status; + } + // Check if the source file exists + if ( !$this->fileExists( $this->params['src'], $predicates ) ) { + $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; + } + // Update file existence predicates + $predicates['exists'][$this->params['src']] = false; + $predicates['exists'][$this->params['dst']] = true; + return $status; + } + + protected function doAttempt() { + $status = Status::newGood(); + // Create a destination backup copy as needed + if ( $this->destAlreadyExists ) { + $status->merge( $this->checkAndBackupDest() ); + if ( !$status->isOK() ) { + return $status; + } + } + if ( !$this->destSameAsSource ) { + // Move the file into the destination + $status->merge( $this->backend->move( $this->params ) ); + } else { + // Create a source backup copy as needed + $status->merge( $this->backupSource() ); + if ( !$status->isOK() ) { + return $status; + } + // Just delete source as the destination needs no changes + $params = array( 'src' => $this->params['src'] ); + $status->merge( $this->backend->delete( $params ) ); + if ( !$status->isOK() ) { + return $status; + } + } + return $status; + } + + protected function doRevert() { + $status = Status::newGood(); + if ( !$this->destSameAsSource ) { + // Move the file back to the source + $params = array( + 'src' => $this->params['dst'], + 'dst' => $this->params['src'] + ); + $status->merge( $this->backend->move( $params ) ); + if ( !$status->isOK() ) { + return $status; // also can't restore any dest file + } + // Restore any file that was at the destination + $status->merge( $this->restoreDest() ); + } else { + // Restore any source file + return $this->restoreSource(); + } + + return $status; + } + + protected function getSourceSha1Base36() { + return $this->getFileSha1Base36( $this->params['src'] ); + } + + public function storagePathsRead() { + return array( $this->params['src'] ); + } + + public function storagePathsChanged() { + return array( $this->params['dst'] ); + } +} + +/** + * Combines files from severals storage paths into a new file in the backend. + * Parameters similar to FileBackend::concatenate(), which include: + * srcs : ordered source storage paths (e.g. chunk1, chunk2, ...) + * dst : destination storage path + * overwriteDest : do nothing and pass if an identical file exists at destination + */ +class ConcatenateFileOp extends FileOp { + protected function allowedParams() { + return array( 'srcs', 'dst', 'overwriteDest' ); + } + + protected function doPrecheck( array &$predicates ) { + $status = Status::newGood(); + // Check if destination file exists + $status->merge( $this->precheckDestExistence( $predicates ) ); + if ( !$status->isOK() ) { + return $status; + } + // Check that source files exists + foreach ( $this->params['srcs'] as $source ) { + if ( !$this->fileExists( $source, $predicates ) ) { + $status->fatal( 'backend-fail-notexists', $source ); + return $status; + } + } + // Update file existence predicates + $predicates['exists'][$this->params['dst']] = true; + return $status; + } + + protected function doAttempt() { + $status = Status::newGood(); + // Create a destination backup copy as needed + if ( $this->destAlreadyExists ) { + $status->merge( $this->checkAndBackupDest() ); + if ( !$status->isOK() ) { + return $status; + } + } + // Concatenate the file at the destination + $status->merge( $this->backend->concatenate( $this->params ) ); + return $status; + } + + protected function doRevert() { + // Restore any file that was at the destination, + // overwritting what was put there in attempt() + return $this->restoreDest(); + } + + protected function getSourceSha1Base36() { + return null; // defer this until we finish building the new file + } + + public function storagePathsRead() { + return $this->params['srcs']; + } + + public function storagePathsChanged() { + return array( $this->params['dst'] ); + } +} + +/** + * Delete a file at the storage path. + * Parameters similar to FileBackend::delete(), which include: + * src : source storage path + * ignoreMissingSource : don't return an error if the file does not exist + */ +class DeleteFileOp extends FileOp { + protected $needsDelete = true; + + protected function allowedParams() { + return array( 'src', 'ignoreMissingSource' ); + } + + protected function doPrecheck( array &$predicates ) { + $status = Status::newGood(); + // Check if the source file exists + if ( !$this->fileExists( $this->params['src'], $predicates ) ) { + if ( !$this->getParam( 'ignoreMissingSource' ) ) { + $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; + } + $this->needsDelete = false; + } + // Update file existence predicates + $predicates['exists'][$this->params['src']] = false; + return $status; + } + + protected function doAttempt() { + $status = Status::newGood(); + if ( $this->needsDelete ) { + // Create a source backup copy as needed + $status->merge( $this->backupSource() ); + if ( !$status->isOK() ) { + return $status; + } + // Delete the source file + $status->merge( $this->backend->delete( $this->params ) ); + if ( !$status->isOK() ) { + return $status; + } + } + return $status; + } + + protected function doRevert() { + // Restore any source file that we deleted + return $this->restoreSource(); + } + + public function storagePathsChanged() { + return array( $this->params['src'] ); + } +} + +/** + * Placeholder operation that has no params and does nothing + */ +class NullFileOp extends FileOp { + protected function doAttempt() { + return Status::newGood(); + } + + protected function doRevert() { + return Status::newGood(); + } +} diff --git a/includes/filerepo/backend/lockmanager/DBLockManager.php b/includes/filerepo/backend/lockmanager/DBLockManager.php new file mode 100644 index 0000000000..94887f1d69 --- /dev/null +++ b/includes/filerepo/backend/lockmanager/DBLockManager.php @@ -0,0 +1,454 @@ + server config array) + /** @var Array Map of bucket indexes to peer DB lists */ + protected $dbsByBucket; // (bucket index => (ldb1, ldb2, ...)) + /** @var BagOStuff */ + protected $statusCache; + + protected $lockExpiry; // integer number of seconds + protected $safeDelay; // integer number of seconds + + protected $session = 0; // random integer + /** @var Array Map of (locked key => lock type => count) */ + protected $locksHeld = array(); + /** @var Array Map Database connections (DB name => Database) */ + protected $conns = array(); + + /** + * Construct a new instance from configuration. + * $config paramaters include: + * 'dbServers' : Associative array of DB names to server configuration. + * Configuration is an associative array that includes: + * 'host' - DB server name + * 'dbname' - DB name + * 'type' - DB type (mysql,postgres,...) + * 'user' - DB user + * 'password' - DB user password + * 'tablePrefix' - DB table prefix + * 'flags' - DB flags (see DatabaseBase) + * 'dbsByBucket' : Array of 1-16 consecutive integer keys, starting from 0, + * each having an odd-numbered list of DB names (peers) as values. + * Any DB named 'localDBMaster' will automatically use the DB master + * settings for this wiki (without the need for a dbServers entry). + * 'lockExpiry' : Lock timeout (seconds) for dropped connections. [optional] + * This tells the DB server how long to wait before assuming + * connection failure and releasing all the locks for a session. + * + * @param Array $config + */ + public function __construct( array $config ) { + $this->dbServers = $config['dbServers']; + // Sanitize dbsByBucket config to prevent PHP errors + $this->dbsByBucket = array_filter( $config['dbsByBucket'], 'is_array' ); + $this->dbsByBucket = array_values( $this->dbsByBucket ); // consecutive + + if ( isset( $config['lockExpiry'] ) ) { + $this->lockExpiry = $config['lockExpiry']; + } else { + $met = ini_get( 'max_execution_time' ); + $this->lockExpiry = $met ? $met : 60; // use some sane amount if 0 + } + $this->safeDelay = ( $this->lockExpiry <= 0 ) + ? 60 // pick a safe-ish number to match DB timeout default + : $this->lockExpiry; // cover worst case + + foreach ( $this->dbsByBucket as $bucket ) { + if ( count( $bucket ) > 1 ) { + // Tracks peers that couldn't be queried recently to avoid lengthy + // connection timeouts. This is useless if each bucket has one peer. + $this->statusCache = wfGetMainCache(); + break; + } + } + + $this->session = ''; + for ( $i = 0; $i < 5; $i++ ) { + $this->session .= mt_rand( 0, 2147483647 ); + } + $this->session = wfBaseConvert( sha1( $this->session ), 16, 36, 31 ); + } + + protected function doLock( array $keys, $type ) { + $status = Status::newGood(); + + $keysToLock = array(); + // Get locks that need to be acquired (buckets => locks)... + foreach ( $keys as $key ) { + if ( isset( $this->locksHeld[$key][$type] ) ) { + ++$this->locksHeld[$key][$type]; + } elseif ( isset( $this->locksHeld[$key][self::LOCK_EX] ) ) { + $this->locksHeld[$key][$type] = 1; + } else { + $bucket = $this->getBucketFromKey( $key ); + $keysToLock[$bucket][] = $key; + } + } + + $lockedKeys = array(); // files locked in this attempt + // Attempt to acquire these locks... + foreach ( $keysToLock as $bucket => $keys ) { + // Try to acquire the locks for this bucket + $res = $this->doLockingQueryAll( $bucket, $keys, $type ); + if ( $res === 'cantacquire' ) { + // Resources already locked by another process. + // Abort and unlock everything we just locked. + $status->fatal( 'lockmanager-fail-acquirelocks', implode( ', ', $keys ) ); + $status->merge( $this->doUnlock( $lockedKeys, $type ) ); + return $status; + } elseif ( $res !== true ) { + // Couldn't contact any DBs for this bucket. + // Abort and unlock everything we just locked. + $status->fatal( 'lockmanager-fail-db-bucket', $bucket ); + $status->merge( $this->doUnlock( $lockedKeys, $type ) ); + return $status; + } + // Record these locks as active + foreach ( $keys as $key ) { + $this->locksHeld[$key][$type] = 1; // locked + } + // Keep track of what locks were made in this attempt + $lockedKeys = array_merge( $lockedKeys, $keys ); + } + + return $status; + } + + protected function doUnlock( array $keys, $type ) { + $status = Status::newGood(); + + foreach ( $keys as $key ) { + if ( !isset( $this->locksHeld[$key] ) ) { + $status->warning( 'lockmanager-notlocked', $key ); + } elseif ( !isset( $this->locksHeld[$key][$type] ) ) { + $status->warning( 'lockmanager-notlocked', $key ); + } else { + --$this->locksHeld[$key][$type]; + if ( $this->locksHeld[$key][$type] <= 0 ) { + unset( $this->locksHeld[$key][$type] ); + } + if ( !count( $this->locksHeld[$key] ) ) { + unset( $this->locksHeld[$key] ); // no SH or EX locks left for key + } + } + } + + // Reference count the locks held and COMMIT when zero + if ( !count( $this->locksHeld ) ) { + $status->merge( $this->finishLockTransactions() ); + } + + return $status; + } + + /** + * Get a connection to a lock DB and acquire locks on $keys. + * This does not use GET_LOCK() per http://bugs.mysql.com/bug.php?id=1118. + * + * @param $lockDb string + * @param $keys Array + * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH + * @return bool Resources able to be locked + * @throws DBError + */ + protected function doLockingQuery( $lockDb, array $keys, $type ) { + if ( $type == self::LOCK_EX ) { // writer locks + $db = $this->getConnection( $lockDb ); + if ( !$db ) { + return false; // bad config + } + $data = array(); + foreach ( $keys as $key ) { + $data[] = array( 'fle_key' => $key ); + } + # Wait on any existing writers and block new ones if we get in + $db->insert( 'filelocks_exclusive', $data, __METHOD__ ); + } + return true; + } + + /** + * Attempt to acquire locks with the peers for a bucket. + * This should avoid throwing any exceptions. + * + * @param $bucket integer + * @param $keys Array List of resource keys to lock + * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH + * @return bool|string One of (true, 'cantacquire', 'dberrors') + */ + protected function doLockingQueryAll( $bucket, array $keys, $type ) { + $yesVotes = 0; // locks made on trustable DBs + $votesLeft = count( $this->dbsByBucket[$bucket] ); // remaining DBs + $quorum = floor( $votesLeft/2 + 1 ); // simple majority + // Get votes for each DB, in order, until we have enough... + foreach ( $this->dbsByBucket[$bucket] as $index => $lockDb ) { + // Check that DB is not *known* to be down + if ( $this->cacheCheckFailures( $lockDb ) ) { + try { + // Attempt to acquire the lock on this DB + if ( !$this->doLockingQuery( $lockDb, $keys, $type ) ) { + return 'cantacquire'; // vetoed; resource locked + } + ++$yesVotes; // success for this peer + if ( $yesVotes >= $quorum ) { + return true; // lock obtained + } + } catch ( DBConnectionError $e ) { + $this->cacheRecordFailure( $lockDb ); + } catch ( DBError $e ) { + if ( $this->lastErrorIndicatesLocked( $lockDb ) ) { + return 'cantacquire'; // vetoed; resource locked + } + } + } + $votesLeft--; + $votesNeeded = $quorum - $yesVotes; + if ( $votesNeeded > $votesLeft ) { + // In "trust cache" mode we don't have to meet the quorum + break; // short-circuit + } + } + // At this point, we must not have meet the quorum + return 'dberrors'; // not enough votes to ensure correctness + } + + /** + * Get (or reuse) a connection to a lock DB + * + * @param $lockDb string + * @return Database + * @throws DBError + */ + protected function getConnection( $lockDb ) { + if ( !isset( $this->conns[$lockDb] ) ) { + $db = null; + if ( $lockDb === 'localDBMaster' ) { + $lb = wfGetLBFactory()->newMainLB(); + $db = $lb->getConnection( DB_MASTER ); + } elseif ( isset( $this->dbServers[$lockDb] ) ) { + $config = $this->dbServers[$lockDb]; + $db = DatabaseBase::factory( $config['type'], $config ); + } + if ( !$db ) { + return null; // config error? + } + $this->conns[$lockDb] = $db; + $this->conns[$lockDb]->clearFlag( DBO_TRX ); + # If the connection drops, try to avoid letting the DB rollback + # and release the locks before the file operations are finished. + # This won't handle the case of DB server restarts however. + $options = array(); + if ( $this->lockExpiry > 0 ) { + $options['connTimeout'] = $this->lockExpiry; + } + $this->conns[$lockDb]->setSessionOptions( $options ); + $this->initConnection( $lockDb, $this->conns[$lockDb] ); + } + if ( !$this->conns[$lockDb]->trxLevel() ) { + $this->conns[$lockDb]->begin(); // start transaction + } + return $this->conns[$lockDb]; + } + + /** + * Do additional initialization for new lock DB connection + * + * @param $lockDb string + * @param $db DatabaseBase + * @return void + * @throws DBError + */ + protected function initConnection( $lockDb, DatabaseBase $db ) {} + + /** + * Commit all changes to lock-active databases. + * This should avoid throwing any exceptions. + * + * @return Status + */ + protected function finishLockTransactions() { + $status = Status::newGood(); + foreach ( $this->conns as $lockDb => $db ) { + if ( $db->trxLevel() ) { // in transaction + try { + $db->rollback(); // finish transaction and kill any rows + } catch ( DBError $e ) { + $status->fatal( 'lockmanager-fail-db-release', $lockDb ); + } + } + } + return $status; + } + + /** + * Check if the last DB error for $lockDb indicates + * that a requested resource was locked by another process. + * This should avoid throwing any exceptions. + * + * @param $lockDb string + * @return bool + */ + protected function lastErrorIndicatesLocked( $lockDb ) { + if ( isset( $this->conns[$lockDb] ) ) { // sanity + $db = $this->conns[$lockDb]; + return ( $db->wasDeadlock() || $db->wasLockTimeout() ); + } + return false; + } + + /** + * Checks if the DB has not recently had connection/query errors. + * This just avoids wasting time on doomed connection attempts. + * + * @param $lockDb string + * @return bool + */ + protected function cacheCheckFailures( $lockDb ) { + if ( $this->statusCache && $this->safeDelay > 0 ) { + $key = $this->getMissKey( $lockDb ); + $misses = $this->statusCache->get( $key ); + return !$misses; + } + return true; + } + + /** + * Log a lock request failure to the cache + * + * @param $lockDb string + * @return bool Success + */ + protected function cacheRecordFailure( $lockDb ) { + if ( $this->statusCache && $this->safeDelay > 0 ) { + $key = $this->getMissKey( $lockDb ); + $misses = $this->statusCache->get( $key ); + if ( $misses ) { + return $this->statusCache->incr( $key ); + } else { + return $this->statusCache->add( $key, 1, $this->safeDelay ); + } + } + return true; + } + + /** + * Get a cache key for recent query misses for a DB + * + * @param $lockDb string + * @return string + */ + protected function getMissKey( $lockDb ) { + return 'lockmanager:querymisses:' . str_replace( ' ', '_', $lockDb ); + } + + /** + * Get the bucket for lock key. + * This should avoid throwing any exceptions. + * + * @param $key string (31 char hex key) + * @return integer + */ + protected function getBucketFromKey( $key ) { + $prefix = substr( $key, 0, 2 ); // first 2 hex chars (8 bits) + return intval( base_convert( $prefix, 16, 10 ) ) % count( $this->dbsByBucket ); + } + + /** + * Make sure remaining locks get cleared for sanity + */ + function __destruct() { + foreach ( $this->conns as $lockDb => $db ) { + if ( $db->trxLevel() ) { // in transaction + try { + $db->rollback(); // finish transaction and kill any rows + } catch ( DBError $e ) { + // oh well + } + } + $db->close(); + } + } +} + +/** + * MySQL version of DBLockManager that supports shared locks. + * All locks are non-blocking, which avoids deadlocks. + * + * @ingroup LockManager + */ +class MySqlLockManager extends DBLockManager { + /** @var Array Mapping of lock types to the type actually used */ + protected $lockTypeMap = array( + self::LOCK_SH => self::LOCK_SH, + self::LOCK_UW => self::LOCK_SH, + self::LOCK_EX => self::LOCK_EX + ); + + protected function initConnection( $lockDb, DatabaseBase $db ) { + # Let this transaction see lock rows from other transactions + $db->query( "SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;" ); + } + + protected function doLockingQuery( $lockDb, array $keys, $type ) { + $db = $this->getConnection( $lockDb ); + if ( !$db ) { + return false; + } + $data = array(); + foreach ( $keys as $key ) { + $data[] = array( 'fls_key' => $key, 'fls_session' => $this->session ); + } + # Block new writers... + $db->insert( 'filelocks_shared', $data, __METHOD__, array( 'IGNORE' ) ); + # Actually do the locking queries... + if ( $type == self::LOCK_SH ) { // reader locks + # Bail if there are any existing writers... + $blocked = $db->selectField( 'filelocks_exclusive', '1', + array( 'fle_key' => $keys ), + __METHOD__ + ); + # Prospective writers that haven't yet updated filelocks_exclusive + # will recheck filelocks_shared after doing so and bail due to our entry. + } else { // writer locks + $encSession = $db->addQuotes( $this->session ); + # Bail if there are any existing writers... + # The may detect readers, but the safe check for them is below. + # Note: if two writers come at the same time, both bail :) + $blocked = $db->selectField( 'filelocks_shared', '1', + array( 'fls_key' => $keys, "fls_session != $encSession" ), + __METHOD__ + ); + if ( !$blocked ) { + $data = array(); + foreach ( $keys as $key ) { + $data[] = array( 'fle_key' => $key ); + } + # Block new readers/writers... + $db->insert( 'filelocks_exclusive', $data, __METHOD__ ); + # Bail if there are any existing readers... + $blocked = $db->selectField( 'filelocks_shared', '1', + array( 'fls_key' => $keys, "fls_session != $encSession" ), + __METHOD__ + ); + } + } + return !$blocked; + } +} diff --git a/includes/filerepo/backend/lockmanager/FSLockManager.php b/includes/filerepo/backend/lockmanager/FSLockManager.php new file mode 100644 index 0000000000..4a5fb296dd --- /dev/null +++ b/includes/filerepo/backend/lockmanager/FSLockManager.php @@ -0,0 +1,199 @@ + self::LOCK_SH, + self::LOCK_UW => self::LOCK_SH, + self::LOCK_EX => self::LOCK_EX + ); + + protected $lockDir; // global dir for all servers + + /** @var Array Map of (locked key => lock type => count) */ + protected $locksHeld = array(); + /** @var Array Map of (locked key => lock type => lock file handle) */ + protected $handles = array(); + + function __construct( array $config ) { + $this->lockDir = $config['lockDirectory']; + } + + protected function doLock( array $keys, $type ) { + $status = Status::newGood(); + + $lockedKeys = array(); // files locked in this attempt + foreach ( $keys as $key ) { + $subStatus = $this->doSingleLock( $key, $type ); + $status->merge( $subStatus ); + if ( $status->isOK() ) { + // Don't append to $lockedKeys if $key is already locked. + // We do NOT want to unlock the key if we have to rollback. + if ( $subStatus->isGood() ) { // no warnings/fatals? + $lockedKeys[] = $key; + } + } else { + // Abort and unlock everything + $status->merge( $this->doUnlock( $lockedKeys, $type ) ); + return $status; + } + } + + return $status; + } + + protected function doUnlock( array $keys, $type ) { + $status = Status::newGood(); + + foreach ( $keys as $key ) { + $status->merge( $this->doSingleUnlock( $key, $type ) ); + } + + return $status; + } + + /** + * Lock a single resource key + * + * @param $key string + * @param $type integer + * @return Status + */ + protected function doSingleLock( $key, $type ) { + $status = Status::newGood(); + + if ( isset( $this->locksHeld[$key][$type] ) ) { + ++$this->locksHeld[$key][$type]; + } elseif ( isset( $this->locksHeld[$key][self::LOCK_EX] ) ) { + $this->locksHeld[$key][$type] = 1; + } else { + wfSuppressWarnings(); + $handle = fopen( $this->getLockPath( $key ), 'a+' ); + wfRestoreWarnings(); + if ( !$handle ) { // lock dir missing? + wfMkdirParents( $this->lockDir ); + wfSuppressWarnings(); + $handle = fopen( $this->getLockPath( $key ), 'a+' ); // try again + wfRestoreWarnings(); + } + if ( $handle ) { + // Either a shared or exclusive lock + $lock = ( $type == self::LOCK_SH ) ? LOCK_SH : LOCK_EX; + if ( flock( $handle, $lock | LOCK_NB ) ) { + // Record this lock as active + $this->locksHeld[$key][$type] = 1; + $this->handles[$key][$type] = $handle; + } else { + fclose( $handle ); + $status->fatal( 'lockmanager-fail-acquirelock', $key ); + } + } else { + $status->fatal( 'lockmanager-fail-openlock', $key ); + } + } + + return $status; + } + + /** + * Unlock a single resource key + * + * @param $key string + * @param $type integer + * @return Status + */ + protected function doSingleUnlock( $key, $type ) { + $status = Status::newGood(); + + if ( !isset( $this->locksHeld[$key] ) ) { + $status->warning( 'lockmanager-notlocked', $key ); + } elseif ( !isset( $this->locksHeld[$key][$type] ) ) { + $status->warning( 'lockmanager-notlocked', $key ); + } else { + $handlesToClose = array(); + --$this->locksHeld[$key][$type]; + if ( $this->locksHeld[$key][$type] <= 0 ) { + unset( $this->locksHeld[$key][$type] ); + // If a LOCK_SH comes in while we have a LOCK_EX, we don't + // actually add a handler, so check for handler existence. + if ( isset( $this->handles[$key][$type] ) ) { + // Mark this handle to be unlocked and closed + $handlesToClose[] = $this->handles[$key][$type]; + unset( $this->handles[$key][$type] ); + } + } + // Unlock handles to release locks and delete + // any lock files that end up with no locks on them... + if ( wfIsWindows() ) { + // Windows: for any process, including this one, + // calling unlink() on a locked file will fail + $status->merge( $this->closeLockHandles( $key, $handlesToClose ) ); + $status->merge( $this->pruneKeyLockFiles( $key ) ); + } else { + // Unix: unlink() can be used on files currently open by this + // process and we must do so in order to avoid race conditions + $status->merge( $this->pruneKeyLockFiles( $key ) ); + $status->merge( $this->closeLockHandles( $key, $handlesToClose ) ); + } + } + + return $status; + } + + private function closeLockHandles( $key, array $handlesToClose ) { + $status = Status::newGood(); + foreach ( $handlesToClose as $handle ) { + wfSuppressWarnings(); + if ( !flock( $handle, LOCK_UN ) ) { + $status->fatal( 'lockmanager-fail-releaselock', $key ); + } + if ( !fclose( $handle ) ) { + $status->warning( 'lockmanager-fail-closelock', $key ); + } + wfRestoreWarnings(); + } + return $status; + } + + private function pruneKeyLockFiles( $key ) { + $status = Status::newGood(); + if ( !count( $this->locksHeld[$key] ) ) { + wfSuppressWarnings(); + # No locks are held for the lock file anymore + if ( !unlink( $this->getLockPath( $key ) ) ) { + $status->warning( 'lockmanager-fail-deletelock', $key ); + } + wfRestoreWarnings(); + unset( $this->locksHeld[$key] ); + unset( $this->handles[$key] ); + } + return $status; + } + + /** + * Get the path to the lock file for a key + * @param $key string + * @return string + */ + protected function getLockPath( $key ) { + return "{$this->lockDir}/{$key}.lock"; + } + + function __destruct() { + // Make sure remaining locks get cleared for sanity + foreach ( $this->locksHeld as $key => $locks ) { + $this->doSingleUnlock( $key, 0 ); + } + } +} diff --git a/includes/filerepo/backend/lockmanager/LSLockManager.php b/includes/filerepo/backend/lockmanager/LSLockManager.php new file mode 100644 index 0000000000..bd6b77259d --- /dev/null +++ b/includes/filerepo/backend/lockmanager/LSLockManager.php @@ -0,0 +1,287 @@ + self::LOCK_SH, + self::LOCK_UW => self::LOCK_SH, + self::LOCK_EX => self::LOCK_EX + ); + + /** @var Array Map of server names to server config */ + protected $lockServers; // (server name => server config array) + /** @var Array Map of bucket indexes to peer server lists */ + protected $srvsByBucket; // (bucket index => (lsrv1, lsrv2, ...)) + + /** @var Array Map of (locked key => lock type => count) */ + protected $locksHeld = array(); + /** @var Array Map Server connections (server name => resource) */ + protected $conns = array(); + + protected $connTimeout; // float number of seconds + protected $session = ''; // random SHA-1 string + + /** + * Construct a new instance from configuration. + * $config paramaters include: + * 'lockServers' : Associative array of server names to configuration. + * Configuration is an associative array that includes: + * 'host' - IP address/hostname + * 'port' - TCP port + * 'authKey' - Secret string the lock server uses + * 'srvsByBucket' : Array of 1-16 consecutive integer keys, starting from 0, + * each having an odd-numbered list of server names (peers) as values. + * 'connTimeout' : Lock server connection attempt timeout. [optional] + * + * @param Array $config + */ + public function __construct( array $config ) { + $this->lockServers = $config['lockServers']; + // Sanitize srvsByBucket config to prevent PHP errors + $this->srvsByBucket = array_filter( $config['srvsByBucket'], 'is_array' ); + $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive + + if ( isset( $config['connTimeout'] ) ) { + $this->connTimeout = $config['connTimeout']; + } else { + $this->connTimeout = 3; // use some sane amount + } + + $this->session = ''; + for ( $i = 0; $i < 5; $i++ ) { + $this->session .= mt_rand( 0, 2147483647 ); + } + $this->session = wfBaseConvert( sha1( $this->session ), 16, 36, 31 ); + } + + protected function doLock( array $keys, $type ) { + $status = Status::newGood(); + + $keysToLock = array(); + // Get locks that need to be acquired (buckets => locks)... + foreach ( $keys as $key ) { + if ( isset( $this->locksHeld[$key][$type] ) ) { + ++$this->locksHeld[$key][$type]; + } elseif ( isset( $this->locksHeld[$key][self::LOCK_EX] ) ) { + $this->locksHeld[$key][$type] = 1; + } else { + $bucket = $this->getBucketFromKey( $key ); + $keysToLock[$bucket][] = $key; + } + } + + $lockedKeys = array(); // files locked in this attempt + // Attempt to acquire these locks... + foreach ( $keysToLock as $bucket => $keys ) { + // Try to acquire the locks for this bucket + $res = $this->doLockingRequestAll( $bucket, $keys, $type ); + if ( $res === 'cantacquire' ) { + // Resources already locked by another process. + // Abort and unlock everything we just locked. + $status->fatal( 'lockmanager-fail-acquirelocks', implode( ', ', $keys ) ); + $status->merge( $this->doUnlock( $lockedKeys, $type ) ); + return $status; + } elseif ( $res !== true ) { + // Couldn't contact any servers for this bucket. + // Abort and unlock everything we just locked. + $status->fatal( 'lockmanager-fail-acquirelocks', implode( ', ', $keys ) ); + $status->merge( $this->doUnlock( $lockedKeys, $type ) ); + return $status; + } + // Record these locks as active + foreach ( $keys as $key ) { + $this->locksHeld[$key][$type] = 1; // locked + } + // Keep track of what locks were made in this attempt + $lockedKeys = array_merge( $lockedKeys, $keys ); + } + + return $status; + } + + protected function doUnlock( array $keys, $type ) { + $status = Status::newGood(); + + foreach ( $keys as $key ) { + if ( !isset( $this->locksHeld[$key] ) ) { + $status->warning( 'lockmanager-notlocked', $key ); + } elseif ( !isset( $this->locksHeld[$key][$type] ) ) { + $status->warning( 'lockmanager-notlocked', $key ); + } else { + --$this->locksHeld[$key][$type]; + if ( $this->locksHeld[$key][$type] <= 0 ) { + unset( $this->locksHeld[$key][$type] ); + } + if ( !count( $this->locksHeld[$key] ) ) { + unset( $this->locksHeld[$key] ); // no SH or EX locks left for key + } + } + } + + // Reference count the locks held and release locks when zero + if ( !count( $this->locksHeld ) ) { + $status->merge( $this->releaseLocks() ); + } + + return $status; + } + + /** + * Get a connection to a lock server and acquire locks on $keys + * + * @param $lockSrv string + * @param $keys Array + * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH + * @return bool Resources able to be locked + */ + protected function doLockingRequest( $lockSrv, array $keys, $type ) { + if ( $type == self::LOCK_SH ) { // reader locks + $type = 'SH'; + } elseif ( $type == self::LOCK_EX ) { // writer locks + $type = 'EX'; + } else { + return true; // ok... + } + + // Send out the command and get the response... + $response = $this->sendCommand( $lockSrv, 'ACQUIRE', $type, $keys ); + + return ( $response === 'ACQUIRED' ); + } + + /** + * Send a command and get back the response + * + * @param $lockSrv string + * @param $action string + * @param $type string + * @param $values Array + * @return string|false + */ + protected function sendCommand( $lockSrv, $action, $type, $values ) { + $conn = $this->getConnection( $lockSrv ); + if ( !$conn ) { + return false; // no connection + } + $authKey = $this->lockServers[$lockSrv]['authKey']; + // Build of the command as a flat string... + $values = implode( '|', $values ); + $key = sha1( $this->session . $action . $type . $values . $authKey ); + // Send out the command... + if ( fwrite( $conn, "{$this->session}:$key:$action:$type:$values\n" ) === false ) { + return false; + } + // Get the response... + $response = fgets( $conn ); + if ( $response === false ) { + return false; + } + return trim( $response ); + } + + /** + * Attempt to acquire locks with the peers for a bucket + * + * @param $bucket integer + * @param $keys Array List of resource keys to lock + * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH + * @return bool|string One of (true, 'cantacquire', 'srverrors') + */ + protected function doLockingRequestAll( $bucket, array $keys, $type ) { + $yesVotes = 0; // locks made on trustable servers + $votesLeft = count( $this->srvsByBucket[$bucket] ); // remaining peers + $quorum = floor( $votesLeft/2 + 1 ); // simple majority + // Get votes for each peer, in order, until we have enough... + foreach ( $this->srvsByBucket[$bucket] as $index => $lockSrv ) { + // Attempt to acquire the lock on this peer + if ( !$this->doLockingRequest( $lockSrv, $keys, $type ) ) { + return 'cantacquire'; // vetoed; resource locked + } + ++$yesVotes; // success for this peer + if ( $yesVotes >= $quorum ) { + return true; // lock obtained + } + $votesLeft--; + $votesNeeded = $quorum - $yesVotes; + if ( $votesNeeded > $votesLeft ) { + // In "trust cache" mode we don't have to meet the quorum + break; // short-circuit + } + } + // At this point, we must not have meet the quorum + return 'srverrors'; // not enough votes to ensure correctness + } + + /** + * Get (or reuse) a connection to a lock server + * + * @param $lockSrv string + * @return resource + */ + protected function getConnection( $lockSrv ) { + if ( !isset( $this->conns[$lockSrv] ) ) { + $cfg = $this->lockServers[$lockSrv]; + wfSuppressWarnings(); + $errno = $errstr = ''; + $conn = fsockopen( $cfg['host'], $cfg['port'], $errno, $errstr, $this->connTimeout ); + wfRestoreWarnings(); + if ( $conn === false ) { + return null; + } + $sec = floor( $this->connTimeout ); + $usec = floor( ( $this->connTimeout - floor( $this->connTimeout ) ) * 1e6 ); + stream_set_timeout( $conn, $sec, $usec ); + $this->conns[$lockSrv] = $conn; + } + return $this->conns[$lockSrv]; + } + + /** + * Release all locks that this session is holding + * + * @return Status + */ + protected function releaseLocks() { + $status = Status::newGood(); + foreach ( $this->conns as $lockSrv => $conn ) { + $response = $this->sendCommand( $lockSrv, 'RELEASE_ALL', '', array() ); + if ( $response !== 'RELEASED_ALL' ) { + $status->fatal( 'lockmanager-fail-svr-release', $lockSrv ); + } + } + return $status; + } + + /** + * Get the bucket for lock key + * + * @param $key string (31 char hex key) + * @return integer + */ + protected function getBucketFromKey( $key ) { + $prefix = substr( $key, 0, 2 ); // first 2 hex chars (8 bits) + return intval( base_convert( $prefix, 16, 10 ) ) % count( $this->srvsByBucket ); + } + + /** + * Make sure remaining locks get cleared for sanity + */ + function __destruct() { + $this->releaseLocks(); + foreach ( $this->conns as $lockSrv => $conn ) { + fclose( $conn ); + } + } +} diff --git a/includes/filerepo/backend/lockmanager/LockManager.php b/includes/filerepo/backend/lockmanager/LockManager.php new file mode 100644 index 0000000000..92dcc38111 --- /dev/null +++ b/includes/filerepo/backend/lockmanager/LockManager.php @@ -0,0 +1,172 @@ + self::LOCK_SH, + self::LOCK_UW => self::LOCK_EX, // subclasses may use self::LOCK_SH + self::LOCK_EX => self::LOCK_EX + ); + + /** + * Construct a new instance from configuration + * + * @param $config Array + */ + public function __construct( array $config ) {} + + /** + * Lock the resources at the given abstract paths + * + * @param $paths Array List of resource names + * @param $type integer LockManager::LOCK_* constant + * @return Status + */ + final public function lock( array $paths, $type = self::LOCK_EX ) { + $keys = array_unique( array_map( 'LockManager::sha1Base36', $paths ) ); + return $this->doLock( $keys, $this->lockTypeMap[$type] ); + } + + /** + * Unlock the resources at the given abstract paths + * + * @param $paths Array List of storage paths + * @param $type integer LockManager::LOCK_* constant + * @return Status + */ + final public function unlock( array $paths, $type = self::LOCK_EX ) { + $keys = array_unique( array_map( 'LockManager::sha1Base36', $paths ) ); + return $this->doUnlock( $keys, $this->lockTypeMap[$type] ); + } + + /** + * Get the base 36 SHA-1 of a string, padded to 31 digits + * + * @param $path string + * @return string + */ + final protected static function sha1Base36( $path ) { + return wfBaseConvert( sha1( $path ), 16, 36, 31 ); + } + + /** + * Lock resources with the given keys and lock type + * + * @param $key Array List of keys to lock (40 char hex hashes) + * @param $type integer LockManager::LOCK_* constant + * @return string + */ + abstract protected function doLock( array $keys, $type ); + + /** + * Unlock resources with the given keys and lock type + * + * @param $key Array List of keys to unlock (40 char hex hashes) + * @param $type integer LockManager::LOCK_* constant + * @return string + */ + abstract protected function doUnlock( array $keys, $type ); +} + +/** + * LockManager helper class to handle scoped locks, which + * release when an object is destroyed or goes out of scope. + * + * @ingroup LockManager + * @since 1.19 + */ +class ScopedLock { + /** @var LockManager */ + protected $manager; + /** @var Status */ + protected $status; + /** @var Array List of resource paths*/ + protected $paths; + + protected $type; // integer lock type + + /** + * @param $manager LockManager + * @param $paths Array List of storage paths + * @param $type integer LockManager::LOCK_* constant + * @param $status Status + */ + protected function __construct( + LockManager $manager, array $paths, $type, Status $status + ) { + $this->manager = $manager; + $this->paths = $paths; + $this->status = $status; + $this->type = $type; + } + + protected function __clone() {} + + /** + * Get a ScopedLock object representing a lock on resource paths. + * Any locks are released once this object goes out of scope. + * The status object is updated with any errors or warnings. + * + * @param $manager LockManager + * @param $paths Array List of storage paths + * @param $type integer LockManager::LOCK_* constant + * @param $status Status + * @return ScopedLock|null Returns null on failure + */ + public static function factory( + LockManager $manager, array $paths, $type, Status $status + ) { + $lockStatus = $manager->lock( $paths, $type ); + $status->merge( $lockStatus ); + if ( $lockStatus->isOK() ) { + return new self( $manager, $paths, $type, $status ); + } + return null; + } + + function __destruct() { + $wasOk = $this->status->isOK(); + $this->status->merge( $this->manager->unlock( $this->paths, $this->type ) ); + if ( $wasOk ) { + // Make sure status is OK, despite any unlockFiles() fatals + $this->status->setResult( true, $this->status->value ); + } + } +} + +/** + * Simple version of LockManager that does nothing + */ +class NullLockManager extends LockManager { + protected function doLock( array $keys, $type ) { + return Status::newGood(); + } + + protected function doUnlock( array $keys, $type ) { + return Status::newGood(); + } +} diff --git a/includes/filerepo/backend/lockmanager/LockManagerGroup.php b/includes/filerepo/backend/lockmanager/LockManagerGroup.php new file mode 100644 index 0000000000..cbd616de12 --- /dev/null +++ b/includes/filerepo/backend/lockmanager/LockManagerGroup.php @@ -0,0 +1,68 @@ + ('class' =>, 'config' =>, 'instance' =>)) */ + protected $managers = array(); + + protected function __construct() {} + protected function __clone() {} + + public static function singleton() { + if ( self::$instance == null ) { + self::$instance = new self(); + } + return self::$instance; + } + + /** + * Register an array of file lock manager configurations + * + * @param $configs Array + * @return void + * @throws MWException + */ + public function register( array $configs ) { + foreach ( $configs as $config ) { + if ( !isset( $config['name'] ) ) { + throw new MWException( "Cannot register a lock manager with no name." ); + } + $name = $config['name']; + if ( !isset( $config['class'] ) ) { + throw new MWException( "Cannot register lock manager `{$name}` with no class." ); + } + $class = $config['class']; + unset( $config['class'] ); // lock manager won't need this + $this->managers[$name] = array( + 'class' => $class, + 'config' => $config, + 'instance' => null + ); + } + } + + /** + * Get the lock manager object with a given name + * + * @param $name string + * @return LockManager + * @throws MWException + */ + public function get( $name ) { + if ( !isset( $this->managers[$name] ) ) { + throw new MWException( "No lock manager defined with the name `$name`." ); + } + // Lazy-load the actual lock manager instance + if ( !isset( $this->managers[$name]['instance'] ) ) { + $class = $this->managers[$name]['class']; + $config = $this->managers[$name]['config']; + $this->managers[$name]['instance'] = new $class( $config ); + } + return $this->managers[$name]['instance']; + } +} diff --git a/includes/filerepo/file/FSFile.php b/includes/filerepo/file/FSFile.php new file mode 100644 index 0000000000..72fddaa57b --- /dev/null +++ b/includes/filerepo/file/FSFile.php @@ -0,0 +1,211 @@ +path = $path; + } + + /** + * Returns the file system path + * + * @return String + */ + public function getPath() { + return $this->path; + } + + /** + * Checks if the file exists + * + * @return bool + */ + public function exists() { + return is_file( $this->path ); + } + + /** + * Get the file size in bytes + * + * @return int|false + */ + public function getSize() { + return filesize( $this->path ); + } + + /** + * Get the file's last-modified timestamp + * + * @return string|false TS_MW timestamp or false on failure + */ + public function getTimestamp() { + wfSuppressWarnings(); + $timestamp = filemtime( $this->path ); + wfRestoreWarnings(); + if ( $timestamp !== false ) { + $timestamp = wfTimestamp( TS_MW, $timestamp ); + } + return $timestamp; + } + + /** + * Get an associative array containing information about + * a file with the given storage path. + * + * @param $ext Mixed: the file extension, or true to extract it from the filename. + * Set it to false to ignore the extension. + * + * @return array + */ + public function getProps( $ext = true ) { + wfProfileIn( __METHOD__ ); + wfDebug( __METHOD__.": Getting file info for $this->path\n" ); + + $info = self::placeholderProps(); + $info['fileExists'] = $this->exists(); + + if ( $info['fileExists'] ) { + $magic = MimeMagic::singleton(); + + # get the file extension + if ( $ext === true ) { + $i = strrpos( $this->path, '.' ); + $ext = strtolower( $i ? substr( $this->path, $i + 1 ) : '' ); + } + + # mime type according to file contents + $info['file-mime'] = $magic->guessMimeType( $this->path, false ); + # logical mime type + $info['mime'] = $magic->improveTypeFromExtension( $info['file-mime'], $ext ); + + list( $info['major_mime'], $info['minor_mime'] ) = File::splitMime( $info['mime'] ); + $info['media_type'] = $magic->getMediaType( $this->path, $info['mime'] ); + + # Get size in bytes + $info['size'] = $this->getSize(); + + # Height, width and metadata + $handler = MediaHandler::getHandler( $info['mime'] ); + if ( $handler ) { + $tempImage = (object)array(); + $info['metadata'] = $handler->getMetadata( $tempImage, $this->path ); + $gis = $handler->getImageSize( $tempImage, $this->path, $info['metadata'] ); + if ( is_array( $gis ) ) { + $info = $this->extractImageSizeInfo( $gis ) + $info; + } + } + $info['sha1'] = $this->getSha1Base36(); + + wfDebug(__METHOD__.": $this->path loaded, {$info['size']} bytes, {$info['mime']}.\n"); + } else { + wfDebug(__METHOD__.": $this->path NOT FOUND!\n"); + } + + wfProfileOut( __METHOD__ ); + return $info; + } + + /** + * Placeholder file properties to use for files that don't exist + * + * @return Array + */ + public static function placeholderProps() { + $info = array(); + $info['fileExists'] = false; + $info['mime'] = null; + $info['media_type'] = MEDIATYPE_UNKNOWN; + $info['metadata'] = ''; + $info['sha1'] = ''; + $info['width'] = 0; + $info['height'] = 0; + $info['bits'] = 0; + return $info; + } + + protected function extractImageSizeInfo( array $gis ) { + $info = array(); + # NOTE: $gis[2] contains a code for the image type. This is no longer used. + $info['width'] = $gis[0]; + $info['height'] = $gis[1]; + if ( isset( $gis['bits'] ) ) { + $info['bits'] = $gis['bits']; + } else { + $info['bits'] = 0; + } + return $info; + } + + /** + * Get a SHA-1 hash of a file in the local filesystem, in base-36 lower case + * encoding, zero padded to 31 digits. + * + * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36 + * fairly neatly. + * + * @return false|string False on failure + */ + public function getSha1Base36() { + wfProfileIn( __METHOD__ ); + + wfSuppressWarnings(); + $hash = sha1_file( $this->path ); + wfRestoreWarnings(); + if ( $hash !== false ) { + $hash = wfBaseConvert( $hash, 16, 36, 31 ); + } + + wfProfileOut( __METHOD__ ); + return $hash; + } + + /** + * Get an associative array containing information about a file in the local filesystem. + * + * @param $path String: absolute local filesystem path + * @param $ext Mixed: the file extension, or true to extract it from the filename. + * Set it to false to ignore the extension. + * + * @return array + */ + static function getPropsFromPath( $path, $ext = true ) { + $fsFile = new self( $path ); + return $fsFile->getProps( $ext ); + } + + /** + * Get a SHA-1 hash of a file in the local filesystem, in base-36 lower case + * encoding, zero padded to 31 digits. + * + * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36 + * fairly neatly. + * + * @param $path string + * + * @return false|string False on failure + */ + static function getSha1Base36FromPath( $path ) { + $fsFile = new self( $path ); + return $fsFile->getSha1Base36(); + } +} diff --git a/includes/filerepo/file/File.php b/includes/filerepo/file/File.php index 57e85684c1..b857c8e9ee 100644 --- a/includes/filerepo/file/File.php +++ b/includes/filerepo/file/File.php @@ -72,11 +72,19 @@ abstract class File { var $lastError, $redirected, $redirectedTitle; + /** + * @var FSFile|false + */ + protected $fsFile; + /** * @var MediaHandler */ protected $handler; + /** + * @var string + */ protected $url, $extension, $name, $path, $hashPath, $pageCount, $transformScript; /** @@ -110,6 +118,7 @@ abstract class File { /** * Given a string or Title object return either a * valid Title object with namespace NS_FILE or null + * * @param $title Title|string * @param $exception string|false Use 'exception' to throw an error on bad titles * @return Title|null @@ -179,8 +188,7 @@ abstract class File { static function checkExtensionCompatibility( File $old, $new ) { $oldMime = $old->getMimeType(); $n = strrpos( $new, '.' ); - $newExt = self::normalizeExtension( - $n ? substr( $new, $n + 1 ) : '' ); + $newExt = self::normalizeExtension( $n ? substr( $new, $n + 1 ) : '' ); $mimeMagic = MimeMagic::singleton(); return $mimeMagic->isMatchingExtension( $newExt, $oldMime ); } @@ -236,9 +244,12 @@ abstract class File { /** * Return the associated title object + * * @return Title|false */ - public function getTitle() { return $this->title; } + public function getTitle() { + return $this->title; + } /** * Return the title used to find this file @@ -300,7 +311,7 @@ abstract class File { } /** - * Return the full filesystem path to the file. Note that this does + * Return the storage path to the file. Note that this does * not mean that a file actually exists under that location. * * This path depends on whether directory hashing is active or not, @@ -320,6 +331,26 @@ abstract class File { return $this->path; } + /** + * Get an FS copy or original of this file and return the path. + * Returns false on failure. Callers must not alter the file. + * Temporary files are cleared automatically. + * + * @return string|false + */ + public function getLocalRefPath() { + $this->assertRepoDefined(); + if ( !isset( $this->fsFile ) ) { + $this->fsFile = $this->repo->getLocalReference( $this->getPath() ); + if ( !$this->fsFile ) { + $this->fsFile = false; // null => false; cache negative hits + } + } + return ( $this->fsFile ) + ? $this->fsFile->getPath() + : false; + } + /** * Return the width of the image. Returns false if the width is unknown * or undefined. @@ -409,7 +440,7 @@ abstract class File { public function convertMetadataVersion($metadata, $version) { $handler = $this->getHandler(); if ( !is_array( $metadata ) ) { - //just to make the return type consistant + // Just to make the return type consistent $metadata = unserialize( $metadata ); } if ( $handler ) { @@ -454,7 +485,9 @@ abstract class File { * Overridden by LocalFile, * STUB */ - function getMediaType() { return MEDIATYPE_UNKNOWN; } + function getMediaType() { + return MEDIATYPE_UNKNOWN; + } /** * Checks if the output of transform() for this file is likely @@ -540,6 +573,8 @@ abstract class File { * @return bool */ protected function _getIsSafeFile() { + global $wgTrustedMediaFormats; + if ( $this->allowInlineDisplay() ) { return true; } @@ -547,8 +582,6 @@ abstract class File { return true; } - global $wgTrustedMediaFormats; - $type = $this->getMediaType(); $mime = $this->getMimeType(); #wfDebug("LocalFile::isSafeFile: type= $type, mime= $mime\n"); @@ -584,7 +617,7 @@ abstract class File { * @return bool */ function isTrustedFile() { - #this could be implemented to check a flag in the databas, + #this could be implemented to check a flag in the database, #look for signatures, etc return false; } @@ -597,7 +630,7 @@ abstract class File { * @return boolean Whether file exists in the repository. */ public function exists() { - return $this->getPath() && file_exists( $this->path ); + return $this->getPath() && $this->repo->fileExists( $this->path ); } /** @@ -669,7 +702,8 @@ abstract class File { return null; } $extension = $this->getExtension(); - list( $thumbExt, $thumbMime ) = $this->handler->getThumbType( $extension, $this->getMimeType(), $params ); + list( $thumbExt, $thumbMime ) = $this->handler->getThumbType( + $extension, $this->getMimeType(), $params ); $thumbName = $this->handler->makeParamString( $params ) . '-' . $name; if ( $thumbExt != $extension ) { $thumbName .= ".$thumbExt"; @@ -700,7 +734,9 @@ abstract class File { $params['height'] = $height; } $thumb = $this->transform( $params ); - if( is_null( $thumb ) || $thumb->isError() ) return ''; + if ( is_null( $thumb ) || $thumb->isError() ) { + return ''; + } return $thumb->getUrl(); } @@ -714,45 +750,64 @@ abstract class File { * Typical keys are width, height and page. * @param $flags Integer: a bitfield, may contain self::RENDER_NOW to force rendering * - * @return MediaTransformOutput | false + * @return MediaTransformOutput|null */ protected function maybeDoTransform( $thumbName, $thumbUrl, $params, $flags = 0 ) { global $wgIgnoreImageErrors, $wgThumbnailEpoch; - $thumbPath = $this->getThumbPath( $thumbName ); - if ( $this->repo && $this->repo->canTransformVia404() && !($flags & self::RENDER_NOW ) ) { + $thumbPath = $this->getThumbPath( $thumbName ); // final thumb path + + if ( $this->repo && $this->repo->canTransformVia404() && !( $flags & self::RENDER_NOW ) ) { wfDebug( __METHOD__ . " transformation deferred." ); - return $this->handler->getTransform( $this, $thumbPath, $thumbUrl, $params ); + return $this->handler->getTransform( $this, false, $thumbUrl, $params ); } wfDebug( __METHOD__.": Doing stat for $thumbPath\n" ); $this->migrateThumbFile( $thumbName ); - if ( file_exists( $thumbPath ) && !($flags & self::RENDER_FORCE) ) { - $thumbTime = filemtime( $thumbPath ); - if ( $thumbTime !== FALSE && - gmdate( 'YmdHis', $thumbTime ) >= $wgThumbnailEpoch ) { - - return $this->handler->getTransform( $this, $thumbPath, $thumbUrl, $params ); + if ( $this->repo->fileExists( $thumbPath ) && !( $flags & self::RENDER_FORCE ) ) { + $timestamp = $this->repo->getFileTimestamp( $thumbPath ); + if ( $timestamp !== false && $timestamp >= $wgThumbnailEpoch ) { + return $this->handler->getTransform( $this, false, $thumbUrl, $params ); } - } elseif( $flags & self::RENDER_FORCE ) { + } elseif ( $flags & self::RENDER_FORCE ) { wfDebug( __METHOD__ . " forcing rendering per flag File::RENDER_FORCE\n" ); } - $thumb = $this->handler->doTransform( $this, $thumbPath, $thumbUrl, $params ); + + // Create a temp FS file with the same extension + $tmpFile = TempFSFile::factory( 'transform_', $this->getExtension() ); + if ( !$tmpFile ) { + return new MediaTransformError( 'thumbnail_error', + $params['width'], 0, wfMsg( 'thumbnail-temp-create' ) ); + } + $tmpThumbPath = $tmpFile->getPath(); // path of 0-byte temp file + + // Actually render the thumbnail + $thumb = $this->handler->doTransform( $this, $tmpThumbPath, $thumbUrl, $params ); + $tmpFile->bind( $thumb ); // keep alive with $thumb // Ignore errors if requested if ( !$thumb ) { $thumb = null; } elseif ( $thumb->isError() ) { $this->lastError = $thumb->toText(); - if ( $wgIgnoreImageErrors && !($flags & self::RENDER_NOW) ) { - $thumb = $this->handler->getTransform( $this, $thumbPath, $thumbUrl, $params ); + if ( $wgIgnoreImageErrors && !( $flags & self::RENDER_NOW ) ) { + $thumb = $this->handler->getTransform( $this, $tmpThumbPath, $thumbUrl, $params ); + } + } elseif ( $thumb->hasFile() && !$thumb->fileIsSource() ) { + // @TODO: use a FileRepo store function + $op = array( 'op' => 'store', + 'src' => $tmpThumbPath, 'dst' => $thumbPath, 'overwriteDest' => true ); + // Copy any thumbnail from the FS into storage at $dstpath + $opts = array( 'ignoreErrors' => true, 'nonLocking' => true ); // performance + if ( !$this->getRepo()->getBackend()->doOperation( $op, $opts )->isOK() ) { + return new MediaTransformError( 'thumbnail_error', + $params['width'], 0, wfMsg( 'thumbnail-dest-create' ) ); } } return $thumb; } - /** * Transform a media file * @@ -797,8 +852,10 @@ abstract class File { // Purge. Useful in the event of Core -> Squid connection failure or squid // purge collisions from elsewhere during failure. Don't keep triggering for // "thumbs" which have the main image URL though (bug 13776) - if ( $wgUseSquid && ( !$thumb || $thumb->isError() || $thumb->getUrl() != $this->getURL()) ) { - SquidUpdate::purge( array( $thumbUrl ) ); + if ( $wgUseSquid ) { + if ( !$thumb || $thumb->isError() || $thumb->getUrl() != $this->getURL() ) { + SquidUpdate::purge( array( $thumbUrl ) ); + } } } while (false); @@ -815,6 +872,7 @@ abstract class File { /** * Get a MediaHandler instance for this file + * * @return MediaHandler */ function getHandler() { @@ -826,16 +884,17 @@ abstract class File { /** * Get a ThumbnailImage representing a file type icon + * * @return ThumbnailImage */ function iconThumb() { global $wgStylePath, $wgStyleDirectory; $try = array( 'fileicon-' . $this->getExtension() . '.png', 'fileicon.png' ); - foreach( $try as $icon ) { + foreach ( $try as $icon ) { $path = '/common/images/icons/' . $icon; $filepath = $wgStyleDirectory . $path; - if( file_exists( $filepath ) ) { + if ( file_exists( $filepath ) ) { // always FS return new ThumbnailImage( $this, $wgStylePath . $path, 120, 120 ); } } @@ -1091,7 +1150,7 @@ abstract class File { */ function getThumbUrl( $suffix = false ) { $this->assertRepoDefined(); - $path = $this->repo->getZoneUrl('thumb') . '/' . $this->getUrlRel(); + $path = $this->repo->getZoneUrl( 'thumb' ) . '/' . $this->getUrlRel(); if ( $suffix !== false ) { $path .= '/' . rawurlencode( $suffix ); } @@ -1099,49 +1158,49 @@ abstract class File { } /** - * Get the virtual URL for an archived file's thumbs, or a specific thumb. + * Get the public zone virtual URL for a current version source file * * @param $suffix bool|string if not false, the name of a thumbnail file * * @return string */ - function getArchiveVirtualUrl( $suffix = false ) { + function getVirtualUrl( $suffix = false ) { $this->assertRepoDefined(); - $path = $this->repo->getVirtualUrl() . '/public/archive/' . $this->getHashPath(); - if ( $suffix === false ) { - $path = substr( $path, 0, -1 ); - } else { - $path .= rawurlencode( $suffix ); + $path = $this->repo->getVirtualUrl() . '/public/' . $this->getUrlRel(); + if ( $suffix !== false ) { + $path .= '/' . rawurlencode( $suffix ); } return $path; } /** - * Get the virtual URL for a thumbnail file or directory + * Get the public zone virtual URL for an archived version source file * * @param $suffix bool|string if not false, the name of a thumbnail file * * @return string */ - function getThumbVirtualUrl( $suffix = false ) { + function getArchiveVirtualUrl( $suffix = false ) { $this->assertRepoDefined(); - $path = $this->repo->getVirtualUrl() . '/thumb/' . $this->getUrlRel(); - if ( $suffix !== false ) { - $path .= '/' . rawurlencode( $suffix ); + $path = $this->repo->getVirtualUrl() . '/public/archive/' . $this->getHashPath(); + if ( $suffix === false ) { + $path = substr( $path, 0, -1 ); + } else { + $path .= rawurlencode( $suffix ); } return $path; } /** - * Get the virtual URL for the file itself + * Get the virtual URL for a thumbnail file or directory * * @param $suffix bool|string if not false, the name of a thumbnail file * * @return string */ - function getVirtualUrl( $suffix = false ) { + function getThumbVirtualUrl( $suffix = false ) { $this->assertRepoDefined(); - $path = $this->repo->getVirtualUrl() . '/public/' . $this->getUrlRel(); + $path = $this->repo->getVirtualUrl() . '/thumb/' . $this->getUrlRel(); if ( $suffix !== false ) { $path .= '/' . rawurlencode( $suffix ); } @@ -1449,17 +1508,13 @@ abstract class File { } /** - * Get the 14-character timestamp of the file upload, or false if - * it doesn't exist + * Get the 14-character timestamp of the file upload * - * @return string + * @return string|false TS_MW timestamp or false on failure */ function getTimestamp() { - $path = $this->getPath(); - if ( !file_exists( $path ) ) { - return false; - } - return wfTimestamp( TS_MW, filemtime( $path ) ); + $this->assertRepoDefined(); + return $this->repo->getFileTimestamp( $this->getPath() ); } /** @@ -1468,7 +1523,8 @@ abstract class File { * @return string */ function getSha1() { - return self::sha1Base36( $this->getPath() ); + $this->assertRepoDefined(); + return $this->repo->getFileSha1( $this->getPath() ); } /** @@ -1508,67 +1564,11 @@ abstract class File { * @return array */ static function getPropsFromPath( $path, $ext = true ) { - wfProfileIn( __METHOD__ ); wfDebug( __METHOD__.": Getting file info for $path\n" ); - $info = array( - 'fileExists' => file_exists( $path ) && !is_dir( $path ) - ); - $gis = false; - if ( $info['fileExists'] ) { - $magic = MimeMagic::singleton(); - - if ( $ext === true ) { - $i = strrpos( $path, '.' ); - $ext = strtolower( $i ? substr( $path, $i + 1 ) : '' ); - } - - # mime type according to file contents - $info['file-mime'] = $magic->guessMimeType( $path, false ); - # logical mime type - $info['mime'] = $magic->improveTypeFromExtension( $info['file-mime'], $ext ); - - list( $info['major_mime'], $info['minor_mime'] ) = self::splitMime( $info['mime'] ); - $info['media_type'] = $magic->getMediaType( $path, $info['mime'] ); + wfDeprecated( __METHOD__, '1.19' ); - # Get size in bytes - $info['size'] = filesize( $path ); - - # Height, width and metadata - $handler = MediaHandler::getHandler( $info['mime'] ); - if ( $handler ) { - $tempImage = (object)array(); - $info['metadata'] = $handler->getMetadata( $tempImage, $path ); - $gis = $handler->getImageSize( $tempImage, $path, $info['metadata'] ); - } else { - $gis = false; - $info['metadata'] = ''; - } - $info['sha1'] = self::sha1Base36( $path ); - - wfDebug(__METHOD__.": $path loaded, {$info['size']} bytes, {$info['mime']}.\n"); - } else { - $info['mime'] = null; - $info['media_type'] = MEDIATYPE_UNKNOWN; - $info['metadata'] = ''; - $info['sha1'] = ''; - wfDebug(__METHOD__.": $path NOT FOUND!\n"); - } - if( $gis ) { - # NOTE: $gis[2] contains a code for the image type. This is no longer used. - $info['width'] = $gis[0]; - $info['height'] = $gis[1]; - if ( isset( $gis['bits'] ) ) { - $info['bits'] = $gis['bits']; - } else { - $info['bits'] = 0; - } - } else { - $info['width'] = 0; - $info['height'] = 0; - $info['bits'] = 0; - } - wfProfileOut( __METHOD__ ); - return $info; + $fsFile = new FSFile( $path ); + return $fsFile->getProps(); } /** @@ -1583,14 +1583,10 @@ abstract class File { * @return false|string False on failure */ static function sha1Base36( $path ) { - wfSuppressWarnings(); - $hash = sha1_file( $path ); - wfRestoreWarnings(); - if ( $hash === false ) { - return false; - } else { - return wfBaseConvert( $hash, 16, 36, 31 ); - } + wfDeprecated( __METHOD__, '1.19' ); + + $fsFile = new FSFile( $path ); + return $fsFile->getSha1Base36(); } /** diff --git a/includes/filerepo/file/ForeignAPIFile.php b/includes/filerepo/file/ForeignAPIFile.php index 281687b1d1..bd895393ad 100644 --- a/includes/filerepo/file/ForeignAPIFile.php +++ b/includes/filerepo/file/ForeignAPIFile.php @@ -13,7 +13,6 @@ * @ingroup FileRepo */ class ForeignAPIFile extends File { - private $mExists; protected $repoClass = 'ForeignApiRepo'; @@ -149,16 +148,16 @@ class ForeignAPIFile extends File { } function getSha1() { - return isset( $this->mInfo['sha1'] ) ? - wfBaseConvert( strval( $this->mInfo['sha1'] ), 16, 36, 31 ) : - null; + return isset( $this->mInfo['sha1'] ) + ? wfBaseConvert( strval( $this->mInfo['sha1'] ), 16, 36, 31 ) + : null; } function getTimestamp() { return wfTimestamp( TS_MW, - isset( $this->mInfo['timestamp'] ) ? - strval( $this->mInfo['timestamp'] ) : - null + isset( $this->mInfo['timestamp'] ) + ? strval( $this->mInfo['timestamp'] ) + : null ); } @@ -198,19 +197,14 @@ class ForeignAPIFile extends File { } function getThumbnails() { - $files = array(); $dir = $this->getThumbPath( $this->getName() ); - if ( is_dir( $dir ) ) { - $handle = opendir( $dir ); - if ( $handle ) { - while ( false !== ( $file = readdir($handle) ) ) { - if ( $file[0] != '.' ) { - $files[] = $file; - } - } - closedir( $handle ); - } + $iter = $this->repo->getBackend()->getFileList( array( 'dir' => $dir ) ); + + $files = array(); + foreach ( $iter as $file ) { + $files[] = $file; } + return $files; } @@ -224,15 +218,20 @@ class ForeignAPIFile extends File { function purgeDescriptionPage() { global $wgMemc, $wgContLang; + $url = $this->repo->getDescriptionRenderUrl( $this->getName(), $wgContLang->getCode() ); $key = $this->repo->getLocalCacheKey( 'RemoteFileDescription', 'url', md5($url) ); + $wgMemc->delete( $key ); } function purgeThumbnails( $options = array() ) { global $wgMemc; + $key = $this->repo->getLocalCacheKey( 'ForeignAPIRepo', 'ThumbUrl', $this->getName() ); $wgMemc->delete( $key ); + + $backend = $this->repo->getBackend(); $files = $this->getThumbnails(); // Give media handler a chance to filter the purge list $handler = $this->getHandler(); @@ -242,10 +241,9 @@ class ForeignAPIFile extends File { $dir = $this->getThumbPath( $this->getName() ); foreach ( $files as $file ) { - unlink( $dir . $file ); - } - if ( is_dir( $dir ) ) { - rmdir( $dir ); // Might have already gone away, spews errors if we don't. + $op = array( 'op' => 'delete', 'src' => "{$dir}{$file}" ); + $backend->doOperation( $op ); } + $backend->clean( array( 'dir' => $dir ) ); } } diff --git a/includes/filerepo/file/LocalFile.php b/includes/filerepo/file/LocalFile.php index d58c6837e6..653caa9783 100644 --- a/includes/filerepo/file/LocalFile.php +++ b/includes/filerepo/file/LocalFile.php @@ -234,7 +234,8 @@ class LocalFile extends File { * Load metadata from the file itself */ function loadFromFile() { - $this->setProps( self::getPropsFromPath( $this->getPath() ) ); + $props = $this->repo->getFileProps( $this->getVirtualUrl() ); + $this->setProps( $props ); } function getCacheFields( $prefix = 'img_' ) { @@ -582,8 +583,9 @@ class LocalFile extends File { */ function migrateThumbFile( $thumbName ) { $thumbDir = $this->getThumbPath(); - $thumbPath = "$thumbDir/$thumbName"; + /* Old code for bug 2532 + $thumbPath = "$thumbDir/$thumbName"; if ( is_dir( $thumbPath ) ) { // Directory where file should be // This happened occasionally due to broken migration code in 1.5 @@ -598,12 +600,12 @@ class LocalFile extends File { // Doesn't exist anymore clearstatcache(); } + */ - if ( is_file( $thumbDir ) ) { + if ( $this->repo->fileExists( $thumbDir, FileRepo::FILES_ONLY ) ) { // File where directory should be - unlink( $thumbDir ); - // Doesn't exist anymore - clearstatcache(); + $op = array( 'op' => 'delete', 'src' => $thumbDir ); + $this->repo->getBackend()->doOperation( $op ); } } @@ -624,21 +626,12 @@ class LocalFile extends File { } else { $dir = $this->getThumbPath(); } - $files = array(); - $files[] = $dir; - - if ( is_dir( $dir ) ) { - $handle = opendir( $dir ); - - if ( $handle ) { - while ( false !== ( $file = readdir( $handle ) ) ) { - if ( $file { 0 } != '.' ) { - $files[] = $file; - } - } - closedir( $handle ); - } + $backend = $this->repo->getBackend(); + $files = array( $dir ); + $iterator = $backend->getFileList( array( 'dir' => $dir ) ); + foreach ( $iterator as $file ) { + $files[] = $file; } return $files; @@ -719,7 +712,6 @@ class LocalFile extends File { } } - /** * Delete cached transformed files for the current version only. */ @@ -728,7 +720,7 @@ class LocalFile extends File { // Delete thumbnails $files = $this->getThumbnails(); - + // Give media handler a chance to filter the purge list if ( !empty( $options['forRefresh'] ) ) { $handler = $this->getHandler(); @@ -736,7 +728,7 @@ class LocalFile extends File { $handler->filterThumbnailPurgeList( $files, $options ); } } - + $dir = array_shift( $files ); $this->purgeThumbList( $dir, $files ); @@ -758,22 +750,24 @@ class LocalFile extends File { * @param $dir string base dir of the files. * @param $files array of strings: relative filenames (to $dir) */ - protected function purgeThumbList($dir, $files) { + protected function purgeThumbList( $dir, $files ) { $fileListDebug = strtr( var_export( $files, true ), array("\n"=>'') ); wfDebug( __METHOD__ . ": $fileListDebug\n" ); + $backend = $this->repo->getBackend(); foreach ( $files as $file ) { # Check that the base file name is part of the thumb name # This is a basic sanity check to avoid erasing unrelated directories if ( strpos( $file, $this->getName() ) !== false ) { - wfSuppressWarnings(); - unlink( "$dir/$file" ); - wfRestoreWarnings(); + $op = array( 'op' => 'delete', 'src' => "{$dir}/{$file}" ); + $backend->doOperation( $op ); } } + # Clear out directory if empty + $backend->clean( array( 'dir' => $dir ) ); } /** purgeDescription inherited */ @@ -892,7 +886,7 @@ class LocalFile extends File { /** * Upload a file and record it in the DB - * @param $srcPath String: source path or virtual URL + * @param $srcPath String: source storage path or virtual URL * @param $comment String: upload description * @param $pageText String: text to use for the new description page, * if a new description page is created @@ -1012,8 +1006,10 @@ class LocalFile extends File { ); if ( $dbw->affectedRows() == 0 ) { + if ( $oldver == '' ) { + throw new MWException( "Empty oi_archive_name. Database and storage out of sync?" ); + } $reupload = true; - # Collision, this is an update of a file # Insert previous contents into oldimage $dbw->insertSelect( 'oldimage', 'image', @@ -1374,7 +1370,8 @@ class LocalFile extends File { $this->load(); // Initialise now if necessary if ( $this->sha1 == '' && $this->fileExists ) { - $this->sha1 = File::sha1Base36( $this->getPath() ); + $tmpPath = $this->getLocalRefPath(); + $this->sha1 = FSFile::sha1Base36( $tmpPath ); if ( !wfReadOnly() && strval( $this->sha1 ) != '' ) { $dbw = $this->repo->getMasterDB(); $dbw->update( 'image', diff --git a/includes/filerepo/file/OldLocalFile.php b/includes/filerepo/file/OldLocalFile.php index a22da16fb1..419c395747 100644 --- a/includes/filerepo/file/OldLocalFile.php +++ b/includes/filerepo/file/OldLocalFile.php @@ -262,7 +262,7 @@ class OldLocalFile extends LocalFile { $dbw->begin(); $dstPath = $this->repo->getZonePath( 'public' ) . '/' . $this->getRel(); - $props = self::getPropsFromPath( $dstPath ); + $props = $this->repo->getFileProps( $dstPath ); if ( !$props['fileExists'] ) { return false; } diff --git a/includes/filerepo/file/TempFSFile.php b/includes/filerepo/file/TempFSFile.php new file mode 100644 index 0000000000..bee00fa784 --- /dev/null +++ b/includes/filerepo/file/TempFSFile.php @@ -0,0 +1,96 @@ += 15 ) { + return null; // give up + } + } + $tmpFile = new self( $path ); + if ( php_sapi_name() != 'cli' ) { + self::$instances[] = $tmpFile; // defer purge till shutdown + } + return $tmpFile; + } + + /** + * Purge this file off the file system + * + * @return bool Success + */ + public function purge() { + $this->canDelete = false; // done + wfSuppressWarnings(); + $ok = unlink( $this->path ); + wfRestoreWarnings(); + return $ok; + } + + /** + * Clean up the temporary file only after an object goes out of scope + * + * @param $object Object + * @return void + */ + public function bind( $object ) { + if ( is_object( $object ) ) { + $object->tempFSFileReferences[] = $this; + } + } + + /** + * Set flag to not clean up after the temporary file + * + * @return void + */ + public function preserve() { + $this->canDelete = false; + } + + /** + * Cleans up after the temporary file by deleting it + */ + function __destruct() { + if ( $this->canDelete ) { + wfSuppressWarnings(); + unlink( $this->path ); + wfRestoreWarnings(); + } + } +} diff --git a/includes/filerepo/file/UnregisteredLocalFile.php b/includes/filerepo/file/UnregisteredLocalFile.php index 6a0e0979c9..14987b7af6 100644 --- a/includes/filerepo/file/UnregisteredLocalFile.php +++ b/includes/filerepo/file/UnregisteredLocalFile.php @@ -27,8 +27,8 @@ class UnregisteredLocalFile extends File { var $handler; /** - * @param $path - * @param $mime + * @param $path string Storage path + * @param $mime string * @return UnregisteredLocalFile */ static function newFromPath( $path, $mime ) { @@ -69,6 +69,7 @@ class UnregisteredLocalFile extends File { if ( $path ) { $this->path = $path; } else { + $this->assertRepoDefined(); $this->path = $repo->getRootDirectory() . '/' . $repo->getHashPath( $this->name ) . $this->name; } @@ -101,7 +102,7 @@ class UnregisteredLocalFile extends File { function getMimeType() { if ( !isset( $this->mime ) ) { $magic = MimeMagic::singleton(); - $this->mime = $magic->guessMimeType( $this->getPath() ); + $this->mime = $magic->guessMimeType( $this->getLocalRefPath() ); } return $this->mime; } @@ -110,7 +111,7 @@ class UnregisteredLocalFile extends File { if ( !$this->getHandler() ) { return false; } - return $this->handler->getImageSize( $this, $this->getPath() ); + return $this->handler->getImageSize( $this, $this->getLocalRefPath() ); } function getMetadata() { @@ -118,7 +119,7 @@ class UnregisteredLocalFile extends File { if ( !$this->getHandler() ) { $this->metadata = false; } else { - $this->metadata = $this->handler->getMetadata( $this, $this->getPath() ); + $this->metadata = $this->handler->getMetadata( $this, $this->getLocalRefPath() ); } } return $this->metadata; @@ -134,10 +135,11 @@ class UnregisteredLocalFile extends File { } function getSize() { - if ( file_exists( $this->path ) ) { - return filesize( $this->path ); - } else { - return false; + $this->assertRepoDefined(); + $props = $this->repo->getFileProps( $this->path ); + if ( isset( $props['size'] ) ) { + return $props['size']; } + return false; // doesn't exist } } diff --git a/includes/media/Bitmap.php b/includes/media/Bitmap.php index dd1f7f1d0a..fa9067d8c9 100644 --- a/includes/media/Bitmap.php +++ b/includes/media/Bitmap.php @@ -125,7 +125,7 @@ class BitmapHandler extends ImageHandler { 'srcWidth' => $image->getWidth(), 'srcHeight' => $image->getHeight(), 'mimeType' => $image->getMimeType(), - 'srcPath' => $image->getPath(), + 'srcPath' => $image->getLocalRefPath(), 'dstPath' => $dstPath, 'dstUrl' => $dstUrl, ); diff --git a/includes/media/Bitmap_ClientOnly.php b/includes/media/Bitmap_ClientOnly.php index 506792292b..3c5d973874 100644 --- a/includes/media/Bitmap_ClientOnly.php +++ b/includes/media/Bitmap_ClientOnly.php @@ -38,6 +38,6 @@ class BitmapHandler_ClientOnly extends BitmapHandler { return new TransformParameterError( $params ); } return new ThumbnailImage( $image, $image->getURL(), $params['width'], - $params['height'], $image->getPath() ); + $params['height'], $image->getLocalRefPath() ); } } diff --git a/includes/media/DjVu.php b/includes/media/DjVu.php index f4553b3afd..6e6505c8f1 100644 --- a/includes/media/DjVu.php +++ b/includes/media/DjVu.php @@ -131,7 +131,7 @@ class DjVuHandler extends ImageHandler { } $width = $params['width']; $height = $params['height']; - $srcPath = $image->getPath(); + $srcPath = $image->getLocalRefPath(); $page = $params['page']; if ( $page > $this->pageCount( $image ) ) { return new MediaTransformError( 'thumbnail_error', $width, $height, wfMsg( 'djvu_page_error' ) ); diff --git a/includes/media/ExifBitmap.php b/includes/media/ExifBitmap.php index 05ce161b60..1b0ab78a53 100644 --- a/includes/media/ExifBitmap.php +++ b/includes/media/ExifBitmap.php @@ -137,7 +137,7 @@ class ExifBitmapHandler extends BitmapHandler { global $wgEnableAutoRotation; $gis = parent::getImageSize( $image, $path ); - // Don't just call $image->getMetadata(); File::getPropsFromPath() calls us with a bogus object. + // Don't just call $image->getMetadata(); FSFile::getPropsFromPath() calls us with a bogus object. // This may mean we read EXIF data twice on initial upload. if ( $wgEnableAutoRotation ) { $meta = $this->getMetadata( $image, $path ); diff --git a/includes/media/Generic.php b/includes/media/Generic.php index 8684af552e..4c36720f62 100644 --- a/includes/media/Generic.php +++ b/includes/media/Generic.php @@ -97,7 +97,7 @@ abstract class MediaHandler { * Get handler-specific metadata which will be saved in the img_metadata field. * * @param $image File: the image object, or false if there isn't one. - * Warning, File::getPropsFromPath might pass an (object)array() instead (!) + * Warning, FSFile::getPropsFromPath might pass an (object)array() instead (!) * @param $path String: the filename * @return String */ @@ -187,7 +187,7 @@ abstract class MediaHandler { * @param $dstUrl String: Destination URL to use in output HTML * @param $params Array: Arbitrary set of parameters validated by $this->validateParam() */ - function getTransform( $image, $dstPath, $dstUrl, $params ) { + final function getTransform( $image, $dstPath, $dstUrl, $params ) { return $this->doTransform( $image, $dstPath, $dstUrl, $params, self::TRANSFORM_LATER ); } @@ -260,7 +260,7 @@ abstract class MediaHandler { * @param $image File */ function getPageDimensions( $image, $page ) { - $gis = $this->getImageSize( $image, $image->getPath() ); + $gis = $this->getImageSize( $image, $image->getLocalRefPath() ); return array( 'width' => $gis[0], 'height' => $gis[1] @@ -642,13 +642,6 @@ abstract class ImageHandler extends MediaHandler { return true; } - /** - * Get a transform output object without actually doing the transform - */ - function getTransform( $image, $dstPath, $dstUrl, $params ) { - return $this->doTransform( $image, $dstPath, $dstUrl, $params, self::TRANSFORM_LATER ); - } - /** * Validate thumbnail parameters and fill in the correct height * diff --git a/includes/media/MediaTransformOutput.php b/includes/media/MediaTransformOutput.php index 1e1b4598be..f7c527086d 100644 --- a/includes/media/MediaTransformOutput.php +++ b/includes/media/MediaTransformOutput.php @@ -22,31 +22,24 @@ abstract class MediaTransformOutput { /** * Get the width of the output box */ - function getWidth() { + public function getWidth() { return $this->width; } /** * Get the height of the output box */ - function getHeight() { + public function getHeight() { return $this->height; } /** * @return string The thumbnail URL */ - function getUrl() { + public function getUrl() { return $this->url; } - /** - * @return String: destination file path (local filesystem) - */ - function getPath() { - return $this->path; - } - /** * Fetch HTML for this transform output * @@ -67,15 +60,57 @@ abstract class MediaTransformOutput { * * @return string */ - abstract function toHtml( $options = array() ); + abstract public function toHtml( $options = array() ); /** * This will be overridden to return true in error classes */ - function isError() { + public function isError() { return false; } + /** + * Check if an output thumbnail file was actually made. + * This will return false if there was an error, the + * thumnail is to be handled client-side only, or if + * transformation was deferred via TRANSFORM_LATER. + * + * @return Bool + */ + public function hasFile() { + // If TRANSFORM_LATER, $this->path will be false + return ( !$this->isError() && $this->path ); + } + + /** + * Check if the output thumbnail file is the same as the source. + * This can occur if the requested width was bigger than the source. + * + * @return Bool + */ + public function fileIsSource() { + return ( !$this->isError() && $this->path === $this->file->getLocalRefPath() ); + } + + /** + * Get the path of a file system copy of the thumbnail + * + * @return string|false Returns false if there isn't one + */ + public function getLocalCopyPath() { + return $this->path; + } + + /** + * Stream the file if there were no errors + * + * @param $headers Array Additional HTTP headers to send on success + * @return Bool success + */ + public function streamFile( $headers = array() ) { + return $this->path && StreamFile::stream( $this->path, $headers ); + } + /** * Wrap some XHTML text in an anchor tag with the given attributes * @@ -97,7 +132,7 @@ abstract class MediaTransformOutput { * @param $params array * @return array */ - function getDescLinkAttribs( $title = null, $params = '' ) { + public function getDescLinkAttribs( $title = null, $params = '' ) { $query = $this->page ? ( 'page=' . urlencode( $this->page ) ) : ''; if( $params ) { $query .= $query ? '&'.$params : $params; diff --git a/includes/media/SVG.php b/includes/media/SVG.php index 986c164ca2..aac838e165 100644 --- a/includes/media/SVG.php +++ b/includes/media/SVG.php @@ -93,7 +93,7 @@ class SvgHandler extends ImageHandler { $clientHeight = $params['height']; $physicalWidth = $params['physicalWidth']; $physicalHeight = $params['physicalHeight']; - $srcPath = $image->getPath(); + $srcPath = $image->getLocalRefPath(); if ( $flags & self::TRANSFORM_LATER ) { return new ThumbnailImage( $image, $dstUrl, $clientWidth, $clientHeight, $dstPath ); diff --git a/includes/specials/SpecialRevisiondelete.php b/includes/specials/SpecialRevisiondelete.php index dce79d3f38..cf06a2f780 100644 --- a/includes/specials/SpecialRevisiondelete.php +++ b/includes/specials/SpecialRevisiondelete.php @@ -313,7 +313,7 @@ class SpecialRevisionDelete extends UnlistedSpecialPage { $key = $oimage->getStorageKey(); $path = $repo->getZonePath( 'deleted' ) . '/' . $repo->getDeletedHashPath( $key ) . $key; - StreamFile::stream( $path ); + $repo->streamFile( $path ); } /** diff --git a/includes/specials/SpecialUndelete.php b/includes/specials/SpecialUndelete.php index 68e35b777d..c50855ea9e 100644 --- a/includes/specials/SpecialUndelete.php +++ b/includes/specials/SpecialUndelete.php @@ -996,7 +996,7 @@ class SpecialUndelete extends SpecialPage { $repo = RepoGroup::singleton()->getLocalRepo(); $path = $repo->getZonePath( 'deleted' ) . '/' . $repo->getDeletedHashPath( $key ) . $key; - StreamFile::stream( $path ); + $repo->streamFile( $path ); } private function showHistory() { diff --git a/includes/upload/UploadBase.php b/includes/upload/UploadBase.php index 8d4cb5db6a..8d09bf51f5 100644 --- a/includes/upload/UploadBase.php +++ b/includes/upload/UploadBase.php @@ -163,6 +163,9 @@ abstract class UploadBase { */ public function initializePathInfo( $name, $tempPath, $fileSize, $removeTempFile = false ) { $this->mDesiredDestName = $name; + if ( FileBackend::isStoragePath( $tempPath ) ) { + throw new MWException( __METHOD__ . " given storage path `$tempPath`." ); + } $this->mTempPath = $tempPath; $this->mFileSize = $fileSize; $this->mRemoveTempFile = $removeTempFile; @@ -196,40 +199,6 @@ abstract class UploadBase { return $this->mFileSize; } - /** - * Append a file to the Repo file - * - * @deprecated since 1.19 - * - * @param $srcPath String: path to source file - * @param $toAppendPath String: path to the Repo file that will be appended to. - * @return Status Status - */ - protected function appendToUploadFile( $srcPath, $toAppendPath ) { - wfDeprecated( __METHOD__, '1.19' ); - - $repo = RepoGroup::singleton()->getLocalRepo(); - $status = $repo->append( $srcPath, $toAppendPath ); - return $status; - } - - /** - * Finish appending to the Repo file - * - * @deprecated since 1.19 - * - * @param $toAppendPath String: path to the Repo file that will be appended to. - * @return Status Status - */ - protected function appendFinish( $toAppendPath ) { - wfDeprecated( __METHOD__, '1.19' ); - - $repo = RepoGroup::singleton()->getLocalRepo(); - $status = $repo->appendFinish( $toAppendPath ); - return $status; - } - - /** * @param $srcPath String: the source path * @return the real path if it was a virtual URL @@ -237,7 +206,11 @@ abstract class UploadBase { function getRealPath( $srcPath ) { $repo = RepoGroup::singleton()->getLocalRepo(); if ( $repo->isVirtualUrl( $srcPath ) ) { - return $repo->resolveVirtualUrl( $srcPath ); + // @TODO: just make uploads work with storage paths + // UploadFromStash loads files via virtuals URLs + $tmpFile = $repo->getLocalCopy( $srcPath ); + $tmpFile->bind( $this ); // keep alive with $thumb + return $tmpFile->getPath(); } return $srcPath; } @@ -370,7 +343,7 @@ abstract class UploadBase { # we need to populate mFinalExtension $this->getTitle(); - $this->mFileProps = File::getPropsFromPath( $this->mTempPath, $this->mFinalExtension ); + $this->mFileProps = FSFile::getPropsFromPath( $this->mTempPath, $this->mFinalExtension ); # check mime type, if desired $mime = $this->mFileProps[ 'file-mime' ]; @@ -556,7 +529,7 @@ abstract class UploadBase { } // Check dupes against existing files - $hash = File::sha1Base36( $this->mTempPath ); + $hash = FSFile::getSha1Base36FromPath( $this->mTempPath ); $dupes = RepoGroup::singleton()->findBySha1( $hash ); $title = $this->getTitle(); // Remove all matches against self diff --git a/includes/upload/UploadFromChunks.php b/includes/upload/UploadFromChunks.php index b0916191d4..01bc111ee6 100644 --- a/includes/upload/UploadFromChunks.php +++ b/includes/upload/UploadFromChunks.php @@ -95,8 +95,8 @@ class UploadFromChunks extends UploadFromFile { $fileList[] = $this->getVirtualChunkLocation( $i ); } - // Concatinate into the mVirtualTempPath location; - $status = $this->repo->concatenate( $fileList, $this->mVirtualTempPath, FileRepo::DELETE_SOURCE ); + // Concatenate into the mVirtualTempPath location; + $status = $this->repo->concatenate( $fileList, $this->mVirtualTempPath, FileRepo::DELETE_SOURCE ); if( !$status->isOk() ){ return $status; } @@ -104,6 +104,21 @@ class UploadFromChunks extends UploadFromFile { $this->mTempPath = $this->getRealPath( $this->mVirtualTempPath ); return $status; } + + /** + * Perform the upload, then remove the temp copy afterward + * @param $comment string + * @param $pageText string + * @param $watch bool + * @param $user User + * @return Status + */ + public function performUpload( $comment, $pageText, $watch, $user ) { + $rv = parent::performUpload( $comment, $pageText, $watch, $user ); + $this->repo->freeTemp( $this->mVirtualTempPath ); + return $rv; + } + /** * Returns the virtual chunk location: * @param unknown_type $index diff --git a/includes/upload/UploadStash.php b/includes/upload/UploadStash.php index 217b84dc92..6aaa1fe04a 100644 --- a/includes/upload/UploadStash.php +++ b/includes/upload/UploadStash.php @@ -48,7 +48,7 @@ class UploadStash { * * @param $repo FileRepo */ - public function __construct( $repo, $user = null ) { + public function __construct( FileRepo $repo, $user = null ) { // this might change based on wiki's configuration. $this->repo = $repo; @@ -106,10 +106,7 @@ class UploadStash { // fetch fileprops $path = $this->fileMetadata[$key]['us_path']; - if ( $this->repo->isVirtualUrl( $path ) ) { - $path = $this->repo->resolveVirtualUrl( $path ); - } - $this->fileProps[$key] = File::getPropsFromPath( $path ); + $this->fileProps[$key] = $this->repo->getFileProps( $path ); } if ( ! $this->files[$key]->exists() ) { @@ -163,7 +160,7 @@ class UploadStash { wfDebug( __METHOD__ . " tried to stash file at '$path', but it doesn't exist\n" ); throw new UploadStashBadPathException( "path doesn't exist" ); } - $fileProps = File::getPropsFromPath( $path ); + $fileProps = FSFile::getPropsFromPath( $path ); wfDebug( __METHOD__ . " stashing file at '$path'\n" ); // we will be initializing from some tmpnam files that don't have extensions. @@ -215,7 +212,7 @@ class UploadStash { $error = array( 'unknown', 'no error recorded' ); } } - throw new UploadStashFileException( "error storing file in '$path': " . implode( '; ', $error ) ); + throw new UploadStashFileException( "Error storing file in '$path': " . implode( '; ', $error ) ); } $stashPath = $storeStatus->value; @@ -233,7 +230,7 @@ class UploadStash { 'us_user' => $this->userId, 'us_key' => $key, 'us_orig_path' => $path, - 'us_path' => $stashPath, + 'us_path' => $stashPath, // virtual URL 'us_size' => $fileProps['size'], 'us_sha1' => $fileProps['sha1'], 'us_mime' => $fileProps['mime'], diff --git a/languages/messages/MessagesEn.php b/languages/messages/MessagesEn.php index 7c9f08a125..949d6a9fe3 100644 --- a/languages/messages/MessagesEn.php +++ b/languages/messages/MessagesEn.php @@ -2253,6 +2253,36 @@ It cannot be properly checked for security.', 'uploadstash-refresh' => 'Refresh the list of files', 'invalid-chunk-offset' => 'Invalid chunk offset', +# file backend +'backend-fail-stream' => 'Could not stream file $1.', +'backend-fail-backup' => 'Could not backup file $1.', +'backend-fail-notexists' => 'The file $1 does not exist.', +'backend-fail-hashes' => 'Could not get file hashes for comparison.', +'backend-fail-notsame' => 'A non-identical file already exists at $1.', +'backend-fail-invalidpath' => '$1 is not a valid storage path.', +'backend-fail-delete' => 'Could not delete file $1.', +'backend-fail-alreadyexists' => 'The file $1 already exists.', +'backend-fail-store' => 'Could not store file $1 at $2', +'backend-fail-copy' => 'Could not copy file $1 to $2', +'backend-fail-move' => 'Could not move file $1 to $2', +'backend-fail-opentemp' => 'Could not open temporary file.', +'backend-fail-writetemp' => 'Could not write to temporary file.', +'backend-fail-closetemp' => 'Could not close temporary file.', +'backend-fail-read' => 'Could not read file $1', +'backend-fail-create' => 'Could not create file $1', + +# lock manager +'lockmanager-notlocked' => 'Could not unlock key "$1"; it is not locked.', +'lockmanager-fail-closelock' => 'Could not close lock file for key "$1".', +'lockmanager-fail-deletelock' => 'Could not delete lock file for key "$1".', +'lockmanager-fail-openlock' => 'Could not open lock file for key "$1".', +'lockmanager-fail-acquirelock' => 'Could not acquire lock for key "$1".', +'lockmanager-fail-releaselock' => 'Could not release lock for key "$1".', +'lockmanager-fail-acquirelocks' => 'Could not acquire locks for keys "$1".', +'lockmanager-fail-db-bucket' => 'Could not contact enough lock databases in bucket $1', +'lockmanager-fail-db-release' => 'Could not release locks on database $1', +'lockmanager-fail-svr-release' => 'Could not release locks on server $1', + # img_auth script messages 'img-auth-accessdenied' => 'Access denied', 'img-auth-nopathinfo' => 'Missing PATH_INFO. @@ -3311,6 +3341,8 @@ Please visit [//www.mediawiki.org/wiki/Localisation MediaWiki Localisation] and 'thumbnail_error' => 'Error creating thumbnail: $1', 'djvu_page_error' => 'DjVu page out of range', 'djvu_no_xml' => 'Unable to fetch XML for DjVu file', +'thumbnail-temp-create' => 'Unable to create temporary thumbnail file', +'thumbnail-dest-create' => 'Unable to save thumbnail to destination', 'thumbnail_invalid_params' => 'Invalid thumbnail parameters', 'thumbnail_dest_directory' => 'Unable to create destination directory', 'thumbnail_image-type' => 'Image type not supported', diff --git a/maintenance/language/messages.inc b/maintenance/language/messages.inc index 52da426d7f..ac388e8bb4 100644 --- a/maintenance/language/messages.inc +++ b/maintenance/language/messages.inc @@ -1346,6 +1346,38 @@ $wgMessageStructure = array( 'upload-http-error', ), + 'filebackend-errors' => array( + 'backend-fail-stream', + 'backend-fail-backup', + 'backend-fail-notexists', + 'backend-fail-hashes', + 'backend-fail-notsame', + 'backend-fail-invalidpath', + 'backend-fail-delete', + 'backend-fail-alreadyexists', + 'backend-fail-store', + 'backend-fail-copy', + 'backend-fail-move', + 'backend-fail-opentemp', + 'backend-fail-writetemp', + 'backend-fail-closetemp', + 'backend-fail-read', + 'backend-fail-create', + ), + + 'lockmanger-errors' => array( + 'lockmanager-notlocked', + 'lockmanager-fail-closelock', + 'lockmanager-fail-deletelock', + 'lockmanager-fail-acquirelock', + 'lockmanager-fail-openlock', + 'lockmanager-fail-releaselock', + 'lockmanager-fail-acquirelocks', + 'lockmanager-fail-db-bucket', + 'lockmanager-fail-db-release', + 'lockmanager-fail-svr-release' + ), + 'zip' => array( 'zip-file-open-error', 'zip-wrong-format', diff --git a/maintenance/locking/LockServerDaemon.php b/maintenance/locking/LockServerDaemon.php new file mode 100644 index 0000000000..cf9d948b72 --- /dev/null +++ b/maintenance/locking/LockServerDaemon.php @@ -0,0 +1,427 @@ +main(); + +/** + * Simple lock server daemon that accepts lock/unlock requests. + * This should not require MediaWiki setup or PHP files. + */ +class LockServerDaemon { + /** @var resource */ + protected $sock; // socket to listen/accept on + /** @var Array */ + protected $shLocks = array(); // (key => session => 1) + /** @var Array */ + protected $exLocks = array(); // (key => session) + /** @var Array */ + protected $sessions = array(); // (session => resource) + /** @var Array */ + protected $deadSessions = array(); // (session => UNIX timestamp) + + /** @var Array */ + protected $sessionIndexSh = array(); // (session => key => 1) + /** @var Array */ + protected $sessionIndexEx = array(); // (session => key => 1) + + protected $address; // string (IP/hostname) + protected $port; // integer + protected $authKey; // string key + protected $connTimeout; // array ( 'sec' => integer, 'usec' => integer ) + protected $lockTimeout; // integer number of seconds + protected $maxLocks; // integer + protected $maxClients; // integer + protected $maxBacklog; // integer + + protected $startTime; // integer UNIX timestamp + protected $lockCount = 0; // integer + protected $ticks = 0; // integer counter + + protected static $instance = null; + + /** + * @params $config Array + * @return LockServerDaemon + */ + public function init( array $config ) { + if ( self::$instance ) { + throw new Exception( 'LockServer already initialized.' ); + } + self::$instance = new self( $config ); + return self::$instance; + } + + /** + * @params $config Array + */ + protected function __construct( array $config ) { + $required = array( 'address', 'port', 'authKey' ); + foreach ( $required as $par ) { + if ( !isset( $config[$par] ) ) { + throw new Exception( "Parameter '$par' must be specified." ); + } + } + + $this->address = $config['address']; + $this->port = $config['port']; + $this->authKey = $config['authKey']; + + $connTimeout = isset( $config['connTimeout'] ) + ? $config['connTimeout'] + : 1.5; + $this->connTimeout = array( + 'sec' => floor( $connTimeout ), + 'usec' => floor( ( $connTimeout - floor( $connTimeout ) ) * 1e6 ) + ); + $this->lockTimeout = isset( $config['lockTimeout'] ) + ? $config['lockTimeout'] + : 60; + $this->maxLocks = isset( $config['maxLocks'] ) + ? $config['maxLocks'] + : 5000; + $this->maxClients = isset( $config['maxClients'] ) + ? $config['maxClients'] + : 100; + $this->maxBacklog = isset( $config['maxBacklog'] ) + ? $config['maxBacklog'] + : 10; + } + + /** + * @return void + */ + protected function setupSocket() { + if ( !function_exists( 'socket_create' ) ) { + throw new Exception( "PHP sockets extension missing from PHP CLI mode." ); + } + $sock = socket_create( AF_INET, SOCK_STREAM, SOL_TCP ); + if ( $sock === false ) { + throw new Exception( "socket_create(): " . socket_strerror( socket_last_error() ) ); + } + socket_set_option( $sock, SOL_SOCKET, SO_REUSEADDR, 1 ); // bypass 2MLS + if ( socket_bind( $sock, $this->address, $this->port ) === false ) { + throw new Exception( "socket_bind(): " . + socket_strerror( socket_last_error( $sock ) ) ); + } elseif ( socket_listen( $sock, $this->maxBacklog ) === false ) { + throw new Exception( "socket_listen(): " . + socket_strerror( socket_last_error( $sock ) ) ); + } + $this->sock = $sock; + + $this->startTime = time(); + } + + /** + * @return void + */ + public function main() { + // Setup socket and start listing + $this->setupSocket(); + // Create a list of all the clients that will be connected to us. + $clients = array( $this->sock ); // start off with listening socket + do { + // Create a copy, so $clients doesn't get modified by socket_select() + $read = $clients; // clients-with-data + // Get a list of all the clients that have data to be read from + $changed = socket_select( $read, $write = NULL, $except = NULL, NULL ); + if ( $changed === false ) { + trigger_error( 'socket_listen(): ' . socket_strerror( socket_last_error() ) ); + continue; + } elseif ( $changed < 1 ) { + continue; // wait + } + // Check if there is a client trying to connect... + if ( in_array( $this->sock, $read ) && count( $clients ) < $this->maxClients ) { + // Accept the new client... + $newsock = socket_accept( $this->sock ); + socket_set_option( $newsock, SOL_SOCKET, SO_RCVTIMEO, $this->connTimeout ); + socket_set_option( $newsock, SOL_SOCKET, SO_SNDTIMEO, $this->connTimeout ); + $clients[] = $newsock; + // Remove the listening socket from the clients-with-data array... + $key = array_search( $this->sock, $read ); + unset( $read[$key] ); + } + // Loop through all the clients that have data to read... + foreach ( $read as $read_sock ) { + // Read until newline or 65535 bytes are recieved. + // socket_read show errors when the client is disconnected. + $data = @socket_read( $read_sock, 65535, PHP_NORMAL_READ ); + // Check if the client is disconnected + if ( $data === false ) { + // Remove client from $clients list + $key = array_search( $read_sock, $clients ); + unset( $clients[$key] ); + // Remove socket's session from tracking (if it exists) + $session = array_search( $read_sock, $this->sessions ); + if ( $session !== false ) { + unset( $this->sessions[$session] ); + // Record recently killed sessions that still have locks + if ( isset( $this->sessionIndexSh[$session] ) + || isset( $this->sessionIndexEx[$session] ) ) + { + $this->deadSessions[$session] = time(); + } + } + } else { + // Perform the requested command... + $response = $this->doCommand( trim( $data ), $read_sock ); + // Send the response to the client... + if ( socket_write( $read_sock, "$response\n" ) === false ) { + trigger_error( 'socket_write(): ' . + socket_strerror( socket_last_error( $read_sock ) ) ); + } + } + } + // Prune dead locks every 10 socket events... + if ( ++$this->ticks >= 9 ) { + $this->ticks = 0; + $this->purgeExpiredLocks(); + } + } while ( true ); + } + + /** + * @param $data string + * @param $sourceSock resource + * @return string + */ + protected function doCommand( $data, $sourceSock ) { + $cmdArr = $this->getCommand( $data ); + if ( is_string( $cmdArr ) ) { + return $cmdArr; // error + } + list( $function, $session, $type, $resources ) = $cmdArr; + // On first command, track the session => sock correspondence + if ( !isset( $this->sessions[$session] ) ) { + $this->sessions[$session] = $sourceSock; + } + if ( $function === 'ACQUIRE' ) { + return $this->lock( $session, $type, $resources ); + } elseif ( $function === 'RELEASE' ) { + return $this->unlock( $session, $type, $resources ); + } elseif ( $function === 'RELEASE_ALL' ) { + return $this->release( $session ); + } elseif ( $function === 'STAT' ) { + return $this->stat(); + } + return 'INTERNAL_ERROR'; + } + + /** + * @param $data string + * @return Array + */ + protected function getCommand( $data ) { + $m = explode( ':', $data ); // + if ( count( $m ) == 5 ) { + list( $session, $key, $command, $type, $values ) = $m; + if ( sha1( $session . $command . $type . $values . $this->authKey ) !== $key ) { + return 'BAD_KEY'; + } elseif ( strlen( $session ) !== 31 ) { + return 'BAD_SESSION'; + } + $values = explode( '|', $values ); + if ( $command === 'ACQUIRE' ) { + $needsLockArgs = true; + } elseif ( $command === 'RELEASE' ) { + $needsLockArgs = true; + } elseif ( $command === 'RELEASE_ALL' ) { + $needsLockArgs = false; + } elseif ( $command === 'STAT' ) { + $needsLockArgs = false; + } else { + return 'BAD_COMMAND'; + } + if ( $needsLockArgs ) { + if ( $type !== 'SH' && $type !== 'EX' ) { + return 'BAD_TYPE'; + } + foreach ( $values as $value ) { + if ( strlen( $value ) !== 31 ) { + return 'BAD_FORMAT'; + } + } + } + return array( $command, $session, $type, $values ); + } + return 'BAD_FORMAT'; + } + + /** + * @param $session string + * @param $type string + * @param $keys Array + * @return string + */ + protected function lock( $session, $type, $keys ) { + if ( $this->lockCount >= $this->maxLocks ) { + return 'TOO_MANY_LOCKS'; + } + if ( $type === 'SH' ) { + // Check if any keys are already write-locked... + foreach ( $keys as $key ) { + if ( isset( $this->exLocks[$key] ) && $this->exLocks[$key] !== $session ) { + return 'CANT_ACQUIRE'; + } + } + // Acquire the read-locks... + foreach ( $keys as $key ) { + $this->set_sh_lock( $key, $session ); + } + return 'ACQUIRED'; + } elseif ( $type === 'EX' ) { + // Check if any keys are already read-locked or write-locked... + foreach ( $keys as $key ) { + if ( isset( $this->exLocks[$key] ) && $this->exLocks[$key] !== $session ) { + return 'CANT_ACQUIRE'; + } + if ( isset( $this->shLocks[$key] ) ) { + foreach ( $this->shLocks[$key] as $otherSession => $x ) { + if ( $otherSession !== $session ) { + return 'CANT_ACQUIRE'; + } + } + } + } + // Acquire the write-locks... + foreach ( $keys as $key ) { + $this->set_ex_lock( $key, $session ); + } + return 'ACQUIRED'; + } + return 'INTERNAL_ERROR'; + } + + /** + * @param $session string + * @param $type string + * @param $keys Array + * @return string + */ + protected function unlock( $session, $type, $keys ) { + if ( $type === 'SH' ) { + foreach ( $keys as $key ) { + $this->unset_sh_lock( $key, $session ); + } + return 'RELEASED'; + } elseif ( $type === 'EX' ) { + foreach ( $keys as $key ) { + $this->unset_ex_lock( $key, $session ); + } + return 'RELEASED'; + } + return 'INTERNAL_ERROR'; + } + + /** + * @param $session string + * @return string + */ + protected function release( $session ) { + if ( isset( $this->sessionIndexSh[$session] ) ) { + foreach ( $this->sessionIndexSh[$session] as $key => $x ) { + $this->unset_sh_lock( $key, $session ); + } + } + if ( isset( $this->sessionIndexEx[$session] ) ) { + foreach ( $this->sessionIndexEx[$session] as $key => $x ) { + $this->unset_ex_lock( $key, $session ); + } + } + return 'RELEASED_ALL'; + } + + /** + * @return string + */ + protected function stat() { + return ( time() - $this->startTime ) . ':' . memory_get_usage(); + } + + /** + * Clear locks for sessions that have been dead for a while + * + * @return void + */ + protected function purgeExpiredLocks() { + $now = time(); + foreach ( $this->deadSessions as $session => $timestamp ) { + if ( ( $now - $timestamp ) > $this->lockTimeout ) { + $this->release( $session ); + unset( $this->deadSessions[$session] ); + } + } + } + + /** + * @param $key string + * @param $session string + * @return void + */ + protected function set_sh_lock( $key, $session ) { + if ( !isset( $this->shLocks[$key][$session] ) ) { + $this->shLocks[$key][$session] = 1; + $this->sessionIndexSh[$session][$key] = 1; + ++$this->lockCount; // we are adding a lock + } + } + + /** + * @param $key string + * @param $session string + * @return void + */ + protected function set_ex_lock( $key, $session ) { + if ( !isset( $this->exLocks[$key][$session] ) ) { + $this->exLocks[$key] = $session; + $this->sessionIndexEx[$session][$key] = 1; + ++$this->lockCount; // we are adding a lock + } + } + + /** + * @param $key string + * @param $session string + * @return void + */ + protected function unset_sh_lock( $key, $session ) { + if ( isset( $this->shLocks[$key][$session] ) ) { + unset( $this->shLocks[$key][$session] ); + if ( !count( $this->shLocks[$key] ) ) { + unset( $this->shLocks[$key] ); + } + unset( $this->sessionIndexSh[$session][$key] ); + if ( !count( $this->sessionIndexSh[$session] ) ) { + unset( $this->sessionIndexSh[$session] ); + } + --$this->lockCount; + } + } + + /** + * @param $key string + * @param $session string + * @return void + */ + protected function unset_ex_lock( $key, $session ) { + if ( isset( $this->exLocks[$key] ) && $this->exLocks[$key] === $session ) { + unset( $this->exLocks[$key] ); + unset( $this->sessionIndexEx[$session][$key] ); + if ( !count( $this->sessionIndexEx[$session] ) ) { + unset( $this->sessionIndexEx[$session] ); + } + --$this->lockCount; + } + } +} diff --git a/maintenance/locking/file_locks.sql b/maintenance/locking/file_locks.sql new file mode 100644 index 0000000000..f51d06b3f6 --- /dev/null +++ b/maintenance/locking/file_locks.sql @@ -0,0 +1,11 @@ +-- Table to handle resource locking (EX) with row-level locking +CREATE TABLE /*_*/filelocks_exclusive ( + fle_key binary(31) NOT NULL PRIMARY KEY +) ENGINE=InnoDB, CHECKSUM=0; + +-- Table to handle resource locking (SH) with row-level locking +CREATE TABLE /*_*/filelocks_shared ( + fls_key binary(31) NOT NULL, + fls_session binary(31) NOT NULL, + PRIMARY KEY (fls_key,fls_session) +) ENGINE=InnoDB, CHECKSUM=0; diff --git a/tests/parser/parserTest.inc b/tests/parser/parserTest.inc index e7478f2510..7df086858d 100644 --- a/tests/parser/parserTest.inc +++ b/tests/parser/parserTest.inc @@ -149,14 +149,23 @@ class ParserTest { $wgStylePath = '/skins'; $wgExtensionAssetsPath = '/extensions'; $wgThumbnailScriptPath = false; + $backend = new FSFileBackend( array( + 'name' => 'local-backend', + 'lockManager' => 'fsLockManager', + 'containerPaths' => array( + 'images-public' => wfTempDir() . '/test-repo/public', + 'images-thumb' => wfTempDir() . '/test-repo/thumb', + 'images-temp' => wfTempDir() . '/test-repo/temp', + 'images-deleted' => wfTempDir() . '/test-repo/delete', + ) + ) ); $wgLocalFileRepo = array( - 'class' => 'LocalRepo', - 'name' => 'local', - 'directory' => wfTempDir() . '/test-repo', - 'url' => 'http://example.com/images', - 'deletedDir' => wfTempDir() . '/test-repo/delete', - 'hashLevels' => 2, + 'class' => 'LocalRepo', + 'name' => 'local', + 'url' => 'http://example.com/images', + 'hashLevels' => 2, 'transformVia404' => false, + 'backend' => $backend ); $wgNamespaceProtection[NS_MEDIAWIKI] = 'editinterface'; $wgNamespaceAliases['Image'] = NS_FILE; diff --git a/tests/phpunit/includes/LocalFileTest.php b/tests/phpunit/includes/LocalFileTest.php index e08d4d7e42..cae2e10878 100644 --- a/tests/phpunit/includes/LocalFileTest.php +++ b/tests/phpunit/includes/LocalFileTest.php @@ -10,12 +10,22 @@ class LocalFileTest extends MediaWikiTestCase { global $wgCapitalLinks; $wgCapitalLinks = true; + + $backend = new FSFileBackend( array( + 'name' => 'local-backend', + 'lockManager' => 'fsLockManager', + 'containerPaths' => array( + 'cont1' => "/testdir/local-backend/tempimages/cont1", + 'cont2' => "/testdir/local-backend/tempimages/cont2" + ) + ) ); $info = array( - 'name' => 'test', - 'directory' => '/testdir', - 'url' => '/testurl', - 'hashLevels' => 2, + 'name' => 'test', + 'directory' => '/testdir', + 'url' => '/testurl', + 'hashLevels' => 2, 'transformVia404' => false, + 'backend' => $backend ); $this->repo_hl0 = new LocalRepo( array( 'hashLevels' => 0 ) + $info ); $this->repo_hl2 = new LocalRepo( array( 'hashLevels' => 2 ) + $info ); @@ -44,17 +54,17 @@ class LocalFileTest extends MediaWikiTestCase { } function testGetArchivePath() { - $this->assertEquals( '/testdir/archive', $this->file_hl0->getArchivePath() ); - $this->assertEquals( '/testdir/archive/a/a2', $this->file_hl2->getArchivePath() ); - $this->assertEquals( '/testdir/archive/!', $this->file_hl0->getArchivePath( '!' ) ); - $this->assertEquals( '/testdir/archive/a/a2/!', $this->file_hl2->getArchivePath( '!' ) ); + $this->assertEquals( 'mwstore://local-backend/images-public/archive', $this->file_hl0->getArchivePath() ); + $this->assertEquals( 'mwstore://local-backend/images-public/archive/a/a2', $this->file_hl2->getArchivePath() ); + $this->assertEquals( 'mwstore://local-backend/images-public/archive/!', $this->file_hl0->getArchivePath( '!' ) ); + $this->assertEquals( 'mwstore://local-backend/images-public/archive/a/a2/!', $this->file_hl2->getArchivePath( '!' ) ); } function testGetThumbPath() { - $this->assertEquals( '/testdir/thumb/Test!', $this->file_hl0->getThumbPath() ); - $this->assertEquals( '/testdir/thumb/a/a2/Test!', $this->file_hl2->getThumbPath() ); - $this->assertEquals( '/testdir/thumb/Test!/x', $this->file_hl0->getThumbPath( 'x' ) ); - $this->assertEquals( '/testdir/thumb/a/a2/Test!/x', $this->file_hl2->getThumbPath( 'x' ) ); + $this->assertEquals( 'mwstore://local-backend/images-thumb/Test!', $this->file_hl0->getThumbPath() ); + $this->assertEquals( 'mwstore://local-backend/images-thumb/a/a2/Test!', $this->file_hl2->getThumbPath() ); + $this->assertEquals( 'mwstore://local-backend/images-thumb/Test!/x', $this->file_hl0->getThumbPath( 'x' ) ); + $this->assertEquals( 'mwstore://local-backend/images-thumb/a/a2/Test!/x', $this->file_hl2->getThumbPath( 'x' ) ); } function testGetArchiveUrl() { diff --git a/tests/phpunit/includes/api/ApiTestCaseUpload.php b/tests/phpunit/includes/api/ApiTestCaseUpload.php index 3d57946072..a155a8c389 100644 --- a/tests/phpunit/includes/api/ApiTestCaseUpload.php +++ b/tests/phpunit/includes/api/ApiTestCaseUpload.php @@ -60,7 +60,7 @@ abstract class ApiTestCaseUpload extends ApiTestCase { * @param $filePath String: path to file on the filesystem */ public function deleteFileByContent( $filePath ) { - $hash = File::sha1Base36( $filePath ); + $hash = FSFile::getSha1Base36FromPath( $filePath ); $dupes = RepoGroup::singleton()->findBySha1( $hash ); $success = true; foreach ( $dupes as $dupe ) { diff --git a/tests/phpunit/includes/api/ApiUploadTest.php b/tests/phpunit/includes/api/ApiUploadTest.php index 95568e4860..869634ac9c 100644 --- a/tests/phpunit/includes/api/ApiUploadTest.php +++ b/tests/phpunit/includes/api/ApiUploadTest.php @@ -417,7 +417,7 @@ class ApiUploadTest extends ApiTestCaseUpload { } $this->assertTrue( isset( $result['upload'] ) ); $this->assertEquals( 'Success', $result['upload']['result'] ); - $this->assertFalse( $exception ); + $this->assertFalse( $exception, "No UsageException exception." ); // clean up $this->deleteFileByFilename( $fileName ); diff --git a/tests/phpunit/includes/filerepo/FileBackendTest.php b/tests/phpunit/includes/filerepo/FileBackendTest.php new file mode 100644 index 0000000000..d94f10bfb4 --- /dev/null +++ b/tests/phpunit/includes/filerepo/FileBackendTest.php @@ -0,0 +1,592 @@ +backend = new FSFileBackend( array( + 'name' => 'localtesting', + 'lockManager' => 'fsLockManager', + 'containerPaths' => array( + 'cont1' => wfTempDir() . '/localtesting/cont1', + 'cont2' => wfTempDir() . '/localtesting/cont2' ) + ) ); + $this->multiBackend = new FileBackendMultiWrite( array( + 'name' => 'localtestingmulti', + 'lockManager' => 'fsLockManager', + 'backends' => array( + array( + 'name' => 'localmutlitesting1', + 'class' => 'FSFileBackend', + 'lockManager' => 'nullLockManager', + 'containerPaths' => array( + 'cont1' => wfTempDir() . '/localtestingmulti1/cont1', + 'cont2' => wfTempDir() . '/localtestingmulti1/cont2' ), + 'isMultiMaster' => false + ), + array( + 'name' => 'localmutlitesting2', + 'class' => 'FSFileBackend', + 'lockManager' => 'nullLockManager', + 'containerPaths' => array( + 'cont1' => wfTempDir() . '/localtestingmulti2/cont1', + 'cont2' => wfTempDir() . '/localtestingmulti2/cont2' ), + 'isMultiMaster' => true + ) + ) + ) ); + $this->filesToPrune = $this->pathsToPrune = array(); + } + + private function singleBasePath() { + return 'mwstore://localtesting'; + } + + /** + * @dataProvider provider_testStore + */ + public function testStore( $op, $source, $dest ) { + $this->filesToPrune[] = $source; + $this->pathsToPrune[] = $dest; + + file_put_contents( $source, "Unit test file" ); + $status = $this->backend->doOperation( $op ); + + $this->assertEquals( true, $status->isOK(), + "Store from $source to $dest succeeded." ); + $this->assertEquals( true, $status->isGood(), + "Store from $source to $dest succeeded without warnings." ); + $this->assertEquals( true, file_exists( $source ), + "Source file $source still exists." ); + $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $dest ) ), + "Destination file $dest exists." ); + + $props1 = FSFile::getPropsFromPath( $source ); + $props2 = $this->backend->getFileProps( array( 'src' => $dest ) ); + $this->assertEquals( $props1, $props2, + "Source and destination have the same props." ); + } + + public function provider_testStore() { + $cases = array(); + + $tmpName = TempFSFile::factory( "unittests_", 'txt' )->getPath(); + $toPath = $this->singleBasePath() . '/cont1/fun/obj1.txt'; + $op = array( 'op' => 'store', 'src' => $tmpName, 'dst' => $toPath ); + $cases[] = array( + $op, // operation + $tmpName, // source + $toPath, // dest + ); + + $op['overwriteDest'] = true; + $cases[] = array( + $op, // operation + $tmpName, // source + $toPath, // dest + ); + + return $cases; + } + + /** + * @dataProvider provider_testCopy + */ + public function testCopy( $op, $source, $dest ) { + $this->pathsToPrune[] = $source; + $this->pathsToPrune[] = $dest; + + $status = $this->backend->doOperation( + array( 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ) ); + $this->assertEquals( true, $status->isOK(), "Creation of file at $source succeeded." ); + + $status = $this->backend->doOperation( $op ); + $this->assertEquals( true, $status->isOK(), + "Copy from $source to $dest succeeded." ); + $this->assertEquals( true, $status->isGood(), + "Copy from $source to $dest succeeded without warnings." ); + $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $source ) ), + "Source file $source still exists." ); + $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $dest ) ), + "Destination file $dest exists after copy." ); + + $props1 = $this->backend->getFileProps( array( 'src' => $source ) ); + $props2 = $this->backend->getFileProps( array( 'src' => $dest ) ); + $this->assertEquals( $props1, $props2, + "Source and destination have the same props." ); + } + + public function provider_testCopy() { + $cases = array(); + + $source = $this->singleBasePath() . '/cont1/file.txt'; + $dest = $this->singleBasePath() . '/cont2/fileMoved.txt'; + + $op = array( 'op' => 'copy', 'src' => $source, 'dst' => $dest ); + $cases[] = array( + $op, // operation + $source, // source + $dest, // dest + ); + + $op['overwriteDest'] = true; + $cases[] = array( + $op, // operation + $source, // source + $dest, // dest + ); + + return $cases; + } + + /** + * @dataProvider provider_testMove + */ + public function testMove( $op, $source, $dest ) { + $this->pathsToPrune[] = $source; + $this->pathsToPrune[] = $dest; + + $status = $this->backend->doOperation( + array( 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ) ); + $this->assertEquals( true, $status->isOK(), "Creation of file at $source succeeded." ); + + $status = $this->backend->doOperation( $op ); + $this->assertEquals( true, $status->isOK(), + "Move from $source to $dest succeeded." ); + $this->assertEquals( true, $status->isGood(), + "Move from $source to $dest succeeded without warnings." ); + $this->assertEquals( false, $this->backend->fileExists( array( 'src' => $source ) ), + "Source file $source does not still exists." ); + $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $dest ) ), + "Destination file $dest exists after move." ); + + $props1 = $this->backend->getFileProps( array( 'src' => $source ) ); + $props2 = $this->backend->getFileProps( array( 'src' => $dest ) ); + $this->assertEquals( false, $props1['fileExists'], + "Source file does not exist accourding to props." ); + $this->assertEquals( true, $props2['fileExists'], + "Destination file exists accourding to props." ); + } + + public function provider_testMove() { + $cases = array(); + + $source = $this->singleBasePath() . '/cont1/file.txt'; + $dest = $this->singleBasePath() . '/cont2/fileMoved.txt'; + + $op = array( 'op' => 'move', 'src' => $source, 'dst' => $dest ); + $cases[] = array( + $op, // operation + $source, // source + $dest, // dest + ); + + $op['overwriteDest'] = true; + $cases[] = array( + $op, // operation + $source, // source + $dest, // dest + ); + + return $cases; + } + + /** + * @dataProvider provider_testDelete + */ + public function testDelete( $op, $source, $withSource, $okStatus ) { + $this->pathsToPrune[] = $source; + + if ( $withSource ) { + $status = $this->backend->doOperation( + array( 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ) ); + $this->assertEquals( true, $status->isOK(), "Creation of file at $source succeeded." ); + } + + $status = $this->backend->doOperation( $op ); + if ( $okStatus ) { + $this->assertEquals( true, $status->isOK(), "Deletion of file at $source succeeded." ); + } else { + $this->assertEquals( false, $status->isOK(), "Deletion of file at $source failed." ); + } + + $this->assertEquals( false, $this->backend->fileExists( array( 'src' => $source ) ), + "Source file $source does not exist after move." ); + + $props1 = $this->backend->getFileProps( array( 'src' => $source ) ); + $this->assertEquals( false, $props1['fileExists'], + "Source file $source does not exist according to props." ); + } + + public function provider_testDelete() { + $cases = array(); + + $source = $this->singleBasePath() . '/cont1/myfacefile.txt'; + + $op = array( 'op' => 'delete', 'src' => $source ); + $cases[] = array( + $op, // operation + $source, // source + true, // with source + true // succeeds + ); + + $cases[] = array( + $op, // operation + $source, // source + false, // without source + false // fails + ); + + $op['ignoreMissingSource'] = true; + $cases[] = array( + $op, // operation + $source, // source + false, // without source + true // succeeds + ); + + return $cases; + } + + /** + * @dataProvider provider_testCreate + */ + public function testCreate( $op, $dest, $alreadyExists, $okStatus, $newSize ) { + $this->pathsToPrune[] = $dest; + + $oldText = 'blah...blah...waahwaah'; + if ( $alreadyExists ) { + $status = $this->backend->doOperation( + array( 'op' => 'create', 'content' => $oldText, 'dst' => $dest ) ); + $this->assertEquals( true, $status->isOK(), "Creation of file at $dest succeeded." ); + } + + $status = $this->backend->doOperation( $op ); + if ( $okStatus ) { + $this->assertEquals( true, $status->isOK(), "Creation of file at $dest succeeded." ); + } else { + $this->assertEquals( false, $status->isOK(), "Creation of file at $dest failed." ); + } + + $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $dest ) ), + "Dest file $dest exists after creation." ); + + $props1 = $this->backend->getFileProps( array( 'src' => $dest ) ); + $this->assertEquals( true, $props1['fileExists'], + "Dest file $dest exists according to props." ); + if ( $okStatus ) { // file content is what we saved + $this->assertEquals( $newSize, $props1['size'], + "Dest file $dest has expected size according to props." ); + } else { // file content is some other previous text + $this->assertEquals( strlen( $oldText ), $props1['size'], + "Dest file $dest has different size that given text according to props." ); + } + } + + /** + * @dataProvider provider_testCreate + */ + public function provider_testCreate() { + $cases = array(); + + $source = $this->singleBasePath() . '/cont2/myspacefile.txt'; + + $dummyText = 'hey hey'; + $op = array( 'op' => 'create', 'content' => $dummyText, 'dst' => $source ); + $cases[] = array( + $op, // operation + $source, // source + false, // no dest already exists + true, // succeeds + strlen( $dummyText ) + ); + + $cases[] = array( + $op, // operation + $source, // source + true, // dest already exists + false, // fails + strlen( $dummyText ) + ); + + $op['overwriteDest'] = true; + $cases[] = array( + $op, // operation + $source, // source + true, // dest already exists + true, // succeeds + strlen( $dummyText ) + ); + + return $cases; + } + + /** + * @dataProvider provider_testConcatenate + */ + public function testConcatenate( $op, $srcs, $srcsContent, $alreadyExists, $okStatus ) { + $this->pathsToPrune = array_merge( $this->pathsToPrune, $srcs ); + $this->pathsToPrune[] = $op['dst']; + + $expContent = ''; + // Create sources + $ops = array(); + foreach ( $srcs as $i => $source ) { + $ops[] = array( + 'op' => 'create', // operation + 'dst' => $source, // source + 'content' => $srcsContent[$i] + ); + $expContent .= $srcsContent[$i]; + } + $status = $this->backend->doOperations( $ops ); + + $this->assertEquals( true, $status->isOK(), "Creation of source files succeeded." ); + + $dest = $op['dst']; + if ( $alreadyExists ) { + $oldText = 'blah...blah...waahwaah'; + $status = $this->backend->doOperation( + array( 'op' => 'create', 'content' => $oldText, 'dst' => $dest ) ); + $this->assertEquals( true, $status->isOK(), "Creation of file at $dest succeeded." ); + } + + // Combine them + $status = $this->backend->doOperation( $op ); + if ( $okStatus ) { + $this->assertEquals( true, $status->isOK(), "Creation of concat file at $dest succeeded." ); + } else { + $this->assertEquals( false, $status->isOK(), "Creation of concat file at $dest failed." ); + } + + if ( $okStatus ) { + $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $dest ) ), + "Dest concat file $dest exists after creation." ); + } else { + $this->assertEquals( true, $this->backend->fileExists( array( 'src' => $dest ) ), + "Dest concat file $dest exists after failed creation." ); + } + + $tmpFile = $this->backend->getLocalCopy( array( 'src' => $dest ) ); + $this->assertNotNull( $tmpFile, "Creation of local copy of $dest succeeded." ); + + $contents = file_get_contents( $tmpFile->getPath() ); + $this->assertNotEquals( false, $contents, "Local copy of $dest exists." ); + + if ( $okStatus ) { + $this->assertEquals( $expContent, $contents, "Concat file at $dest has correct contents." ); + } else { + $this->assertNotEquals( $expContent, $contents, "Concat file at $dest has correct contents." ); + } + } + + function provider_testConcatenate() { + $cases = array(); + + $dest = $this->singleBasePath() . '/cont1/full_file.txt'; + $srcs = array( + $this->singleBasePath() . '/cont1/file1.txt', + $this->singleBasePath() . '/cont1/file2.txt', + $this->singleBasePath() . '/cont1/file3.txt', + $this->singleBasePath() . '/cont1/file4.txt', + $this->singleBasePath() . '/cont1/file5.txt', + $this->singleBasePath() . '/cont1/file6.txt', + $this->singleBasePath() . '/cont1/file7.txt', + $this->singleBasePath() . '/cont1/file8.txt', + $this->singleBasePath() . '/cont1/file9.txt', + $this->singleBasePath() . '/cont1/file10.txt' + ); + $content = array( + 'egfage', + 'ageageag', + 'rhokohlr', + 'shgmslkg', + 'kenga', + 'owagmal', + 'kgmae', + 'g eak;g', + 'lkaem;a', + 'legma' + ); + $op = array( 'op' => 'concatenate', 'srcs' => $srcs, 'dst' => $dest ); + + $cases[] = array( + $op, // operation + $srcs, // sources + $content, // content for each source + false, // no dest already exists + true, // succeeds + ); + + $cases[] = array( + $op, // operation + $srcs, // sources + $content, // content for each source + true, // no dest already exists + false, // succeeds + ); + + $op['overwriteDest'] = true; + $cases[] = array( + $op, // operation + $srcs, // sources + $content, // content for each source + true, // no dest already exists + true, // succeeds + ); + + return $cases; + } + + /** + * @dataProvider provider_testGetLocalCopy + */ + public function testGetLocalCopy( $src, $content ) { + $this->pathsToPrune[] = $src; + + $status = $this->backend->doOperation( + array( 'op' => 'create', 'content' => $content, 'dst' => $src ) ); + $this->assertEquals( true, $status->isOK(), "Creation of file at $src succeeded." ); + + $tmpFile = $this->backend->getLocalCopy( array( 'src' => $src ) ); + $this->assertNotNull( $tmpFile, "Creation of local copy of $src succeeded." ); + + $contents = file_get_contents( $tmpFile->getPath() ); + $this->assertNotEquals( false, $contents, "Local copy of $src exists." ); + } + + function provider_testGetLocalCopy() { + $cases = array(); + + $base = $this->singleBasePath(); + $cases[] = array( "$base/cont1/a/z/some_file.txt", "some file contents" ); + $cases[] = array( "$base/cont1/a/some-other_file.txt", "more file contents" ); + + return $cases; + } + + /** + * @dataProvider provider_testGetReference + */ + public function testGetLocalReference( $src, $content ) { + $this->pathsToPrune[] = $src; + + $status = $this->backend->doOperation( + array( 'op' => 'create', 'content' => $content, 'dst' => $src ) ); + $this->assertEquals( true, $status->isOK(), "Creation of file at $src succeeded." ); + + $tmpFile = $this->backend->getLocalReference( array( 'src' => $src ) ); + $this->assertNotNull( $tmpFile, "Creation of local copy of $src succeeded." ); + + $contents = file_get_contents( $tmpFile->getPath() ); + $this->assertNotEquals( false, $contents, "Local copy of $src exists." ); + } + + function provider_testGetReference() { + $cases = array(); + + $base = $this->singleBasePath(); + $cases[] = array( "$base/cont1/a/z/some_file.txt", "some file contents" ); + $cases[] = array( "$base/cont1/a/some-other_file.txt", "more file contents" ); + + return $cases; + } + + // @TODO: testPrepare + + // @TODO: testSecure + + // @TODO: testClean + + // @TODO: testDoOperations + + public function testGetFileList() { + $base = $this->singleBasePath(); + $files = array( + "$base/cont1/test1.txt", + "$base/cont1/test2.txt", + "$base/cont1/test3.txt", + "$base/cont1/subdir1/test1.txt", + "$base/cont1/subdir1/test2.txt", + "$base/cont1/subdir2/test3.txt", + "$base/cont1/subdir2/test4.txt", + "$base/cont1/subdir2/subdir/test1.txt", + "$base/cont1/subdir2/subdir/test2.txt", + "$base/cont1/subdir2/subdir/test3.txt", + "$base/cont1/subdir2/subdir/test4.txt", + "$base/cont1/subdir2/subdir/test5.txt", + "$base/cont1/subdir2/subdir/sub/test0.txt", + "$base/cont1/subdir2/subdir/sub/120-px-file.txt", + ); + $this->pathsToPrune = array_merge( $this->pathsToPrune, $files ); + + // Add the files + $ops = array(); + foreach ( $files as $file ) { + $ops[] = array( 'op' => 'create', 'content' => 'xxy', 'dst' => $file ); + } + $status = $this->backend->doOperations( $ops ); + $this->assertEquals( true, $status->isOK(), "Creation of files succeeded." ); + + // Expected listing + $expected = array( + "test1.txt", + "test2.txt", + "test3.txt", + "subdir1/test1.txt", + "subdir1/test2.txt", + "subdir2/test3.txt", + "subdir2/test1.txt", + "subdir2/subdir/test1.txt", + "subdir2/subdir/test2.txt", + "subdir2/subdir/test3.txt", + "subdir2/subdir/test4.txt", + "subdir2/subdir/test5.txt", + "subdir2/subdir/sub/test0.txt", + "subdir2/subdir/sub/120-px-file.txt", + ); + $expected = sort( $expected ); + + // Actual listing (no trailing slash) + $list = array(); + $iter = $this->backend->getFileList( array( 'dir' => "$base/cont1" ) ); + foreach ( $iter as $file ) { + $list[] = $file; + } + + $this->assertEquals( $expected, sort( $list ), "Correct file listing." ); + + // Actual listing (with trailing slash) + $list = array(); + $iter = $this->backend->getFileList( array( 'dir' => "$base/cont1/" ) ); + foreach ( $iter as $file ) { + $list[] = $file; + } + + $this->assertEquals( $expected, sort( $list ), "Correct file listing." ); + + foreach ( $files as $file ) { + $this->backend->delete( array( 'src' => "$base/$files" ) ); + } + + $iter = $this->backend->getFileList( array( 'dir' => "$base/cont1/not/exists" ) ); + foreach ( $iter as $iter ) {} // no errors + } + + function tearDown() { + parent::tearDown(); + foreach ( $this->filesToPrune as $file ) { + @unlink( $file ); + } + foreach ( $this->pathsToPrune as $file ) { + $this->backend->doOperation( array( 'op' => 'delete', 'src' => $file ) ); + $this->multiBackend->doOperation( array( 'op' => 'delete', 'src' => $file ) ); + } + $this->backend = $this->multiBackend = null; + $this->filesToPrune = $this->pathsToPrune = array(); + } +} diff --git a/tests/phpunit/includes/media/ExifRotationTest.php b/tests/phpunit/includes/media/ExifRotationTest.php index 639091d04f..6cabd9a557 100644 --- a/tests/phpunit/includes/media/ExifRotationTest.php +++ b/tests/phpunit/includes/media/ExifRotationTest.php @@ -7,13 +7,19 @@ class ExifRotationTest extends MediaWikiTestCase { function setUp() { parent::setUp(); - $this->filePath = dirname( __FILE__ ) . '/../../data/media/'; $this->handler = new BitmapHandler(); - $this->repo = new FSRepo(array( - 'name' => 'temp', - 'directory' => wfTempDir() . '/exif-test-' . time() . '-' . mt_rand(), - 'url' => 'http://localhost/thumbtest' - )); + $filePath = dirname( __FILE__ ) . '/../../data/media'; + $tmpDir = wfTempDir() . '/exif-test-' . time() . '-' . mt_rand(); + $this->backend = new FSFileBackend( array( + 'name' => 'localtesting', + 'lockManager' => 'nullLockManager', + 'containerPaths' => array( 'images-thumb' => $tmpDir, 'data' => $filePath ) + ) ); + $this->repo = new FSRepo( array( + 'name' => 'temp', + 'url' => 'http://localhost/thumbtest', + 'backend' => $this->backend + ) ); if ( !wfDl( 'exif' ) ) { $this->markTestSkipped( "This test needs the exif extension." ); } @@ -39,7 +45,7 @@ class ExifRotationTest extends MediaWikiTestCase { if ( !BitmapHandler::canRotate() ) { $this->markTestSkipped( "This test needs a rasterizer that can auto-rotate." ); } - $file = UnregisteredLocalFile::newFromPath( $this->filePath . $name, $type ); + $file = $this->dataFile( $name, $type ); $this->assertEquals( $info['width'], $file->getWidth(), "$name: width check" ); $this->assertEquals( $info['height'], $file->getHeight(), "$name: height check" ); } @@ -66,13 +72,13 @@ class ExifRotationTest extends MediaWikiTestCase { throw new MWException('bogus test data format ' . $size); } - $file = $this->localFile( $name, $type ); - $thumb = $file->transform( $params, File::RENDER_NOW ); + $file = $this->dataFile( $name, $type ); + $thumb = $file->transform( $params, File::RENDER_NOW | File::RENDER_FORCE ); $this->assertEquals( $out[0], $thumb->getWidth(), "$name: thumb reported width check for $size" ); $this->assertEquals( $out[1], $thumb->getHeight(), "$name: thumb reported height check for $size" ); - $gis = getimagesize( $thumb->getPath() ); + $gis = getimagesize( $thumb->getLocalCopyPath() ); if ($out[0] > $info['width']) { // Physical image won't be scaled bigger than the original. $this->assertEquals( $info['width'], $gis[0], "$name: thumb actual width check for $size"); @@ -84,8 +90,9 @@ class ExifRotationTest extends MediaWikiTestCase { } } - private function localFile( $name, $type ) { - return new UnregisteredLocalFile( false, $this->repo, $this->filePath . $name, $type ); + private function dataFile( $name, $type ) { + return new UnregisteredLocalFile( false, $this->repo, + "mwstore://localtesting/data/$name", $type ); } function providerFiles() { @@ -129,7 +136,7 @@ class ExifRotationTest extends MediaWikiTestCase { global $wgEnableAutoRotation; $wgEnableAutoRotation = false; - $file = UnregisteredLocalFile::newFromPath( $this->filePath . $name, $type ); + $file = $this->dataFile( $name, $type ); $this->assertEquals( $info['width'], $file->getWidth(), "$name: width check" ); $this->assertEquals( $info['height'], $file->getHeight(), "$name: height check" ); @@ -158,13 +165,13 @@ class ExifRotationTest extends MediaWikiTestCase { throw new MWException('bogus test data format ' . $size); } - $file = $this->localFile( $name, $type ); - $thumb = $file->transform( $params, File::RENDER_NOW ); + $file = $this->dataFile( $name, $type ); + $thumb = $file->transform( $params, File::RENDER_NOW | File::RENDER_FORCE ); $this->assertEquals( $out[0], $thumb->getWidth(), "$name: thumb reported width check for $size" ); $this->assertEquals( $out[1], $thumb->getHeight(), "$name: thumb reported height check for $size" ); - $gis = getimagesize( $thumb->getPath() ); + $gis = getimagesize( $thumb->getLocalCopyPath() ); if ($out[0] > $info['width']) { // Physical image won't be scaled bigger than the original. $this->assertEquals( $info['width'], $gis[0], "$name: thumb actual width check for $size"); @@ -242,7 +249,7 @@ class ExifRotationTest extends MediaWikiTestCase { array( 270, array( self::TEST_HEIGHT, self::TEST_WIDTH ) - ), + ), ); } } diff --git a/tests/phpunit/includes/media/FormatMetadataTest.php b/tests/phpunit/includes/media/FormatMetadataTest.php index 276c000092..8a632f527c 100644 --- a/tests/phpunit/includes/media/FormatMetadataTest.php +++ b/tests/phpunit/includes/media/FormatMetadataTest.php @@ -4,6 +4,17 @@ class FormatMetadataTest extends MediaWikiTestCase { if ( !wfDl( 'exif' ) ) { $this->markTestSkipped( "This test needs the exif extension." ); } + $filePath = dirname( __FILE__ ) . '/../../data/media'; + $this->backend = new FSFileBackend( array( + 'name' => 'localtesting', + 'lockManager' => 'nullLockManager', + 'containerPaths' => array( 'data' => $filePath ) + ) ); + $this->repo = new FSRepo( array( + 'name' => 'temp', + 'url' => 'http://localhost/thumbtest', + 'backend' => $this->backend + ) ); global $wgShowEXIF; $this->show = $wgShowEXIF; $wgShowEXIF = true; @@ -14,8 +25,7 @@ class FormatMetadataTest extends MediaWikiTestCase { } public function testInvalidDate() { - $file = UnregisteredLocalFile::newFromPath( dirname( __FILE__ ) . - '/../../data/media/broken_exif_date.jpg', 'image/jpeg' ); + $file = $this->dataFile( 'broken_exif_date.jpg', 'image/jpeg' ); // Throws an error if bug hit $meta = $file->formatMetadata(); @@ -34,4 +44,9 @@ class FormatMetadataTest extends MediaWikiTestCase { $meta['visible'][$dateIndex]['value'], 'File with invalid date metadata (bug 29471)' ); } + + private function dataFile( $name, $type ) { + return new UnregisteredLocalFile( false, $this->repo, + "mwstore://localtesting/data/$name", $type ); + } } diff --git a/tests/phpunit/includes/media/GIFTest.php b/tests/phpunit/includes/media/GIFTest.php index 42c25ca56d..3665835804 100644 --- a/tests/phpunit/includes/media/GIFTest.php +++ b/tests/phpunit/includes/media/GIFTest.php @@ -2,12 +2,22 @@ class GIFHandlerTest extends MediaWikiTestCase { public function setUp() { - $this->filePath = dirname( __FILE__ ) . '/../../data/media/'; + $this->filePath = dirname( __FILE__ ) . '/../../data/media'; + $this->backend = new FSFileBackend( array( + 'name' => 'localtesting', + 'lockManager' => 'nullLockManager', + 'containerPaths' => array( 'data' => $this->filePath ) + ) ); + $this->repo = new FSRepo( array( + 'name' => 'temp', + 'url' => 'http://localhost/thumbtest', + 'backend' => $this->backend + ) ); $this->handler = new GIFHandler(); } public function testInvalidFile() { - $res = $this->handler->getMetadata( null, $this->filePath . 'README' ); + $res = $this->handler->getMetadata( null, $this->filePath . '/README' ); $this->assertEquals( GIFHandler::BROKEN_FILE, $res ); } /** @@ -16,8 +26,7 @@ class GIFHandlerTest extends MediaWikiTestCase { * @dataProvider dataIsAnimated */ public function testIsAnimanted( $filename, $expected ) { - $file = UnregisteredLocalFile::newFromPath( $this->filePath . $filename, - 'image/gif' ); + $file = $this->dataFile( $filename, 'image/gif' ); $actual = $this->handler->isAnimatedImage( $file ); $this->assertEquals( $expected, $actual ); } @@ -34,8 +43,7 @@ class GIFHandlerTest extends MediaWikiTestCase { * @dataProvider dataGetImageArea */ public function testGetImageArea( $filename, $expected ) { - $file = UnregisteredLocalFile::newFromPath( $this->filePath . $filename, - 'image/gif' ); + $file = $this->dataFile( $filename, 'image/gif' ); $actual = $this->handler->getImageArea( $file, $file->getWidth(), $file->getHeight() ); $this->assertEquals( $expected, $actual ); } @@ -71,15 +79,20 @@ class GIFHandlerTest extends MediaWikiTestCase { * @dataProvider dataGetMetadata */ public function testGetMetadata( $filename, $expected ) { - $file = UnregisteredLocalFile::newFromPath( $this->filePath . $filename, - 'image/gif' ); - $actual = $this->handler->getMetadata( $file, $this->filePath . $filename ); + $file = $this->dataFile( $filename, 'image/gif' ); + $actual = $this->handler->getMetadata( $file, "$this->filePath/$filename" ); $this->assertEquals( unserialize( $expected ), unserialize( $actual ) ); } + public function dataGetMetadata() { return array( array( 'nonanimated.gif', 'a:4:{s:10:"frameCount";i:1;s:6:"looped";b:0;s:8:"duration";d:0.1000000000000000055511151231257827021181583404541015625;s:8:"metadata";a:2:{s:14:"GIFFileComment";a:1:{i:0;s:35:"GIF test file ⁕ Created with GIMP";}s:15:"_MW_GIF_VERSION";i:1;}}' ), array( 'animated-xmp.gif', 'a:4:{s:10:"frameCount";i:4;s:6:"looped";b:1;s:8:"duration";d:2.399999999999999911182158029987476766109466552734375;s:8:"metadata";a:5:{s:6:"Artist";s:7:"Bawolff";s:16:"ImageDescription";a:2:{s:9:"x-default";s:18:"A file to test GIF";s:5:"_type";s:4:"lang";}s:15:"SublocationDest";s:13:"The interwebs";s:14:"GIFFileComment";a:1:{i:0;s:16:"GIƒ·test·file";}s:15:"_MW_GIF_VERSION";i:1;}}' ), ); } + + private function dataFile( $name, $type ) { + return new UnregisteredLocalFile( false, $this->repo, + "mwstore://localtesting/data/$name", $type ); + } } diff --git a/tests/phpunit/includes/media/PNGTest.php b/tests/phpunit/includes/media/PNGTest.php index b782918cb6..b6f911fd8a 100644 --- a/tests/phpunit/includes/media/PNGTest.php +++ b/tests/phpunit/includes/media/PNGTest.php @@ -2,12 +2,22 @@ class PNGHandlerTest extends MediaWikiTestCase { public function setUp() { - $this->filePath = dirname( __FILE__ ) . '/../../data/media/'; + $this->filePath = dirname( __FILE__ ) . '/../../data/media'; + $this->backend = new FSFileBackend( array( + 'name' => 'localtesting', + 'lockManager' => 'nullLockManager', + 'containerPaths' => array( 'data' => $this->filePath ) + ) ); + $this->repo = new FSRepo( array( + 'name' => 'temp', + 'url' => 'http://localhost/thumbtest', + 'backend' => $this->backend + ) ); $this->handler = new PNGHandler(); } public function testInvalidFile() { - $res = $this->handler->getMetadata( null, $this->filePath . 'README' ); + $res = $this->handler->getMetadata( null, $this->filePath . '/README' ); $this->assertEquals( PNGHandler::BROKEN_FILE, $res ); } /** @@ -16,8 +26,7 @@ class PNGHandlerTest extends MediaWikiTestCase { * @dataProvider dataIsAnimated */ public function testIsAnimanted( $filename, $expected ) { - $file = UnregisteredLocalFile::newFromPath( $this->filePath . $filename, - 'image/png' ); + $file = $this->dataFile( $filename, 'image/png' ); $actual = $this->handler->isAnimatedImage( $file ); $this->assertEquals( $expected, $actual ); } @@ -34,8 +43,7 @@ class PNGHandlerTest extends MediaWikiTestCase { * @dataProvider dataGetImageArea */ public function testGetImageArea( $filename, $expected ) { - $file = UnregisteredLocalFile::newFromPath( $this->filePath . $filename, - 'image/png' ); + $file = $this->dataFile($filename, 'image/png' ); $actual = $this->handler->getImageArea( $file, $file->getWidth(), $file->getHeight() ); $this->assertEquals( $expected, $actual ); } @@ -73,9 +81,8 @@ class PNGHandlerTest extends MediaWikiTestCase { * @dataProvider dataGetMetadata */ public function testGetMetadata( $filename, $expected ) { - $file = UnregisteredLocalFile::newFromPath( $this->filePath . $filename, - 'image/png' ); - $actual = $this->handler->getMetadata( $file, $this->filePath . $filename ); + $file = $this->dataFile( $filename, 'image/png' ); + $actual = $this->handler->getMetadata( $file, "$this->filePath/$filename" ); // $this->assertEquals( unserialize( $expected ), unserialize( $actual ) ); $this->assertEquals( ( $expected ), ( $actual ) ); } @@ -85,4 +92,9 @@ class PNGHandlerTest extends MediaWikiTestCase { array( 'xmp.png', 'a:6:{s:10:"frameCount";i:0;s:9:"loopCount";i:1;s:8:"duration";d:0;s:8:"bitDepth";i:1;s:9:"colorType";s:14:"index-coloured";s:8:"metadata";a:2:{s:12:"SerialNumber";s:9:"123456789";s:15:"_MW_PNG_VERSION";i:1;}}' ), ); } + + private function dataFile( $name, $type ) { + return new UnregisteredLocalFile( false, $this->repo, + "mwstore://localtesting/data/$name", $type ); + } } diff --git a/tests/phpunit/includes/parser/NewParserTest.php b/tests/phpunit/includes/parser/NewParserTest.php index 37a0486fb7..a939e91e51 100644 --- a/tests/phpunit/includes/parser/NewParserTest.php +++ b/tests/phpunit/includes/parser/NewParserTest.php @@ -32,10 +32,6 @@ class NewParserTest extends MediaWikiTestCase { protected $file = false; - /*function __construct($a = null,$b = array(),$c = null ) { - parent::__construct($a,$b,$c); - }*/ - function setUp() { global $wgContLang, $wgNamespaceProtection, $wgNamespaceAliases; global $wgHooks, $IP; @@ -60,15 +56,14 @@ class NewParserTest extends MediaWikiTestCase { $tmpGlobals['wgStylePath'] = '/skins'; $tmpGlobals['wgThumbnailScriptPath'] = false; $tmpGlobals['wgLocalFileRepo'] = array( - 'class' => 'LocalRepo', - 'name' => 'local', - 'directory' => wfTempDir() . '/test-repo', - 'url' => 'http://example.com/images', - 'deletedDir' => wfTempDir() . '/test-repo/delete', - 'hashLevels' => 2, + 'class' => 'LocalRepo', + 'name' => 'local', + 'url' => 'http://example.com/images', + 'hashLevels' => 2, 'transformVia404' => false, + 'backend' => 'local-backend' ); - + $tmpGlobals['wgForeignFileRepos'] = array(); $tmpGlobals['wgEnableParserCache'] = false; $tmpGlobals['wgHooks'] = $wgHooks; $tmpGlobals['wgDeferredUpdateList'] = array(); @@ -104,7 +99,6 @@ class NewParserTest extends MediaWikiTestCase { $wgNamespaceProtection[NS_MEDIAWIKI] = 'editinterface'; $wgNamespaceAliases['Image'] = NS_FILE; $wgNamespaceAliases['Image_talk'] = NS_FILE_TALK; - } public function tearDown() { @@ -118,9 +112,15 @@ class NewParserTest extends MediaWikiTestCase { $wgNamespaceProtection[NS_MEDIAWIKI] = $this->savedWeirdGlobals['mw_namespace_protection']; $wgNamespaceAliases['Image'] = $this->savedWeirdGlobals['image_alias']; $wgNamespaceAliases['Image_talk'] = $this->savedWeirdGlobals['image_talk_alias']; + + // Restore backends + FileBackendGroup::destroySingleton(); + FileBackendGroup::singleton()->register( $GLOBALS['wgFileBackends'] ); + RepoGroup::destroySingleton(); } function addDBData() { + $this->tablesUsed[] = 'image'; # Hack: insert a few Wikipedia in-project interwiki prefixes, # for testing inter-language links $this->db->insert( 'interwiki', array( @@ -234,12 +234,12 @@ class NewParserTest extends MediaWikiTestCase { 'wgExtensionAssetsPath' => '/extensions', 'wgActionPaths' => array(), 'wgLocalFileRepo' => array( - 'class' => 'LocalRepo', - 'name' => 'local', - 'directory' => $this->uploadDir, - 'url' => 'http://example.com/images', - 'hashLevels' => 2, + 'class' => 'LocalRepo', + 'name' => 'local', + 'url' => 'http://example.com/images', + 'hashLevels' => 2, 'transformVia404' => false, + 'backend' => 'local-backend' ), 'wgEnableUploads' => self::getOptionValue( 'wgEnableUploads', $opts, true ), 'wgStylePath' => '/skins', @@ -315,12 +315,24 @@ class NewParserTest extends MediaWikiTestCase { $GLOBALS['wgOut'] = $context->getOutput(); $GLOBALS['wgUser'] = $context->getUser(); + FileBackendGroup::destroySingleton(); // reset + $backend = array( + 'name' => 'local-backend', + 'class' => 'FSFileBackend', + 'lockManager' => 'nullLockManager', + 'containerPaths' => array( + 'images-public' => $this->uploadDir, + 'images-thumb' => $this->uploadDir . '/thumb' ) + ); + FileBackendGroup::singleton()->register( array( $backend ) ); + global $wgHooks; $wgHooks['ParserTestParser'][] = 'ParserTestParserHook::setup'; $wgHooks['ParserGetVariableValueTs'][] = 'ParserTest::getFakeTimestamp'; MagicWord::clearCache(); + RepoGroup::destroySingleton(); # Publish the articles after we have the final language set $this->publishTestArticles(); @@ -370,13 +382,14 @@ class NewParserTest extends MediaWikiTestCase { * after each test runs. */ protected function teardownGlobals() { - RepoGroup::destroySingleton(); - LinkCache::singleton()->clear(); - foreach ( $this->savedGlobals as $var => $val ) { $GLOBALS[$var] = $val; } + RepoGroup::destroySingleton(); + LinkCache::singleton()->clear(); + FileBackendGroup::destroySingleton(); + $this->teardownUploadDir( $this->uploadDir ); } diff --git a/tests/phpunit/suites/UploadFromUrlTestSuite.php b/tests/phpunit/suites/UploadFromUrlTestSuite.php index c9f413eda8..a42048ddf3 100644 --- a/tests/phpunit/suites/UploadFromUrlTestSuite.php +++ b/tests/phpunit/suites/UploadFromUrlTestSuite.php @@ -26,14 +26,23 @@ class UploadFromUrlTestSuite extends PHPUnit_Framework_TestSuite { $wgStyleSheetPath = '/skins'; $wgStylePath = '/skins'; $wgThumbnailScriptPath = false; + $backend = new FSFileBackend( array( + 'name' => 'local-backend', + 'lockManager' => 'fsLockManager', + 'containerPaths' => array( + 'images-public' => wfTempDir() . '/test-repo/public', + 'images-thumb' => wfTempDir() . '/test-repo/thumb', + 'images-temp' => wfTempDir() . '/test-repo/temp', + 'images-deleted' => wfTempDir() . '/test-repo/delete', + ) + ) ); $wgLocalFileRepo = array( - 'class' => 'LocalRepo', - 'name' => 'local', - 'directory' => wfTempDir() . '/test-repo', - 'url' => 'http://example.com/images', - 'deletedDir' => wfTempDir() . '/test-repo/delete', - 'hashLevels' => 2, + 'class' => 'LocalRepo', + 'name' => 'local', + 'url' => 'http://example.com/images', + 'hashLevels' => 2, 'transformVia404' => false, + 'backend' => $backend ); $wgNamespaceProtection[NS_MEDIAWIKI] = 'editinterface'; $wgNamespaceAliases['Image'] = NS_FILE; diff --git a/thumb.php b/thumb.php index 74cc71123c..15574b8438 100644 --- a/thumb.php +++ b/thumb.php @@ -128,6 +128,7 @@ function wfStreamThumb( array $params ) { $headers[] = 'Vary: Cookie'; } + // Check the source file storage path if ( !$img ) { wfThumbError( 404, wfMsg( 'badtitletext' ) ); wfProfileOut( __METHOD__ ); @@ -153,9 +154,9 @@ function wfStreamThumb( array $params ) { // Calculate time wfSuppressWarnings(); $imsUnix = strtotime( $imsString ); - $stat = stat( $sourcePath ); wfRestoreWarnings(); - if ( $stat['mtime'] <= $imsUnix ) { + $sourceTsUnix = wfTimestamp( TS_UNIX, $img->getTimestamp() ); + if ( $sourceTsUnix <= $imsUnix ) { header( 'HTTP/1.1 304 Not Modified' ); wfProfileOut( __METHOD__ ); return; @@ -165,10 +166,10 @@ function wfStreamThumb( array $params ) { // Stream the file if it exists already... try { $thumbName = $img->thumbName( $params ); - if ( $thumbName !== false ) { // valid params? + if ( strlen( $thumbName ) ) { // valid params? $thumbPath = $img->getThumbPath( $thumbName ); - if ( is_file( $thumbPath ) ) { - StreamFile::stream( $thumbPath, $headers ); + if ( $img->getRepo()->fileExists( $thumbPath ) ) { + $img->getRepo()->streamFile( $thumbPath, $headers ); wfProfileOut( __METHOD__ ); return; } @@ -182,7 +183,7 @@ function wfStreamThumb( array $params ) { // Thumbnail isn't already there, so create the new thumbnail... try { $thumb = $img->transform( $params, File::RENDER_NOW ); - } catch( Exception $ex ) { + } catch ( Exception $ex ) { // Tried to select a page on a non-paged file? $thumb = false; } @@ -193,18 +194,18 @@ function wfStreamThumb( array $params ) { $errorMsg = wfMsgHtml( 'thumbnail_error', 'File::transform() returned false' ); } elseif ( $thumb->isError() ) { $errorMsg = $thumb->getHtmlMsg(); - } elseif ( !$thumb->getPath() ) { + } elseif ( !$thumb->hasFile() ) { $errorMsg = wfMsgHtml( 'thumbnail_error', 'No path supplied in thumbnail object' ); - } elseif ( $thumb->getPath() == $img->getPath() ) { - $errorMsg = wfMsgHtml( 'thumbnail_error', 'Image was not scaled, ' . - 'is the requested width bigger than the source?' ); + } elseif ( $thumb->fileIsSource() ) { + $errorMsg = wfMsgHtml( 'thumbnail_error', + 'Image was not scaled, is the requested width bigger than the source?' ); } if ( $errorMsg !== false ) { wfThumbError( 500, $errorMsg ); } else { // Stream the file if there were no errors - StreamFile::stream( $thumb->getPath(), $headers ); + $thumb->streamFile( $headers ); } wfProfileOut( __METHOD__ ); @@ -298,4 +299,3 @@ $debug EOT; } - -- 2.20.1