From 66e1422948585f16d7120c4b3ac2ed68af51282c Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Thu, 12 Jan 2012 20:01:54 +0000 Subject: [PATCH] * Merged (added) SwiftFileBackend class from branch. * Added i18n messages used by the new class. --- includes/AutoLoader.php | 1 + .../filerepo/backend/SwiftFileBackend.php | 697 ++++++++++++++++++ languages/messages/MessagesEn.php | 2 + maintenance/language/messages.inc | 4 +- 4 files changed, 703 insertions(+), 1 deletion(-) create mode 100644 includes/filerepo/backend/SwiftFileBackend.php diff --git a/includes/AutoLoader.php b/includes/AutoLoader.php index 61ae868501..d1304178f2 100644 --- a/includes/AutoLoader.php +++ b/includes/AutoLoader.php @@ -491,6 +491,7 @@ $wgAutoloadLocalClasses = array( 'FileBackend' => 'includes/filerepo/backend/FileBackend.php', 'FileBackendMultiWrite' => 'includes/filerepo/backend/FileBackendMultiWrite.php', 'FSFileBackend' => 'includes/filerepo/backend/FSFileBackend.php', + 'SwiftFileBackend' => 'includes/filerepo/backend/SwiftFileBackend.php', 'FSFileIterator' => 'includes/filerepo/backend/FSFileBackend.php', 'LockManagerGroup' => 'includes/filerepo/backend/lockmanager/LockManagerGroup.php', 'LockManager' => 'includes/filerepo/backend/lockmanager/LockManager.php', diff --git a/includes/filerepo/backend/SwiftFileBackend.php b/includes/filerepo/backend/SwiftFileBackend.php new file mode 100644 index 0000000000..88381857ca --- /dev/null +++ b/includes/filerepo/backend/SwiftFileBackend.php @@ -0,0 +1,697 @@ +auth = new CF_Authentication( + $config['swiftUser'], $config['swiftKey'], null, $config['swiftAuthUrl'] ); + // Optional settings + $this->connTTL = isset( $config['connTTL'] ) + ? $config['connTTL'] + : 60; // some sane number + $this->swiftProxyUser = isset( $config['swiftProxyUser'] ) + ? $config['swiftProxyUser'] + : ''; + $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] ) + ? $config['shardViaHashLevels'] + : ''; + } + + /** + * @see FileBackend::resolveContainerPath() + */ + protected function resolveContainerPath( $container, $relStoragePath ) { + if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) { + return null; // too long for swift + } + return $relStoragePath; + } + + /** + * @see FileBackend::doCopyInternal() + */ + protected function doCreateInternal( array $params ) { + $status = Status::newGood(); + + list( $dstCont, $destRel ) = $this->resolveStoragePathReal( $params['dst'] ); + if ( $destRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; + } + + // (a) Get a swift proxy connection + $conn = $this->getConnection(); + if ( !$conn ) { + $status->fatal( 'backend-fail-connect', $this->name ); + return $status; + } + + // (b) Check the destination container + try { + $dContObj = $conn->get_container( $dstCont ); + } catch ( NoSuchContainerException $e ) { + $status->fatal( 'backend-fail-create', $params['dst'] ); + return $status; + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + return $status; + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-internal' ); + $this->logException( $e, __METHOD__, $params ); + return $status; + } + + // (c) Check if the destination object already exists + try { + $dContObj->get_object( $destRel ); // throws NoSuchObjectException + // NoSuchObjectException not thrown: file must exist + if ( empty( $params['overwriteDest'] ) ) { + $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); + return $status; + } + } catch ( NoSuchObjectException $e ) { + // NoSuchObjectException thrown: file does not exist + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + return $status; + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-internal' ); + $this->logException( $e, __METHOD__, $params ); + return $status; + } + + // (d) Get a SHA-1 hash of the object + $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 ); + + // (e) Actually create the object + try { + $obj = $dContObj->create_object( $destRel ); + // Note: metadata keys stored as [Upper case char][[Lower case char]...] + $obj->metadata = array( 'Sha1base36' => $sha1Hash ); + $obj->write( $params['content'] ); + } catch ( BadContentTypeException $e ) { + $status->fatal( 'backend-fail-contenttype', $params['dst'] ); + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-internal' ); + $this->logException( $e, __METHOD__, $params ); + } + + return $status; + } + + /** + * @see FileBackend::doStoreInternal() + */ + protected function doStoreInternal( array $params ) { + $status = Status::newGood(); + + list( $dstCont, $destRel ) = $this->resolveStoragePathReal( $params['dst'] ); + if ( $destRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; + } + + // (a) Get a swift proxy connection + $conn = $this->getConnection(); + if ( !$conn ) { + $status->fatal( 'backend-fail-connect', $this->name ); + return $status; + } + + // (b) Check the destination container + try { + $dContObj = $conn->get_container( $dstCont ); + } catch ( NoSuchContainerException $e ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + return $status; + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + return $status; + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-internal' ); + $this->logException( $e, __METHOD__, $params ); + return $status; + } + + // (c) Check if the destination object already exists + try { + $dContObj->get_object( $destRel ); // throws NoSuchObjectException + // NoSuchObjectException not thrown: file must exist + if ( empty( $params['overwriteDest'] ) ) { + $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); + return $status; + } + } catch ( NoSuchObjectException $e ) { + // NoSuchObjectException thrown: file does not exist + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + return $status; + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-internal' ); + $this->logException( $e, __METHOD__, $params ); + return $status; + } + + // (d) Get a SHA-1 hash of the object + $sha1Hash = sha1_file( $params['src'] ); + if ( $sha1Hash === false ) { // source doesn't exist? + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + return $status; + } + $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 ); + + // (e) Actually store the object + try { + $obj = $dContObj->create_object( $destRel ); + // Note: metadata keys stored as [Upper case char][[Lower case char]...] + $obj->metadata = array( 'Sha1base36' => $sha1Hash ); + $obj->load_from_filename( $params['src'], True ); // calls $obj->write() + } catch ( BadContentTypeException $e ) { + $status->fatal( 'backend-fail-contenttype', $params['dst'] ); + } catch ( IOException $e ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-internal' ); + $this->logException( $e, __METHOD__, $params ); + } + + return $status; + } + + /** + * @see FileBackend::doCopyInternal() + */ + protected function doCopyInternal( array $params ) { + $status = Status::newGood(); + + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; + } + + list( $dstCont, $destRel ) = $this->resolveStoragePathReal( $params['dst'] ); + if ( $destRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; + } + + // (a) Get a swift proxy connection + $conn = $this->getConnection(); + if ( !$conn ) { + $status->fatal( 'backend-fail-connect', $this->name ); + return $status; + } + + // (b) Check the source and destination containers + try { + $sContObj = $conn->get_container( $srcCont ); + $dContObj = $conn->get_container( $dstCont ); + } catch ( NoSuchContainerException $e ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + return $status; + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + return $status; + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-internal' ); + $this->logException( $e, __METHOD__, $params ); + return $status; + } + + // (c) Check if the destination object already exists + try { + $dContObj->get_object( $destRel ); // throws NoSuchObjectException + // NoSuchObjectException not thrown: file must exist + if ( empty( $params['overwriteDest'] ) ) { + $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); + return $status; + } + } catch ( NoSuchObjectException $e ) { + // NoSuchObjectException thrown: file does not exist + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + return $status; + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-internal' ); + $this->logException( $e, __METHOD__, $params ); + return $status; + } + + // (d) Actually copy the file to the destination + try { + $sContObj->copy_object_to( $srcRel, $dContObj, $destRel ); + } catch ( NoSuchObjectException $e ) { // source object does not exist + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-internal' ); + $this->logException( $e, __METHOD__, $params ); + } + + return $status; + } + + /** + * @see FileBackend::doDeleteInternal() + */ + protected function doDeleteInternal( array $params ) { + $status = Status::newGood(); + + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; + } + + // (a) Get a swift proxy connection + $conn = $this->getConnection(); + if ( !$conn ) { + $status->fatal( 'backend-fail-connect', $this->name ); + return $status; + } + + // (b) Check the source container + try { + $sContObj = $conn->get_container( $srcCont ); + } catch ( NoSuchContainerException $e ) { + $status->fatal( 'backend-fail-delete', $params['src'] ); + return $status; + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + return $status; + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-internal' ); + $this->logException( $e, __METHOD__, $params ); + return $status; + } + + // (c) Actually delete the object + try { + $sContObj->delete_object( $srcRel ); + } catch ( NoSuchObjectException $e ) { + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-delete', $params['src'] ); + } + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-internal' ); + $this->logException( $e, __METHOD__, $params ); + } + + return $status; + } + + /** + * @see FileBackend::doPrepareInternal() + */ + protected function doPrepareInternal( $fullCont, $dir, array $params ) { + $status = Status::newGood(); + + // (a) Get a swift proxy connection + $conn = $this->getConnection(); + if ( !$conn ) { + $status->fatal( 'backend-fail-connect', $this->name ); + return $status; + } + + // (b) Create the destination container + try { + $conn->create_container( $fullCont ); + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-internal' ); + $this->logException( $e, __METHOD__, $params ); + } + + return $status; + } + + /** + * @see FileBackend::doSecureInternal() + */ + protected function doSecureInternal( $fullCont, $dir, array $params ) { + $status = Status::newGood(); + // @TODO: restrict container from $this->swiftProxyUser + return $status; // badgers? We don't need no steenking badgers! + } + + /** + * @see FileBackend::doFileExists() + */ + protected function doGetFileStat( array $params ) { + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + return false; // invalid storage path + } + + $conn = $this->getConnection(); + if ( !$conn ) { + return null; + } + + $stat = false; + try { + $container = $conn->get_container( $srcCont ); + $obj = $container->get_object( $srcRel ); + // Convert "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW + $date = DateTime::createFromFormat( 'D, d F Y G:i:s e', $obj->last_modified ); + if ( $date ) { + $stat = array( + 'mtime' => $date->format( 'YmdHis' ), + 'size' => $obj->content_length, + 'sha1' => $obj->metadata['Sha1base36'] + ); + } else { // exception will be caught below + throw new Exception( "Could not parse date for object {$srcRel}" ); + } + } catch ( NoSuchContainerException $e ) { + } catch ( NoSuchObjectException $e ) { + } catch ( InvalidResponseException $e ) { + $stat = null; + } catch ( Exception $e ) { // some other exception? + $stat = null; + $this->logException( $e, __METHOD__, $params ); + } + + return $stat; + } + + /** + * @see FileBackendBase::getFileContents() + */ + public function getFileContents( array $params ) { + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + return false; // invalid storage path + } + + $conn = $this->getConnection(); + if ( !$conn ) { + return false; + } + + $data = false; + try { + $container = $conn->get_container( $srcCont ); + $obj = $container->get_object( $srcRel ); + $data = $obj->read(); + } catch ( NoSuchContainerException $e ) { + } catch ( NoSuchObjectException $e ) { + } catch ( InvalidResponseException $e ) { + } catch ( Exception $e ) { // some other exception? + $this->logException( $e, __METHOD__, $params ); + } + + return $data; + } + + /** + * @see FileBackend::getFileListInternal() + */ + public function getFileListInternal( $fullCont, $dir, array $params ) { + return new SwiftFileIterator( $this, $fullCont, $dir ); + } + + /** + * Do not call this function outside of SwiftFileIterator + * + * @param $fullCont string Resolved container name + * @param $dir string Resolved storage directory + * @param $after string Storage path of file to list items after + * @param $limit integer Max number of items to list + * @return Array + */ + public function getFileListPageInternal( $fullCont, $dir, $after, $limit ) { + $conn = $this->getConnection(); + if ( !$conn ) { + return null; + } + + $files = array(); + try { + $container = $conn->get_container( $fullCont ); + $files = $container->list_objects( $limit, $after, $dir ); + } catch ( NoSuchContainerException $e ) { + } catch ( NoSuchObjectException $e ) { + } catch ( InvalidResponseException $e ) { + } catch ( Exception $e ) { // some other exception? + $this->logException( $e, __METHOD__, $params ); + } + + return $files; + } + + /** + * @see FileBackend::doGetFileSha1base36() + */ + public function doGetFileSha1base36( array $params ) { + $stat = $this->getFileStat( $params ); + if ( $stat ) { + return $stat['sha1']; + } else { + return false; + } + } + + /** + * @see FileBackend::doStreamFile() + */ + protected function doStreamFile( array $params ) { + $status = Status::newGood(); + + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + } + + $conn = $this->getConnection(); + if ( !$conn ) { + $status->fatal( 'backend-fail-connect', $this->name ); + } + + try { + $cont = $conn->get_container( $srcCont ); + $obj = $cont->get_object( $srcRel ); + } catch ( NoSuchContainerException $e ) { + $status->fatal( 'backend-fail-stream', $params['src'] ); + return $status; + } catch ( NoSuchObjectException $e ) { + $status->fatal( 'backend-fail-stream', $params['src'] ); + return $status; + } catch ( IOException $e ) { + $status->fatal( 'backend-fail-stream', $params['src'] ); + return $status; + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-stream', $params['src'] ); + $this->logException( $e, __METHOD__, $params ); + return $status; + } + + try { + $output = fopen("php://output", "w"); + $obj->stream( $output ); + } catch ( InvalidResponseException $e ) { + $status->fatal( 'backend-fail-connect', $this->name ); + } catch ( Exception $e ) { // some other exception? + $status->fatal( 'backend-fail-stream', $params['src'] ); + $this->logException( $e, __METHOD__, $params ); + } + + return $status; + } + + /** + * @see FileBackend::getLocalCopy() + */ + public function getLocalCopy( array $params ) { + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + return null; + } + + $conn = $this->getConnection(); + if ( !$conn ) { + return null; + } + + // Get source file extension + $ext = FileBackend::extensionFromPath( $srcRel ); + // Create a new temporary file... + $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext ); + if ( !$tmpFile ) { + return null; + } + + try { + $cont = $conn->get_container( $srcCont ); + $obj = $cont->get_object( $srcRel ); + $obj->save_to_filename( $tmpFile->getPath() ); + } catch ( NoSuchContainerException $e ) { + $tmpFile = null; + } catch ( NoSuchObjectException $e ) { + $tmpFile = null; + } catch ( IOException $e ) { + $tmpFile = null; + } catch ( InvalidResponseException $e ) { + $tmpFile = null; + } catch ( Exception $e ) { // some other exception? + $tmpFile = null; + $this->logException( $e, __METHOD__, $params ); + } + + return $tmpFile; + } + + /** + * Get a connection to the swift proxy + * + * @return CF_Connection|false + */ + protected function getConnection() { + if ( $this->conn === false ) { + return false; // failed last attempt + } + // Authenticate with proxy and get a session key. + // Session keys expire after a while, so we renew them periodically. + if ( $this->conn === null || ( time() - $this->connStarted ) > $this->connTTL ) { + try { + $this->auth->authenticate(); + $this->conn = new CF_Connection( $this->auth ); + $this->connStarted = time(); + } catch ( AuthenticationException $e ) { + $this->conn = false; // don't keep re-trying + } catch ( InvalidResponseException $e ) { + $this->conn = false; // don't keep re-trying + } + } + return $this->conn; + } + + /** + * Log an unexpected exception for this backend + * + * @param $e Exception + * @param $func string + * @param $params Array + * @return void + */ + protected function logException( Exception $e, $func, array $params ) { + wfDebugLog( 'SwiftBackend', + get_class( $e ) . " in '{$this->name}': '{$func}' with " . serialize( $params ) + ); + } +} + +/** + * SwiftFileBackend helper class to page through object listings. + * Swift also has a listing limit of 10,000 objects for sanity. + * + * @ingroup FileBackend + */ +class SwiftFileIterator implements Iterator { + /** @var Array */ + protected $bufferIter = array(); + protected $bufferAfter = null; // string; list items *after* this path + protected $pos = 0; // integer + + /** @var SwiftFileBackend */ + protected $backend; + protected $container; // + protected $dir; // string storage directory + + const PAGE_SIZE = 5000; // file listing buffer size + + /** + * Get an FSFileIterator from a file system directory + * + * @param $backend SwiftFileBackend + * @param $fullCont string Resolved container name + * @param $dir string Resolved relateive path + */ + public function __construct( SwiftFileBackend $backend, $fullCont, $dir ) { + $this->container = $fullCont; + $this->dir = $dir; + $this->backend = $backend; + } + + public function current() { + return current( $this->bufferIter ); + } + + public function key() { + return $this->pos; + } + + public function next() { + // Advance to the next file in the page + next( $this->bufferIter ); + ++$this->pos; + // Check if there are no files left in this page and + // advance to the next page if this page was not empty. + if ( !$this->valid() && count( $this->bufferIter ) ) { + $this->bufferAfter = end( $this->bufferIter ); + $this->bufferIter = $this->backend->getFileListPageInternal( + $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE + ); + } + } + + public function rewind() { + $this->pos = 0; + $this->bufferAfter = null; + $this->bufferIter = $this->backend->getFileListPageInternal( + $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE + ); + } + + public function valid() { + return ( current( $this->bufferIter ) !== false ); // no paths can have this value + } +} diff --git a/languages/messages/MessagesEn.php b/languages/messages/MessagesEn.php index 3ed3a3a846..60675f374d 100644 --- a/languages/messages/MessagesEn.php +++ b/languages/messages/MessagesEn.php @@ -2255,6 +2255,8 @@ If the problem persists, contact an [[Special:ListUsers/sysop|administrator]].', 'backend-fail-create' => 'Could not create file $1.', 'backend-fail-readonly' => 'The backend "$1" is currently read-only. The reason given is: "$2"', 'backend-fail-synced' => 'The file "$1" is in an inconsistent state within the internal backends', +'backend-fail-connect' => 'Could not connect to file backend "$1".', +'backend-fail-internal' => 'An unknown internal file backend error occured.', # Lock manager 'lockmanager-notlocked' => 'Could not unlock "$1"; it is not locked.', diff --git a/maintenance/language/messages.inc b/maintenance/language/messages.inc index 7a04ac7dea..6e5317535c 100644 --- a/maintenance/language/messages.inc +++ b/maintenance/language/messages.inc @@ -1364,7 +1364,9 @@ $wgMessageStructure = array( 'backend-fail-read', 'backend-fail-create', 'backend-fail-readonly', - 'backend-fail-synced' + 'backend-fail-synced', + 'backend-fail-connect', + 'backend-fail-internal' ), 'lockmanager-errors' => array( -- 2.20.1