X-Git-Url: https://git.cyclocoop.org/%242?a=blobdiff_plain;f=includes%2Ffilerepo%2Fbackend%2FSwiftFileBackend.php;h=08a311933777ab9b47d9855eea9be1bbd56d314a;hb=1abcd5e30eea04bb03f01eb84f9c77bca7980729;hp=79662eeddc4a7cad8273bba28bb19f153ce48bb8;hpb=6554e9877dad0d9434d3ce0574c8ae56ae42610e;p=lhc%2Fweb%2Fwiklou.git diff --git a/includes/filerepo/backend/SwiftFileBackend.php b/includes/filerepo/backend/SwiftFileBackend.php index 79662eeddc..08a3119337 100644 --- a/includes/filerepo/backend/SwiftFileBackend.php +++ b/includes/filerepo/backend/SwiftFileBackend.php @@ -1,5 +1,22 @@ auth = new CF_Authentication( - $config['swiftUser'], - $config['swiftKey'], + $config['swiftUser'], + $config['swiftKey'], null, // account; unused - $config['swiftAuthUrl'] + $config['swiftAuthUrl'] ); // Optional settings $this->authTTL = isset( $config['swiftAuthTTL'] ) ? $config['swiftAuthTTL'] - : 120; // some sane number + : 5 * 60; // some sane number $this->swiftAnonUser = isset( $config['swiftAnonUser'] ) ? $config['swiftAnonUser'] : ''; $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] ) ? $config['shardViaHashLevels'] : ''; + // Cache container info to mask latency + $this->memCache = wfGetMainCache(); } /** - * @see FileBackend::resolveContainerPath() + * @see FileBackendStore::resolveContainerPath() + * @return null */ protected function resolveContainerPath( $container, $relStoragePath ) { if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) { @@ -73,7 +101,8 @@ class SwiftFileBackend extends FileBackend { } /** - * @see FileBackend::isPathUsableInternal() + * @see FileBackendStore::isPathUsableInternal() + * @return bool */ public function isPathUsableInternal( $storagePath ) { list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath ); @@ -85,16 +114,16 @@ class SwiftFileBackend extends FileBackend { $this->getContainer( $container ); return true; // container exists } catch ( NoSuchContainerException $e ) { - } catch ( InvalidResponseException $e ) { - } catch ( Exception $e ) { // some other exception? - $this->logException( $e, __METHOD__, array( 'path' => $storagePath ) ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) ); } return false; } /** - * @see FileBackend::doCopyInternal() + * @see FileBackendStore::doCreateInternal() + * @return Status */ protected function doCreateInternal( array $params ) { $status = Status::newGood(); @@ -109,7 +138,7 @@ class SwiftFileBackend extends FileBackend { try { $dContObj = $this->getContainer( $dstCont ); if ( empty( $params['overwrite'] ) && - $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) + $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) { $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); return $status; @@ -117,12 +146,8 @@ class SwiftFileBackend extends FileBackend { } catch ( NoSuchContainerException $e ) { $status->fatal( 'backend-fail-create', $params['dst'] ); return $status; - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - return $status; - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-internal', $this->name ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); return $status; } @@ -141,22 +166,35 @@ class SwiftFileBackend extends FileBackend { $obj->set_etag( md5( $params['content'] ) ); // Use the same content type as StreamFile for security $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] ); - // Actually write the object in Swift - $obj->write( $params['content'] ); + if ( !empty( $params['async'] ) ) { // deferred + $handle = $obj->write_async( $params['content'] ); + $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle ); + } else { // actually write the object in Swift + $obj->write( $params['content'] ); + } } catch ( BadContentTypeException $e ) { $status->fatal( 'backend-fail-contenttype', $params['dst'] ); - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-internal', $this->name ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); } return $status; } /** - * @see FileBackend::doStoreInternal() + * @see SwiftFileBackend::doExecuteOpHandlesInternal() + */ + protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) { + try { + $cfOp->getLastResponse(); + } catch ( BadContentTypeException $e ) { + $status->fatal( 'backend-fail-contenttype', $params['dst'] ); + } + } + + /** + * @see FileBackendStore::doStoreInternal() + * @return Status */ protected function doStoreInternal( array $params ) { $status = Status::newGood(); @@ -171,7 +209,7 @@ class SwiftFileBackend extends FileBackend { try { $dContObj = $this->getContainer( $dstCont ); if ( empty( $params['overwrite'] ) && - $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) + $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) { $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); return $status; @@ -179,12 +217,8 @@ class SwiftFileBackend extends FileBackend { } catch ( NoSuchContainerException $e ) { $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); return $status; - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - return $status; - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-internal', $this->name ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); return $status; } @@ -207,24 +241,47 @@ class SwiftFileBackend extends FileBackend { $obj->set_etag( md5_file( $params['src'] ) ); // Use the same content type as StreamFile for security $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] ); - // Actually write the object in Swift - $obj->load_from_filename( $params['src'], True ); // calls $obj->write() + if ( !empty( $params['async'] ) ) { // deferred + wfSuppressWarnings(); + $fp = fopen( $params['src'], 'rb' ); + wfRestoreWarnings(); + if ( !$fp ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + } else { + $handle = $obj->write_async( $fp, filesize( $params['src'] ), true ); + $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle ); + $status->value->resourcesToClose[] = $fp; + } + } else { // actually write the object in Swift + $obj->load_from_filename( $params['src'], true ); // calls $obj->write() + } } catch ( BadContentTypeException $e ) { $status->fatal( 'backend-fail-contenttype', $params['dst'] ); } catch ( IOException $e ) { $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-internal', $this->name ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); } return $status; } /** - * @see FileBackend::doCopyInternal() + * @see SwiftFileBackend::doExecuteOpHandlesInternal() + */ + protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) { + try { + $cfOp->getLastResponse(); + } catch ( BadContentTypeException $e ) { + $status->fatal( 'backend-fail-contenttype', $params['dst'] ); + } catch ( IOException $e ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + } + } + + /** + * @see FileBackendStore::doCopyInternal() + * @return Status */ protected function doCopyInternal( array $params ) { $status = Status::newGood(); @@ -246,7 +303,7 @@ class SwiftFileBackend extends FileBackend { $sContObj = $this->getContainer( $srcCont ); $dContObj = $this->getContainer( $dstCont ); if ( empty( $params['overwrite'] ) && - $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) + $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) { $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); return $status; @@ -254,32 +311,107 @@ class SwiftFileBackend extends FileBackend { } catch ( NoSuchContainerException $e ) { $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); return $status; - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - return $status; - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-internal', $this->name ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); return $status; } // (b) Actually copy the file to the destination try { - $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel ); + if ( !empty( $params['async'] ) ) { // deferred + $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel ); + $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle ); + } else { // actually write the object in Swift + $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel ); + } } catch ( NoSuchObjectException $e ) { // source object does not exist $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-internal', $this->name ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); } return $status; } /** - * @see FileBackend::doDeleteInternal() + * @see SwiftFileBackend::doExecuteOpHandlesInternal() + */ + protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) { + try { + $cfOp->getLastResponse(); + } catch ( NoSuchObjectException $e ) { // source object does not exist + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + } + } + + /** + * @see FileBackendStore::doMoveInternal() + * @return Status + */ + protected function doMoveInternal( array $params ) { + $status = Status::newGood(); + + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; + } + + list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] ); + if ( $dstRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; + } + + // (a) Check the source/destination containers and destination object + try { + $sContObj = $this->getContainer( $srcCont ); + $dContObj = $this->getContainer( $dstCont ); + if ( empty( $params['overwrite'] ) && + $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) + { + $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); + return $status; + } + } catch ( NoSuchContainerException $e ) { + $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + return $status; + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); + return $status; + } + + // (b) Actually move the file to the destination + try { + if ( !empty( $params['async'] ) ) { // deferred + $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel ); + $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle ); + } else { // actually write the object in Swift + $sContObj->move_object_to( $srcRel, $dContObj, $dstRel ); + } + } catch ( NoSuchObjectException $e ) { // source object does not exist + $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); + } + + return $status; + } + + /** + * @see SwiftFileBackend::doExecuteOpHandlesInternal() + */ + protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) { + try { + $cfOp->getLastResponse(); + } catch ( NoSuchObjectException $e ) { // source object does not exist + $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + } + } + + /** + * @see FileBackendStore::doDeleteInternal() + * @return Status */ protected function doDeleteInternal( array $params ) { $status = Status::newGood(); @@ -292,25 +424,43 @@ class SwiftFileBackend extends FileBackend { try { $sContObj = $this->getContainer( $srcCont ); - $sContObj->delete_object( $srcRel ); + if ( !empty( $params['async'] ) ) { // deferred + $handle = $sContObj->delete_object_async( $srcRel ); + $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle ); + } else { // actually write the object in Swift + $sContObj->delete_object( $srcRel ); + } } catch ( NoSuchContainerException $e ) { $status->fatal( 'backend-fail-delete', $params['src'] ); } catch ( NoSuchObjectException $e ) { if ( empty( $params['ignoreMissingSource'] ) ) { $status->fatal( 'backend-fail-delete', $params['src'] ); } - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-internal', $this->name ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); } return $status; } /** - * @see FileBackend::doPrepareInternal() + * @see SwiftFileBackend::doExecuteOpHandlesInternal() + */ + protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) { + try { + $cfOp->getLastResponse(); + } catch ( NoSuchContainerException $e ) { + $status->fatal( 'backend-fail-delete', $params['src'] ); + } catch ( NoSuchObjectException $e ) { + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-delete', $params['src'] ); + } + } + } + + /** + * @see FileBackendStore::doPrepareInternal() + * @return Status */ protected function doPrepareInternal( $fullCont, $dir, array $params ) { $status = Status::newGood(); @@ -322,12 +472,8 @@ class SwiftFileBackend extends FileBackend { return $status; // already exists } catch ( NoSuchContainerException $e ) { // NoSuchContainerException thrown: container does not exist - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - return $status; - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-internal', $this->name ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); return $status; } @@ -342,12 +488,8 @@ class SwiftFileBackend extends FileBackend { array( $this->auth->username ) // write ) ); } - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - return $status; - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-internal', $this->name ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); return $status; } @@ -355,7 +497,8 @@ class SwiftFileBackend extends FileBackend { } /** - * @see FileBackend::doSecureInternal() + * @see FileBackendStore::doSecureInternal() + * @return Status */ protected function doSecureInternal( $fullCont, $dir, array $params ) { $status = Status::newGood(); @@ -377,11 +520,8 @@ class SwiftFileBackend extends FileBackend { // metadata, we can make use of that to avoid RTTs $contObj->mw_wasSecured = true; // avoid useless RTTs } - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-internal', $this->name ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); } } @@ -389,7 +529,8 @@ class SwiftFileBackend extends FileBackend { } /** - * @see FileBackend::doCleanInternal() + * @see FileBackendStore::doCleanInternal() + * @return Status */ protected function doCleanInternal( $fullCont, $dir, array $params ) { $status = Status::newGood(); @@ -404,12 +545,8 @@ class SwiftFileBackend extends FileBackend { $contObj = $this->getContainer( $fullCont, true ); } catch ( NoSuchContainerException $e ) { return $status; // ok, nothing to do - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - return $status; - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-internal', $this->name ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); return $status; } @@ -419,12 +556,10 @@ class SwiftFileBackend extends FileBackend { $this->deleteContainer( $fullCont ); } catch ( NoSuchContainerException $e ) { return $status; // race? - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - return $status; - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-internal', $this->name ); - $this->logException( $e, __METHOD__, $params ); + } catch ( NonEmptyContainerException $e ) { + return $status; // race? consistency delay? + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); return $status; } } @@ -433,7 +568,8 @@ class SwiftFileBackend extends FileBackend { } /** - * @see FileBackend::doFileExists() + * @see FileBackendStore::doFileExists() + * @return array|bool|null */ protected function doGetFileStat( array $params ) { list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); @@ -454,11 +590,9 @@ class SwiftFileBackend extends FileBackend { ); } catch ( NoSuchContainerException $e ) { } catch ( NoSuchObjectException $e ) { - } catch ( InvalidResponseException $e ) { + } catch ( CloudFilesException $e ) { // some other exception? $stat = null; - } catch ( Exception $e ) { // some other exception? - $stat = null; - $this->logException( $e, __METHOD__, $params ); + $this->handleException( $e, null, __METHOD__, $params ); } return $stat; @@ -466,7 +600,7 @@ class SwiftFileBackend extends FileBackend { /** * Fill in any missing object metadata and save it to Swift - * + * * @param $obj CF_Object * @param $path string Storage path to object * @return bool Success @@ -479,7 +613,8 @@ class SwiftFileBackend extends FileBackend { $status = Status::newGood(); $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status ); if ( $status->isOK() ) { - $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) ); + # Do not stat the file in getLocalCopy() to avoid infinite loops + $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) ); if ( $tmpFile ) { $hash = $tmpFile->getSha1Base36(); if ( $hash !== false ) { @@ -494,7 +629,8 @@ class SwiftFileBackend extends FileBackend { } /** - * @see FileBackendBase::getFileContents() + * @see FileBackend::getFileContents() + * @return bool|null|string */ public function getFileContents( array $params ) { list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); @@ -512,51 +648,157 @@ class SwiftFileBackend extends FileBackend { $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request $data = $obj->read( $this->headersFromParams( $params ) ); } catch ( NoSuchContainerException $e ) { - } catch ( InvalidResponseException $e ) { - } catch ( Exception $e ) { // some other exception? - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, null, __METHOD__, $params ); } return $data; } /** - * @see FileBackend::getFileListInternal() + * @see FileBackendStore::doDirectoryExists() + * @return bool|null + */ + protected function doDirectoryExists( $fullCont, $dir, array $params ) { + try { + $container = $this->getContainer( $fullCont ); + $prefix = ( $dir == '' ) ? null : "{$dir}/"; + return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 ); + } catch ( NoSuchContainerException $e ) { + return false; + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, null, __METHOD__, + array( 'cont' => $fullCont, 'dir' => $dir ) ); + } + + return null; // error + } + + /** + * @see FileBackendStore::getDirectoryListInternal() + * @return SwiftFileBackendDirList + */ + public function getDirectoryListInternal( $fullCont, $dir, array $params ) { + return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params ); + } + + /** + * @see FileBackendStore::getFileListInternal() + * @return SwiftFileBackendFileList */ public function getFileListInternal( $fullCont, $dir, array $params ) { - return new SwiftFileBackendFileList( $this, $fullCont, $dir ); + return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params ); } /** * Do not call this function outside of SwiftFileBackendFileList - * + * * @param $fullCont string Resolved container name * @param $dir string Resolved storage directory with no trailing slash - * @param $after string Storage path of file to list items after + * @param $after string|null Storage path of file to list items after * @param $limit integer Max number of items to list - * @return Array + * @param $params Array Includes flag for 'topOnly' + * @return Array List of relative paths of dirs directly under $dir */ - public function getFileListPageInternal( $fullCont, $dir, $after, $limit ) { + public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { + $dirs = array(); + + try { + $container = $this->getContainer( $fullCont ); + $prefix = ( $dir == '' ) ? null : "{$dir}/"; + // Non-recursive: only list dirs right under $dir + if ( !empty( $params['topOnly'] ) ) { + $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); + foreach ( $objects as $object ) { // files and dirs + if ( substr( $object, -1 ) === '/' ) { + $dirs[] = $object; // directories end in '/' + } + $after = $object; // update last item + } + // Recursive: list all dirs under $dir and its subdirs + } else { + // Get directory from last item of prior page + $lastDir = $this->getParentDir( $after ); // must be first page + $objects = $container->list_objects( $limit, $after, $prefix ); + foreach ( $objects as $object ) { // files + $objectDir = $this->getParentDir( $object ); // directory of object + if ( $objectDir !== false ) { // file has a parent dir + // Swift stores paths in UTF-8, using binary sorting. + // See function "create_container_table" in common/db.py. + // If a directory is not "greater" than the last one, + // then it was already listed by the calling iterator. + if ( $objectDir > $lastDir ) { + $pDir = $objectDir; + do { // add dir and all its parent dirs + $dirs[] = "{$pDir}/"; + $pDir = $this->getParentDir( $pDir ); + } while ( $pDir !== false // sanity + && $pDir > $lastDir // not done already + && strlen( $pDir ) > strlen( $dir ) // within $dir + ); + } + $lastDir = $objectDir; + } + $after = $object; // update last item + } + } + } catch ( NoSuchContainerException $e ) { + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, null, __METHOD__, + array( 'cont' => $fullCont, 'dir' => $dir ) ); + } + + return $dirs; + } + + protected function getParentDir( $path ) { + return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false; + } + + /** + * Do not call this function outside of SwiftFileBackendFileList + * + * @param $fullCont string Resolved container name + * @param $dir string Resolved storage directory with no trailing slash + * @param $after string|null Storage path of file to list items after + * @param $limit integer Max number of items to list + * @param $params Array Includes flag for 'topOnly' + * @return Array List of relative paths of files under $dir + */ + public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { $files = array(); try { $container = $this->getContainer( $fullCont ); $prefix = ( $dir == '' ) ? null : "{$dir}/"; - $files = $container->list_objects( $limit, $after, $prefix ); + // Non-recursive: only list files right under $dir + if ( !empty( $params['topOnly'] ) ) { // files and dirs + $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); + foreach ( $objects as $object ) { + if ( substr( $object, -1 ) !== '/' ) { + $files[] = $object; // directories end in '/' + } + } + // Recursive: list all files under $dir and its subdirs + } else { // files + $files = $container->list_objects( $limit, $after, $prefix ); + } + $after = end( $files ); // update last item + reset( $files ); // reset pointer } catch ( NoSuchContainerException $e ) { - } catch ( NoSuchObjectException $e ) { - } catch ( InvalidResponseException $e ) { - } catch ( Exception $e ) { // some other exception? - $this->logException( $e, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, null, __METHOD__, + array( 'cont' => $fullCont, 'dir' => $dir ) ); } return $files; } /** - * @see FileBackend::doGetFileSha1base36() + * @see FileBackendStore::doGetFileSha1base36() + * @return bool */ - public function doGetFileSha1base36( array $params ) { + protected function doGetFileSha1base36( array $params ) { $stat = $this->getFileStat( $params ); if ( $stat ) { return $stat['sha1']; @@ -566,7 +808,8 @@ class SwiftFileBackend extends FileBackend { } /** - * @see FileBackend::doStreamFile() + * @see FileBackendStore::doStreamFile() + * @return Status */ protected function doStreamFile( array $params ) { $status = Status::newGood(); @@ -581,32 +824,25 @@ class SwiftFileBackend extends FileBackend { } catch ( NoSuchContainerException $e ) { $status->fatal( 'backend-fail-stream', $params['src'] ); return $status; - } catch ( InvalidResponseException $e ) { - $status->fatal( 'backend-fail-connect', $this->name ); - return $status; - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-stream', $params['src'] ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); return $status; } try { - $output = fopen( 'php://output', 'w' ); - // FileBackend::streamFile() already checks existence + $output = fopen( 'php://output', 'wb' ); $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request $obj->stream( $output, $this->headersFromParams( $params ) ); - } catch ( InvalidResponseException $e ) { // 404? connection problem? - $status->fatal( 'backend-fail-stream', $params['src'] ); - } catch ( Exception $e ) { // some other exception? - $status->fatal( 'backend-fail-stream', $params['src'] ); - $this->logException( $e, __METHOD__, $params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); } return $status; } /** - * @see FileBackend::getLocalCopy() + * @see FileBackendStore::getLocalCopy() + * @return null|TempFSFile */ public function getLocalCopy( array $params ) { list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); @@ -614,7 +850,8 @@ class SwiftFileBackend extends FileBackend { return null; } - if ( !$this->fileExists( $params ) ) { + # Check the recursion guard to avoid loops when filling metadata + if ( empty( $params['nostat'] ) && !$this->fileExists( $params ) ) { return null; } @@ -637,23 +874,29 @@ class SwiftFileBackend extends FileBackend { } } catch ( NoSuchContainerException $e ) { $tmpFile = null; - } catch ( InvalidResponseException $e ) { + } catch ( CloudFilesException $e ) { // some other exception? $tmpFile = null; - } catch ( Exception $e ) { // some other exception? - $tmpFile = null; - $this->logException( $e, __METHOD__, $params ); + $this->handleException( $e, null, __METHOD__, $params ); } return $tmpFile; } + /** + * @see FileBackendStore::directoriesAreVirtual() + * @return bool + */ + protected function directoriesAreVirtual() { + return true; + } + /** * Get headers to send to Swift when reading a file based - * on a FileBackend params array, e.g. that of getLocalCopy(). + * on a FileBackend params array, e.g. that of getLocalCopy(). * $params is currently only checked for a 'latest' flag. - * + * * @param $params Array - * @return Array + * @return Array */ protected function headersFromParams( array $params ) { $hdrs = array(); @@ -663,6 +906,39 @@ class SwiftFileBackend extends FileBackend { return $hdrs; } + /** + * @see FileBackendStore::doExecuteOpHandlesInternal() + * @return Array List of corresponding Status objects + */ + protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { + $statuses = array(); + + $cfOps = array(); // list of CF_Async_Op objects + foreach ( $fileOpHandles as $index => $fileOpHandle ) { + $cfOps[$index] = $fileOpHandle->cfOp; + } + $batch = new CF_Async_Op_Batch( $cfOps ); + + $cfOps = $batch->execute(); + foreach ( $cfOps as $index => $cfOp ) { + $status = Status::newGood(); + try { // catch exceptions; update status + $function = '_getResponse' . $fileOpHandles[$index]->call; + $this->$function( $cfOp, $status, $fileOpHandles[$index]->params ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, + __CLASS__ . ":$function", $fileOpHandles[$index]->params ); + } + $statuses[$index] = $status; + } + + foreach ( $fileOpHandles as $fileOpHandle ) { + $fileOpHandle->closeResources(); + } + + return $statuses; + } + /** * Set read/write permissions for a Swift container * @@ -690,12 +966,12 @@ class SwiftFileBackend extends FileBackend { /** * Get a connection to the Swift proxy * - * @return CF_Connection|false - * @throws InvalidResponseException + * @return CF_Connection|bool False on failure + * @throws CloudFilesException */ protected function getConnection() { - if ( $this->conn === false ) { - throw new InvalidResponseException; // failed last attempt + if ( $this->connException instanceof Exception ) { + throw $this->connException; // failed last attempt } // Session keys expire after a while, so we renew them periodically if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) { @@ -703,26 +979,23 @@ class SwiftFileBackend extends FileBackend { $this->conn = null; } // Authenticate with proxy and get a session key... - if ( $this->conn === null ) { + if ( !$this->conn ) { + $this->connStarted = 0; $this->connContainers = array(); try { $this->auth->authenticate(); $this->conn = new CF_Connection( $this->auth ); $this->connStarted = time(); - } catch ( AuthenticationException $e ) { - $this->conn = false; // don't keep re-trying - } catch ( InvalidResponseException $e ) { - $this->conn = false; // don't keep re-trying + } catch ( CloudFilesException $e ) { + $this->connException = $e; // don't keep re-trying + throw $e; // throw it back } } - if ( !$this->conn ) { - throw new InvalidResponseException; // auth/connection problem - } return $this->conn; } /** - * @see FileBackend::doClearCache() + * @see FileBackendStore::doClearCache() */ protected function doClearCache( array $paths = null ) { $this->connContainers = array(); // clear container object cache @@ -733,23 +1006,30 @@ class SwiftFileBackend extends FileBackend { * Use $reCache if the file count or byte count is needed. * * @param $container string Container name - * @param $reCache bool Refresh the process cache + * @param $bypassCache bool Bypass all caches and load from Swift * @return CF_Container + * @throws CloudFilesException */ - protected function getContainer( $container, $reCache = false ) { + protected function getContainer( $container, $bypassCache = false ) { $conn = $this->getConnection(); // Swift proxy connection - if ( $reCache ) { - unset( $this->connContainers[$container] ); // purge cache + if ( $bypassCache ) { // purge cache + unset( $this->connContainers[$container] ); + } elseif ( !isset( $this->connContainers[$container] ) ) { + $this->primeContainerCache( array( $container ) ); // check persistent cache } if ( !isset( $this->connContainers[$container] ) ) { $contObj = $conn->get_container( $container ); // NoSuchContainerException not thrown: container must exist if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache? reset( $this->connContainers ); - $key = key( $this->connContainers ); - unset( $this->connContainers[$key] ); + unset( $this->connContainers[key( $this->connContainers )] ); } $this->connContainers[$container] = $contObj; // cache it + if ( !$bypassCache ) { + $this->setContainerCache( $container, // update persistent cache + array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count ) + ); + } } return $this->connContainers[$container]; } @@ -759,6 +1039,7 @@ class SwiftFileBackend extends FileBackend { * * @param $container string Container name * @return CF_Container + * @throws InvalidResponseException */ protected function createContainer( $container ) { $conn = $this->getConnection(); // Swift proxy connection @@ -772,6 +1053,7 @@ class SwiftFileBackend extends FileBackend { * * @param $container string Container name * @return void + * @throws InvalidResponseException */ protected function deleteContainer( $container ) { $conn = $this->getConnection(); // Swift proxy connection @@ -780,37 +1062,88 @@ class SwiftFileBackend extends FileBackend { } /** - * Log an unexpected exception for this backend - * + * @see FileBackendStore::doPrimeContainerCache() + * @return void + */ + protected function doPrimeContainerCache( array $containerInfo ) { + try { + $conn = $this->getConnection(); // Swift proxy connection + foreach ( $containerInfo as $container => $info ) { + $this->connContainers[$container] = new CF_Container( + $conn->cfs_auth, + $conn->cfs_http, + $container, + $info['count'], + $info['bytes'] + ); + } + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, null, __METHOD__, array() ); + } + } + + /** + * Log an unexpected exception for this backend. + * This also sets the Status object to have a fatal error. + * * @param $e Exception + * @param $status Status|null * @param $func string * @param $params Array * @return void */ - protected function logException( Exception $e, $func, array $params ) { + protected function handleException( Exception $e, $status, $func, array $params ) { + if ( $status instanceof Status ) { + if ( $e instanceof AuthenticationException ) { + $status->fatal( 'backend-fail-connect', $this->name ); + } else { + $status->fatal( 'backend-fail-internal', $this->name ); + } + } + if ( $e->getMessage() ) { + trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING ); + } wfDebugLog( 'SwiftBackend', - get_class( $e ) . " in '{$this->name}': '{$func}' with " . serialize( $params ) + get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" . + ( $e->getMessage() ? ": {$e->getMessage()}" : "" ) ); } } /** - * SwiftFileBackend helper class to page through object listings. + * @see FileBackendStoreOpHandle + */ +class SwiftFileOpHandle extends FileBackendStoreOpHandle { + /** @var CF_Async_Op */ + public $cfOp; + + public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) { + $this->backend = $backend; + $this->params = $params; + $this->call = $call; + $this->cfOp = $cfOp; + } +} + +/** + * SwiftFileBackend helper class to page through listings. * Swift also has a listing limit of 10,000 objects for sanity. * Do not use this class from places outside SwiftFileBackend. * * @ingroup FileBackend */ -class SwiftFileBackendFileList implements Iterator { +abstract class SwiftFileBackendList implements Iterator { /** @var Array */ protected $bufferIter = array(); protected $bufferAfter = null; // string; list items *after* this path protected $pos = 0; // integer + /** @var Array */ + protected $params = array(); /** @var SwiftFileBackend */ - protected $backend; - protected $container; // - protected $dir; // string storage directory + protected $backend; + protected $container; // string; container name + protected $dir; // string; storage directory protected $suffixStart; // integer const PAGE_SIZE = 5000; // file listing buffer size @@ -819,8 +1152,9 @@ class SwiftFileBackendFileList implements Iterator { * @param $backend SwiftFileBackend * @param $fullCont string Resolved container name * @param $dir string Resolved directory relative to container + * @param $params Array */ - public function __construct( SwiftFileBackend $backend, $fullCont, $dir ) { + public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) { $this->backend = $backend; $this->container = $fullCont; $this->dir = $dir; @@ -832,16 +1166,21 @@ class SwiftFileBackendFileList implements Iterator { } else { // dir within container $this->suffixStart = strlen( $this->dir ) + 1; // size of "path/to/dir/" } + $this->params = $params; } - public function current() { - return substr( current( $this->bufferIter ), $this->suffixStart ); - } - + /** + * @see Iterator::key() + * @return integer + */ public function key() { return $this->pos; } + /** + * @see Iterator::next() + * @return void + */ public function next() { // Advance to the next file in the page next( $this->bufferIter ); @@ -849,22 +1188,87 @@ class SwiftFileBackendFileList implements Iterator { // Check if there are no files left in this page and // advance to the next page if this page was not empty. if ( !$this->valid() && count( $this->bufferIter ) ) { - $this->bufferAfter = end( $this->bufferIter ); - $this->bufferIter = $this->backend->getFileListPageInternal( - $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE - ); + $this->bufferIter = $this->pageFromList( + $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params + ); // updates $this->bufferAfter } } + /** + * @see Iterator::rewind() + * @return void + */ public function rewind() { $this->pos = 0; $this->bufferAfter = null; - $this->bufferIter = $this->backend->getFileListPageInternal( - $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE - ); + $this->bufferIter = $this->pageFromList( + $this->container, $this->dir, $this->bufferAfter, self::PAGE_SIZE, $this->params + ); // updates $this->bufferAfter } + /** + * @see Iterator::valid() + * @return bool + */ public function valid() { - return ( current( $this->bufferIter ) !== false ); // no paths can have this value + if ( $this->bufferIter === null ) { + return false; // some failure? + } else { + return ( current( $this->bufferIter ) !== false ); // no paths can have this value + } + } + + /** + * Get the given list portion (page) + * + * @param $container string Resolved container name + * @param $dir string Resolved path relative to container + * @param $after string|null + * @param $limit integer + * @param $params Array + * @return Traversable|Array|null Returns null on failure + */ + abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params ); +} + +/** + * Iterator for listing directories + */ +class SwiftFileBackendDirList extends SwiftFileBackendList { + /** + * @see Iterator::current() + * @return string|bool String (relative path) or false + */ + public function current() { + return substr( current( $this->bufferIter ), $this->suffixStart, -1 ); + } + + /** + * @see SwiftFileBackendList::pageFromList() + * @return Array|null + */ + protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { + return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params ); + } +} + +/** + * Iterator for listing regular files + */ +class SwiftFileBackendFileList extends SwiftFileBackendList { + /** + * @see Iterator::current() + * @return string|bool String (relative path) or false + */ + public function current() { + return substr( current( $this->bufferIter ), $this->suffixStart ); + } + + /** + * @see SwiftFileBackendList::pageFromList() + * @return Array|null + */ + protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { + return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params ); } }