*/
/**
- * @brief Class for an OpenStack Swift based file backend.
+ * @brief Class for an OpenStack Swift (or Ceph RGW) based file backend.
*
* This requires the SwiftCloudFiles MediaWiki extension, which includes
* the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
* @since 1.19
*/
class SwiftFileBackend extends FileBackendStore {
- /** @var CF_Authentication */
- protected $auth; // Swift authentication handler
- protected $authTTL; // integer seconds
- protected $swiftTempUrlKey; // string; shared secret value for making temp urls
- protected $swiftAnonUser; // string; username to handle unauthenticated requests
- protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
- protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
- protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
+ /** @var CF_Authentication Swift authentication handler */
+ protected $auth;
+
+ /** @var int TTL in seconds */
+ protected $authTTL;
+
+ /** @var string Shared secret value for making temp URLs */
+ protected $swiftTempUrlKey;
+
+ /** @var string Username to handle unauthenticated requests */
+ protected $swiftAnonUser;
+
+ /** @var bool Whether CloudFiles CDN is enabled */
+ protected $swiftUseCDN;
+
+ /** @var int How long to cache things in the CDN */
+ protected $swiftCDNExpiry;
+
+ /** @var bool Whether object CDN purging is enabled */
+ protected $swiftCDNPurgable;
// Rados Gateway specific options
- protected $rgwS3AccessKey; // string; S3 access key
- protected $rgwS3SecretKey; // string; S3 authentication key
+ /** @var string S3 access key */
+ protected $rgwS3AccessKey;
+
+ /** @var string S3 authentication key */
+ protected $rgwS3SecretKey;
- /** @var CF_Connection */
- protected $conn; // Swift connection handle
- protected $sessionStarted = 0; // integer UNIX timestamp
+ /** @var CF_Connection Swift connection handle*/
+ protected $conn;
+
+ /** @var int UNIX timestamp */
+ protected $sessionStarted = 0;
/** @var CloudFilesException */
protected $connException;
- protected $connErrorTime = 0; // UNIX timestamp
+
+ /** @var int UNIX timestamp */
+ protected $connErrorTime = 0;
/** @var BagOStuff */
protected $srvCache;
} else {
try { // look for APC, XCache, WinCache, ect...
$this->srvCache = ObjectCache::newAccelerator( array() );
- } catch ( Exception $e ) {}
+ } catch ( Exception $e ) {
+ }
}
}
$this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
/**
* @see FileBackendStore::resolveContainerPath()
- * @return null
+ * @param string $container
+ * @param string $relStoragePath
+ * @return string|null Returns null when the URL encoded storage path is
+ * longer than 1024 characters or not UTF-8 encoded.
*/
protected function resolveContainerPath( $container, $relStoragePath ) {
if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
} elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
return null; // too long for Swift
}
+
return $relStoragePath;
}
try {
$this->getContainer( $container );
+
return true; // container exists
} catch ( NoSuchContainerException $e ) {
} catch ( CloudFilesException $e ) { // some other exception?
if ( isset( $headers['Content-Disposition'] ) ) {
$headers['Content-Disposition'] = $this->truncDisp( $headers['Content-Disposition'] );
}
+
return $headers;
}
break; // too long; sigh
}
}
+
return $res;
}
list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
if ( $dstRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+
return $status;
}
$dContObj = $this->getContainer( $dstCont );
} catch ( NoSuchContainerException $e ) {
$status->fatal( 'backend-fail-create', $params['dst'] );
+
return $status;
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
+
return $status;
}
list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
if ( $dstRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+
return $status;
}
$dContObj = $this->getContainer( $dstCont );
} catch ( NoSuchContainerException $e ) {
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+
return $status;
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
+
return $status;
}
wfRestoreWarnings();
if ( $sha1Hash === false ) { // source doesn't exist?
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+
return $status;
}
$sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
if ( $srcRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
return $status;
}
list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
if ( $dstRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+
return $status;
}
if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) {
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
}
+
return $status;
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
+
return $status;
}
list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
if ( $srcRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
return $status;
}
list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
if ( $dstRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+
return $status;
}
if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) {
$status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
}
+
return $status;
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
+
return $status;
}
list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
if ( $srcRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
return $status;
}
list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
if ( $srcRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
return $status;
}
// (a) Check if container already exists
try {
$this->getContainer( $fullCont );
+
// NoSuchContainerException not thrown: container must exist
return $status; // already exists
} catch ( NoSuchContainerException $e ) {
// NoSuchContainerException thrown: container does not exist
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
+
return $status;
}
// CDN not enabled; nothing to see here
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
+
return $status;
}
/**
* @see FileBackendStore::doSecureInternal()
+ * @param string $fullCont
+ * @param string $dir
+ * @param array $params
* @return Status
*/
protected function doSecureInternal( $fullCont, $dir, array $params ) {
/**
* @see FileBackendStore::doPublishInternal()
+ * @param string $fullCont
+ * @param string $dir
+ * @param array $params
* @return Status
*/
protected function doPublishInternal( $fullCont, $dir, array $params ) {
return $status; // ok, nothing to do
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
+
return $status;
}
return $status; // race? consistency delay?
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
+
return $status;
}
}
$srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
$this->addMissingMetadata( $srcObj, $params['src'] );
$stat = array(
- // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
- 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
+ // Convert various random Swift dates to TS_MW
+ 'mtime' => $this->convertSwiftDate( $srcObj->last_modified, TS_MW ),
'size' => (int)$srcObj->content_length,
'sha1' => $srcObj->getMetadataValue( 'Sha1base36' )
);
return $stat;
}
+ /**
+ * Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
+ * Dates might also come in like "2013-05-11T07:37:27.678360" from Swift listings,
+ * missing the timezone suffix (though Ceph RGW does not appear to have this bug).
+ *
+ * @param string $ts
+ * @param int $format Output format (TS_* constant)
+ * @return string
+ * @throws MWException
+ */
+ protected function convertSwiftDate( $ts, $format = TS_MW ) {
+ $timestamp = new MWTimestamp( $ts );
+
+ return $timestamp->getTimestamp( $format );
+ }
+
/**
* Fill in any missing object metadata and save it to Swift
*
$obj->setMetadataValues( array( 'Sha1base36' => $hash ) );
$obj->sync_metadata(); // save to Swift
wfProfileOut( __METHOD__ );
+
return true; // success
}
}
trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING );
$obj->setMetadataValues( array( 'Sha1base36' => false ) );
wfProfileOut( __METHOD__ );
+
return false; // failed
}
/**
* @see FileBackendStore::doDirectoryExists()
+ * @param string $fullCont
+ * @param string $dir
+ * @param array $params
* @return bool|null
*/
protected function doDirectoryExists( $fullCont, $dir, array $params ) {
try {
$container = $this->getContainer( $fullCont );
$prefix = ( $dir == '' ) ? null : "{$dir}/";
+
return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
} catch ( NoSuchContainerException $e ) {
return false;
/**
* @see FileBackendStore::getDirectoryListInternal()
+ * @param string $fullCont
+ * @param string $dir
+ * @param array $params
* @return SwiftFileBackendDirList
*/
public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
/**
* @see FileBackendStore::getFileListInternal()
+ * @param string $fullCont
+ * @param string $dir
+ * @param array $params
* @return SwiftFileBackendFileList
*/
public function getFileListInternal( $fullCont, $dir, array $params ) {
*
* @param string $fullCont Resolved container name
* @param string $dir Resolved storage directory with no trailing slash
- * @param string|null $after Storage path of file to list items after
- * @param integer $limit Max number of items to list
+ * @param string|null $after Resolved container relative path to list items after
+ * @param int $limit Max number of items to list
* @param array $params Parameters for getDirectoryList()
- * @return Array List of resolved paths of directories directly under $dir
+ * @return array List of container relative resolved paths of directories directly under $dir
* @throws FileBackendError
*/
public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
*
* @param string $fullCont Resolved container name
* @param string $dir Resolved storage directory with no trailing slash
- * @param string|null $after Storage path of file to list items after
- * @param integer $limit Max number of items to list
+ * @param string|null $after Resolved container relative path of file to list items after
+ * @param int $limit Max number of items to list
* @param array $params Parameters for getDirectoryList()
- * @return Array List of resolved paths of files under $dir
+ * @return array List of resolved container relative paths of files under $dir
* @throws FileBackendError
*/
public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
- $files = array();
+ $files = array(); // list of (path, stat array or null) entries
if ( $after === INF ) {
return $files; // nothing more
}
try {
$container = $this->getContainer( $fullCont );
$prefix = ( $dir == '' ) ? null : "{$dir}/";
+
+ // $objects will contain a list of unfiltered names or CF_Object items
// Non-recursive: only list files right under $dir
- if ( !empty( $params['topOnly'] ) ) { // files and dirs
+ if ( !empty( $params['topOnly'] ) ) {
if ( !empty( $params['adviseStat'] ) ) {
- $limit = min( $limit, self::CACHE_CHEAP_SIZE );
// Note: get_objects() does not include directories
- $objects = $this->loadObjectListing( $params, $dir,
- $container->get_objects( $limit, $after, $prefix, null, '/' ) );
- $files = $objects;
+ $objects = $container->get_objects( $limit, $after, $prefix, null, '/' );
} else {
+ // Note: list_objects() includes directories here
$objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
- foreach ( $objects as $object ) { // files and directories
- if ( substr( $object, -1 ) !== '/' ) {
- $files[] = $object; // directories end in '/'
- }
- }
}
+ $files = $this->buildFileObjectListing( $params, $dir, $objects );
// Recursive: list all files under $dir and its subdirs
- } else { // files
+ } else {
+ // Note: get_objects()/list_objects() here only return file objects
if ( !empty( $params['adviseStat'] ) ) {
- $limit = min( $limit, self::CACHE_CHEAP_SIZE );
- $objects = $this->loadObjectListing( $params, $dir,
- $container->get_objects( $limit, $after, $prefix ) );
+ $objects = $container->get_objects( $limit, $after, $prefix );
} else {
$objects = $container->list_objects( $limit, $after, $prefix );
}
- $files = $objects;
+ $files = $this->buildFileObjectListing( $params, $dir, $objects );
}
// Page on the unfiltered object listing (what is returned may be filtered)
if ( count( $objects ) < $limit ) {
$after = INF; // avoid a second RTT
} else {
$after = end( $objects ); // update last item
+ $after = is_object( $after ) ? $after->name : $after;
}
} catch ( NoSuchContainerException $e ) {
} catch ( CloudFilesException $e ) { // some other exception?
}
/**
- * Load a list of objects that belong under $dir into stat cache
- * and return a list of the names of the objects in the same order.
+ * Build a list of file objects, filtering out any directories
+ * and extracting any stat info if provided in $objects (for CF_Objects)
*
* @param array $params Parameters for getDirectoryList()
* @param string $dir Resolved container directory path
- * @param array $cfObjects List of CF_Object items
- * @return array List of object names
+ * @param array $objects List of CF_Object items or object names
+ * @return array List of (names,stat array or null) entries
*/
- private function loadObjectListing( array $params, $dir, array $cfObjects ) {
+ private function buildFileObjectListing( array $params, $dir, array $objects ) {
$names = array();
- $storageDir = rtrim( $params['dir'], '/' );
- $suffixStart = ( $dir === '' ) ? 0 : strlen( $dir ) + 1; // size of "path/to/dir/"
- // Iterate over the list *backwards* as this primes the stat cache, which is LRU.
- // If this fills the cache and the caller stats an uncached file before stating
- // the ones on the listing, there would be zero cache hits if this went forwards.
- for ( end( $cfObjects ); key( $cfObjects ) !== null; prev( $cfObjects ) ) {
- $object = current( $cfObjects );
- $path = "{$storageDir}/" . substr( $object->name, $suffixStart );
- $val = array(
- // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
- 'mtime' => wfTimestamp( TS_MW, $object->last_modified ),
- 'size' => (int)$object->content_length,
- 'latest' => false // eventually consistent
- );
- $this->cheapCache->set( $path, 'stat', $val );
- $names[] = $object->name;
+ foreach ( $objects as $object ) {
+ if ( is_object( $object ) ) {
+ $stat = array(
+ // Convert various random Swift dates to TS_MW
+ 'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
+ 'size' => (int)$object->content_length,
+ 'latest' => false // eventually consistent
+ );
+ $names[] = array( $object->name, $stat );
+ } elseif ( substr( $object, -1 ) !== '/' ) {
+ // Omit directories, which end in '/' in listings
+ $names[] = array( $object, null );
+ }
}
- return array_reverse( $names ); // keep the paths in original order
+
+ return $names;
+ }
+
+ /**
+ * Do not call this function outside of SwiftFileBackendFileList
+ *
+ * @param string $path Storage path
+ * @param array $val Stat value
+ */
+ public function loadListingStatInternal( $path, array $val ) {
+ $this->cheapCache->set( $path, 'stat', $val );
}
protected function doGetFileSha1base36( array $params ) {
$this->clearCache( array( $params['src'] ) );
$stat = $this->getFileStat( $params );
}
+
return $stat['sha1'];
} else {
return false;
$cont = $this->getContainer( $srcCont );
} catch ( NoSuchContainerException $e ) {
$status->fatal( 'backend-fail-stream', $params['src'] );
+
return $status;
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
+
return $status;
}
public function getFileHttpUrl( array $params ) {
if ( $this->swiftTempUrlKey != '' ||
- ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) )
- {
+ ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
+ ) {
list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
if ( $srcRel === null ) {
return null; // invalid path
$this->rgwS3SecretKey,
true // raw
) );
+
// See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
// Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
return wfAppendQuery(
$this->handleException( $e, null, __METHOD__, $params );
}
}
+
return null;
}
* $params is currently only checked for a 'latest' flag.
*
* @param array $params
- * @return Array
+ * @return array
*/
protected function headersFromParams( array $params ) {
$hdrs = array();
if ( !empty( $params['latest'] ) ) {
$hdrs[] = 'X-Newest: true';
}
+
return $hdrs;
}
/**
* Set read/write permissions for a Swift container.
*
- * $readGrps is a list of the possible criteria for a request to have
+ * @see http://swift.openstack.org/misc.html#acls
+ *
+ * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
+ * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
+ *
+ * @param CF_Container $contObj Swift container
+ * @param array $readGrps List of the possible criteria for a request to have
* access to read a container. Each item is one of the following formats:
* - account:user : Grants access if the request is by the given user
* - ".r:<regex>" : Grants access if the request is from a referrer host that
* Setting this to '*' effectively makes a container public.
* -".rlistings:<regex>" : Grants access if the request is from a referrer host that
* matches the expression and the request is for a listing.
- *
- * $writeGrps is a list of the possible criteria for a request to have
+ * @param array $writeGrps A list of the possible criteria for a request to have
* access to write to a container. Each item is of the following format:
* - account:user : Grants access if the request is by the given user
- *
- * @see http://swift.openstack.org/misc.html#acls
- *
- * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
- * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
- *
- * @param CF_Container $contObj Swift container
- * @param array $readGrps List of read access routes
- * @param array $writeGrps List of write access routes
* @return Status
*/
protected function setContainerAccess(
* This is for Rackspace/Akamai CDNs.
*
* @param array $objects List of CF_Object items
- * @return void
*/
public function purgeCDNCache( array $objects ) {
if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
}
$this->conn = new CF_Connection( $this->auth );
}
+
return $this->conn;
}
/**
* Close the connection to the Swift proxy
- *
- * @return void
*/
protected function closeConnection() {
if ( $this->conn ) {
);
}
}
+
return $this->connContainerCache->get( $container, 'obj' );
}
* Delete a Swift container
*
* @param string $container Container name
- * @return void
* @throws CloudFilesException
*/
protected function deleteContainer( $container ) {
* This also sets the Status object to have a fatal error.
*
* @param Exception $e
- * @param Status $status|null
+ * @param Status $status null
* @param string $func
* @param array $params
- * @return void
*/
protected function handleException( Exception $e, $status, $func, array $params ) {
if ( $status instanceof Status ) {
class SwiftFileOpHandle extends FileBackendStoreOpHandle {
/** @var CF_Async_Op */
public $cfOp;
- /** @var Array */
+
+ /** @var array */
public $affectedObjects = array();
/**
* @ingroup FileBackend
*/
abstract class SwiftFileBackendList implements Iterator {
- /** @var Array */
+ /** @var array List of path or (path,stat array) entries */
protected $bufferIter = array();
- protected $bufferAfter = null; // string; list items *after* this path
- protected $pos = 0; // integer
- /** @var Array */
+
+ /** @var string List items *after* this path */
+ protected $bufferAfter = null;
+
+ /** @var int */
+ protected $pos = 0;
+
+ /** @var array */
protected $params = array();
/** @var SwiftFileBackend */
protected $backend;
- protected $container; // string; container name
- protected $dir; // string; storage directory
- protected $suffixStart; // integer
+
+ /** @var string Container name */
+ protected $container;
+
+ /** @var string Storage directory */
+ protected $dir;
+
+ /** @var int */
+ protected $suffixStart;
const PAGE_SIZE = 9000; // file listing buffer size
/**
* @see Iterator::key()
- * @return integer
+ * @return int
*/
public function key() {
return $this->pos;
/**
* @see Iterator::next()
- * @return void
*/
public function next() {
// Advance to the next file in the page
/**
* @see Iterator::rewind()
- * @return void
*/
public function rewind() {
$this->pos = 0;
*
* @param string $container Resolved container name
* @param string $dir Resolved path relative to container
- * @param string $after|null
- * @param integer $limit
+ * @param string $after null
+ * @param int $limit
* @param array $params
- * @return Traversable|Array
+ * @return Traversable|array
*/
abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
}
/**
* @see SwiftFileBackendList::pageFromList()
- * @return Array
+ * @param string $container
+ * @param string $dir
+ * @param string $after
+ * @param int $limit
+ * @param array $params
+ * @return array
*/
protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
* @return string|bool String (relative path) or false
*/
public function current() {
- return substr( current( $this->bufferIter ), $this->suffixStart );
+ list( $path, $stat ) = current( $this->bufferIter );
+ $relPath = substr( $path, $this->suffixStart );
+ if ( is_array( $stat ) ) {
+ $storageDir = rtrim( $this->params['dir'], '/' );
+ $this->backend->loadListingStatInternal( "$storageDir/$relPath", $stat );
+ }
+
+ return $relPath;
}
/**
* @see SwiftFileBackendList::pageFromList()
- * @return Array
+ * @param string $container
+ * @param string $dir
+ * @param string $after
+ * @param int $limit
+ * @param array $params
+ * @return array
*/
protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
dépôts / lhc / web / wiklou.git / blobdiff