/** @var CF_Connection */
protected $conn; // Swift connection handle
- protected $connStarted = 0; // integer UNIX timestamp
- protected $connException; // CloudFiles exception
+ protected $sessionStarted = 0; // integer UNIX timestamp
+
+ /** @var CloudFilesException */
+ protected $connException;
+ protected $connErrorTime = 0; // UNIX timestamp
+
+ /** @var BagOStuff */
+ protected $srvCache;
/** @var ProcessCacheLRU */
protected $connContainerCache; // container object cache
* - levels : the number of hash levels (and digits)
* - repeat : hash subdirectories are prefixed with all the
* parent hash directory names (e.g. "a/ab/abc")
+ * - cacheAuthInfo : Whether to cache authentication tokens in APC, XCache, ect.
+ * If those are not available, then the main cache will be used.
+ * This is probably insecure in shared hosting environments.
*/
public function __construct( array $config ) {
parent::__construct( $config );
: false;
$this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
? $config['swiftCDNExpiry']
- : 3600; // hour
+ : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
$this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
? $config['swiftCDNPurgable']
: true;
- // Cache container info to mask latency
+ // Cache container information to mask latency
$this->memCache = wfGetMainCache();
// Process cache for container info
$this->connContainerCache = new ProcessCacheLRU( 300 );
+ // Cache auth token information to avoid RTTs
+ if ( !empty( $config['cacheAuthInfo'] ) ) {
+ if ( php_sapi_name() === 'cli' ) {
+ $this->srvCache = wfGetMainCache(); // preferrably memcached
+ } else {
+ try { // look for APC, XCache, WinCache, ect...
+ $this->srvCache = ObjectCache::newAccelerator( array() );
+ } catch ( Exception $e ) {}
+ }
+ }
+ $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
}
/**
* @return null
*/
protected function resolveContainerPath( $container, $relStoragePath ) {
- if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
+ if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
+ return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
+ } elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
return null; // too long for Swift
}
return $relStoragePath;
return false;
}
+ /**
+ * @param $disposition string Content-Disposition header value
+ * @return string Truncated Content-Disposition header value to meet Swift limits
+ */
+ protected function truncDisp( $disposition ) {
+ $res = '';
+ foreach ( explode( ';', $disposition ) as $part ) {
+ $part = trim( $part );
+ $new = ( $res === '' ) ? $part : "{$res};{$part}";
+ if ( strlen( $new ) <= 255 ) {
+ $res = $new;
+ } else {
+ break; // too long; sigh
+ }
+ }
+ return $res;
+ }
+
/**
* @see FileBackendStore::doCreateInternal()
* @return Status
if ( !strlen( $obj->content_type ) ) { // special case
$obj->content_type = 'unknown/unknown';
}
+ // Set the Content-Disposition header if requested
+ if ( isset( $params['disposition'] ) ) {
+ $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
+ }
if ( !empty( $params['async'] ) ) { // deferred
- $handle = $obj->write_async( $params['content'] );
- $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle );
- $status->value->affectedObjects[] = $obj;
+ $op = $obj->write_async( $params['content'] );
+ $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op );
+ if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ $status->value->affectedObjects[] = $obj;
+ }
} else { // actually write the object in Swift
$obj->write( $params['content'] );
- $this->purgeCDNCache( array( $obj ) );
+ if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ $this->purgeCDNCache( array( $obj ) );
+ }
}
} catch ( CDNNotEnabledException $e ) {
// CDN not enabled; nothing to see here
if ( !strlen( $obj->content_type ) ) { // special case
$obj->content_type = 'unknown/unknown';
}
+ // Set the Content-Disposition header if requested
+ if ( isset( $params['disposition'] ) ) {
+ $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
+ }
if ( !empty( $params['async'] ) ) { // deferred
wfSuppressWarnings();
$fp = fopen( $params['src'], 'rb' );
if ( !$fp ) {
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
} else {
- $handle = $obj->write_async( $fp, filesize( $params['src'] ), true );
- $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle );
+ $op = $obj->write_async( $fp, filesize( $params['src'] ), true );
+ $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op );
$status->value->resourcesToClose[] = $fp;
- $status->value->affectedObjects[] = $obj;
+ if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ $status->value->affectedObjects[] = $obj;
+ }
}
} else { // actually write the object in Swift
$obj->load_from_filename( $params['src'], true ); // calls $obj->write()
- $this->purgeCDNCache( array( $obj ) );
+ if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ $this->purgeCDNCache( array( $obj ) );
+ }
}
} catch ( CDNNotEnabledException $e ) {
// CDN not enabled; nothing to see here
// (b) Actually copy the file to the destination
try {
$dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
+ $hdrs = array(); // source file headers to override with new values
+ if ( isset( $params['disposition'] ) ) {
+ $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
+ }
if ( !empty( $params['async'] ) ) { // deferred
- $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel );
- $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle );
- $status->value->affectedObjects[] = $dstObj;
+ $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
+ $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op );
+ if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ $status->value->affectedObjects[] = $dstObj;
+ }
} else { // actually write the object in Swift
- $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
- $this->purgeCDNCache( array( $dstObj ) );
+ $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
+ if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ $this->purgeCDNCache( array( $dstObj ) );
+ }
}
} catch ( CDNNotEnabledException $e ) {
// CDN not enabled; nothing to see here
try {
$srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
$dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
+ $hdrs = array(); // source file headers to override with new values
+ if ( isset( $params['disposition'] ) ) {
+ $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] );
+ }
if ( !empty( $params['async'] ) ) { // deferred
- $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel );
- $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle );
+ $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
+ $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op );
$status->value->affectedObjects[] = $srcObj;
- $status->value->affectedObjects[] = $dstObj;
+ if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ $status->value->affectedObjects[] = $dstObj;
+ }
} else { // actually write the object in Swift
- $sContObj->move_object_to( $srcRel, $dContObj, $dstRel );
- $this->purgeCDNCache( array( $srcObj, $dstObj ) );
+ $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
+ $this->purgeCDNCache( array( $srcObj ) );
+ if ( !empty( $params['overwrite'] ) ) { // file possibly mutated
+ $this->purgeCDNCache( array( $dstObj ) );
+ }
}
} catch ( CDNNotEnabledException $e ) {
// CDN not enabled; nothing to see here
$sContObj = $this->getContainer( $srcCont );
$srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
if ( !empty( $params['async'] ) ) { // deferred
- $handle = $sContObj->delete_object_async( $srcRel );
- $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle );
+ $op = $sContObj->delete_object_async( $srcRel );
+ $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $op );
$status->value->affectedObjects[] = $srcObj;
} else { // actually write the object in Swift
$sContObj->delete_object( $srcRel );
if ( isset( $obj->metadata['Sha1base36'] ) ) {
return true; // nothing to do
}
+ wfProfileIn( __METHOD__ );
$status = Status::newGood();
$scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
if ( $status->isOK() ) {
- # Do not stat the file in getLocalCopy() to avoid infinite loops
- $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) );
+ $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
if ( $tmpFile ) {
$hash = $tmpFile->getSha1Base36();
if ( $hash !== false ) {
$obj->metadata['Sha1base36'] = $hash;
$obj->sync_metadata(); // save to Swift
+ wfProfileOut( __METHOD__ );
return true; // success
}
}
}
$obj->metadata['Sha1base36'] = false;
+ wfProfileOut( __METHOD__ );
return false; // failed
}
/**
- * @see FileBackend::getFileContents()
- * @return bool|null|string
+ * @see FileBackendStore::doGetFileContentsMulti()
+ * @return Array
*/
- public function getFileContents( array $params ) {
- list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
- if ( $srcRel === null ) {
- return false; // invalid storage path
- }
-
- if ( !$this->fileExists( $params ) ) {
- return null;
- }
+ protected function doGetFileContentsMulti( array $params ) {
+ $contents = array();
+
+ $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
+ // Blindly create tmp files and stream to them, catching any exception if the file does
+ // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
+ foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
+ $cfOps = array(); // (path => CF_Async_Op)
+
+ foreach ( $pathBatch as $path ) { // each path in this concurrent batch
+ list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
+ if ( $srcRel === null ) {
+ $contents[$path] = false;
+ continue;
+ }
+ $data = false;
+ try {
+ $sContObj = $this->getContainer( $srcCont );
+ $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
+ // Get source file extension
+ $ext = FileBackend::extensionFromPath( $path );
+ // Create a new temporary memory file...
+ $handle = fopen( 'php://temp', 'wb' );
+ if ( $handle ) {
+ $headers = $this->headersFromParams( $params );
+ if ( count( $pathBatch ) > 1 ) {
+ $cfOps[$path] = $obj->stream_async( $handle, $headers );
+ $cfOps[$path]->_file_handle = $handle; // close this later
+ } else {
+ $obj->stream( $handle, $headers );
+ rewind( $handle ); // start from the beginning
+ $data = stream_get_contents( $handle );
+ fclose( $handle );
+ }
+ } else {
+ $data = false;
+ }
+ } catch ( NoSuchContainerException $e ) {
+ $data = false;
+ } catch ( NoSuchObjectException $e ) {
+ $data = false;
+ } catch ( CloudFilesException $e ) { // some other exception?
+ $data = false;
+ $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
+ }
+ $contents[$path] = $data;
+ }
- $data = false;
- try {
- $sContObj = $this->getContainer( $srcCont );
- $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
- $data = $obj->read( $this->headersFromParams( $params ) );
- } catch ( NoSuchContainerException $e ) {
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, null, __METHOD__, $params );
+ $batch = new CF_Async_Op_Batch( $cfOps );
+ $cfOps = $batch->execute();
+ foreach ( $cfOps as $path => $cfOp ) {
+ try {
+ $cfOp->getLastResponse();
+ rewind( $cfOp->_file_handle ); // start from the beginning
+ $contents[$path] = stream_get_contents( $cfOp->_file_handle );
+ } catch ( NoSuchContainerException $e ) {
+ $contents[$path] = false;
+ } catch ( NoSuchObjectException $e ) {
+ $contents[$path] = false;
+ } catch ( CloudFilesException $e ) { // some other exception?
+ $contents[$path] = false;
+ $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
+ }
+ fclose( $cfOp->_file_handle ); // close open handle
+ }
}
- return $data;
+ return $contents;
}
/**
// See function "create_container_table" in common/db.py.
// If a directory is not "greater" than the last one,
// then it was already listed by the calling iterator.
- if ( $objectDir > $lastDir ) {
+ if ( strcmp( $objectDir, $lastDir ) > 0 ) {
$pDir = $objectDir;
do { // add dir and all its parent dirs
$dirs[] = "{$pDir}/";
$pDir = $this->getParentDir( $pDir );
} while ( $pDir !== false // sanity
- && $pDir > $lastDir // not done already
+ && strcmp( $pDir, $lastDir ) > 0 // not done already
&& strlen( $pDir ) > strlen( $dir ) // within $dir
);
}
$output = fopen( 'php://output', 'wb' );
$obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
$obj->stream( $output, $this->headersFromParams( $params ) );
+ } catch ( NoSuchObjectException $e ) {
+ $status->fatal( 'backend-fail-stream', $params['src'] );
} catch ( CloudFilesException $e ) { // some other exception?
$this->handleException( $e, $status, __METHOD__, $params );
}
}
/**
- * @see FileBackendStore::getLocalCopy()
+ * @see FileBackendStore::doGetLocalCopyMulti()
* @return null|TempFSFile
*/
- public function getLocalCopy( array $params ) {
- list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
- if ( $srcRel === null ) {
- return null;
- }
-
- # Check the recursion guard to avoid loops when filling metadata
- if ( empty( $params['nostat'] ) && !$this->fileExists( $params ) ) {
- return null;
- }
+ protected function doGetLocalCopyMulti( array $params ) {
+ $tmpFiles = array();
+
+ $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
+ // Blindly create tmp files and stream to them, catching any exception if the file does
+ // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
+ foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
+ $cfOps = array(); // (path => CF_Async_Op)
+
+ foreach ( $pathBatch as $path ) { // each path in this concurrent batch
+ list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
+ if ( $srcRel === null ) {
+ $tmpFiles[$path] = null;
+ continue;
+ }
+ $tmpFile = null;
+ try {
+ $sContObj = $this->getContainer( $srcCont );
+ $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
+ // Get source file extension
+ $ext = FileBackend::extensionFromPath( $path );
+ // Create a new temporary file...
+ $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
+ if ( $tmpFile ) {
+ $handle = fopen( $tmpFile->getPath(), 'wb' );
+ if ( $handle ) {
+ $headers = $this->headersFromParams( $params );
+ if ( count( $pathBatch ) > 1 ) {
+ $cfOps[$path] = $obj->stream_async( $handle, $headers );
+ $cfOps[$path]->_file_handle = $handle; // close this later
+ } else {
+ $obj->stream( $handle, $headers );
+ fclose( $handle );
+ }
+ } else {
+ $tmpFile = null;
+ }
+ }
+ } catch ( NoSuchContainerException $e ) {
+ $tmpFile = null;
+ } catch ( NoSuchObjectException $e ) {
+ $tmpFile = null;
+ } catch ( CloudFilesException $e ) { // some other exception?
+ $tmpFile = null;
+ $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
+ }
+ $tmpFiles[$path] = $tmpFile;
+ }
- $tmpFile = null;
- try {
- $sContObj = $this->getContainer( $srcCont );
- $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
- // Get source file extension
- $ext = FileBackend::extensionFromPath( $srcRel );
- // Create a new temporary file...
- $tmpFile = TempFSFile::factory( wfBaseName( $srcRel ) . '_', $ext );
- if ( $tmpFile ) {
- $handle = fopen( $tmpFile->getPath(), 'wb' );
- if ( $handle ) {
- $obj->stream( $handle, $this->headersFromParams( $params ) );
- fclose( $handle );
- } else {
- $tmpFile = null; // couldn't open temp file
+ $batch = new CF_Async_Op_Batch( $cfOps );
+ $cfOps = $batch->execute();
+ foreach ( $cfOps as $path => $cfOp ) {
+ try {
+ $cfOp->getLastResponse();
+ } catch ( NoSuchContainerException $e ) {
+ $tmpFiles[$path] = null;
+ } catch ( NoSuchObjectException $e ) {
+ $tmpFiles[$path] = null;
+ } catch ( CloudFilesException $e ) { // some other exception?
+ $tmpFiles[$path] = null;
+ $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
}
+ fclose( $cfOp->_file_handle ); // close open handle
}
- } catch ( NoSuchContainerException $e ) {
- $tmpFile = null;
- } catch ( CloudFilesException $e ) { // some other exception?
- $tmpFile = null;
- $this->handleException( $e, null, __METHOD__, $params );
}
- return $tmpFile;
+ return $tmpFiles;
}
/**
}
/**
- * Get a connection to the Swift proxy
+ * Get an authenticated connection handle to the Swift proxy
*
* @return CF_Connection|bool False on failure
* @throws CloudFilesException
*/
protected function getConnection() {
- if ( $this->connException instanceof Exception ) {
- throw $this->connException; // failed last attempt
+ if ( $this->connException instanceof CloudFilesException ) {
+ if ( ( time() - $this->connErrorTime ) < 60 ) {
+ throw $this->connException; // failed last attempt; don't bother
+ } else { // actually retry this time
+ $this->connException = null;
+ $this->connErrorTime = 0;
+ }
}
// Session keys expire after a while, so we renew them periodically
- if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
- $this->conn->close(); // close active cURL connections
- $this->conn = null;
- }
+ $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL );
// Authenticate with proxy and get a session key...
- if ( !$this->conn ) {
- $this->connStarted = 0;
+ if ( !$this->conn || $reAuth ) {
+ $this->sessionStarted = 0;
$this->connContainerCache->clear();
- try {
- $this->auth->authenticate();
- $this->conn = new CF_Connection( $this->auth );
- $this->connStarted = time();
- } catch ( CloudFilesException $e ) {
- $this->connException = $e; // don't keep re-trying
- throw $e; // throw it back
+ $cacheKey = $this->getCredsCacheKey( $this->auth->username );
+ $creds = $this->srvCache->get( $cacheKey ); // credentials
+ if ( is_array( $creds ) ) { // cache hit
+ $this->auth->load_cached_credentials(
+ $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] );
+ $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case
+ } else { // cache miss
+ try {
+ $this->auth->authenticate();
+ $creds = $this->auth->export_credentials();
+ $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache
+ $this->sessionStarted = time();
+ } catch ( CloudFilesException $e ) {
+ $this->connException = $e; // don't keep re-trying
+ $this->connErrorTime = time();
+ throw $e; // throw it back
+ }
+ }
+ if ( $this->conn ) { // re-authorizing?
+ $this->conn->close(); // close active cURL handles in CF_Http object
}
+ $this->conn = new CF_Connection( $this->auth );
}
return $this->conn;
}
+ /**
+ * Close the connection to the Swift proxy
+ *
+ * @return void
+ */
+ protected function closeConnection() {
+ if ( $this->conn ) {
+ $this->conn->close(); // close active cURL handles in CF_Http object
+ $this->sessionStarted = 0;
+ $this->connContainerCache->clear();
+ }
+ }
+
+ /**
+ * Get the cache key for a container
+ *
+ * @param $username string
+ * @return string
+ */
+ private function getCredsCacheKey( $username ) {
+ return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username );
+ }
+
/**
* @see FileBackendStore::doClearCache()
*/
if ( $e->getMessage() ) {
trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
}
+ if ( $e instanceof InvalidResponseException ) { // possibly a stale token
+ $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) );
+ $this->closeConnection(); // force a re-connect and re-auth next time
+ }
wfDebugLog( 'SwiftBackend',
get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
( $e->getMessage() ? ": {$e->getMessage()}" : "" )
dépôts / lhc / web / wiklou.git / blobdiff