From 908b4d4340fd05bec71734b37fc9201c121e36a7 Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 10 May 2012 17:41:48 -0700 Subject: [PATCH] [FileBackend] Added CDN support to Swift for third-party sites. Change-Id: I220ab8cf8fd97d3a080b8dcc8f010d97fcc09671 --- .../filerepo/backend/SwiftFileBackend.php | 108 ++++++++++++++---- 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/includes/filerepo/backend/SwiftFileBackend.php b/includes/filerepo/backend/SwiftFileBackend.php index 0def61fc04..c0a087fcf0 100644 --- a/includes/filerepo/backend/SwiftFileBackend.php +++ b/includes/filerepo/backend/SwiftFileBackend.php @@ -41,6 +41,7 @@ class SwiftFileBackend extends FileBackendStore { protected $auth; // Swift authentication handler protected $authTTL; // integer seconds protected $swiftAnonUser; // string; username to handle unauthenticated requests + protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled protected $maxContCacheSize = 300; // integer; max containers with entries /** @var CF_Connection */ @@ -56,6 +57,7 @@ class SwiftFileBackend extends FileBackendStore { * swiftKey : Swift authentication key for the above user * swiftAuthTTL : Swift authentication TTL (seconds) * swiftAnonUser : Swift user used for end-user requests (account:username) + * swiftUseCDN : Whether a Cloud Files Content Delivery Network is set up * shardViaHashLevels : Map of container names to sharding config with: * 'base' : base of hash characters, 16 or 36 * 'levels' : the number of hash levels (and digits) @@ -84,6 +86,9 @@ class SwiftFileBackend extends FileBackendStore { $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] ) ? $config['shardViaHashLevels'] : ''; + $this->swiftUseCDN = isset( $config['swiftUseCDN'] ) + ? $config['swiftUseCDN'] + : false; // Cache container info to mask latency $this->memCache = wfGetMainCache(); } @@ -168,9 +173,13 @@ class SwiftFileBackend extends FileBackendStore { if ( !empty( $params['async'] ) ) { // deferred $handle = $obj->write_async( $params['content'] ); $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $handle ); + $status->value->affectedObjects[] = $obj; } else { // actually write the object in Swift $obj->write( $params['content'] ); + $this->purgeCDNCache( array( $obj ) ); } + } catch ( CDNNotEnabledException $e ) { + // CDN not enabled; nothing to see here } catch ( BadContentTypeException $e ) { $status->fatal( 'backend-fail-contenttype', $params['dst'] ); } catch ( CloudFilesException $e ) { // some other exception? @@ -250,10 +259,14 @@ class SwiftFileBackend extends FileBackendStore { $handle = $obj->write_async( $fp, filesize( $params['src'] ), true ); $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $handle ); $status->value->resourcesToClose[] = $fp; + $status->value->affectedObjects[] = $obj; } } else { // actually write the object in Swift $obj->load_from_filename( $params['src'], true ); // calls $obj->write() + $this->purgeCDNCache( array( $obj ) ); } + } catch ( CDNNotEnabledException $e ) { + // CDN not enabled; nothing to see here } catch ( BadContentTypeException $e ) { $status->fatal( 'backend-fail-contenttype', $params['dst'] ); } catch ( IOException $e ) { @@ -317,12 +330,17 @@ class SwiftFileBackend extends FileBackendStore { // (b) Actually copy the file to the destination try { + $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD if ( !empty( $params['async'] ) ) { // deferred $handle = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel ); $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $handle ); + $status->value->affectedObjects[] = $dstObj; } else { // actually write the object in Swift $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel ); + $this->purgeCDNCache( array( $dstObj ) ); } + } catch ( CDNNotEnabledException $e ) { + // CDN not enabled; nothing to see here } catch ( NoSuchObjectException $e ) { // source object does not exist $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); } catch ( CloudFilesException $e ) { // some other exception? @@ -382,12 +400,19 @@ class SwiftFileBackend extends FileBackendStore { // (b) Actually move the file to the destination try { + $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD + $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD if ( !empty( $params['async'] ) ) { // deferred $handle = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel ); $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $handle ); + $status->value->affectedObjects[] = $srcObj; + $status->value->affectedObjects[] = $dstObj; } else { // actually write the object in Swift $sContObj->move_object_to( $srcRel, $dContObj, $dstRel ); + $this->purgeCDNCache( array( $srcObj, $dstObj ) ); } + } catch ( CDNNotEnabledException $e ) { + // CDN not enabled; nothing to see here } catch ( NoSuchObjectException $e ) { // source object does not exist $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); } catch ( CloudFilesException $e ) { // some other exception? @@ -423,12 +448,17 @@ class SwiftFileBackend extends FileBackendStore { try { $sContObj = $this->getContainer( $srcCont ); + $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD if ( !empty( $params['async'] ) ) { // deferred $handle = $sContObj->delete_object_async( $srcRel ); $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $handle ); + $status->value->affectedObjects[] = $srcObj; } else { // actually write the object in Swift $sContObj->delete_object( $srcRel ); + $this->purgeCDNCache( array( $srcObj ) ); } + } catch ( CDNNotEnabledException $e ) { + // CDN not enabled; nothing to see here } catch ( NoSuchContainerException $e ) { $status->fatal( 'backend-fail-delete', $params['src'] ); } catch ( NoSuchObjectException $e ) { @@ -479,14 +509,19 @@ class SwiftFileBackend extends FileBackendStore { // (b) Create container as needed try { $contObj = $this->createContainer( $fullCont ); + // Make container public to end-users... if ( $this->swiftAnonUser != '' ) { - // Make container public to end-users... $status->merge( $this->setContainerAccess( $contObj, array( $this->auth->username, $this->swiftAnonUser ), // read array( $this->auth->username ) // write ) ); } + if ( $this->swiftUseCDN ) { // Rackspace style CDN + $contObj->make_public(); + } + } catch ( CDNNotEnabledException $e ) { + // CDN not enabled; nothing to see here } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, $status, __METHOD__, $params ); return $status; @@ -502,26 +537,31 @@ class SwiftFileBackend extends FileBackendStore { protected function doSecureInternal( $fullCont, $dir, array $params ) { $status = Status::newGood(); - if ( $this->swiftAnonUser != '' ) { - // Restrict container from end-users... - try { - // doPrepareInternal() should have been called, - // so the Swift container should already exist... - $contObj = $this->getContainer( $fullCont ); // normally a cache hit - // NoSuchContainerException not thrown: container must exist - if ( !isset( $contObj->mw_wasSecured ) ) { - $status->merge( $this->setContainerAccess( - $contObj, - array( $this->auth->username ), // read - array( $this->auth->username ) // write - ) ); - // @TODO: when php-cloudfiles supports container - // metadata, we can make use of that to avoid RTTs - $contObj->mw_wasSecured = true; // avoid useless RTTs - } - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, $status, __METHOD__, $params ); + // Restrict container from end-users... + try { + // doPrepareInternal() should have been called, + // so the Swift container should already exist... + $contObj = $this->getContainer( $fullCont ); // normally a cache hit + // NoSuchContainerException not thrown: container must exist + + // Make container private to end-users... + if ( $this->swiftAnonUser != '' && !isset( $contObj->mw_wasSecured ) ) { + $status->merge( $this->setContainerAccess( + $contObj, + array( $this->auth->username ), // read + array( $this->auth->username ) // write + ) ); + // @TODO: when php-cloudfiles supports container + // metadata, we can make use of that to avoid RTTs + $contObj->mw_wasSecured = true; // avoid useless RTTs + } + if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN + $contObj->make_private(); } + } catch ( CDNNotEnabledException $e ) { + // CDN not enabled; nothing to see here + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); } return $status; @@ -643,7 +683,7 @@ class SwiftFileBackend extends FileBackendStore { $data = false; try { $sContObj = $this->getContainer( $srcCont ); - $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request + $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD $data = $obj->read( $this->headersFromParams( $params ) ); } catch ( NoSuchContainerException $e ) { } catch ( CloudFilesException $e ) { // some other exception? @@ -829,7 +869,7 @@ class SwiftFileBackend extends FileBackendStore { try { $output = fopen( 'php://output', 'wb' ); - $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request + $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD $obj->stream( $output, $this->headersFromParams( $params ) ); } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, $status, __METHOD__, $params ); @@ -922,6 +962,7 @@ class SwiftFileBackend extends FileBackendStore { try { // catch exceptions; update status $function = '_getResponse' . $fileOpHandles[$index]->call; $this->$function( $cfOp, $status, $fileOpHandles[$index]->params ); + $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects ); } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, $status, __CLASS__ . ":$function", $fileOpHandles[$index]->params ); @@ -960,6 +1001,27 @@ class SwiftFileBackend extends FileBackendStore { return $req->execute(); // should return 204 } + /** + * Purge the CDN cache of affected objects if CDN caching is enabled + * + * @param $objects Array List of CF_Object items + * @return void + */ + public function purgeCDNCache( array $objects ) { + if ( $this->swiftUseCDN ) { // Rackspace style CDN + foreach ( $objects as $object ) { + try { + $object->purge_from_cdn(); + } catch ( CDNNotEnabledException $e ) { + // CDN not enabled; nothing to see here + } catch ( CloudFilesException $e ) { + $this->handleException( $e, null, __METHOD__, + array( 'cont' => $object->container->name, 'obj' => $object->name ) ); + } + } + } + } + /** * Get a connection to the Swift proxy * @@ -1117,6 +1179,8 @@ class SwiftFileBackend extends FileBackendStore { class SwiftFileOpHandle extends FileBackendStoreOpHandle { /** @var CF_Async_Op */ public $cfOp; + /** @var Array */ + public $affectedObjects = array(); public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) { $this->backend = $backend; -- 2.20.1