10 * @brief Class for an OpenStack Swift based file backend.
12 * This requires the SwiftCloudFiles MediaWiki extension, which includes
13 * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
14 * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
16 * Status messages should avoid mentioning the Swift account name.
17 * Likewise, error suppression should be used to avoid path disclosure.
19 * @ingroup FileBackend
22 class SwiftFileBackend
extends FileBackendStore
{
23 /** @var CF_Authentication */
24 protected $auth; // Swift authentication handler
25 protected $authTTL; // integer seconds
26 protected $swiftAnonUser; // string; username to handle unauthenticated requests
27 protected $maxContCacheSize = 100; // integer; max containers with entries
29 /** @var CF_Connection */
30 protected $conn; // Swift connection handle
31 protected $connStarted = 0; // integer UNIX timestamp
32 protected $connContainers = array(); // container object cache
35 * @see FileBackendStore::__construct()
36 * Additional $config params include:
37 * swiftAuthUrl : Swift authentication server URL
38 * swiftUser : Swift user used by MediaWiki (account:username)
39 * swiftKey : Swift authentication key for the above user
40 * swiftAuthTTL : Swift authentication TTL (seconds)
41 * swiftAnonUser : Swift user used for end-user requests (account:username)
42 * shardViaHashLevels : Map of container names to sharding config with:
43 * 'base' : base of hash characters, 16 or 36
44 * 'levels' : the number of hash levels (and digits)
45 * 'repeat' : hash subdirectories are prefixed with all the
46 * parent hash directory names (e.g. "a/ab/abc")
48 public function __construct( array $config ) {
49 parent
::__construct( $config );
51 $this->auth
= new CF_Authentication(
54 null, // account; unused
55 $config['swiftAuthUrl']
58 $this->authTTL
= isset( $config['swiftAuthTTL'] )
59 ?
$config['swiftAuthTTL']
60 : 120; // some sane number
61 $this->swiftAnonUser
= isset( $config['swiftAnonUser'] )
62 ?
$config['swiftAnonUser']
64 $this->shardViaHashLevels
= isset( $config['shardViaHashLevels'] )
65 ?
$config['shardViaHashLevels']
70 * @see FileBackendStore::resolveContainerPath()
73 protected function resolveContainerPath( $container, $relStoragePath ) {
74 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
75 return null; // too long for Swift
77 return $relStoragePath;
81 * @see FileBackendStore::isPathUsableInternal()
84 public function isPathUsableInternal( $storagePath ) {
85 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
86 if ( $rel === null ) {
87 return false; // invalid
91 $this->getContainer( $container );
92 return true; // container exists
93 } catch ( NoSuchContainerException
$e ) {
94 } catch ( InvalidResponseException
$e ) {
95 } catch ( Exception
$e ) { // some other exception?
96 $this->logException( $e, __METHOD__
, array( 'path' => $storagePath ) );
103 * @see FileBackendStore::doCreateInternal()
106 protected function doCreateInternal( array $params ) {
107 $status = Status
::newGood();
109 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
110 if ( $dstRel === null ) {
111 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
115 // (a) Check the destination container and object
117 $dContObj = $this->getContainer( $dstCont );
118 if ( empty( $params['overwrite'] ) &&
119 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
121 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
124 } catch ( NoSuchContainerException
$e ) {
125 $status->fatal( 'backend-fail-create', $params['dst'] );
127 } catch ( InvalidResponseException
$e ) {
128 $status->fatal( 'backend-fail-connect', $this->name
);
130 } catch ( Exception
$e ) { // some other exception?
131 $status->fatal( 'backend-fail-internal', $this->name
);
132 $this->logException( $e, __METHOD__
, $params );
136 // (b) Get a SHA-1 hash of the object
137 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
139 // (c) Actually create the object
141 // Create a fresh CF_Object with no fields preloaded.
142 // We don't want to preserve headers, metadata, and such.
143 $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
144 // Note: metadata keys stored as [Upper case char][[Lower case char]...]
145 $obj->metadata
= array( 'Sha1base36' => $sha1Hash );
146 // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
147 // The MD5 here will be checked within Swift against its own MD5.
148 $obj->set_etag( md5( $params['content'] ) );
149 // Use the same content type as StreamFile for security
150 $obj->content_type
= StreamFile
::contentTypeFromPath( $params['dst'] );
151 // Actually write the object in Swift
152 $obj->write( $params['content'] );
153 } catch ( BadContentTypeException
$e ) {
154 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
155 } catch ( InvalidResponseException
$e ) {
156 $status->fatal( 'backend-fail-connect', $this->name
);
157 } catch ( Exception
$e ) { // some other exception?
158 $status->fatal( 'backend-fail-internal', $this->name
);
159 $this->logException( $e, __METHOD__
, $params );
166 * @see FileBackendStore::doStoreInternal()
169 protected function doStoreInternal( array $params ) {
170 $status = Status
::newGood();
172 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
173 if ( $dstRel === null ) {
174 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
178 // (a) Check the destination container and object
180 $dContObj = $this->getContainer( $dstCont );
181 if ( empty( $params['overwrite'] ) &&
182 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
184 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
187 } catch ( NoSuchContainerException
$e ) {
188 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
190 } catch ( InvalidResponseException
$e ) {
191 $status->fatal( 'backend-fail-connect', $this->name
);
193 } catch ( Exception
$e ) { // some other exception?
194 $status->fatal( 'backend-fail-internal', $this->name
);
195 $this->logException( $e, __METHOD__
, $params );
199 // (b) Get a SHA-1 hash of the object
200 $sha1Hash = sha1_file( $params['src'] );
201 if ( $sha1Hash === false ) { // source doesn't exist?
202 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
205 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
207 // (c) Actually store the object
209 // Create a fresh CF_Object with no fields preloaded.
210 // We don't want to preserve headers, metadata, and such.
211 $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
212 // Note: metadata keys stored as [Upper case char][[Lower case char]...]
213 $obj->metadata
= array( 'Sha1base36' => $sha1Hash );
214 // The MD5 here will be checked within Swift against its own MD5.
215 $obj->set_etag( md5_file( $params['src'] ) );
216 // Use the same content type as StreamFile for security
217 $obj->content_type
= StreamFile
::contentTypeFromPath( $params['dst'] );
218 // Actually write the object in Swift
219 $obj->load_from_filename( $params['src'], True ); // calls $obj->write()
220 } catch ( BadContentTypeException
$e ) {
221 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
222 } catch ( IOException
$e ) {
223 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
224 } catch ( InvalidResponseException
$e ) {
225 $status->fatal( 'backend-fail-connect', $this->name
);
226 } catch ( Exception
$e ) { // some other exception?
227 $status->fatal( 'backend-fail-internal', $this->name
);
228 $this->logException( $e, __METHOD__
, $params );
235 * @see FileBackendStore::doCopyInternal()
238 protected function doCopyInternal( array $params ) {
239 $status = Status
::newGood();
241 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
242 if ( $srcRel === null ) {
243 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
247 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
248 if ( $dstRel === null ) {
249 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
253 // (a) Check the source/destination containers and destination object
255 $sContObj = $this->getContainer( $srcCont );
256 $dContObj = $this->getContainer( $dstCont );
257 if ( empty( $params['overwrite'] ) &&
258 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
260 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
263 } catch ( NoSuchContainerException
$e ) {
264 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
266 } catch ( InvalidResponseException
$e ) {
267 $status->fatal( 'backend-fail-connect', $this->name
);
269 } catch ( Exception
$e ) { // some other exception?
270 $status->fatal( 'backend-fail-internal', $this->name
);
271 $this->logException( $e, __METHOD__
, $params );
275 // (b) Actually copy the file to the destination
277 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
278 } catch ( NoSuchObjectException
$e ) { // source object does not exist
279 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
280 } catch ( InvalidResponseException
$e ) {
281 $status->fatal( 'backend-fail-connect', $this->name
);
282 } catch ( Exception
$e ) { // some other exception?
283 $status->fatal( 'backend-fail-internal', $this->name
);
284 $this->logException( $e, __METHOD__
, $params );
291 * @see FileBackendStore::doDeleteInternal()
294 protected function doDeleteInternal( array $params ) {
295 $status = Status
::newGood();
297 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
298 if ( $srcRel === null ) {
299 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
304 $sContObj = $this->getContainer( $srcCont );
305 $sContObj->delete_object( $srcRel );
306 } catch ( NoSuchContainerException
$e ) {
307 $status->fatal( 'backend-fail-delete', $params['src'] );
308 } catch ( NoSuchObjectException
$e ) {
309 if ( empty( $params['ignoreMissingSource'] ) ) {
310 $status->fatal( 'backend-fail-delete', $params['src'] );
312 } catch ( InvalidResponseException
$e ) {
313 $status->fatal( 'backend-fail-connect', $this->name
);
314 } catch ( Exception
$e ) { // some other exception?
315 $status->fatal( 'backend-fail-internal', $this->name
);
316 $this->logException( $e, __METHOD__
, $params );
323 * @see FileBackendStore::doPrepareInternal()
326 protected function doPrepareInternal( $fullCont, $dir, array $params ) {
327 $status = Status
::newGood();
329 // (a) Check if container already exists
331 $contObj = $this->getContainer( $fullCont );
332 // NoSuchContainerException not thrown: container must exist
333 return $status; // already exists
334 } catch ( NoSuchContainerException
$e ) {
335 // NoSuchContainerException thrown: container does not exist
336 } catch ( InvalidResponseException
$e ) {
337 $status->fatal( 'backend-fail-connect', $this->name
);
339 } catch ( Exception
$e ) { // some other exception?
340 $status->fatal( 'backend-fail-internal', $this->name
);
341 $this->logException( $e, __METHOD__
, $params );
345 // (b) Create container as needed
347 $contObj = $this->createContainer( $fullCont );
348 if ( $this->swiftAnonUser
!= '' ) {
349 // Make container public to end-users...
350 $status->merge( $this->setContainerAccess(
352 array( $this->auth
->username
, $this->swiftAnonUser
), // read
353 array( $this->auth
->username
) // write
356 } catch ( InvalidResponseException
$e ) {
357 $status->fatal( 'backend-fail-connect', $this->name
);
359 } catch ( Exception
$e ) { // some other exception?
360 $status->fatal( 'backend-fail-internal', $this->name
);
361 $this->logException( $e, __METHOD__
, $params );
369 * @see FileBackendStore::doSecureInternal()
372 protected function doSecureInternal( $fullCont, $dir, array $params ) {
373 $status = Status
::newGood();
375 if ( $this->swiftAnonUser
!= '' ) {
376 // Restrict container from end-users...
378 // doPrepareInternal() should have been called,
379 // so the Swift container should already exist...
380 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
381 // NoSuchContainerException not thrown: container must exist
382 if ( !isset( $contObj->mw_wasSecured
) ) {
383 $status->merge( $this->setContainerAccess(
385 array( $this->auth
->username
), // read
386 array( $this->auth
->username
) // write
388 // @TODO: when php-cloudfiles supports container
389 // metadata, we can make use of that to avoid RTTs
390 $contObj->mw_wasSecured
= true; // avoid useless RTTs
392 } catch ( InvalidResponseException
$e ) {
393 $status->fatal( 'backend-fail-connect', $this->name
);
394 } catch ( Exception
$e ) { // some other exception?
395 $status->fatal( 'backend-fail-internal', $this->name
);
396 $this->logException( $e, __METHOD__
, $params );
404 * @see FileBackendStore::doCleanInternal()
407 protected function doCleanInternal( $fullCont, $dir, array $params ) {
408 $status = Status
::newGood();
410 // Only containers themselves can be removed, all else is virtual
412 return $status; // nothing to do
415 // (a) Check the container
417 $contObj = $this->getContainer( $fullCont, true );
418 } catch ( NoSuchContainerException
$e ) {
419 return $status; // ok, nothing to do
420 } catch ( InvalidResponseException
$e ) {
421 $status->fatal( 'backend-fail-connect', $this->name
);
423 } catch ( Exception
$e ) { // some other exception?
424 $status->fatal( 'backend-fail-internal', $this->name
);
425 $this->logException( $e, __METHOD__
, $params );
429 // (b) Delete the container if empty
430 if ( $contObj->object_count
== 0 ) {
432 $this->deleteContainer( $fullCont );
433 } catch ( NoSuchContainerException
$e ) {
434 return $status; // race?
435 } catch ( InvalidResponseException
$e ) {
436 $status->fatal( 'backend-fail-connect', $this->name
);
438 } catch ( Exception
$e ) { // some other exception?
439 $status->fatal( 'backend-fail-internal', $this->name
);
440 $this->logException( $e, __METHOD__
, $params );
449 * @see FileBackendStore::doFileExists()
450 * @return array|bool|null
452 protected function doGetFileStat( array $params ) {
453 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
454 if ( $srcRel === null ) {
455 return false; // invalid storage path
460 $contObj = $this->getContainer( $srcCont );
461 $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
462 $this->addMissingMetadata( $srcObj, $params['src'] );
464 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
465 'mtime' => wfTimestamp( TS_MW
, $srcObj->last_modified
),
466 'size' => $srcObj->content_length
,
467 'sha1' => $srcObj->metadata
['Sha1base36']
469 } catch ( NoSuchContainerException
$e ) {
470 } catch ( NoSuchObjectException
$e ) {
471 } catch ( InvalidResponseException
$e ) {
473 } catch ( Exception
$e ) { // some other exception?
475 $this->logException( $e, __METHOD__
, $params );
482 * Fill in any missing object metadata and save it to Swift
484 * @param $obj CF_Object
485 * @param $path string Storage path to object
486 * @return bool Success
487 * @throws Exception cloudfiles exceptions
489 protected function addMissingMetadata( CF_Object
$obj, $path ) {
490 if ( isset( $obj->metadata
['Sha1base36'] ) ) {
491 return true; // nothing to do
493 $status = Status
::newGood();
494 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager
::LOCK_UW
, $status );
495 if ( $status->isOK() ) {
496 $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
498 $hash = $tmpFile->getSha1Base36();
499 if ( $hash !== false ) {
500 $obj->metadata
['Sha1base36'] = $hash;
501 $obj->sync_metadata(); // save to Swift
502 return true; // success
506 $obj->metadata
['Sha1base36'] = false;
507 return false; // failed
511 * @see FileBackend::getFileContents()
512 * @return bool|null|string
514 public function getFileContents( array $params ) {
515 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
516 if ( $srcRel === null ) {
517 return false; // invalid storage path
520 if ( !$this->fileExists( $params ) ) {
526 $sContObj = $this->getContainer( $srcCont );
527 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request
528 $data = $obj->read( $this->headersFromParams( $params ) );
529 } catch ( NoSuchContainerException
$e ) {
530 } catch ( InvalidResponseException
$e ) {
531 } catch ( Exception
$e ) { // some other exception?
532 $this->logException( $e, __METHOD__
, $params );
539 * @see FileBackendStore::doDirectoryExists()
542 protected function doDirectoryExists( $fullCont, $dir, array $params ) {
544 $container = $this->getContainer( $fullCont );
545 $prefix = ( $dir == '' ) ?
null : "{$dir}/";
546 return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
547 } catch ( NoSuchContainerException
$e ) {
549 } catch ( InvalidResponseException
$e ) {
550 } catch ( Exception
$e ) { // some other exception?
551 $this->logException( $e, __METHOD__
, array( 'cont' => $fullCont, 'dir' => $dir ) );
554 return null; // error
558 * @see FileBackendStore::getDirectoryListInternal()
559 * @return SwiftFileBackendDirList
561 public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
562 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
566 * @see FileBackendStore::getFileListInternal()
567 * @return SwiftFileBackendFileList
569 public function getFileListInternal( $fullCont, $dir, array $params ) {
570 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
574 * Do not call this function outside of SwiftFileBackendFileList
576 * @param $fullCont string Resolved container name
577 * @param $dir string Resolved storage directory with no trailing slash
578 * @param $after string|null Storage path of file to list items after
579 * @param $limit integer Max number of items to list
580 * @param $params Array Includes flag for 'topOnly'
581 * @return Array List of relative paths of dirs directly under $dir
583 public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
587 $container = $this->getContainer( $fullCont );
588 $prefix = ( $dir == '' ) ?
null : "{$dir}/";
589 // Non-recursive: only list dirs right under $dir
590 if ( !empty( $params['topOnly'] ) ) {
591 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
592 foreach ( $objects as $object ) { // files and dirs
593 if ( substr( $object, -1 ) === '/' ) {
594 $dirs[] = $object; // directories end in '/'
596 $after = $object; // update last item
598 // Recursive: list all dirs under $dir and its subdirs
600 // Get directory from last item of prior page
601 $lastDir = $this->getParentDir( $after ); // must be first page
602 $objects = $container->list_objects( $limit, $after, $prefix );
603 foreach ( $objects as $object ) { // files
604 $objectDir = $this->getParentDir( $object ); // directory of object
605 if ( $objectDir !== false ) { // file has a parent dir
606 // Swift stores paths in UTF-8, using binary sorting.
607 // See function "create_container_table" in common/db.py.
608 // If a directory is not "greater" than the last one,
609 // then it was already listed by the calling iterator.
610 if ( $objectDir > $lastDir ) {
612 do { // add dir and all its parent dirs
613 $dirs[] = "{$pDir}/";
614 $pDir = $this->getParentDir( $pDir );
615 } while ( $pDir !== false // sanity
616 && $pDir > $lastDir // not done already
617 && strlen( $pDir ) > strlen( $dir ) // within $dir
620 $lastDir = $objectDir;
622 $after = $object; // update last item
625 } catch ( NoSuchContainerException
$e ) {
626 } catch ( InvalidResponseException
$e ) {
627 } catch ( Exception
$e ) { // some other exception?
628 $this->logException( $e, __METHOD__
, array( 'cont' => $fullCont, 'dir' => $dir ) );
634 protected function getParentDir( $path ) {
635 return ( strpos( $path, '/' ) !== false ) ?
dirname( $path ) : false;
639 * Do not call this function outside of SwiftFileBackendFileList
641 * @param $fullCont string Resolved container name
642 * @param $dir string Resolved storage directory with no trailing slash
643 * @param $after string|null Storage path of file to list items after
644 * @param $limit integer Max number of items to list
645 * @param $params Array Includes flag for 'topOnly'
646 * @return Array List of relative paths of files under $dir
648 public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
652 $container = $this->getContainer( $fullCont );
653 $prefix = ( $dir == '' ) ?
null : "{$dir}/";
654 // Non-recursive: only list files right under $dir
655 if ( !empty( $params['topOnly'] ) ) { // files and dirs
656 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
657 foreach ( $objects as $object ) {
658 if ( substr( $object, -1 ) !== '/' ) {
659 $files[] = $object; // directories end in '/'
662 // Recursive: list all files under $dir and its subdirs
664 $files = $container->list_objects( $limit, $after, $prefix );
666 $after = end( $files ); // update last item
667 reset( $files ); // reset pointer
668 } catch ( NoSuchContainerException
$e ) {
669 } catch ( InvalidResponseException
$e ) {
670 } catch ( Exception
$e ) { // some other exception?
671 $this->logException( $e, __METHOD__
, array( 'cont' => $fullCont, 'dir' => $dir ) );
678 * @see FileBackendStore::doGetFileSha1base36()
681 protected function doGetFileSha1base36( array $params ) {
682 $stat = $this->getFileStat( $params );
684 return $stat['sha1'];
691 * @see FileBackendStore::doStreamFile()
694 protected function doStreamFile( array $params ) {
695 $status = Status
::newGood();
697 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
698 if ( $srcRel === null ) {
699 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
703 $cont = $this->getContainer( $srcCont );
704 } catch ( NoSuchContainerException
$e ) {
705 $status->fatal( 'backend-fail-stream', $params['src'] );
707 } catch ( InvalidResponseException
$e ) {
708 $status->fatal( 'backend-fail-connect', $this->name
);
710 } catch ( Exception
$e ) { // some other exception?
711 $status->fatal( 'backend-fail-stream', $params['src'] );
712 $this->logException( $e, __METHOD__
, $params );
717 $output = fopen( 'php://output', 'wb' );
718 $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
719 $obj->stream( $output, $this->headersFromParams( $params ) );
720 } catch ( InvalidResponseException
$e ) { // 404? connection problem?
721 $status->fatal( 'backend-fail-stream', $params['src'] );
722 } catch ( Exception
$e ) { // some other exception?
723 $status->fatal( 'backend-fail-stream', $params['src'] );
724 $this->logException( $e, __METHOD__
, $params );
731 * @see FileBackendStore::getLocalCopy()
732 * @return null|TempFSFile
734 public function getLocalCopy( array $params ) {
735 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
736 if ( $srcRel === null ) {
740 if ( !$this->fileExists( $params ) ) {
746 $sContObj = $this->getContainer( $srcCont );
747 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
748 // Get source file extension
749 $ext = FileBackend
::extensionFromPath( $srcRel );
750 // Create a new temporary file...
751 $tmpFile = TempFSFile
::factory( wfBaseName( $srcRel ) . '_', $ext );
753 $handle = fopen( $tmpFile->getPath(), 'wb' );
755 $obj->stream( $handle, $this->headersFromParams( $params ) );
758 $tmpFile = null; // couldn't open temp file
761 } catch ( NoSuchContainerException
$e ) {
763 } catch ( InvalidResponseException
$e ) {
765 } catch ( Exception
$e ) { // some other exception?
767 $this->logException( $e, __METHOD__
, $params );
774 * Get headers to send to Swift when reading a file based
775 * on a FileBackend params array, e.g. that of getLocalCopy().
776 * $params is currently only checked for a 'latest' flag.
778 * @param $params Array
781 protected function headersFromParams( array $params ) {
783 if ( !empty( $params['latest'] ) ) {
784 $hdrs[] = 'X-Newest: true';
790 * Set read/write permissions for a Swift container
792 * @param $contObj CF_Container Swift container
793 * @param $readGrps Array Swift users who can read (account:user)
794 * @param $writeGrps Array Swift users who can write (account:user)
797 protected function setContainerAccess(
798 CF_Container
$contObj, array $readGrps, array $writeGrps
800 $creds = $contObj->cfs_auth
->export_credentials();
802 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name
);
804 // Note: 10 second timeout consistent with php-cloudfiles
805 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
806 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
807 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
808 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
810 return $req->execute(); // should return 204
814 * Get a connection to the Swift proxy
816 * @return CF_Connection|bool False on failure
817 * @throws InvalidResponseException
819 protected function getConnection() {
820 if ( $this->conn
=== false ) {
821 throw new InvalidResponseException
; // failed last attempt
823 // Session keys expire after a while, so we renew them periodically
824 if ( $this->conn
&& ( time() - $this->connStarted
) > $this->authTTL
) {
825 $this->conn
->close(); // close active cURL connections
828 // Authenticate with proxy and get a session key...
829 if ( $this->conn
=== null ) {
830 $this->connContainers
= array();
832 $this->auth
->authenticate();
833 $this->conn
= new CF_Connection( $this->auth
);
834 $this->connStarted
= time();
835 } catch ( AuthenticationException
$e ) {
836 $this->conn
= false; // don't keep re-trying
837 } catch ( InvalidResponseException
$e ) {
838 $this->conn
= false; // don't keep re-trying
841 if ( !$this->conn
) {
842 throw new InvalidResponseException
; // auth/connection problem
848 * @see FileBackendStore::doClearCache()
850 protected function doClearCache( array $paths = null ) {
851 $this->connContainers
= array(); // clear container object cache
855 * Get a Swift container object, possibly from process cache.
856 * Use $reCache if the file count or byte count is needed.
858 * @param $container string Container name
859 * @param $reCache bool Refresh the process cache
860 * @return CF_Container
862 protected function getContainer( $container, $reCache = false ) {
863 $conn = $this->getConnection(); // Swift proxy connection
865 unset( $this->connContainers
[$container] ); // purge cache
867 if ( !isset( $this->connContainers
[$container] ) ) {
868 $contObj = $conn->get_container( $container );
869 // NoSuchContainerException not thrown: container must exist
870 if ( count( $this->connContainers
) >= $this->maxContCacheSize
) { // trim cache?
871 reset( $this->connContainers
);
872 $key = key( $this->connContainers
);
873 unset( $this->connContainers
[$key] );
875 $this->connContainers
[$container] = $contObj; // cache it
877 return $this->connContainers
[$container];
881 * Create a Swift container
883 * @param $container string Container name
884 * @return CF_Container
886 protected function createContainer( $container ) {
887 $conn = $this->getConnection(); // Swift proxy connection
888 $contObj = $conn->create_container( $container );
889 $this->connContainers
[$container] = $contObj; // cache it
894 * Delete a Swift container
896 * @param $container string Container name
899 protected function deleteContainer( $container ) {
900 $conn = $this->getConnection(); // Swift proxy connection
901 $conn->delete_container( $container );
902 unset( $this->connContainers
[$container] ); // purge cache
906 * Log an unexpected exception for this backend
908 * @param $e Exception
909 * @param $func string
910 * @param $params Array
913 protected function logException( Exception
$e, $func, array $params ) {
914 wfDebugLog( 'SwiftBackend',
915 get_class( $e ) . " in '{$func}' (given '" . FormatJson
::encode( $params ) . "')" .
916 ( $e instanceof InvalidResponseException
917 ?
": {$e->getMessage()}"
925 * SwiftFileBackend helper class to page through listings.
926 * Swift also has a listing limit of 10,000 objects for sanity.
927 * Do not use this class from places outside SwiftFileBackend.
929 * @ingroup FileBackend
931 abstract class SwiftFileBackendList
implements Iterator
{
933 protected $bufferIter = array();
934 protected $bufferAfter = null; // string; list items *after* this path
935 protected $pos = 0; // integer
937 protected $params = array();
939 /** @var SwiftFileBackend */
941 protected $container; // string; container name
942 protected $dir; // string; storage directory
943 protected $suffixStart; // integer
945 const PAGE_SIZE
= 5000; // file listing buffer size
948 * @param $backend SwiftFileBackend
949 * @param $fullCont string Resolved container name
950 * @param $dir string Resolved directory relative to container
951 * @param $params Array
953 public function __construct( SwiftFileBackend
$backend, $fullCont, $dir, array $params ) {
954 $this->backend
= $backend;
955 $this->container
= $fullCont;
957 if ( substr( $this->dir
, -1 ) === '/' ) {
958 $this->dir
= substr( $this->dir
, 0, -1 ); // remove trailing slash
960 if ( $this->dir
== '' ) { // whole container
961 $this->suffixStart
= 0;
962 } else { // dir within container
963 $this->suffixStart
= strlen( $this->dir
) +
1; // size of "path/to/dir/"
965 $this->params
= $params;
969 * @see Iterator::key()
972 public function key() {
977 * @see Iterator::next()
980 public function next() {
981 // Advance to the next file in the page
982 next( $this->bufferIter
);
984 // Check if there are no files left in this page and
985 // advance to the next page if this page was not empty.
986 if ( !$this->valid() && count( $this->bufferIter
) ) {
987 $this->bufferIter
= $this->pageFromList(
988 $this->container
, $this->dir
, $this->bufferAfter
, self
::PAGE_SIZE
, $this->params
989 ); // updates $this->bufferAfter
994 * @see Iterator::rewind()
997 public function rewind() {
999 $this->bufferAfter
= null;
1000 $this->bufferIter
= $this->pageFromList(
1001 $this->container
, $this->dir
, $this->bufferAfter
, self
::PAGE_SIZE
, $this->params
1002 ); // updates $this->bufferAfter
1006 * @see Iterator::valid()
1009 public function valid() {
1010 return ( current( $this->bufferIter
) !== false ); // no paths can have this value
1014 * Get the given list portion (page)
1016 * @param $container string Resolved container name
1017 * @param $dir string Resolved path relative to container
1018 * @param $after string|null
1019 * @param $limit integer
1020 * @param $params Array
1021 * @return Traversable|Array|null
1023 abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1027 * Iterator for listing directories
1029 class SwiftFileBackendDirList
extends SwiftFileBackendList
{
1031 * @see Iterator::current()
1032 * @return string|bool String (relative path) or false
1034 public function current() {
1035 return substr( current( $this->bufferIter
), $this->suffixStart
, -1 );
1039 * @see SwiftFileBackendList::pageFromList()
1042 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1043 return $this->backend
->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1048 * Iterator for listing regular files
1050 class SwiftFileBackendFileList
extends SwiftFileBackendList
{
1052 * @see Iterator::current()
1053 * @return string|bool String (relative path) or false
1055 public function current() {
1056 return substr( current( $this->bufferIter
), $this->suffixStart
);
1060 * @see SwiftFileBackendList::pageFromList()
1063 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1064 return $this->backend
->getFileListPageInternal( $container, $dir, $after, $limit, $params );