10 * @brief Class for an OpenStack Swift based file backend.
12 * This requires the SwiftCloudFiles MediaWiki extension, which includes
13 * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
14 * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
16 * Status messages should avoid mentioning the Swift account name.
17 * Likewise, error suppression should be used to avoid path disclosure.
19 * @ingroup FileBackend
22 class SwiftFileBackend
extends FileBackendStore
{
23 /** @var CF_Authentication */
24 protected $auth; // Swift authentication handler
25 protected $authTTL; // integer seconds
26 protected $swiftAnonUser; // string; username to handle unauthenticated requests
27 protected $maxContCacheSize = 100; // integer; max containers with entries
29 /** @var CF_Connection */
30 protected $conn; // Swift connection handle
31 protected $connStarted = 0; // integer UNIX timestamp
32 protected $connContainers = array(); // container object cache
35 * @see FileBackendStore::__construct()
36 * Additional $config params include:
37 * swiftAuthUrl : Swift authentication server URL
38 * swiftUser : Swift user used by MediaWiki (account:username)
39 * swiftKey : Swift authentication key for the above user
40 * swiftAuthTTL : Swift authentication TTL (seconds)
41 * swiftAnonUser : Swift user used for end-user requests (account:username)
42 * shardViaHashLevels : Map of container names to sharding config with:
43 * 'base' : base of hash characters, 16 or 36
44 * 'levels' : the number of hash levels (and digits)
45 * 'repeat' : hash subdirectories are prefixed with all the
46 * parent hash directory names (e.g. "a/ab/abc")
48 public function __construct( array $config ) {
49 parent
::__construct( $config );
51 $this->auth
= new CF_Authentication(
54 null, // account; unused
55 $config['swiftAuthUrl']
58 $this->authTTL
= isset( $config['swiftAuthTTL'] )
59 ?
$config['swiftAuthTTL']
60 : 120; // some sane number
61 $this->swiftAnonUser
= isset( $config['swiftAnonUser'] )
62 ?
$config['swiftAnonUser']
64 $this->shardViaHashLevels
= isset( $config['shardViaHashLevels'] )
65 ?
$config['shardViaHashLevels']
67 // Cache container info to mask latency
68 $this->memCache
= wfGetMainCache();
72 * @see FileBackendStore::resolveContainerPath()
75 protected function resolveContainerPath( $container, $relStoragePath ) {
76 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
77 return null; // too long for Swift
79 return $relStoragePath;
83 * @see FileBackendStore::isPathUsableInternal()
86 public function isPathUsableInternal( $storagePath ) {
87 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
88 if ( $rel === null ) {
89 return false; // invalid
93 $this->getContainer( $container );
94 return true; // container exists
95 } catch ( NoSuchContainerException
$e ) {
96 } catch ( InvalidResponseException
$e ) {
97 } catch ( Exception
$e ) { // some other exception?
98 $this->logException( $e, __METHOD__
, array( 'path' => $storagePath ) );
105 * @see FileBackendStore::doCreateInternal()
108 protected function doCreateInternal( array $params ) {
109 $status = Status
::newGood();
111 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
112 if ( $dstRel === null ) {
113 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
117 // (a) Check the destination container and object
119 $dContObj = $this->getContainer( $dstCont );
120 if ( empty( $params['overwrite'] ) &&
121 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
123 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
126 } catch ( NoSuchContainerException
$e ) {
127 $status->fatal( 'backend-fail-create', $params['dst'] );
129 } catch ( InvalidResponseException
$e ) {
130 $status->fatal( 'backend-fail-connect', $this->name
);
132 } catch ( Exception
$e ) { // some other exception?
133 $status->fatal( 'backend-fail-internal', $this->name
);
134 $this->logException( $e, __METHOD__
, $params );
138 // (b) Get a SHA-1 hash of the object
139 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
141 // (c) Actually create the object
143 // Create a fresh CF_Object with no fields preloaded.
144 // We don't want to preserve headers, metadata, and such.
145 $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
146 // Note: metadata keys stored as [Upper case char][[Lower case char]...]
147 $obj->metadata
= array( 'Sha1base36' => $sha1Hash );
148 // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
149 // The MD5 here will be checked within Swift against its own MD5.
150 $obj->set_etag( md5( $params['content'] ) );
151 // Use the same content type as StreamFile for security
152 $obj->content_type
= StreamFile
::contentTypeFromPath( $params['dst'] );
153 // Actually write the object in Swift
154 $obj->write( $params['content'] );
155 } catch ( BadContentTypeException
$e ) {
156 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
157 } catch ( InvalidResponseException
$e ) {
158 $status->fatal( 'backend-fail-connect', $this->name
);
159 } catch ( Exception
$e ) { // some other exception?
160 $status->fatal( 'backend-fail-internal', $this->name
);
161 $this->logException( $e, __METHOD__
, $params );
168 * @see FileBackendStore::doStoreInternal()
171 protected function doStoreInternal( array $params ) {
172 $status = Status
::newGood();
174 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
175 if ( $dstRel === null ) {
176 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
180 // (a) Check the destination container and object
182 $dContObj = $this->getContainer( $dstCont );
183 if ( empty( $params['overwrite'] ) &&
184 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
186 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
189 } catch ( NoSuchContainerException
$e ) {
190 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
192 } catch ( InvalidResponseException
$e ) {
193 $status->fatal( 'backend-fail-connect', $this->name
);
195 } catch ( Exception
$e ) { // some other exception?
196 $status->fatal( 'backend-fail-internal', $this->name
);
197 $this->logException( $e, __METHOD__
, $params );
201 // (b) Get a SHA-1 hash of the object
202 $sha1Hash = sha1_file( $params['src'] );
203 if ( $sha1Hash === false ) { // source doesn't exist?
204 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
207 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
209 // (c) Actually store the object
211 // Create a fresh CF_Object with no fields preloaded.
212 // We don't want to preserve headers, metadata, and such.
213 $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
214 // Note: metadata keys stored as [Upper case char][[Lower case char]...]
215 $obj->metadata
= array( 'Sha1base36' => $sha1Hash );
216 // The MD5 here will be checked within Swift against its own MD5.
217 $obj->set_etag( md5_file( $params['src'] ) );
218 // Use the same content type as StreamFile for security
219 $obj->content_type
= StreamFile
::contentTypeFromPath( $params['dst'] );
220 // Actually write the object in Swift
221 $obj->load_from_filename( $params['src'], True ); // calls $obj->write()
222 } catch ( BadContentTypeException
$e ) {
223 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
224 } catch ( IOException
$e ) {
225 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
226 } catch ( InvalidResponseException
$e ) {
227 $status->fatal( 'backend-fail-connect', $this->name
);
228 } catch ( Exception
$e ) { // some other exception?
229 $status->fatal( 'backend-fail-internal', $this->name
);
230 $this->logException( $e, __METHOD__
, $params );
237 * @see FileBackendStore::doCopyInternal()
240 protected function doCopyInternal( array $params ) {
241 $status = Status
::newGood();
243 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
244 if ( $srcRel === null ) {
245 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
249 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
250 if ( $dstRel === null ) {
251 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
255 // (a) Check the source/destination containers and destination object
257 $sContObj = $this->getContainer( $srcCont );
258 $dContObj = $this->getContainer( $dstCont );
259 if ( empty( $params['overwrite'] ) &&
260 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
262 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
265 } catch ( NoSuchContainerException
$e ) {
266 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
268 } catch ( InvalidResponseException
$e ) {
269 $status->fatal( 'backend-fail-connect', $this->name
);
271 } catch ( Exception
$e ) { // some other exception?
272 $status->fatal( 'backend-fail-internal', $this->name
);
273 $this->logException( $e, __METHOD__
, $params );
277 // (b) Actually copy the file to the destination
279 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
280 } catch ( NoSuchObjectException
$e ) { // source object does not exist
281 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
282 } catch ( InvalidResponseException
$e ) {
283 $status->fatal( 'backend-fail-connect', $this->name
);
284 } catch ( Exception
$e ) { // some other exception?
285 $status->fatal( 'backend-fail-internal', $this->name
);
286 $this->logException( $e, __METHOD__
, $params );
293 * @see FileBackendStore::doDeleteInternal()
296 protected function doDeleteInternal( array $params ) {
297 $status = Status
::newGood();
299 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
300 if ( $srcRel === null ) {
301 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
306 $sContObj = $this->getContainer( $srcCont );
307 $sContObj->delete_object( $srcRel );
308 } catch ( NoSuchContainerException
$e ) {
309 $status->fatal( 'backend-fail-delete', $params['src'] );
310 } catch ( NoSuchObjectException
$e ) {
311 if ( empty( $params['ignoreMissingSource'] ) ) {
312 $status->fatal( 'backend-fail-delete', $params['src'] );
314 } catch ( InvalidResponseException
$e ) {
315 $status->fatal( 'backend-fail-connect', $this->name
);
316 } catch ( Exception
$e ) { // some other exception?
317 $status->fatal( 'backend-fail-internal', $this->name
);
318 $this->logException( $e, __METHOD__
, $params );
325 * @see FileBackendStore::doPrepareInternal()
328 protected function doPrepareInternal( $fullCont, $dir, array $params ) {
329 $status = Status
::newGood();
331 // (a) Check if container already exists
333 $contObj = $this->getContainer( $fullCont );
334 // NoSuchContainerException not thrown: container must exist
335 return $status; // already exists
336 } catch ( NoSuchContainerException
$e ) {
337 // NoSuchContainerException thrown: container does not exist
338 } catch ( InvalidResponseException
$e ) {
339 $status->fatal( 'backend-fail-connect', $this->name
);
341 } catch ( Exception
$e ) { // some other exception?
342 $status->fatal( 'backend-fail-internal', $this->name
);
343 $this->logException( $e, __METHOD__
, $params );
347 // (b) Create container as needed
349 $contObj = $this->createContainer( $fullCont );
350 if ( $this->swiftAnonUser
!= '' ) {
351 // Make container public to end-users...
352 $status->merge( $this->setContainerAccess(
354 array( $this->auth
->username
, $this->swiftAnonUser
), // read
355 array( $this->auth
->username
) // write
358 } catch ( InvalidResponseException
$e ) {
359 $status->fatal( 'backend-fail-connect', $this->name
);
361 } catch ( Exception
$e ) { // some other exception?
362 $status->fatal( 'backend-fail-internal', $this->name
);
363 $this->logException( $e, __METHOD__
, $params );
371 * @see FileBackendStore::doSecureInternal()
374 protected function doSecureInternal( $fullCont, $dir, array $params ) {
375 $status = Status
::newGood();
377 if ( $this->swiftAnonUser
!= '' ) {
378 // Restrict container from end-users...
380 // doPrepareInternal() should have been called,
381 // so the Swift container should already exist...
382 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
383 // NoSuchContainerException not thrown: container must exist
384 if ( !isset( $contObj->mw_wasSecured
) ) {
385 $status->merge( $this->setContainerAccess(
387 array( $this->auth
->username
), // read
388 array( $this->auth
->username
) // write
390 // @TODO: when php-cloudfiles supports container
391 // metadata, we can make use of that to avoid RTTs
392 $contObj->mw_wasSecured
= true; // avoid useless RTTs
394 } catch ( InvalidResponseException
$e ) {
395 $status->fatal( 'backend-fail-connect', $this->name
);
396 } catch ( Exception
$e ) { // some other exception?
397 $status->fatal( 'backend-fail-internal', $this->name
);
398 $this->logException( $e, __METHOD__
, $params );
406 * @see FileBackendStore::doCleanInternal()
409 protected function doCleanInternal( $fullCont, $dir, array $params ) {
410 $status = Status
::newGood();
412 // Only containers themselves can be removed, all else is virtual
414 return $status; // nothing to do
417 // (a) Check the container
419 $contObj = $this->getContainer( $fullCont, true );
420 } catch ( NoSuchContainerException
$e ) {
421 return $status; // ok, nothing to do
422 } catch ( InvalidResponseException
$e ) {
423 $status->fatal( 'backend-fail-connect', $this->name
);
425 } catch ( Exception
$e ) { // some other exception?
426 $status->fatal( 'backend-fail-internal', $this->name
);
427 $this->logException( $e, __METHOD__
, $params );
431 // (b) Delete the container if empty
432 if ( $contObj->object_count
== 0 ) {
434 $this->deleteContainer( $fullCont );
435 } catch ( NoSuchContainerException
$e ) {
436 return $status; // race?
437 } catch ( InvalidResponseException
$e ) {
438 $status->fatal( 'backend-fail-connect', $this->name
);
440 } catch ( Exception
$e ) { // some other exception?
441 $status->fatal( 'backend-fail-internal', $this->name
);
442 $this->logException( $e, __METHOD__
, $params );
451 * @see FileBackendStore::doFileExists()
452 * @return array|bool|null
454 protected function doGetFileStat( array $params ) {
455 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
456 if ( $srcRel === null ) {
457 return false; // invalid storage path
462 $contObj = $this->getContainer( $srcCont );
463 $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
464 $this->addMissingMetadata( $srcObj, $params['src'] );
466 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
467 'mtime' => wfTimestamp( TS_MW
, $srcObj->last_modified
),
468 'size' => $srcObj->content_length
,
469 'sha1' => $srcObj->metadata
['Sha1base36']
471 } catch ( NoSuchContainerException
$e ) {
472 } catch ( NoSuchObjectException
$e ) {
473 } catch ( InvalidResponseException
$e ) {
475 } catch ( Exception
$e ) { // some other exception?
477 $this->logException( $e, __METHOD__
, $params );
484 * Fill in any missing object metadata and save it to Swift
486 * @param $obj CF_Object
487 * @param $path string Storage path to object
488 * @return bool Success
489 * @throws Exception cloudfiles exceptions
491 protected function addMissingMetadata( CF_Object
$obj, $path ) {
492 if ( isset( $obj->metadata
['Sha1base36'] ) ) {
493 return true; // nothing to do
495 $status = Status
::newGood();
496 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager
::LOCK_UW
, $status );
497 if ( $status->isOK() ) {
498 $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
500 $hash = $tmpFile->getSha1Base36();
501 if ( $hash !== false ) {
502 $obj->metadata
['Sha1base36'] = $hash;
503 $obj->sync_metadata(); // save to Swift
504 return true; // success
508 $obj->metadata
['Sha1base36'] = false;
509 return false; // failed
513 * @see FileBackend::getFileContents()
514 * @return bool|null|string
516 public function getFileContents( array $params ) {
517 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
518 if ( $srcRel === null ) {
519 return false; // invalid storage path
522 if ( !$this->fileExists( $params ) ) {
528 $sContObj = $this->getContainer( $srcCont );
529 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request
530 $data = $obj->read( $this->headersFromParams( $params ) );
531 } catch ( NoSuchContainerException
$e ) {
532 } catch ( InvalidResponseException
$e ) {
533 } catch ( Exception
$e ) { // some other exception?
534 $this->logException( $e, __METHOD__
, $params );
541 * @see FileBackendStore::doDirectoryExists()
544 protected function doDirectoryExists( $fullCont, $dir, array $params ) {
546 $container = $this->getContainer( $fullCont );
547 $prefix = ( $dir == '' ) ?
null : "{$dir}/";
548 return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
549 } catch ( NoSuchContainerException
$e ) {
551 } catch ( InvalidResponseException
$e ) {
552 } catch ( Exception
$e ) { // some other exception?
553 $this->logException( $e, __METHOD__
, array( 'cont' => $fullCont, 'dir' => $dir ) );
556 return null; // error
560 * @see FileBackendStore::getDirectoryListInternal()
561 * @return SwiftFileBackendDirList
563 public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
564 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
568 * @see FileBackendStore::getFileListInternal()
569 * @return SwiftFileBackendFileList
571 public function getFileListInternal( $fullCont, $dir, array $params ) {
572 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
576 * Do not call this function outside of SwiftFileBackendFileList
578 * @param $fullCont string Resolved container name
579 * @param $dir string Resolved storage directory with no trailing slash
580 * @param $after string|null Storage path of file to list items after
581 * @param $limit integer Max number of items to list
582 * @param $params Array Includes flag for 'topOnly'
583 * @return Array List of relative paths of dirs directly under $dir
585 public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
589 $container = $this->getContainer( $fullCont );
590 $prefix = ( $dir == '' ) ?
null : "{$dir}/";
591 // Non-recursive: only list dirs right under $dir
592 if ( !empty( $params['topOnly'] ) ) {
593 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
594 foreach ( $objects as $object ) { // files and dirs
595 if ( substr( $object, -1 ) === '/' ) {
596 $dirs[] = $object; // directories end in '/'
598 $after = $object; // update last item
600 // Recursive: list all dirs under $dir and its subdirs
602 // Get directory from last item of prior page
603 $lastDir = $this->getParentDir( $after ); // must be first page
604 $objects = $container->list_objects( $limit, $after, $prefix );
605 foreach ( $objects as $object ) { // files
606 $objectDir = $this->getParentDir( $object ); // directory of object
607 if ( $objectDir !== false ) { // file has a parent dir
608 // Swift stores paths in UTF-8, using binary sorting.
609 // See function "create_container_table" in common/db.py.
610 // If a directory is not "greater" than the last one,
611 // then it was already listed by the calling iterator.
612 if ( $objectDir > $lastDir ) {
614 do { // add dir and all its parent dirs
615 $dirs[] = "{$pDir}/";
616 $pDir = $this->getParentDir( $pDir );
617 } while ( $pDir !== false // sanity
618 && $pDir > $lastDir // not done already
619 && strlen( $pDir ) > strlen( $dir ) // within $dir
622 $lastDir = $objectDir;
624 $after = $object; // update last item
627 } catch ( NoSuchContainerException
$e ) {
628 } catch ( InvalidResponseException
$e ) {
629 } catch ( Exception
$e ) { // some other exception?
630 $this->logException( $e, __METHOD__
, array( 'cont' => $fullCont, 'dir' => $dir ) );
636 protected function getParentDir( $path ) {
637 return ( strpos( $path, '/' ) !== false ) ?
dirname( $path ) : false;
641 * Do not call this function outside of SwiftFileBackendFileList
643 * @param $fullCont string Resolved container name
644 * @param $dir string Resolved storage directory with no trailing slash
645 * @param $after string|null Storage path of file to list items after
646 * @param $limit integer Max number of items to list
647 * @param $params Array Includes flag for 'topOnly'
648 * @return Array List of relative paths of files under $dir
650 public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
654 $container = $this->getContainer( $fullCont );
655 $prefix = ( $dir == '' ) ?
null : "{$dir}/";
656 // Non-recursive: only list files right under $dir
657 if ( !empty( $params['topOnly'] ) ) { // files and dirs
658 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
659 foreach ( $objects as $object ) {
660 if ( substr( $object, -1 ) !== '/' ) {
661 $files[] = $object; // directories end in '/'
664 // Recursive: list all files under $dir and its subdirs
666 $files = $container->list_objects( $limit, $after, $prefix );
668 $after = end( $files ); // update last item
669 reset( $files ); // reset pointer
670 } catch ( NoSuchContainerException
$e ) {
671 } catch ( InvalidResponseException
$e ) {
672 } catch ( Exception
$e ) { // some other exception?
673 $this->logException( $e, __METHOD__
, array( 'cont' => $fullCont, 'dir' => $dir ) );
680 * @see FileBackendStore::doGetFileSha1base36()
683 protected function doGetFileSha1base36( array $params ) {
684 $stat = $this->getFileStat( $params );
686 return $stat['sha1'];
693 * @see FileBackendStore::doStreamFile()
696 protected function doStreamFile( array $params ) {
697 $status = Status
::newGood();
699 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
700 if ( $srcRel === null ) {
701 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
705 $cont = $this->getContainer( $srcCont );
706 } catch ( NoSuchContainerException
$e ) {
707 $status->fatal( 'backend-fail-stream', $params['src'] );
709 } catch ( InvalidResponseException
$e ) {
710 $status->fatal( 'backend-fail-connect', $this->name
);
712 } catch ( Exception
$e ) { // some other exception?
713 $status->fatal( 'backend-fail-stream', $params['src'] );
714 $this->logException( $e, __METHOD__
, $params );
719 $output = fopen( 'php://output', 'wb' );
720 $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
721 $obj->stream( $output, $this->headersFromParams( $params ) );
722 } catch ( InvalidResponseException
$e ) { // 404? connection problem?
723 $status->fatal( 'backend-fail-stream', $params['src'] );
724 } catch ( Exception
$e ) { // some other exception?
725 $status->fatal( 'backend-fail-stream', $params['src'] );
726 $this->logException( $e, __METHOD__
, $params );
733 * @see FileBackendStore::getLocalCopy()
734 * @return null|TempFSFile
736 public function getLocalCopy( array $params ) {
737 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
738 if ( $srcRel === null ) {
742 if ( !$this->fileExists( $params ) ) {
748 $sContObj = $this->getContainer( $srcCont );
749 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
750 // Get source file extension
751 $ext = FileBackend
::extensionFromPath( $srcRel );
752 // Create a new temporary file...
753 $tmpFile = TempFSFile
::factory( wfBaseName( $srcRel ) . '_', $ext );
755 $handle = fopen( $tmpFile->getPath(), 'wb' );
757 $obj->stream( $handle, $this->headersFromParams( $params ) );
760 $tmpFile = null; // couldn't open temp file
763 } catch ( NoSuchContainerException
$e ) {
765 } catch ( InvalidResponseException
$e ) {
767 } catch ( Exception
$e ) { // some other exception?
769 $this->logException( $e, __METHOD__
, $params );
776 * @see FileBackendStore::directoriesAreVirtual()
779 protected function directoriesAreVirtual() {
784 * Get headers to send to Swift when reading a file based
785 * on a FileBackend params array, e.g. that of getLocalCopy().
786 * $params is currently only checked for a 'latest' flag.
788 * @param $params Array
791 protected function headersFromParams( array $params ) {
793 if ( !empty( $params['latest'] ) ) {
794 $hdrs[] = 'X-Newest: true';
800 * Set read/write permissions for a Swift container
802 * @param $contObj CF_Container Swift container
803 * @param $readGrps Array Swift users who can read (account:user)
804 * @param $writeGrps Array Swift users who can write (account:user)
807 protected function setContainerAccess(
808 CF_Container
$contObj, array $readGrps, array $writeGrps
810 $creds = $contObj->cfs_auth
->export_credentials();
812 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name
);
814 // Note: 10 second timeout consistent with php-cloudfiles
815 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
816 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
817 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
818 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
820 return $req->execute(); // should return 204
824 * Get a connection to the Swift proxy
826 * @return CF_Connection|bool False on failure
827 * @throws InvalidResponseException
829 protected function getConnection() {
830 if ( $this->conn
=== false ) {
831 throw new InvalidResponseException
; // failed last attempt
833 // Session keys expire after a while, so we renew them periodically
834 if ( $this->conn
&& ( time() - $this->connStarted
) > $this->authTTL
) {
835 $this->conn
->close(); // close active cURL connections
838 // Authenticate with proxy and get a session key...
839 if ( $this->conn
=== null ) {
840 $this->connContainers
= array();
842 $this->auth
->authenticate();
843 $this->conn
= new CF_Connection( $this->auth
);
844 $this->connStarted
= time();
845 } catch ( AuthenticationException
$e ) {
846 $this->conn
= false; // don't keep re-trying
847 } catch ( InvalidResponseException
$e ) {
848 $this->conn
= false; // don't keep re-trying
851 if ( !$this->conn
) {
852 throw new InvalidResponseException
; // auth/connection problem
858 * @see FileBackendStore::doClearCache()
860 protected function doClearCache( array $paths = null ) {
861 $this->connContainers
= array(); // clear container object cache
865 * Get a Swift container object, possibly from process cache.
866 * Use $reCache if the file count or byte count is needed.
868 * @param $container string Container name
869 * @param $bypassCache bool Bypass all caches and load from Swift
870 * @return CF_Container
871 * @throws InvalidResponseException
873 protected function getContainer( $container, $bypassCache = false ) {
874 $conn = $this->getConnection(); // Swift proxy connection
875 if ( $bypassCache ) { // purge cache
876 unset( $this->connContainers
[$container] );
877 } elseif ( !isset( $this->connContainers
[$container] ) ) {
878 $this->primeContainerCache( array( $container ) ); // check persistent cache
880 if ( !isset( $this->connContainers
[$container] ) ) {
881 $contObj = $conn->get_container( $container );
882 // NoSuchContainerException not thrown: container must exist
883 if ( count( $this->connContainers
) >= $this->maxContCacheSize
) { // trim cache?
884 reset( $this->connContainers
);
885 unset( $this->connContainers
[key( $this->connContainers
)] );
887 $this->connContainers
[$container] = $contObj; // cache it
888 if ( !$bypassCache ) {
889 $this->setContainerCache( $container, // update persistent cache
890 array( 'bytes' => $contObj->bytes_used
, 'count' => $contObj->object_count
)
894 return $this->connContainers
[$container];
898 * Create a Swift container
900 * @param $container string Container name
901 * @return CF_Container
902 * @throws InvalidResponseException
904 protected function createContainer( $container ) {
905 $conn = $this->getConnection(); // Swift proxy connection
906 $contObj = $conn->create_container( $container );
907 $this->connContainers
[$container] = $contObj; // cache it
912 * Delete a Swift container
914 * @param $container string Container name
916 * @throws InvalidResponseException
918 protected function deleteContainer( $container ) {
919 $conn = $this->getConnection(); // Swift proxy connection
920 $conn->delete_container( $container );
921 unset( $this->connContainers
[$container] ); // purge cache
925 * @see FileBackendStore::doPrimeContainerCache()
928 protected function doPrimeContainerCache( array $containerInfo ) {
930 $conn = $this->getConnection(); // Swift proxy connection
931 foreach ( $containerInfo as $container => $info ) {
932 $this->connContainers
[$container] = new CF_Container(
940 } catch ( InvalidResponseException
$e ) {
941 } catch ( Exception
$e ) { // some other exception?
942 $this->logException( $e, __METHOD__
, array() );
947 * Log an unexpected exception for this backend
949 * @param $e Exception
950 * @param $func string
951 * @param $params Array
954 protected function logException( Exception
$e, $func, array $params ) {
955 wfDebugLog( 'SwiftBackend',
956 get_class( $e ) . " in '{$func}' (given '" . FormatJson
::encode( $params ) . "')" .
957 ( $e instanceof InvalidResponseException
958 ?
": {$e->getMessage()}"
966 * SwiftFileBackend helper class to page through listings.
967 * Swift also has a listing limit of 10,000 objects for sanity.
968 * Do not use this class from places outside SwiftFileBackend.
970 * @ingroup FileBackend
972 abstract class SwiftFileBackendList
implements Iterator
{
974 protected $bufferIter = array();
975 protected $bufferAfter = null; // string; list items *after* this path
976 protected $pos = 0; // integer
978 protected $params = array();
980 /** @var SwiftFileBackend */
982 protected $container; // string; container name
983 protected $dir; // string; storage directory
984 protected $suffixStart; // integer
986 const PAGE_SIZE
= 5000; // file listing buffer size
989 * @param $backend SwiftFileBackend
990 * @param $fullCont string Resolved container name
991 * @param $dir string Resolved directory relative to container
992 * @param $params Array
994 public function __construct( SwiftFileBackend
$backend, $fullCont, $dir, array $params ) {
995 $this->backend
= $backend;
996 $this->container
= $fullCont;
998 if ( substr( $this->dir
, -1 ) === '/' ) {
999 $this->dir
= substr( $this->dir
, 0, -1 ); // remove trailing slash
1001 if ( $this->dir
== '' ) { // whole container
1002 $this->suffixStart
= 0;
1003 } else { // dir within container
1004 $this->suffixStart
= strlen( $this->dir
) +
1; // size of "path/to/dir/"
1006 $this->params
= $params;
1010 * @see Iterator::key()
1013 public function key() {
1018 * @see Iterator::next()
1021 public function next() {
1022 // Advance to the next file in the page
1023 next( $this->bufferIter
);
1025 // Check if there are no files left in this page and
1026 // advance to the next page if this page was not empty.
1027 if ( !$this->valid() && count( $this->bufferIter
) ) {
1028 $this->bufferIter
= $this->pageFromList(
1029 $this->container
, $this->dir
, $this->bufferAfter
, self
::PAGE_SIZE
, $this->params
1030 ); // updates $this->bufferAfter
1035 * @see Iterator::rewind()
1038 public function rewind() {
1040 $this->bufferAfter
= null;
1041 $this->bufferIter
= $this->pageFromList(
1042 $this->container
, $this->dir
, $this->bufferAfter
, self
::PAGE_SIZE
, $this->params
1043 ); // updates $this->bufferAfter
1047 * @see Iterator::valid()
1050 public function valid() {
1051 return ( current( $this->bufferIter
) !== false ); // no paths can have this value
1055 * Get the given list portion (page)
1057 * @param $container string Resolved container name
1058 * @param $dir string Resolved path relative to container
1059 * @param $after string|null
1060 * @param $limit integer
1061 * @param $params Array
1062 * @return Traversable|Array|null
1064 abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1068 * Iterator for listing directories
1070 class SwiftFileBackendDirList
extends SwiftFileBackendList
{
1072 * @see Iterator::current()
1073 * @return string|bool String (relative path) or false
1075 public function current() {
1076 return substr( current( $this->bufferIter
), $this->suffixStart
, -1 );
1080 * @see SwiftFileBackendList::pageFromList()
1083 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1084 return $this->backend
->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1089 * Iterator for listing regular files
1091 class SwiftFileBackendFileList
extends SwiftFileBackendList
{
1093 * @see Iterator::current()
1094 * @return string|bool String (relative path) or false
1096 public function current() {
1097 return substr( current( $this->bufferIter
), $this->suffixStart
);
1101 * @see SwiftFileBackendList::pageFromList()
1104 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1105 return $this->backend
->getFileListPageInternal( $container, $dir, $after, $limit, $params );