3 * OpenStack Swift based file backend.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @ingroup FileBackend
23 * @author Aaron Schulz
27 * @brief Class for an OpenStack Swift based file backend.
29 * This requires the SwiftCloudFiles MediaWiki extension, which includes
30 * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
31 * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
33 * Status messages should avoid mentioning the Swift account name.
34 * Likewise, error suppression should be used to avoid path disclosure.
36 * @ingroup FileBackend
39 class SwiftFileBackend
extends FileBackendStore
{
40 /** @var CF_Authentication */
41 protected $auth; // Swift authentication handler
42 protected $authTTL; // integer seconds
43 protected $swiftAnonUser; // string; username to handle unauthenticated requests
44 protected $maxContCacheSize = 300; // integer; max containers with entries
46 /** @var CF_Connection */
47 protected $conn; // Swift connection handle
48 protected $connStarted = 0; // integer UNIX timestamp
49 protected $connContainers = array(); // container object cache
52 * @see FileBackendStore::__construct()
53 * Additional $config params include:
54 * swiftAuthUrl : Swift authentication server URL
55 * swiftUser : Swift user used by MediaWiki (account:username)
56 * swiftKey : Swift authentication key for the above user
57 * swiftAuthTTL : Swift authentication TTL (seconds)
58 * swiftAnonUser : Swift user used for end-user requests (account:username)
59 * shardViaHashLevels : Map of container names to sharding config with:
60 * 'base' : base of hash characters, 16 or 36
61 * 'levels' : the number of hash levels (and digits)
62 * 'repeat' : hash subdirectories are prefixed with all the
63 * parent hash directory names (e.g. "a/ab/abc")
65 public function __construct( array $config ) {
66 parent
::__construct( $config );
68 $this->auth
= new CF_Authentication(
71 null, // account; unused
72 $config['swiftAuthUrl']
75 $this->authTTL
= isset( $config['swiftAuthTTL'] )
76 ?
$config['swiftAuthTTL']
77 : 5 * 60; // some sane number
78 $this->swiftAnonUser
= isset( $config['swiftAnonUser'] )
79 ?
$config['swiftAnonUser']
81 $this->shardViaHashLevels
= isset( $config['shardViaHashLevels'] )
82 ?
$config['shardViaHashLevels']
84 // Cache container info to mask latency
85 $this->memCache
= wfGetMainCache();
89 * @see FileBackendStore::resolveContainerPath()
92 protected function resolveContainerPath( $container, $relStoragePath ) {
93 if ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
94 return null; // too long for Swift
96 return $relStoragePath;
100 * @see FileBackendStore::isPathUsableInternal()
103 public function isPathUsableInternal( $storagePath ) {
104 list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath );
105 if ( $rel === null ) {
106 return false; // invalid
110 $this->getContainer( $container );
111 return true; // container exists
112 } catch ( NoSuchContainerException
$e ) {
113 } catch ( InvalidResponseException
$e ) {
114 } catch ( Exception
$e ) { // some other exception?
115 $this->logException( $e, __METHOD__
, array( 'path' => $storagePath ) );
122 * @see FileBackendStore::doCreateInternal()
125 protected function doCreateInternal( array $params ) {
126 $status = Status
::newGood();
128 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
129 if ( $dstRel === null ) {
130 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
134 // (a) Check the destination container and object
136 $dContObj = $this->getContainer( $dstCont );
137 if ( empty( $params['overwrite'] ) &&
138 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
140 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
143 } catch ( NoSuchContainerException
$e ) {
144 $status->fatal( 'backend-fail-create', $params['dst'] );
146 } catch ( InvalidResponseException
$e ) {
147 $status->fatal( 'backend-fail-connect', $this->name
);
149 } catch ( Exception
$e ) { // some other exception?
150 $status->fatal( 'backend-fail-internal', $this->name
);
151 $this->logException( $e, __METHOD__
, $params );
155 // (b) Get a SHA-1 hash of the object
156 $sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
158 // (c) Actually create the object
160 // Create a fresh CF_Object with no fields preloaded.
161 // We don't want to preserve headers, metadata, and such.
162 $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
163 // Note: metadata keys stored as [Upper case char][[Lower case char]...]
164 $obj->metadata
= array( 'Sha1base36' => $sha1Hash );
165 // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
166 // The MD5 here will be checked within Swift against its own MD5.
167 $obj->set_etag( md5( $params['content'] ) );
168 // Use the same content type as StreamFile for security
169 $obj->content_type
= StreamFile
::contentTypeFromPath( $params['dst'] );
170 // Actually write the object in Swift
171 $obj->write( $params['content'] );
172 } catch ( BadContentTypeException
$e ) {
173 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
174 } catch ( InvalidResponseException
$e ) {
175 $status->fatal( 'backend-fail-connect', $this->name
);
176 } catch ( Exception
$e ) { // some other exception?
177 $status->fatal( 'backend-fail-internal', $this->name
);
178 $this->logException( $e, __METHOD__
, $params );
185 * @see FileBackendStore::doStoreInternal()
188 protected function doStoreInternal( array $params ) {
189 $status = Status
::newGood();
191 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
192 if ( $dstRel === null ) {
193 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
197 // (a) Check the destination container and object
199 $dContObj = $this->getContainer( $dstCont );
200 if ( empty( $params['overwrite'] ) &&
201 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
203 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
206 } catch ( NoSuchContainerException
$e ) {
207 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
209 } catch ( InvalidResponseException
$e ) {
210 $status->fatal( 'backend-fail-connect', $this->name
);
212 } catch ( Exception
$e ) { // some other exception?
213 $status->fatal( 'backend-fail-internal', $this->name
);
214 $this->logException( $e, __METHOD__
, $params );
218 // (b) Get a SHA-1 hash of the object
219 $sha1Hash = sha1_file( $params['src'] );
220 if ( $sha1Hash === false ) { // source doesn't exist?
221 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
224 $sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
226 // (c) Actually store the object
228 // Create a fresh CF_Object with no fields preloaded.
229 // We don't want to preserve headers, metadata, and such.
230 $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
231 // Note: metadata keys stored as [Upper case char][[Lower case char]...]
232 $obj->metadata
= array( 'Sha1base36' => $sha1Hash );
233 // The MD5 here will be checked within Swift against its own MD5.
234 $obj->set_etag( md5_file( $params['src'] ) );
235 // Use the same content type as StreamFile for security
236 $obj->content_type
= StreamFile
::contentTypeFromPath( $params['dst'] );
237 // Actually write the object in Swift
238 $obj->load_from_filename( $params['src'], True ); // calls $obj->write()
239 } catch ( BadContentTypeException
$e ) {
240 $status->fatal( 'backend-fail-contenttype', $params['dst'] );
241 } catch ( IOException
$e ) {
242 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
243 } catch ( InvalidResponseException
$e ) {
244 $status->fatal( 'backend-fail-connect', $this->name
);
245 } catch ( Exception
$e ) { // some other exception?
246 $status->fatal( 'backend-fail-internal', $this->name
);
247 $this->logException( $e, __METHOD__
, $params );
254 * @see FileBackendStore::doCopyInternal()
257 protected function doCopyInternal( array $params ) {
258 $status = Status
::newGood();
260 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
261 if ( $srcRel === null ) {
262 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
266 list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
267 if ( $dstRel === null ) {
268 $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
272 // (a) Check the source/destination containers and destination object
274 $sContObj = $this->getContainer( $srcCont );
275 $dContObj = $this->getContainer( $dstCont );
276 if ( empty( $params['overwrite'] ) &&
277 $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) )
279 $status->fatal( 'backend-fail-alreadyexists', $params['dst'] );
282 } catch ( NoSuchContainerException
$e ) {
283 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
285 } catch ( InvalidResponseException
$e ) {
286 $status->fatal( 'backend-fail-connect', $this->name
);
288 } catch ( Exception
$e ) { // some other exception?
289 $status->fatal( 'backend-fail-internal', $this->name
);
290 $this->logException( $e, __METHOD__
, $params );
294 // (b) Actually copy the file to the destination
296 $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel );
297 } catch ( NoSuchObjectException
$e ) { // source object does not exist
298 $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
299 } catch ( InvalidResponseException
$e ) {
300 $status->fatal( 'backend-fail-connect', $this->name
);
301 } catch ( Exception
$e ) { // some other exception?
302 $status->fatal( 'backend-fail-internal', $this->name
);
303 $this->logException( $e, __METHOD__
, $params );
310 * @see FileBackendStore::doDeleteInternal()
313 protected function doDeleteInternal( array $params ) {
314 $status = Status
::newGood();
316 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
317 if ( $srcRel === null ) {
318 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
323 $sContObj = $this->getContainer( $srcCont );
324 $sContObj->delete_object( $srcRel );
325 } catch ( NoSuchContainerException
$e ) {
326 $status->fatal( 'backend-fail-delete', $params['src'] );
327 } catch ( NoSuchObjectException
$e ) {
328 if ( empty( $params['ignoreMissingSource'] ) ) {
329 $status->fatal( 'backend-fail-delete', $params['src'] );
331 } catch ( InvalidResponseException
$e ) {
332 $status->fatal( 'backend-fail-connect', $this->name
);
333 } catch ( Exception
$e ) { // some other exception?
334 $status->fatal( 'backend-fail-internal', $this->name
);
335 $this->logException( $e, __METHOD__
, $params );
342 * @see FileBackendStore::doPrepareInternal()
345 protected function doPrepareInternal( $fullCont, $dir, array $params ) {
346 $status = Status
::newGood();
348 // (a) Check if container already exists
350 $contObj = $this->getContainer( $fullCont );
351 // NoSuchContainerException not thrown: container must exist
352 return $status; // already exists
353 } catch ( NoSuchContainerException
$e ) {
354 // NoSuchContainerException thrown: container does not exist
355 } catch ( InvalidResponseException
$e ) {
356 $status->fatal( 'backend-fail-connect', $this->name
);
358 } catch ( Exception
$e ) { // some other exception?
359 $status->fatal( 'backend-fail-internal', $this->name
);
360 $this->logException( $e, __METHOD__
, $params );
364 // (b) Create container as needed
366 $contObj = $this->createContainer( $fullCont );
367 if ( $this->swiftAnonUser
!= '' ) {
368 // Make container public to end-users...
369 $status->merge( $this->setContainerAccess(
371 array( $this->auth
->username
, $this->swiftAnonUser
), // read
372 array( $this->auth
->username
) // write
375 } catch ( InvalidResponseException
$e ) {
376 $status->fatal( 'backend-fail-connect', $this->name
);
378 } catch ( Exception
$e ) { // some other exception?
379 $status->fatal( 'backend-fail-internal', $this->name
);
380 $this->logException( $e, __METHOD__
, $params );
388 * @see FileBackendStore::doSecureInternal()
391 protected function doSecureInternal( $fullCont, $dir, array $params ) {
392 $status = Status
::newGood();
394 if ( $this->swiftAnonUser
!= '' ) {
395 // Restrict container from end-users...
397 // doPrepareInternal() should have been called,
398 // so the Swift container should already exist...
399 $contObj = $this->getContainer( $fullCont ); // normally a cache hit
400 // NoSuchContainerException not thrown: container must exist
401 if ( !isset( $contObj->mw_wasSecured
) ) {
402 $status->merge( $this->setContainerAccess(
404 array( $this->auth
->username
), // read
405 array( $this->auth
->username
) // write
407 // @TODO: when php-cloudfiles supports container
408 // metadata, we can make use of that to avoid RTTs
409 $contObj->mw_wasSecured
= true; // avoid useless RTTs
411 } catch ( InvalidResponseException
$e ) {
412 $status->fatal( 'backend-fail-connect', $this->name
);
413 } catch ( Exception
$e ) { // some other exception?
414 $status->fatal( 'backend-fail-internal', $this->name
);
415 $this->logException( $e, __METHOD__
, $params );
423 * @see FileBackendStore::doCleanInternal()
426 protected function doCleanInternal( $fullCont, $dir, array $params ) {
427 $status = Status
::newGood();
429 // Only containers themselves can be removed, all else is virtual
431 return $status; // nothing to do
434 // (a) Check the container
436 $contObj = $this->getContainer( $fullCont, true );
437 } catch ( NoSuchContainerException
$e ) {
438 return $status; // ok, nothing to do
439 } catch ( InvalidResponseException
$e ) {
440 $status->fatal( 'backend-fail-connect', $this->name
);
442 } catch ( Exception
$e ) { // some other exception?
443 $status->fatal( 'backend-fail-internal', $this->name
);
444 $this->logException( $e, __METHOD__
, $params );
448 // (b) Delete the container if empty
449 if ( $contObj->object_count
== 0 ) {
451 $this->deleteContainer( $fullCont );
452 } catch ( NoSuchContainerException
$e ) {
453 return $status; // race?
454 } catch ( InvalidResponseException
$e ) {
455 $status->fatal( 'backend-fail-connect', $this->name
);
457 } catch ( Exception
$e ) { // some other exception?
458 $status->fatal( 'backend-fail-internal', $this->name
);
459 $this->logException( $e, __METHOD__
, $params );
468 * @see FileBackendStore::doFileExists()
469 * @return array|bool|null
471 protected function doGetFileStat( array $params ) {
472 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
473 if ( $srcRel === null ) {
474 return false; // invalid storage path
479 $contObj = $this->getContainer( $srcCont );
480 $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
481 $this->addMissingMetadata( $srcObj, $params['src'] );
483 // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
484 'mtime' => wfTimestamp( TS_MW
, $srcObj->last_modified
),
485 'size' => $srcObj->content_length
,
486 'sha1' => $srcObj->metadata
['Sha1base36']
488 } catch ( NoSuchContainerException
$e ) {
489 } catch ( NoSuchObjectException
$e ) {
490 } catch ( InvalidResponseException
$e ) {
492 } catch ( Exception
$e ) { // some other exception?
494 $this->logException( $e, __METHOD__
, $params );
501 * Fill in any missing object metadata and save it to Swift
503 * @param $obj CF_Object
504 * @param $path string Storage path to object
505 * @return bool Success
506 * @throws Exception cloudfiles exceptions
508 protected function addMissingMetadata( CF_Object
$obj, $path ) {
509 if ( isset( $obj->metadata
['Sha1base36'] ) ) {
510 return true; // nothing to do
512 $status = Status
::newGood();
513 $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager
::LOCK_UW
, $status );
514 if ( $status->isOK() ) {
515 $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) );
517 $hash = $tmpFile->getSha1Base36();
518 if ( $hash !== false ) {
519 $obj->metadata
['Sha1base36'] = $hash;
520 $obj->sync_metadata(); // save to Swift
521 return true; // success
525 $obj->metadata
['Sha1base36'] = false;
526 return false; // failed
530 * @see FileBackend::getFileContents()
531 * @return bool|null|string
533 public function getFileContents( array $params ) {
534 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
535 if ( $srcRel === null ) {
536 return false; // invalid storage path
539 if ( !$this->fileExists( $params ) ) {
545 $sContObj = $this->getContainer( $srcCont );
546 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD request
547 $data = $obj->read( $this->headersFromParams( $params ) );
548 } catch ( NoSuchContainerException
$e ) {
549 } catch ( InvalidResponseException
$e ) {
550 } catch ( Exception
$e ) { // some other exception?
551 $this->logException( $e, __METHOD__
, $params );
558 * @see FileBackendStore::doDirectoryExists()
561 protected function doDirectoryExists( $fullCont, $dir, array $params ) {
563 $container = $this->getContainer( $fullCont );
564 $prefix = ( $dir == '' ) ?
null : "{$dir}/";
565 return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
566 } catch ( NoSuchContainerException
$e ) {
568 } catch ( InvalidResponseException
$e ) {
569 } catch ( Exception
$e ) { // some other exception?
570 $this->logException( $e, __METHOD__
, array( 'cont' => $fullCont, 'dir' => $dir ) );
573 return null; // error
577 * @see FileBackendStore::getDirectoryListInternal()
578 * @return SwiftFileBackendDirList
580 public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
581 return new SwiftFileBackendDirList( $this, $fullCont, $dir, $params );
585 * @see FileBackendStore::getFileListInternal()
586 * @return SwiftFileBackendFileList
588 public function getFileListInternal( $fullCont, $dir, array $params ) {
589 return new SwiftFileBackendFileList( $this, $fullCont, $dir, $params );
593 * Do not call this function outside of SwiftFileBackendFileList
595 * @param $fullCont string Resolved container name
596 * @param $dir string Resolved storage directory with no trailing slash
597 * @param $after string|null Storage path of file to list items after
598 * @param $limit integer Max number of items to list
599 * @param $params Array Includes flag for 'topOnly'
600 * @return Array List of relative paths of dirs directly under $dir
602 public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
606 $container = $this->getContainer( $fullCont );
607 $prefix = ( $dir == '' ) ?
null : "{$dir}/";
608 // Non-recursive: only list dirs right under $dir
609 if ( !empty( $params['topOnly'] ) ) {
610 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
611 foreach ( $objects as $object ) { // files and dirs
612 if ( substr( $object, -1 ) === '/' ) {
613 $dirs[] = $object; // directories end in '/'
615 $after = $object; // update last item
617 // Recursive: list all dirs under $dir and its subdirs
619 // Get directory from last item of prior page
620 $lastDir = $this->getParentDir( $after ); // must be first page
621 $objects = $container->list_objects( $limit, $after, $prefix );
622 foreach ( $objects as $object ) { // files
623 $objectDir = $this->getParentDir( $object ); // directory of object
624 if ( $objectDir !== false ) { // file has a parent dir
625 // Swift stores paths in UTF-8, using binary sorting.
626 // See function "create_container_table" in common/db.py.
627 // If a directory is not "greater" than the last one,
628 // then it was already listed by the calling iterator.
629 if ( $objectDir > $lastDir ) {
631 do { // add dir and all its parent dirs
632 $dirs[] = "{$pDir}/";
633 $pDir = $this->getParentDir( $pDir );
634 } while ( $pDir !== false // sanity
635 && $pDir > $lastDir // not done already
636 && strlen( $pDir ) > strlen( $dir ) // within $dir
639 $lastDir = $objectDir;
641 $after = $object; // update last item
644 } catch ( NoSuchContainerException
$e ) {
645 } catch ( InvalidResponseException
$e ) {
646 } catch ( Exception
$e ) { // some other exception?
647 $this->logException( $e, __METHOD__
, array( 'cont' => $fullCont, 'dir' => $dir ) );
653 protected function getParentDir( $path ) {
654 return ( strpos( $path, '/' ) !== false ) ?
dirname( $path ) : false;
658 * Do not call this function outside of SwiftFileBackendFileList
660 * @param $fullCont string Resolved container name
661 * @param $dir string Resolved storage directory with no trailing slash
662 * @param $after string|null Storage path of file to list items after
663 * @param $limit integer Max number of items to list
664 * @param $params Array Includes flag for 'topOnly'
665 * @return Array List of relative paths of files under $dir
667 public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
671 $container = $this->getContainer( $fullCont );
672 $prefix = ( $dir == '' ) ?
null : "{$dir}/";
673 // Non-recursive: only list files right under $dir
674 if ( !empty( $params['topOnly'] ) ) { // files and dirs
675 $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
676 foreach ( $objects as $object ) {
677 if ( substr( $object, -1 ) !== '/' ) {
678 $files[] = $object; // directories end in '/'
681 // Recursive: list all files under $dir and its subdirs
683 $files = $container->list_objects( $limit, $after, $prefix );
685 $after = end( $files ); // update last item
686 reset( $files ); // reset pointer
687 } catch ( NoSuchContainerException
$e ) {
688 } catch ( InvalidResponseException
$e ) {
689 } catch ( Exception
$e ) { // some other exception?
690 $this->logException( $e, __METHOD__
, array( 'cont' => $fullCont, 'dir' => $dir ) );
697 * @see FileBackendStore::doGetFileSha1base36()
700 protected function doGetFileSha1base36( array $params ) {
701 $stat = $this->getFileStat( $params );
703 return $stat['sha1'];
710 * @see FileBackendStore::doStreamFile()
713 protected function doStreamFile( array $params ) {
714 $status = Status
::newGood();
716 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
717 if ( $srcRel === null ) {
718 $status->fatal( 'backend-fail-invalidpath', $params['src'] );
722 $cont = $this->getContainer( $srcCont );
723 } catch ( NoSuchContainerException
$e ) {
724 $status->fatal( 'backend-fail-stream', $params['src'] );
726 } catch ( InvalidResponseException
$e ) {
727 $status->fatal( 'backend-fail-connect', $this->name
);
729 } catch ( Exception
$e ) { // some other exception?
730 $status->fatal( 'backend-fail-stream', $params['src'] );
731 $this->logException( $e, __METHOD__
, $params );
736 $output = fopen( 'php://output', 'wb' );
737 $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD request
738 $obj->stream( $output, $this->headersFromParams( $params ) );
739 } catch ( InvalidResponseException
$e ) { // 404? connection problem?
740 $status->fatal( 'backend-fail-stream', $params['src'] );
741 } catch ( Exception
$e ) { // some other exception?
742 $status->fatal( 'backend-fail-stream', $params['src'] );
743 $this->logException( $e, __METHOD__
, $params );
750 * @see FileBackendStore::getLocalCopy()
751 * @return null|TempFSFile
753 public function getLocalCopy( array $params ) {
754 list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
755 if ( $srcRel === null ) {
759 if ( !$this->fileExists( $params ) ) {
765 $sContObj = $this->getContainer( $srcCont );
766 $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
767 // Get source file extension
768 $ext = FileBackend
::extensionFromPath( $srcRel );
769 // Create a new temporary file...
770 $tmpFile = TempFSFile
::factory( wfBaseName( $srcRel ) . '_', $ext );
772 $handle = fopen( $tmpFile->getPath(), 'wb' );
774 $obj->stream( $handle, $this->headersFromParams( $params ) );
777 $tmpFile = null; // couldn't open temp file
780 } catch ( NoSuchContainerException
$e ) {
782 } catch ( InvalidResponseException
$e ) {
784 } catch ( Exception
$e ) { // some other exception?
786 $this->logException( $e, __METHOD__
, $params );
793 * @see FileBackendStore::directoriesAreVirtual()
796 protected function directoriesAreVirtual() {
801 * Get headers to send to Swift when reading a file based
802 * on a FileBackend params array, e.g. that of getLocalCopy().
803 * $params is currently only checked for a 'latest' flag.
805 * @param $params Array
808 protected function headersFromParams( array $params ) {
810 if ( !empty( $params['latest'] ) ) {
811 $hdrs[] = 'X-Newest: true';
817 * Set read/write permissions for a Swift container
819 * @param $contObj CF_Container Swift container
820 * @param $readGrps Array Swift users who can read (account:user)
821 * @param $writeGrps Array Swift users who can write (account:user)
824 protected function setContainerAccess(
825 CF_Container
$contObj, array $readGrps, array $writeGrps
827 $creds = $contObj->cfs_auth
->export_credentials();
829 $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name
);
831 // Note: 10 second timeout consistent with php-cloudfiles
832 $req = new CurlHttpRequest( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
833 $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
834 $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
835 $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
837 return $req->execute(); // should return 204
841 * Get a connection to the Swift proxy
843 * @return CF_Connection|bool False on failure
844 * @throws InvalidResponseException
846 protected function getConnection() {
847 if ( $this->conn
=== false ) {
848 throw new InvalidResponseException
; // failed last attempt
850 // Session keys expire after a while, so we renew them periodically
851 if ( $this->conn
&& ( time() - $this->connStarted
) > $this->authTTL
) {
852 $this->conn
->close(); // close active cURL connections
855 // Authenticate with proxy and get a session key...
856 if ( $this->conn
=== null ) {
857 $this->connContainers
= array();
859 $this->auth
->authenticate();
860 $this->conn
= new CF_Connection( $this->auth
);
861 $this->connStarted
= time();
862 } catch ( AuthenticationException
$e ) {
863 $this->conn
= false; // don't keep re-trying
864 } catch ( InvalidResponseException
$e ) {
865 $this->conn
= false; // don't keep re-trying
868 if ( !$this->conn
) {
869 throw new InvalidResponseException
; // auth/connection problem
875 * @see FileBackendStore::doClearCache()
877 protected function doClearCache( array $paths = null ) {
878 $this->connContainers
= array(); // clear container object cache
882 * Get a Swift container object, possibly from process cache.
883 * Use $reCache if the file count or byte count is needed.
885 * @param $container string Container name
886 * @param $bypassCache bool Bypass all caches and load from Swift
887 * @return CF_Container
888 * @throws InvalidResponseException
890 protected function getContainer( $container, $bypassCache = false ) {
891 $conn = $this->getConnection(); // Swift proxy connection
892 if ( $bypassCache ) { // purge cache
893 unset( $this->connContainers
[$container] );
894 } elseif ( !isset( $this->connContainers
[$container] ) ) {
895 $this->primeContainerCache( array( $container ) ); // check persistent cache
897 if ( !isset( $this->connContainers
[$container] ) ) {
898 $contObj = $conn->get_container( $container );
899 // NoSuchContainerException not thrown: container must exist
900 if ( count( $this->connContainers
) >= $this->maxContCacheSize
) { // trim cache?
901 reset( $this->connContainers
);
902 unset( $this->connContainers
[key( $this->connContainers
)] );
904 $this->connContainers
[$container] = $contObj; // cache it
905 if ( !$bypassCache ) {
906 $this->setContainerCache( $container, // update persistent cache
907 array( 'bytes' => $contObj->bytes_used
, 'count' => $contObj->object_count
)
911 return $this->connContainers
[$container];
915 * Create a Swift container
917 * @param $container string Container name
918 * @return CF_Container
919 * @throws InvalidResponseException
921 protected function createContainer( $container ) {
922 $conn = $this->getConnection(); // Swift proxy connection
923 $contObj = $conn->create_container( $container );
924 $this->connContainers
[$container] = $contObj; // cache it
929 * Delete a Swift container
931 * @param $container string Container name
933 * @throws InvalidResponseException
935 protected function deleteContainer( $container ) {
936 $conn = $this->getConnection(); // Swift proxy connection
937 $conn->delete_container( $container );
938 unset( $this->connContainers
[$container] ); // purge cache
942 * @see FileBackendStore::doPrimeContainerCache()
945 protected function doPrimeContainerCache( array $containerInfo ) {
947 $conn = $this->getConnection(); // Swift proxy connection
948 foreach ( $containerInfo as $container => $info ) {
949 $this->connContainers
[$container] = new CF_Container(
957 } catch ( InvalidResponseException
$e ) {
958 } catch ( Exception
$e ) { // some other exception?
959 $this->logException( $e, __METHOD__
, array() );
964 * Log an unexpected exception for this backend
966 * @param $e Exception
967 * @param $func string
968 * @param $params Array
971 protected function logException( Exception
$e, $func, array $params ) {
972 wfDebugLog( 'SwiftBackend',
973 get_class( $e ) . " in '{$func}' (given '" . FormatJson
::encode( $params ) . "')" .
974 ( $e instanceof InvalidResponseException
975 ?
": {$e->getMessage()}"
983 * SwiftFileBackend helper class to page through listings.
984 * Swift also has a listing limit of 10,000 objects for sanity.
985 * Do not use this class from places outside SwiftFileBackend.
987 * @ingroup FileBackend
989 abstract class SwiftFileBackendList
implements Iterator
{
991 protected $bufferIter = array();
992 protected $bufferAfter = null; // string; list items *after* this path
993 protected $pos = 0; // integer
995 protected $params = array();
997 /** @var SwiftFileBackend */
999 protected $container; // string; container name
1000 protected $dir; // string; storage directory
1001 protected $suffixStart; // integer
1003 const PAGE_SIZE
= 5000; // file listing buffer size
1006 * @param $backend SwiftFileBackend
1007 * @param $fullCont string Resolved container name
1008 * @param $dir string Resolved directory relative to container
1009 * @param $params Array
1011 public function __construct( SwiftFileBackend
$backend, $fullCont, $dir, array $params ) {
1012 $this->backend
= $backend;
1013 $this->container
= $fullCont;
1015 if ( substr( $this->dir
, -1 ) === '/' ) {
1016 $this->dir
= substr( $this->dir
, 0, -1 ); // remove trailing slash
1018 if ( $this->dir
== '' ) { // whole container
1019 $this->suffixStart
= 0;
1020 } else { // dir within container
1021 $this->suffixStart
= strlen( $this->dir
) +
1; // size of "path/to/dir/"
1023 $this->params
= $params;
1027 * @see Iterator::key()
1030 public function key() {
1035 * @see Iterator::next()
1038 public function next() {
1039 // Advance to the next file in the page
1040 next( $this->bufferIter
);
1042 // Check if there are no files left in this page and
1043 // advance to the next page if this page was not empty.
1044 if ( !$this->valid() && count( $this->bufferIter
) ) {
1045 $this->bufferIter
= $this->pageFromList(
1046 $this->container
, $this->dir
, $this->bufferAfter
, self
::PAGE_SIZE
, $this->params
1047 ); // updates $this->bufferAfter
1052 * @see Iterator::rewind()
1055 public function rewind() {
1057 $this->bufferAfter
= null;
1058 $this->bufferIter
= $this->pageFromList(
1059 $this->container
, $this->dir
, $this->bufferAfter
, self
::PAGE_SIZE
, $this->params
1060 ); // updates $this->bufferAfter
1064 * @see Iterator::valid()
1067 public function valid() {
1068 if ( $this->bufferIter
=== null ) {
1069 return false; // some failure?
1071 return ( current( $this->bufferIter
) !== false ); // no paths can have this value
1076 * Get the given list portion (page)
1078 * @param $container string Resolved container name
1079 * @param $dir string Resolved path relative to container
1080 * @param $after string|null
1081 * @param $limit integer
1082 * @param $params Array
1083 * @return Traversable|Array|null Returns null on failure
1085 abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
1089 * Iterator for listing directories
1091 class SwiftFileBackendDirList
extends SwiftFileBackendList
{
1093 * @see Iterator::current()
1094 * @return string|bool String (relative path) or false
1096 public function current() {
1097 return substr( current( $this->bufferIter
), $this->suffixStart
, -1 );
1101 * @see SwiftFileBackendList::pageFromList()
1102 * @return Array|null
1104 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1105 return $this->backend
->getDirListPageInternal( $container, $dir, $after, $limit, $params );
1110 * Iterator for listing regular files
1112 class SwiftFileBackendFileList
extends SwiftFileBackendList
{
1114 * @see Iterator::current()
1115 * @return string|bool String (relative path) or false
1117 public function current() {
1118 return substr( current( $this->bufferIter
), $this->suffixStart
);
1122 * @see SwiftFileBackendList::pageFromList()
1123 * @return Array|null
1125 protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
1126 return $this->backend
->getFileListPageInternal( $container, $dir, $after, $limit, $params );