From ac14c1c8a65d154f7a9bc47d1b51f6db67188f86 Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Mon, 30 Sep 2013 00:12:10 -0700 Subject: [PATCH] filebackend: Added supported for retrieving file metadata/headers * This can be useful for carrying over metadata when copying files around * Also fixed a bug in sanitizeHdrs() for Swift (broken content-disposition) Change-Id: I4534e9acac2b306086797b3677f85c05b98e39fc --- includes/filebackend/FileBackend.php | 45 +++++++++++++ .../filebackend/FileBackendMultiWrite.php | 9 +++ includes/filebackend/FileBackendStore.php | 65 ++++++++++++++++++- includes/filebackend/SwiftFileBackend.php | 22 ++++++- .../includes/filebackend/FileBackendTest.php | 38 ++++++++++- 5 files changed, 173 insertions(+), 6 deletions(-) diff --git a/includes/filebackend/FileBackend.php b/includes/filebackend/FileBackend.php index bb21f1b5e4..f5d63b9df7 100644 --- a/includes/filebackend/FileBackend.php +++ b/includes/filebackend/FileBackend.php @@ -104,6 +104,10 @@ abstract class FileBackend { /** @var FileJournal */ protected $fileJournal; + /** Flags for supported features */ + const ATTR_HEADERS = 1; + const ATTR_METADATA = 2; + /** * Create a new backend instance from configuration. * This should only be called from within FileBackendGroup. @@ -200,6 +204,27 @@ abstract class FileBackend { return ( $this->readOnly != '' ) ? $this->readOnly : false; } + /** + * Get the a bitfield of extra features supported by the backend medium + * + * @return integer Bitfield of FileBackend::ATTR_* flags + * @since 1.23 + */ + public function getFeatures() { + return 0; + } + + /** + * Check if the backend medium supports a field of extra features + * + * @return integer Bitfield of FileBackend::ATTR_* flags + * @return bool + * @since 1.23 + */ + final public function hasFeatures( $bitfield ) { + return ( $this->getFeatures() & $bitfield ) === $bitfield; + } + /** * This is the main entry point into the backend for write operations. * Callers supply an ordered list of operations to perform as a transaction. @@ -901,6 +926,26 @@ abstract class FileBackend { */ abstract public function getFileContentsMulti( array $params ); + /** + * Get metadata about a file at a storage path in the backend. + * If the file does not exist, then this returns false. + * Otherwise, the result is an associative array that includes: + * - headers : map of HTTP headers used for GET/HEAD requests (name => value) + * - metadata : map of file metadata (name => value) + * Metadata keys and headers names will be returned in all lower-case. + * Additional values may be included for internal use only. + * + * Use FileBackend::hasFeatures() to check how well this is supported. + * + * @param array $params + * $params include: + * - src : source storage path + * - latest : use the latest available data + * @return Array|bool Returns false on failure + * @since 1.23 + */ + abstract public function getFileXAttributes( array $params ); + /** * Get the size (bytes) of a file at a storage path in the backend. * diff --git a/includes/filebackend/FileBackendMultiWrite.php b/includes/filebackend/FileBackendMultiWrite.php index 1c9832de74..1b2860a201 100644 --- a/includes/filebackend/FileBackendMultiWrite.php +++ b/includes/filebackend/FileBackendMultiWrite.php @@ -567,6 +567,11 @@ class FileBackendMultiWrite extends FileBackend { return $this->backends[$this->masterIndex]->getFileStat( $realParams ); } + public function getFileXAttributes( array $params ) { + $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + return $this->backends[$this->masterIndex]->getFileXAttributes( $realParams ); + } + public function getFileContentsMulti( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); $contentsM = $this->backends[$this->masterIndex]->getFileContentsMulti( $realParams ); @@ -645,6 +650,10 @@ class FileBackendMultiWrite extends FileBackend { return $this->backends[$this->masterIndex]->getFileList( $realParams ); } + public function getFeatures() { + return $this->backends[$this->masterIndex]->getFeatures(); + } + public function clearCache( array $paths = null ) { foreach ( $this->backends as $backend ) { $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null; diff --git a/includes/filebackend/FileBackendStore.php b/includes/filebackend/FileBackendStore.php index fe3a068456..7980d10067 100644 --- a/includes/filebackend/FileBackendStore.php +++ b/includes/filebackend/FileBackendStore.php @@ -655,10 +655,15 @@ abstract class FileBackendStore extends FileBackend { $this->cheapCache->set( $path, 'sha1', array( 'hash' => $stat['sha1'], 'latest' => $latest ) ); } + if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata + $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] ); + $this->cheapCache->set( $path, 'xattr', + array( 'map' => $stat['xattr'], 'latest' => $latest ) ); + } } elseif ( $stat === false ) { // file does not exist $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' ); - $this->cheapCache->set( $path, 'sha1', // the SHA-1 must be false too - array( 'hash' => false, 'latest' => $latest ) ); + $this->cheapCache->set( $path, 'xattr', array( 'map' => false, 'latest' => $latest ) ); + $this->cheapCache->set( $path, 'sha1', array( 'hash' => false, 'latest' => $latest ) ); wfDebug( __METHOD__ . ": File $path does not exist.\n" ); } else { // an error occurred wfDebug( __METHOD__ . ": Could not stat file $path.\n" ); @@ -697,6 +702,39 @@ abstract class FileBackendStore extends FileBackend { return $contents; } + final public function getFileXAttributes( array $params ) { + $path = self::normalizeStoragePath( $params['src'] ); + if ( $path === null ) { + return false; // invalid storage path + } + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); + $latest = !empty( $params['latest'] ); // use latest data? + if ( $this->cheapCache->has( $path, 'xattr', self::CACHE_TTL ) ) { + $stat = $this->cheapCache->get( $path, 'xattr' ); + // If we want the latest data, check that this cached + // value was in fact fetched with the latest available data. + if ( !$latest || $stat['latest'] ) { + return $stat['map']; + } + } + wfProfileIn( __METHOD__ . '-miss' ); + wfProfileIn( __METHOD__ . '-miss-' . $this->name ); + $fields = $this->doGetFileXAttributes( $params ); + $fields = is_array( $fields ) ? self::normalizeXAttributes( $fields ) : false; + wfProfileOut( __METHOD__ . '-miss-' . $this->name ); + wfProfileOut( __METHOD__ . '-miss' ); + $this->cheapCache->set( $path, 'xattr', array( 'map' => $fields, 'latest' => $latest ) ); + return $fields; + } + + /** + * @see FileBackendStore::getFileXAttributes() + * @return bool|string + */ + protected function doGetFileXAttributes( array $params ) { + return array( 'headers' => array(), 'metadata' => array() ); // not supported + } + final public function getFileSha1Base36( array $params ) { $path = self::normalizeStoragePath( $params['src'] ); if ( $path === null ) { @@ -1625,10 +1663,33 @@ abstract class FileBackendStore extends FileBackend { $this->cheapCache->set( $path, 'sha1', array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) ); } + if ( isset( $val['xattr'] ) ) { // some backends store headers/metadata + $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] ); + $this->cheapCache->set( $path, 'xattr', + array( 'map' => $val['xattr'], 'latest' => $val['latest'] ) ); + } } } } + /** + * Normalize file headers/metadata to the FileBackend::getFileXAttributes() format + * + * @param array $xattr + * @return array + * @since 1.22 + */ + final protected static function normalizeXAttributes( array $xattr ) { + $newXAttr = array( 'headers' => array(), 'metadata' => array() ); + foreach ( $xattr['headers'] as $name => $value ) { + $newXAttr['headers'][strtolower( $name )] = $value; + } + foreach ( $xattr['metadata'] as $name => $value ) { + $newXAttr['metadata'][strtolower( $name )] = $value; + } + return $newXAttr; + } + /** * Set the 'concurrency' option from a list of operation options * diff --git a/includes/filebackend/SwiftFileBackend.php b/includes/filebackend/SwiftFileBackend.php index 528889b69b..037b1c3c30 100644 --- a/includes/filebackend/SwiftFileBackend.php +++ b/includes/filebackend/SwiftFileBackend.php @@ -142,6 +142,10 @@ class SwiftFileBackend extends FileBackendStore { $this->srvCache = $this->srvCache ?: new EmptyBagOStuff(); } + public function getFeatures() { + return ( FileBackend::ATTR_HEADERS | FileBackend::ATTR_METADATA ); + } + protected function resolveContainerPath( $container, $relStoragePath ) { if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF return null; // not UTF-8, makes it hard to use CF and the swift HTTP API @@ -179,6 +183,8 @@ class SwiftFileBackend extends FileBackendStore { continue; // blacklisted } elseif ( preg_match( '/^(x-)?content-/', $name ) ) { $headers[$name] = $value; // allowed + } elseif ( preg_match( '/^content-(disposition)/', $name ) ) { + $headers[$name] = $value; // allowed } } } @@ -189,7 +195,7 @@ class SwiftFileBackend extends FileBackendStore { $part = trim( $part ); $new = ( $disposition === '' ) ? $part : "{$disposition};{$part}"; if ( strlen( $new ) <= 255 ) { - $res = $new; + $disposition = $new; } else { break; // too long; sigh } @@ -986,6 +992,20 @@ class SwiftFileBackend extends FileBackendStore { $this->cheapCache->set( $path, 'stat', $val ); } + protected function doGetFileXAttributes( array $params ) { + $stat = $this->getFileStat( $params ); + if ( $stat ) { + if ( !isset( $stat['xattr'] ) ) { + // Stat entries filled by file listings don't include metadata/headers + $this->clearCache( array( $params['src'] ) ); + $stat = $this->getFileStat( $params ); + } + return $stat['xattr']; + } else { + return false; + } + } + protected function doGetFileSha1base36( array $params ) { $stat = $this->getFileStat( $params ); if ( $stat ) { diff --git a/tests/phpunit/includes/filebackend/FileBackendTest.php b/tests/phpunit/includes/filebackend/FileBackendTest.php index c48fdc9efc..4590856ae3 100644 --- a/tests/phpunit/includes/filebackend/FileBackendTest.php +++ b/tests/phpunit/includes/filebackend/FileBackendTest.php @@ -656,9 +656,25 @@ class FileBackendTest extends MediaWikiTestCase { if ( $withSource ) { $status = $this->backend->doOperation( - array( 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ) ); + array( 'op' => 'create', 'content' => 'blahblah', 'dst' => $source, + 'headers' => array( 'Content-Disposition' => 'xxx' ) ) ); $this->assertGoodStatus( $status, "Creation of file at $source succeeded ($backendName)." ); + if ( $this->backend->hasFeatures( FileBackend::ATTR_HEADERS ) ) { + $attr = $this->backend->getFileXAttributes( array( 'src' => $source ) ); + $this->assertHasHeaders( array( 'Content-Disposition' => 'xxx' ), $attr ); + } + + $status = $this->backend->describe( array( 'src' => $source, + 'headers' => array( 'Content-Disposition' => '' ) ) ); // remove + $this->assertGoodStatus( $status, + "Removal of header for $source succeeded ($backendName)." ); + + if ( $this->backend->hasFeatures( FileBackend::ATTR_HEADERS ) ) { + $attr = $this->backend->getFileXAttributes( array( 'src' => $source ) ); + $this->assertFalse( isset( $attr['headers']['content-disposition'] ), + "File 'Content-Disposition' header removed." ); + } } $status = $this->backend->doOperation( $op ); @@ -669,6 +685,9 @@ class FileBackendTest extends MediaWikiTestCase { "Describe of file at $source succeeded ($backendName)." ); $this->assertEquals( array( 0 => true ), $status->success, "Describe of file at $source has proper 'success' field in Status ($backendName)." ); + if ( $this->backend->hasFeatures( FileBackend::ATTR_HEADERS ) ) { + $this->assertHasHeaders( $op['headers'], $attr ); + } } else { $this->assertEquals( false, $status->isOK(), "Describe of file at $source failed ($backendName)." ); @@ -677,14 +696,27 @@ class FileBackendTest extends MediaWikiTestCase { $this->assertBackendPathsConsistent( array( $source ) ); } + private function assertHasHeaders( array $headers, array $attr ) { + foreach ( $headers as $n => $v ) { + if ( $n !== '' ) { + $this->assertTrue( isset( $attr['headers'][strtolower( $n )] ), + "File has '$n' header." ); + $this->assertEquals( $v, $attr['headers'][strtolower( $n )], + "File has '$n' header value." ); + } else { + $this->assertFalse( isset( $attr['headers'][strtolower( $n )] ), + "File does not have '$n' header." ); + } + } + } + public static function provider_testDescribe() { $cases = array(); $source = self::baseStorePath() . '/unittest-cont1/e/myfacefile.txt'; $op = array( 'op' => 'describe', 'src' => $source, - 'headers' => array( 'X-Content-Length' => '91.3', 'Content-Old-Header' => '' ), - 'disposition' => 'inline' ); + 'headers' => array( 'Content-Disposition' => 'inline' ), ); $cases[] = array( $op, // operation true, // with source -- 2.20.1