From: Aaron Schulz <aschulz@wikimedia.org>
Date: Sat, 26 May 2012 05:42:33 +0000 (-0700)
Subject: [FileBackend] Process cache fixes and cleanups.
X-Git-Tag: 1.31.0-rc.0~23302^2
X-Git-Url: http://git.cyclocoop.org/%22%20.%20generer_url_ecrire%28%22messagerie%22%29%20.%20%22?a=commitdiff_plain;h=a998d3ff0906934fe3dd891887131ea48df2748b;p=lhc%2Fweb%2Fwiklou.git

[FileBackend] Process cache fixes and cleanups.

* Made getLocalReference() and getFileSha1Base36() respect latest wrt to cache.
* Made Made getLocalReference() and getFileSha1Base36() normalize the paths for cache.
* Avoid misleading (suggesting cache misses) getFileSha1Base36() profile calls for Swift.

Change-Id: I192a4f00c123029cfa338ec74cd08a355e29896b
---

diff --git a/includes/filerepo/backend/FileBackendStore.php b/includes/filerepo/backend/FileBackendStore.php
index f02724f2fa..b658efc9fd 100644
--- a/includes/filerepo/backend/FileBackendStore.php
+++ b/includes/filerepo/backend/FileBackendStore.php
@@ -532,14 +532,12 @@ abstract class FileBackendStore extends FileBackend {
 	 * @return bool
 	 */
 	final public function getFileStat( array $params ) {
-		wfProfileIn( __METHOD__ );
-		wfProfileIn( __METHOD__ . '-' . $this->name );
 		$path = self::normalizeStoragePath( $params['src'] );
 		if ( $path === null ) {
-			wfProfileOut( __METHOD__ . '-' . $this->name );
-			wfProfileOut( __METHOD__ );
 			return false; // invalid storage path
 		}
+		wfProfileIn( __METHOD__ );
+		wfProfileIn( __METHOD__ . '-' . $this->name );
 		$latest = !empty( $params['latest'] ); // use latest data?
 		if ( !isset( $this->cache[$path]['stat'] ) ) {
 			$this->primeFileCache( array( $path ) ); // check persistent cache
@@ -564,6 +562,11 @@ abstract class FileBackendStore extends FileBackend {
 			$this->trimCache(); // limit memory
 			$this->cache[$path]['stat'] = $stat;
 			$this->setFileCache( $path, $stat ); // update persistent cache
+			if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
+				$this->trimCache(); // limit memory
+				$this->cache[$path]['sha1'] =
+					array( 'hash' => $stat['sha1'], 'latest' => $latest );
+			}
 		} else {
 			wfDebug( __METHOD__ . ": File $path does not exist.\n" );
 		}
@@ -603,14 +606,22 @@ abstract class FileBackendStore extends FileBackend {
 	 * @return bool|string
 	 */
 	final public function getFileSha1Base36( array $params ) {
+		$path = self::normalizeStoragePath( $params['src'] );
+		if ( $path === null ) {
+			return false; // invalid storage path
+		}
 		wfProfileIn( __METHOD__ );
 		wfProfileIn( __METHOD__ . '-' . $this->name );
-		$path = $params['src'];
+		$latest = !empty( $params['latest'] ); // use latest data?
 		if ( isset( $this->cache[$path]['sha1'] ) ) {
-			$this->pingCache( $path ); // LRU
-			wfProfileOut( __METHOD__ . '-' . $this->name );
-			wfProfileOut( __METHOD__ );
-			return $this->cache[$path]['sha1'];
+			// If we want the latest data, check that this cached
+			// value was in fact fetched with the latest available data.
+			if ( !$latest || $this->cache[$path]['sha1']['latest'] ) {
+				$this->pingCache( $path ); // LRU
+				wfProfileOut( __METHOD__ . '-' . $this->name );
+				wfProfileOut( __METHOD__ );
+				return $this->cache[$path]['sha1']['hash'];
+			}
 		}
 		wfProfileIn( __METHOD__ . '-miss' );
 		wfProfileIn( __METHOD__ . '-miss-' . $this->name );
@@ -619,7 +630,7 @@ abstract class FileBackendStore extends FileBackend {
 		wfProfileOut( __METHOD__ . '-miss' );
 		if ( $hash ) { // don't cache negatives
 			$this->trimCache(); // limit memory
-			$this->cache[$path]['sha1'] = $hash;
+			$this->cache[$path]['sha1'] = array( 'hash' => $hash, 'latest' => $latest );
 		}
 		wfProfileOut( __METHOD__ . '-' . $this->name );
 		wfProfileOut( __METHOD__ );
@@ -628,7 +639,7 @@ abstract class FileBackendStore extends FileBackend {
 
 	/**
 	 * @see FileBackendStore::getFileSha1Base36()
-	 * @return bool
+	 * @return bool|string
 	 */
 	protected function doGetFileSha1Base36( array $params ) {
 		$fsFile = $this->getLocalReference( $params );
@@ -658,19 +669,28 @@ abstract class FileBackendStore extends FileBackend {
 	 * @return TempFSFile|null
 	 */
 	public function getLocalReference( array $params ) {
+		$path = self::normalizeStoragePath( $params['src'] );
+		if ( $path === null ) {
+			return null; // invalid storage path
+		}
 		wfProfileIn( __METHOD__ );
 		wfProfileIn( __METHOD__ . '-' . $this->name );
-		$path = $params['src'];
+		$latest = !empty( $params['latest'] ); // use latest data?
 		if ( isset( $this->expensiveCache[$path]['localRef'] ) ) {
-			$this->pingExpensiveCache( $path );
-			wfProfileOut( __METHOD__ . '-' . $this->name );
-			wfProfileOut( __METHOD__ );
-			return $this->expensiveCache[$path]['localRef'];
+			// If we want the latest data, check that this cached
+			// value was in fact fetched with the latest available data.
+			if ( !$latest || $this->expensiveCache[$path]['localRef']['latest'] ) {
+				$this->pingExpensiveCache( $path );
+				wfProfileOut( __METHOD__ . '-' . $this->name );
+				wfProfileOut( __METHOD__ );
+				return $this->expensiveCache[$path]['localRef']['object'];
+			}
 		}
 		$tmpFile = $this->getLocalCopy( $params );
 		if ( $tmpFile ) { // don't cache negatives
 			$this->trimExpensiveCache(); // limit memory
-			$this->expensiveCache[$path]['localRef'] = $tmpFile;
+			$this->expensiveCache[$path]['localRef'] =
+				array( 'object' => $tmpFile, 'latest' => $latest );
 		}
 		wfProfileOut( __METHOD__ . '-' . $this->name );
 		wfProfileOut( __METHOD__ );
@@ -1492,8 +1512,14 @@ abstract class FileBackendStore extends FileBackend {
 		$values = $this->memCache->getMulti( array_keys( $pathNames ) );
 		foreach ( $values as $cacheKey => $val ) {
 			if ( is_array( $val ) ) {
+				$path = $pathNames[$cacheKey];
 				$this->trimCache(); // limit memory
-				$this->cache[$pathNames[$cacheKey]]['stat'] = $val;
+				$this->cache[$path]['stat'] = $val;
+				if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
+					$this->trimCache(); // limit memory
+					$this->cache[$path]['sha1'] =
+						array( 'hash' => $val['sha1'], 'latest' => $val['latest'] );
+				}
 			}
 		}
 
diff --git a/includes/filerepo/backend/SwiftFileBackend.php b/includes/filerepo/backend/SwiftFileBackend.php
index 36d4334e7f..74646d3d9f 100644
--- a/includes/filerepo/backend/SwiftFileBackend.php
+++ b/includes/filerepo/backend/SwiftFileBackend.php
@@ -625,7 +625,7 @@ class SwiftFileBackend extends FileBackendStore {
 			$stat = array(
 				// Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
 				'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
-				'size'  => $srcObj->content_length,
+				'size'  => (int)$srcObj->content_length,
 				'sha1'  => $srcObj->metadata['Sha1base36']
 			);
 		} catch ( NoSuchContainerException $e ) {