From: Aaron Schulz Date: Thu, 24 Jan 2013 03:20:57 +0000 (-0800) Subject: [FileRepo] Lazy load large metadata from the DB. X-Git-Tag: 1.31.0-rc.0~20869 X-Git-Url: http://git.cyclocoop.org/%24self?a=commitdiff_plain;h=dcbdcf07810580e820e53b24b6c3b9d4c3ca37a3;p=lhc%2Fweb%2Fwiklou.git [FileRepo] Lazy load large metadata from the DB. * Previously, large values that could not fit into memcached would not be cached and the master DB would be hit for any file loading (djvu files hit this often). Change-Id: I3ed8ad4a85d6e3084330b56c3b48ee76103bd2b8 --- diff --git a/includes/filerepo/file/LocalFile.php b/includes/filerepo/file/LocalFile.php index 6c20eb1109..c08d192590 100644 --- a/includes/filerepo/file/LocalFile.php +++ b/includes/filerepo/file/LocalFile.php @@ -24,7 +24,7 @@ /** * Bump this number when serialized cache records may be incompatible. */ -define( 'MW_FILE_VERSION', 8 ); +define( 'MW_FILE_VERSION', 9 ); /** * Class to represent a local file in the wiki's own database @@ -67,7 +67,8 @@ class LocalFile extends File { $sha1, # SHA-1 base 36 content hash $user, $user_text, # User, who uploaded the file $description, # Description of current revision of the file - $dataLoaded, # Whether or not all this has been loaded from the database (loadFromXxx) + $dataLoaded, # Whether or not core data has been loaded from the database (loadFromXxx) + $extraDataLoaded, # Whether or not lazy-loaded data has been loaded from the database $upgraded, # Whether the row was upgraded on load $locked, # True if the image row is locked $missing, # True if file is not present in file system. Not to be cached in memcached @@ -82,6 +83,8 @@ class LocalFile extends File { protected $repoClass = 'LocalRepo'; + const LOAD_ALL = 1; // integer; load all the lazy fields too (like metadata) + /** * Create a LocalFile from a title * Do not call this except from inside a repo class. @@ -175,6 +178,7 @@ class LocalFile extends File { $this->historyLine = 0; $this->historyRes = null; $this->dataLoaded = false; + $this->extraDataLoaded = false; $this->assertRepoDefined(); $this->assertTitleDefined(); @@ -200,6 +204,7 @@ class LocalFile extends File { wfProfileIn( __METHOD__ ); $this->dataLoaded = false; + $this->extraDataLoaded = false; $key = $this->getCacheKey(); if ( !$key ) { @@ -210,13 +215,17 @@ class LocalFile extends File { $cachedValues = $wgMemc->get( $key ); // Check if the key existed and belongs to this version of MediaWiki - if ( isset( $cachedValues['version'] ) && ( $cachedValues['version'] == MW_FILE_VERSION ) ) { + if ( isset( $cachedValues['version'] ) && $cachedValues['version'] == MW_FILE_VERSION ) { wfDebug( "Pulling file metadata from cache key $key\n" ); $this->fileExists = $cachedValues['fileExists']; if ( $this->fileExists ) { $this->setProps( $cachedValues ); } $this->dataLoaded = true; + $this->extraDataLoaded = true; + foreach ( $this->getLazyCacheFields( '' ) as $field ) { + $this->extraDataLoaded = $this->extraDataLoaded && isset( $cachedValues[$field] ); + } } if ( $this->dataLoaded ) { @@ -252,6 +261,15 @@ class LocalFile extends File { } } + // Strip off excessive entries from the subset of fields that can become large. + // If the cache value gets to large it will not fit in memcached and nothing will + // get cached at all, causing master queries for any file access. + foreach ( $this->getLazyCacheFields( '' ) as $field ) { + if ( isset( $cache[$field] ) && strlen( $cache[$field] ) > 100*1024 ) { + unset( $cache[$field] ); // don't let the value get too big + } + } + // Cache presence for 1 week and negatives for 1 day $wgMemc->set( $key, $cache, $this->fileExists ? 86400 * 7 : 86400 ); } @@ -288,6 +306,28 @@ class LocalFile extends File { return $results[$prefix]; } + /** + * @return array + */ + function getLazyCacheFields( $prefix = 'img_' ) { + static $fields = array( 'metadata' ); + static $results = array(); + + if ( $prefix == '' ) { + return $fields; + } + + if ( !isset( $results[$prefix] ) ) { + $prefixedFields = array(); + foreach ( $fields as $field ) { + $prefixedFields[] = $prefix . $field; + } + $results[$prefix] = $prefixedFields; + } + + return $results[$prefix]; + } + /** * Load file metadata from the DB */ @@ -298,9 +338,9 @@ class LocalFile extends File { # Unconditionally set loaded=true, we don't want the accessors constantly rechecking $this->dataLoaded = true; + $this->extraDataLoaded = true; $dbr = $this->repo->getMasterDB(); - $row = $dbr->selectRow( 'image', $this->getCacheFields( 'img_' ), array( 'img_name' => $this->getName() ), $fname ); @@ -314,14 +354,45 @@ class LocalFile extends File { } /** - * Decode a row from the database (either object or array) to an array - * with timestamps and MIME types decoded, and the field prefix removed. - * @param $row + * Load lazy file metadata from the DB. + * This covers fields that are sometimes not cached. + */ + protected function loadExtraFromDB() { + # Polymorphic function name to distinguish foreign and local fetches + $fname = get_class( $this ) . '::' . __FUNCTION__; + wfProfileIn( $fname ); + + # Unconditionally set loaded=true, we don't want the accessors constantly rechecking + $this->extraDataLoaded = true; + + $dbr = $this->repo->getSlaveDB(); + // In theory the file could have just been renamed/deleted...oh well + $row = $dbr->selectRow( 'image', $this->getLazyCacheFields( 'img_' ), + array( 'img_name' => $this->getName() ), $fname ); + + if ( !$row ) { // fallback to master + $dbr = $this->repo->getMasterDB(); + $row = $dbr->selectRow( 'image', $this->getLazyCacheFields( 'img_' ), + array( 'img_name' => $this->getName() ), $fname ); + } + + if ( $row ) { + foreach ( $this->unprefixRow( $row, 'img_' ) as $name => $value ) { + $this->$name = $value; + } + } else { + throw new MWException( "Could not find data for image '{$this->getName()}'." ); + } + + wfProfileOut( $fname ); + } + + /** + * @param Row $row * @param $prefix string - * @throws MWException - * @return array + * @return Array */ - function decodeRow( $row, $prefix = 'img_' ) { + protected function unprefixRow( $row, $prefix = 'img_' ) { $array = (array)$row; $prefixLength = strlen( $prefix ); @@ -331,10 +402,22 @@ class LocalFile extends File { } $decoded = array(); - foreach ( $array as $name => $value ) { $decoded[substr( $name, $prefixLength )] = $value; } + return $decoded; + } + + /** + * Decode a row from the database (either object or array) to an array + * with timestamps and MIME types decoded, and the field prefix removed. + * @param $row + * @param $prefix string + * @throws MWException + * @return array + */ + function decodeRow( $row, $prefix = 'img_' ) { + $decoded = $this->unprefixRow( $row, $prefix ); $decoded['timestamp'] = wfTimestamp( TS_MW, $decoded['timestamp'] ); @@ -358,6 +441,8 @@ class LocalFile extends File { */ function loadFromRow( $row, $prefix = 'img_' ) { $this->dataLoaded = true; + $this->extraDataLoaded = true; + $array = $this->decodeRow( $row, $prefix ); foreach ( $array as $name => $value ) { @@ -370,8 +455,9 @@ class LocalFile extends File { /** * Load file metadata from cache or DB, unless already loaded + * @param integer $flags */ - function load() { + function load( $flags = 0 ) { if ( !$this->dataLoaded ) { if ( !$this->loadFromCache() ) { $this->loadFromDB(); @@ -379,6 +465,9 @@ class LocalFile extends File { } $this->dataLoaded = true; } + if ( ( $flags & self::LOAD_ALL ) && !$this->extraDataLoaded ) { + $this->loadExtraFromDB(); + } } /** @@ -398,7 +487,7 @@ class LocalFile extends File { } else { $handler = $this->getHandler(); if ( $handler ) { - $validity = $handler->isMetadataValid( $this, $this->metadata ); + $validity = $handler->isMetadataValid( $this, $this->getMetadata() ); if ( $validity === MediaHandler::METADATA_BAD || ( $validity === MediaHandler::METADATA_COMPATIBLE && $wgUpdateCompatibleMetadata ) ) { @@ -465,6 +554,7 @@ class LocalFile extends File { /** * Set properties in this object to be equal to those given in the * associative array $info. Only cacheable fields can be set. + * All fields *must* be set in $info except for getLazyCacheFields(). * * If 'mime' is given, it will be split into major_mime/minor_mime. * If major_mime/minor_mime are given, $this->mime will also be set. @@ -571,7 +661,7 @@ class LocalFile extends File { * @return string */ function getMetadata() { - $this->load(); + $this->load( self::LOAD_ALL ); // large metadata is loaded in another step return $this->metadata; } diff --git a/includes/filerepo/file/OldLocalFile.php b/includes/filerepo/file/OldLocalFile.php index 40d7dca7bb..3180835e46 100644 --- a/includes/filerepo/file/OldLocalFile.php +++ b/includes/filerepo/file/OldLocalFile.php @@ -169,13 +169,14 @@ class OldLocalFile extends LocalFile { function loadFromDB() { wfProfileIn( __METHOD__ ); + $this->dataLoaded = true; $dbr = $this->repo->getSlaveDB(); $conds = array( 'oi_name' => $this->getName() ); if ( is_null( $this->requestedTime ) ) { $conds['oi_archive_name'] = $this->archive_name; } else { - $conds[] = 'oi_timestamp = ' . $dbr->addQuotes( $dbr->timestamp( $this->requestedTime ) ); + $conds['oi_timestamp'] = $dbr->timestamp( $this->requestedTime ); } $row = $dbr->selectRow( 'oldimage', $this->getCacheFields( 'oi_' ), $conds, __METHOD__, array( 'ORDER BY' => 'oi_timestamp DESC' ) ); @@ -184,6 +185,42 @@ class OldLocalFile extends LocalFile { } else { $this->fileExists = false; } + + wfProfileOut( __METHOD__ ); + } + + /** + * Load lazy file metadata from the DB + */ + protected function loadExtraFromDB() { + wfProfileIn( __METHOD__ ); + + $this->extraDataLoaded = true; + $dbr = $this->repo->getSlaveDB(); + $conds = array( 'oi_name' => $this->getName() ); + if ( is_null( $this->requestedTime ) ) { + $conds['oi_archive_name'] = $this->archive_name; + } else { + $conds['oi_timestamp'] = $dbr->timestamp( $this->requestedTime ); + } + // In theory the file could have just been renamed/deleted...oh well + $row = $dbr->selectRow( 'oldimage', $this->getLazyCacheFields( 'oi_' ), + $conds, __METHOD__, array( 'ORDER BY' => 'oi_timestamp DESC' ) ); + + if ( !$row ) { // fallback to master + $dbr = $this->repo->getMasterDB(); + $row = $dbr->selectRow( 'oldimage', $this->getLazyCacheFields( 'oi_' ), + $conds, __METHOD__, array( 'ORDER BY' => 'oi_timestamp DESC' ) ); + } + + if ( $row ) { + foreach ( $this->unprefixRow( $row, 'oi_' ) as $name => $value ) { + $this->$name = $value; + } + } else { + throw new MWException( "Could not find data for image '{$this->archive_name}'." ); + } + wfProfileOut( __METHOD__ ); }