From: Brion VIBBER Date: Thu, 26 Apr 2012 23:40:12 +0000 (+0000) Subject: Merge "[FileBackend] Some documentation and minor cleanups." X-Git-Tag: 1.31.0-rc.0~23804 X-Git-Url: https://git.cyclocoop.org/%27.WWW_URL.%27admin/?a=commitdiff_plain;h=c9480d2eeed7c50c5ca554106706cbd289de0993;hp=-c;p=lhc%2Fweb%2Fwiklou.git Merge "[FileBackend] Some documentation and minor cleanups." --- c9480d2eeed7c50c5ca554106706cbd289de0993 diff --combined includes/filerepo/backend/FileBackend.php index 52baf1e1e4,aea2cd988c..544c9c28dc --- a/includes/filerepo/backend/FileBackend.php +++ b/includes/filerepo/backend/FileBackend.php @@@ -176,8 -176,9 +176,9 @@@ abstract class FileBackend * contents as the new contents to be written there. * * $opts is an associative of boolean flags, including: - * 'force' : Errors that would normally cause a rollback do not. - * The remaining operations are still attempted if any fail. + * 'force' : Operation precondition errors no longer trigger an abort. + * Any remaining operations are still attempted. Unexpected + * failures may still cause remaning operations to be aborted. * 'nonLocking' : No locks are acquired for the operations. * This can increase performance for non-critical writes. * This has no effect unless the 'force' flag is set. @@@ -315,8 -316,8 +316,8 @@@ * otherwise safe from modification from other processes. Normally, * the file will be a new temp file, which should be adequate. * $params include: - * srcs : ordered source storage paths (e.g. chunk1, chunk2, ...) - * dst : file system path to 0-byte temp file + * srcs : ordered source storage paths (e.g. chunk1, chunk2, ...) + * dst : file system path to 0-byte temp file * * @param $params Array Operation parameters * @return Status @@@ -383,8 -384,7 +384,8 @@@ * is that of an empty container, in which case it should be deleted. * * $params include: - * dir : storage directory + * dir : storage directory + * recursive : recursively delete empty subdirectories first (@since 1.20) * * @param $params Array * @return Status @@@ -573,7 -573,7 +574,7 @@@ * * $params include: * dir : storage directory - * topOnly : only return direct child directories of the directory + * topOnly : only return direct child dirs of the directory * * @return Traversable|Array|null Returns null on failure * @since 1.20 @@@ -608,7 -608,7 +609,7 @@@ * * $params include: * dir : storage directory - * topOnly : only return direct child files of the directory + * topOnly : only return direct child files of the directory (@since 1.20) * * @return Traversable|Array|null Returns null on failure */ diff --combined includes/filerepo/backend/FileBackendStore.php index 803f966a02,ba6271f8eb..b87a69f28a --- a/includes/filerepo/backend/FileBackendStore.php +++ b/includes/filerepo/backend/FileBackendStore.php @@@ -19,31 -19,18 +19,31 @@@ * @since 1.19 */ abstract class FileBackendStore extends FileBackend { + /** @var BagOStuff */ + protected $memCache; + /** @var Array Map of paths to small (RAM/disk) cache items */ protected $cache = array(); // (storage path => key => value) - protected $maxCacheSize = 100; // integer; max paths with entries + protected $maxCacheSize = 300; // integer; max paths with entries /** @var Array Map of paths to large (RAM/disk) cache items */ protected $expensiveCache = array(); // (storage path => key => value) - protected $maxExpensiveCacheSize = 10; // integer; max paths with entries + protected $maxExpensiveCacheSize = 5; // integer; max paths with entries /** @var Array Map of container names to sharding settings */ protected $shardViaHashLevels = array(); // (container name => config array) protected $maxFileSize = 4294967296; // integer bytes (4GiB) + /** + * @see FileBackend::__construct() + * + * @param $config Array + */ + public function __construct( array $config ) { + parent::__construct( $config ); + $this->memCache = new EmptyBagOStuff(); // disabled by default + } + /** * Get the maximum allowable file size given backend * medium restrictions and basic performance constraints. @@@ -384,17 -371,6 +384,17 @@@ wfProfileIn( __METHOD__ . '-' . $this->name ); $status = Status::newGood(); + // Recursive: first delete all empty subdirs recursively + if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) { + $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) ); + if ( $subDirsRel !== null ) { // no errors + foreach ( $subDirsRel as $subDirRel ) { + $subDir = $params['dir'] . "/{$subDirRel}"; // full path + $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) ); + } + } + } + list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); @@@ -414,13 -390,11 +414,13 @@@ if ( $shard !== null ) { // confined to a single container/shard $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) ); + $this->deleteContainerCache( $fullCont ); // purge cache } else { // directory is on several shards wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) ); + $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache } } @@@ -711,7 -685,7 +711,7 @@@ /** * @see FileBackend::getDirectoryList() - * @return Array|null|Traversable + * @return Traversable|Array|null Returns null on failure */ final public function getDirectoryList( array $params ) { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); @@@ -738,13 -712,13 +738,13 @@@ * @param $container string Resolved container name * @param $dir string Resolved path relative to container * @param $params Array - * @return Traversable|Array|null + * @return Traversable|Array|null Returns null on failure */ abstract public function getDirectoryListInternal( $container, $dir, array $params ); /** * @see FileBackend::getFileList() - * @return Array|null|Traversable + * @return Traversable|Array|null Returns null on failure */ final public function getFileList( array $params ) { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); @@@ -771,7 -745,7 +771,7 @@@ * @param $container string Resolved container name * @param $dir string Resolved path relative to container * @param $params Array - * @return Traversable|Array|null + * @return Traversable|Array|null Returns null on failure */ abstract public function getFileListInternal( $container, $dir, array $params ); @@@ -872,12 -846,9 +872,12 @@@ } } - // Clear any cache entries (after locks acquired) + // Clear any file cache entries (after locks acquired) $this->clearCache(); + // Load from the persistent container cache + $this->primeContainerCache( $performOps ); + // Actually attempt the operation batch... $subStatus = FileOp::attemptBatch( $performOps, $opts, $this->fileJournal ); @@@ -920,19 -891,11 +920,20 @@@ */ protected function doClearCache( array $paths = null ) {} + /** + * Is this a key/value store where directories are just virtual? + * Virtual directories exists in so much as files exists that are + * prefixed with the directory path followed by a forward slash. + * + * @return bool + */ + abstract protected function directoriesAreVirtual(); + /** * Move a cache entry to the top (such as when accessed) * * @param $path string Storage path + * @return void */ protected function pingCache( $path ) { if ( isset( $this->cache[$path] ) ) { @@@ -958,6 -921,7 +959,7 @@@ * Move a cache entry to the top (such as when accessed) * * @param $path string Storage path + * @return void */ protected function pingExpensiveCache( $path ) { if ( isset( $this->expensiveCache[$path] ) ) { @@@ -1178,87 -1142,6 +1180,87 @@@ protected function resolveContainerPath( $container, $relStoragePath ) { return $relStoragePath; } + + /** + * Get the cache key for a container + * + * @param $container Resolved container name + * @return string + */ + private function containerCacheKey( $container ) { + return wfMemcKey( 'backend', $this->getName(), 'container', $container ); + } + + /** + * Set the cached info for a container + * + * @param $container Resolved container name + * @param $val mixed Information to cache + * @return void + */ + final protected function setContainerCache( $container, $val ) { + $this->memCache->set( $this->containerCacheKey( $container ), $val, 7*86400 ); + } + + /** + * Delete the cached info for a container + * + * @param $container Resolved container name + * @return void + */ + final protected function deleteContainerCache( $container ) { + $this->memCache->delete( $this->containerCacheKey( $container ) ); + } + + /** + * Do a batch lookup from cache for container stats for all containers + * used in a list of container names, storage paths, or FileOp objects. + * + * @param $items Array List of storage paths or FileOps + * @return void + */ + final protected function primeContainerCache( array $items ) { + $paths = array(); // list of storage paths + $contNames = array(); // (cache key => resolved container name) + // Get all the paths/containers from the items... + foreach ( $items as $item ) { + if ( $item instanceof FileOp ) { + $paths = array_merge( $paths, $item->storagePathsRead() ); + $paths = array_merge( $paths, $item->storagePathsChanged() ); + } elseif ( self::isStoragePath( $item ) ) { + $paths[] = $item; + } elseif ( is_string( $item ) ) { // full container name + $contNames[$this->containerCacheKey( $item )] = $item; + } + } + // Get all the corresponding cache keys for paths... + foreach ( $paths as $path ) { + list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path ); + if ( $fullCont !== null ) { // valid path for this backend + $contNames[$this->containerCacheKey( $fullCont )] = $fullCont; + } + } + + $contInfo = array(); // (resolved container name => cache value) + // Get all cache entries for these container cache keys... + $values = $this->memCache->getBatch( array_keys( $contNames ) ); + foreach ( $values as $cacheKey => $val ) { + $contInfo[$contNames[$cacheKey]] = $val; + } + + // Populate the container process cache for the backend... + $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) ); + } + + /** + * Fill the backend-specific process cache given an array of + * resolved container names and their corresponding cached info. + * Only containers that actually exist should appear in the map. + * + * @param $containerInfo Array Map of resolved container names to cached info + * @return void + */ + protected function doPrimeContainerCache( array $containerInfo ) {} } /** @@@ -1358,7 -1241,7 +1360,7 @@@ abstract class FileBackendStoreShardLis * @return bool */ public function valid() { - if ( $this->iter == null ) { + if ( $this->iter === null ) { return false; // some failure? } elseif ( is_array( $this->iter ) ) { return ( current( $this->iter ) !== false ); // no paths can have this value diff --combined includes/filerepo/backend/SwiftFileBackend.php index 435931c790,16a37dc68e..41d30dd697 --- a/includes/filerepo/backend/SwiftFileBackend.php +++ b/includes/filerepo/backend/SwiftFileBackend.php @@@ -24,7 -24,7 +24,7 @@@ class SwiftFileBackend extends FileBack protected $auth; // Swift authentication handler protected $authTTL; // integer seconds protected $swiftAnonUser; // string; username to handle unauthenticated requests - protected $maxContCacheSize = 100; // integer; max containers with entries + protected $maxContCacheSize = 300; // integer; max containers with entries /** @var CF_Connection */ protected $conn; // Swift connection handle @@@ -57,15 -57,13 +57,15 @@@ // Optional settings $this->authTTL = isset( $config['swiftAuthTTL'] ) ? $config['swiftAuthTTL'] - : 120; // some sane number + : 5 * 60; // some sane number $this->swiftAnonUser = isset( $config['swiftAnonUser'] ) ? $config['swiftAnonUser'] : ''; $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] ) ? $config['shardViaHashLevels'] : ''; + // Cache container info to mask latency + $this->memCache = wfGetMainCache(); } /** @@@ -772,14 -770,6 +772,14 @@@ return $tmpFile; } + /** + * @see FileBackendStore::directoriesAreVirtual() + * @return bool + */ + protected function directoriesAreVirtual() { + return true; + } + /** * Get headers to send to Swift when reading a file based * on a FileBackend params array, e.g. that of getLocalCopy(). @@@ -866,30 -856,23 +866,30 @@@ * Use $reCache if the file count or byte count is needed. * * @param $container string Container name - * @param $reCache bool Refresh the process cache + * @param $bypassCache bool Bypass all caches and load from Swift * @return CF_Container + * @throws InvalidResponseException */ - protected function getContainer( $container, $reCache = false ) { + protected function getContainer( $container, $bypassCache = false ) { $conn = $this->getConnection(); // Swift proxy connection - if ( $reCache ) { - unset( $this->connContainers[$container] ); // purge cache + if ( $bypassCache ) { // purge cache + unset( $this->connContainers[$container] ); + } elseif ( !isset( $this->connContainers[$container] ) ) { + $this->primeContainerCache( array( $container ) ); // check persistent cache } if ( !isset( $this->connContainers[$container] ) ) { $contObj = $conn->get_container( $container ); // NoSuchContainerException not thrown: container must exist if ( count( $this->connContainers ) >= $this->maxContCacheSize ) { // trim cache? reset( $this->connContainers ); - $key = key( $this->connContainers ); - unset( $this->connContainers[$key] ); + unset( $this->connContainers[key( $this->connContainers )] ); } $this->connContainers[$container] = $contObj; // cache it + if ( !$bypassCache ) { + $this->setContainerCache( $container, // update persistent cache + array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count ) + ); + } } return $this->connContainers[$container]; } @@@ -899,7 -882,6 +899,7 @@@ * * @param $container string Container name * @return CF_Container + * @throws InvalidResponseException */ protected function createContainer( $container ) { $conn = $this->getConnection(); // Swift proxy connection @@@ -913,7 -895,6 +913,7 @@@ * * @param $container string Container name * @return void + * @throws InvalidResponseException */ protected function deleteContainer( $container ) { $conn = $this->getConnection(); // Swift proxy connection @@@ -921,28 -902,6 +921,28 @@@ unset( $this->connContainers[$container] ); // purge cache } + /** + * @see FileBackendStore::doPrimeContainerCache() + * @return void + */ + protected function doPrimeContainerCache( array $containerInfo ) { + try { + $conn = $this->getConnection(); // Swift proxy connection + foreach ( $containerInfo as $container => $info ) { + $this->connContainers[$container] = new CF_Container( + $conn->cfs_auth, + $conn->cfs_http, + $container, + $info['count'], + $info['bytes'] + ); + } + } catch ( InvalidResponseException $e ) { + } catch ( Exception $e ) { // some other exception? + $this->logException( $e, __METHOD__, array() ); + } + } + /** * Log an unexpected exception for this backend * @@@ -1048,7 -1007,11 +1048,11 @@@ abstract class SwiftFileBackendList imp * @return bool */ public function valid() { - return ( current( $this->bufferIter ) !== false ); // no paths can have this value + if ( $this->bufferIter === null ) { + return false; // some failure? + } else { + return ( current( $this->bufferIter ) !== false ); // no paths can have this value + } } /** @@@ -1059,7 -1022,7 +1063,7 @@@ * @param $after string|null * @param $limit integer * @param $params Array - * @return Traversable|Array|null + * @return Traversable|Array|null Returns null on failure */ abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params ); } @@@ -1078,7 -1041,7 +1082,7 @@@ class SwiftFileBackendDirList extends S /** * @see SwiftFileBackendList::pageFromList() - * @return Array + * @return Array|null */ protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params ); @@@ -1099,7 -1062,7 +1103,7 @@@ class SwiftFileBackendFileList extends /** * @see SwiftFileBackendList::pageFromList() - * @return Array + * @return Array|null */ protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );