From: Aaron Schulz Date: Mon, 10 Dec 2012 20:43:35 +0000 (-0800) Subject: Moved ExternalStore stuff to a /externalstore subdir. X-Git-Tag: 1.31.0-rc.0~21342 X-Git-Url: http://git.cyclocoop.org/%22.%24h.%22?a=commitdiff_plain;h=cbc28d41ae116c68704b4daf55155142beab988e;p=lhc%2Fweb%2Fwiklou.git Moved ExternalStore stuff to a /externalstore subdir. Change-Id: If631040d8242354734280403258ed5d51542728d --- diff --git a/includes/AutoLoader.php b/includes/AutoLoader.php index 3d0c27ace0..f71857dbcc 100644 --- a/includes/AutoLoader.php +++ b/includes/AutoLoader.php @@ -90,9 +90,9 @@ $wgAutoloadLocalClasses = array( 'ErrorPageError' => 'includes/Exception.php', 'ExplodeIterator' => 'includes/StringUtils.php', 'ExternalEdit' => 'includes/ExternalEdit.php', - 'ExternalStore' => 'includes/ExternalStore.php', - 'ExternalStoreDB' => 'includes/ExternalStoreDB.php', - 'ExternalStoreHttp' => 'includes/ExternalStoreHttp.php', + 'ExternalStore' => 'includes/externalstore/ExternalStore.php', + 'ExternalStoreDB' => 'includes/externalstore/ExternalStoreDB.php', + 'ExternalStoreHttp' => 'includes/externalstore/ExternalStoreHttp.php', 'ExternalUser' => 'includes/ExternalUser.php', 'FakeTitle' => 'includes/FakeTitle.php', 'Fallback' => 'includes/Fallback.php', diff --git a/includes/ExternalStore.php b/includes/ExternalStore.php deleted file mode 100644 index 1b7c29dbbb..0000000000 --- a/includes/ExternalStore.php +++ /dev/null @@ -1,182 +0,0 @@ -mParams = $params; - } - - /** - * Fetch data from given URL - * - * @param $url String: The URL of the text to get - * @param $params Array: associative array of parameters for the ExternalStore object. - * @return string|bool The text stored or false on error - */ - static function fetchFromURL( $url, $params = array() ) { - global $wgExternalStores; - - if( !$wgExternalStores ) { - return false; - } - - $parts = explode( '://', $url, 2 ); - - if ( count( $parts ) != 2 ) { - return false; - } - - list( $proto, $path ) = $parts; - - if ( $path == '' ) { // Bad URL - return false; - } - - $store = self::getStoreObject( $proto, $params ); - if ( $store === false ) { - return false; - } - - return $store->fetchFromURL( $url ); - } - - /** - * Get an external store object of the given type, with the given parameters - * - * @param $proto String: type of external storage, should be a value in $wgExternalStores - * @param $params Array: associative array of parameters for the ExternalStore object. - * @return ExternalStore|bool ExternalStore class or false on error - */ - static function getStoreObject( $proto, $params = array() ) { - global $wgExternalStores; - if( !$wgExternalStores ) { - return false; - } - - /* Protocol not enabled */ - if( !in_array( $proto, $wgExternalStores ) ) { - return false; - } - - $class = 'ExternalStore' . ucfirst( $proto ); - /* Any custom modules should be added to $wgAutoLoadClasses for on-demand loading */ - if( !MWInit::classExists( $class ) ) { - return false; - } - - return new $class($params); - } - - /** - * Store a data item to an external store, identified by a partial URL - * The protocol part is used to identify the class, the rest is passed to the - * class itself as a parameter. - * @param $url - * @param $data - * @param $params array - * @return string|bool The URL of the stored data item, or false on error - */ - static function insert( $url, $data, $params = array() ) { - list( $proto, $params ) = explode( '://', $url, 2 ); - $store = self::getStoreObject( $proto, $params ); - if ( $store === false ) { - return false; - } else { - return $store->store( $params, $data ); - } - } - - /** - * Like insert() above, but does more of the work for us. - * This function does not need a url param, it builds it by - * itself. It also fails-over to the next possible clusters. - * - * @param $data String - * @param $storageParams Array: associative array of parameters for the ExternalStore object. - * @throws MWException|DBConnectionError|DBQueryError - * @return string|bool The URL of the stored data item, or false on error - */ - public static function insertToDefault( $data, $storageParams = array() ) { - global $wgDefaultExternalStore; - $tryStores = (array)$wgDefaultExternalStore; - $error = false; - while ( count( $tryStores ) > 0 ) { - $index = mt_rand(0, count( $tryStores ) - 1); - $storeUrl = $tryStores[$index]; - wfDebug( __METHOD__.": trying $storeUrl\n" ); - list( $proto, $params ) = explode( '://', $storeUrl, 2 ); - $store = self::getStoreObject( $proto, $storageParams ); - if ( $store === false ) { - throw new MWException( "Invalid external storage protocol - $storeUrl" ); - } - try { - $url = $store->store( $params, $data ); // Try to save the object - } catch ( DBConnectionError $error ) { - $url = false; - } catch( DBQueryError $error ) { - $url = false; - } - if ( $url ) { - return $url; // Done! - } else { - unset( $tryStores[$index] ); // Don't try this one again! - $tryStores = array_values( $tryStores ); // Must have consecutive keys - wfDebugLog( 'ExternalStorage', "Unable to store text to external storage $storeUrl" ); - } - } - // All stores failed - if ( $error ) { - // Rethrow the last connection error - throw $error; - } else { - throw new MWException( "Unable to store text to external storage" ); - } - } - - /** - * @param $data - * @param $wiki - * - * @return string - */ - public static function insertToForeignDefault( $data, $wiki ) { - return self::insertToDefault( $data, array( 'wiki' => $wiki ) ); - } -} diff --git a/includes/ExternalStoreDB.php b/includes/ExternalStoreDB.php deleted file mode 100644 index 37b1b9335d..0000000000 --- a/includes/ExternalStoreDB.php +++ /dev/null @@ -1,189 +0,0 @@ -mParams = $params; - } - - /** - * Get a LoadBalancer for the specified cluster - * - * @param $cluster String: cluster name - * @return LoadBalancer object - */ - function &getLoadBalancer( $cluster ) { - $wiki = isset($this->mParams['wiki']) ? $this->mParams['wiki'] : false; - - return wfGetLBFactory()->getExternalLB( $cluster, $wiki ); - } - - /** - * Get a slave database connection for the specified cluster - * - * @param $cluster String: cluster name - * @return DatabaseBase object - */ - function &getSlave( $cluster ) { - global $wgDefaultExternalStore; - - $wiki = isset($this->mParams['wiki']) ? $this->mParams['wiki'] : false; - $lb =& $this->getLoadBalancer( $cluster ); - - if ( !in_array( "DB://" . $cluster, (array)$wgDefaultExternalStore ) ) { - wfDebug( "read only external store" ); - $lb->allowLagged(true); - } else { - wfDebug( "writable external store" ); - } - - return $lb->getConnection( DB_SLAVE, array(), $wiki ); - } - - /** - * Get a master database connection for the specified cluster - * - * @param $cluster String: cluster name - * @return DatabaseBase object - */ - function &getMaster( $cluster ) { - $wiki = isset($this->mParams['wiki']) ? $this->mParams['wiki'] : false; - $lb =& $this->getLoadBalancer( $cluster ); - return $lb->getConnection( DB_MASTER, array(), $wiki ); - } - - /** - * Get the 'blobs' table name for this database - * - * @param $db DatabaseBase - * @return String: table name ('blobs' by default) - */ - function getTable( &$db ) { - $table = $db->getLBInfo( 'blobs table' ); - if ( is_null( $table ) ) { - $table = 'blobs'; - } - return $table; - } - - /** - * Fetch data from given URL - * @param $url String: an url of the form DB://cluster/id or DB://cluster/id/itemid for concatened storage. - * @return mixed - */ - function fetchFromURL( $url ) { - $path = explode( '/', $url ); - $cluster = $path[2]; - $id = $path[3]; - if ( isset( $path[4] ) ) { - $itemID = $path[4]; - } else { - $itemID = false; - } - - $ret =& $this->fetchBlob( $cluster, $id, $itemID ); - - if ( $itemID !== false && $ret !== false ) { - return $ret->getItem( $itemID ); - } - return $ret; - } - - /** - * Fetch a blob item out of the database; a cache of the last-loaded - * blob will be kept so that multiple loads out of a multi-item blob - * can avoid redundant database access and decompression. - * @param $cluster - * @param $id - * @param $itemID - * @return mixed - * @private - */ - function &fetchBlob( $cluster, $id, $itemID ) { - /** - * One-step cache variable to hold base blobs; operations that - * pull multiple revisions may often pull multiple times from - * the same blob. By keeping the last-used one open, we avoid - * redundant unserialization and decompression overhead. - */ - static $externalBlobCache = array(); - - $cacheID = ( $itemID === false ) ? "$cluster/$id" : "$cluster/$id/"; - if( isset( $externalBlobCache[$cacheID] ) ) { - wfDebug( "ExternalStoreDB::fetchBlob cache hit on $cacheID\n" ); - return $externalBlobCache[$cacheID]; - } - - wfDebug( "ExternalStoreDB::fetchBlob cache miss on $cacheID\n" ); - - $dbr =& $this->getSlave( $cluster ); - $ret = $dbr->selectField( $this->getTable( $dbr ), 'blob_text', array( 'blob_id' => $id ), __METHOD__ ); - if ( $ret === false ) { - wfDebugLog( 'ExternalStoreDB', "ExternalStoreDB::fetchBlob master fallback on $cacheID\n" ); - // Try the master - $dbw =& $this->getMaster( $cluster ); - $ret = $dbw->selectField( $this->getTable( $dbw ), 'blob_text', array( 'blob_id' => $id ), __METHOD__ ); - if( $ret === false) { - wfDebugLog( 'ExternalStoreDB', "ExternalStoreDB::fetchBlob master failed to find $cacheID\n" ); - } - } - if( $itemID !== false && $ret !== false ) { - // Unserialise object; caller extracts item - $ret = unserialize( $ret ); - } - - $externalBlobCache = array( $cacheID => &$ret ); - return $ret; - } - - /** - * Insert a data item into a given cluster - * - * @param $cluster String: the cluster name - * @param $data String: the data item - * @throws MWException - * @return string URL - */ - function store( $cluster, $data ) { - $dbw = $this->getMaster( $cluster ); - $id = $dbw->nextSequenceValue( 'blob_blob_id_seq' ); - $dbw->insert( $this->getTable( $dbw ), - array( 'blob_id' => $id, 'blob_text' => $data ), - __METHOD__ ); - $id = $dbw->insertId(); - if ( !$id ) { - throw new MWException( __METHOD__.': no insert ID' ); - } - if ( $dbw->getFlag( DBO_TRX ) ) { - $dbw->commit( __METHOD__ ); - } - return "DB://$cluster/$id"; - } -} diff --git a/includes/ExternalStoreHttp.php b/includes/ExternalStoreHttp.php deleted file mode 100644 index 311e32b38c..0000000000 --- a/includes/ExternalStoreHttp.php +++ /dev/null @@ -1,45 +0,0 @@ -mParams = $params; + } + + /** + * Fetch data from given URL + * + * @param $url String: The URL of the text to get + * @param $params Array: associative array of parameters for the ExternalStore object. + * @return string|bool The text stored or false on error + */ + static function fetchFromURL( $url, $params = array() ) { + global $wgExternalStores; + + if( !$wgExternalStores ) { + return false; + } + + $parts = explode( '://', $url, 2 ); + + if ( count( $parts ) != 2 ) { + return false; + } + + list( $proto, $path ) = $parts; + + if ( $path == '' ) { // Bad URL + return false; + } + + $store = self::getStoreObject( $proto, $params ); + if ( $store === false ) { + return false; + } + + return $store->fetchFromURL( $url ); + } + + /** + * Get an external store object of the given type, with the given parameters + * + * @param $proto String: type of external storage, should be a value in $wgExternalStores + * @param $params Array: associative array of parameters for the ExternalStore object. + * @return ExternalStore|bool ExternalStore class or false on error + */ + static function getStoreObject( $proto, $params = array() ) { + global $wgExternalStores; + if( !$wgExternalStores ) { + return false; + } + + /* Protocol not enabled */ + if( !in_array( $proto, $wgExternalStores ) ) { + return false; + } + + $class = 'ExternalStore' . ucfirst( $proto ); + /* Any custom modules should be added to $wgAutoLoadClasses for on-demand loading */ + if( !MWInit::classExists( $class ) ) { + return false; + } + + return new $class($params); + } + + /** + * Store a data item to an external store, identified by a partial URL + * The protocol part is used to identify the class, the rest is passed to the + * class itself as a parameter. + * @param $url + * @param $data + * @param $params array + * @return string|bool The URL of the stored data item, or false on error + */ + static function insert( $url, $data, $params = array() ) { + list( $proto, $params ) = explode( '://', $url, 2 ); + $store = self::getStoreObject( $proto, $params ); + if ( $store === false ) { + return false; + } else { + return $store->store( $params, $data ); + } + } + + /** + * Like insert() above, but does more of the work for us. + * This function does not need a url param, it builds it by + * itself. It also fails-over to the next possible clusters. + * + * @param $data String + * @param $storageParams Array: associative array of parameters for the ExternalStore object. + * @throws MWException|DBConnectionError|DBQueryError + * @return string|bool The URL of the stored data item, or false on error + */ + public static function insertToDefault( $data, $storageParams = array() ) { + global $wgDefaultExternalStore; + $tryStores = (array)$wgDefaultExternalStore; + $error = false; + while ( count( $tryStores ) > 0 ) { + $index = mt_rand(0, count( $tryStores ) - 1); + $storeUrl = $tryStores[$index]; + wfDebug( __METHOD__.": trying $storeUrl\n" ); + list( $proto, $params ) = explode( '://', $storeUrl, 2 ); + $store = self::getStoreObject( $proto, $storageParams ); + if ( $store === false ) { + throw new MWException( "Invalid external storage protocol - $storeUrl" ); + } + try { + $url = $store->store( $params, $data ); // Try to save the object + } catch ( DBConnectionError $error ) { + $url = false; + } catch( DBQueryError $error ) { + $url = false; + } + if ( $url ) { + return $url; // Done! + } else { + unset( $tryStores[$index] ); // Don't try this one again! + $tryStores = array_values( $tryStores ); // Must have consecutive keys + wfDebugLog( 'ExternalStorage', "Unable to store text to external storage $storeUrl" ); + } + } + // All stores failed + if ( $error ) { + // Rethrow the last connection error + throw $error; + } else { + throw new MWException( "Unable to store text to external storage" ); + } + } + + /** + * @param $data + * @param $wiki + * + * @return string + */ + public static function insertToForeignDefault( $data, $wiki ) { + return self::insertToDefault( $data, array( 'wiki' => $wiki ) ); + } +} diff --git a/includes/externalstore/ExternalStoreDB.php b/includes/externalstore/ExternalStoreDB.php new file mode 100644 index 0000000000..37b1b9335d --- /dev/null +++ b/includes/externalstore/ExternalStoreDB.php @@ -0,0 +1,189 @@ +mParams = $params; + } + + /** + * Get a LoadBalancer for the specified cluster + * + * @param $cluster String: cluster name + * @return LoadBalancer object + */ + function &getLoadBalancer( $cluster ) { + $wiki = isset($this->mParams['wiki']) ? $this->mParams['wiki'] : false; + + return wfGetLBFactory()->getExternalLB( $cluster, $wiki ); + } + + /** + * Get a slave database connection for the specified cluster + * + * @param $cluster String: cluster name + * @return DatabaseBase object + */ + function &getSlave( $cluster ) { + global $wgDefaultExternalStore; + + $wiki = isset($this->mParams['wiki']) ? $this->mParams['wiki'] : false; + $lb =& $this->getLoadBalancer( $cluster ); + + if ( !in_array( "DB://" . $cluster, (array)$wgDefaultExternalStore ) ) { + wfDebug( "read only external store" ); + $lb->allowLagged(true); + } else { + wfDebug( "writable external store" ); + } + + return $lb->getConnection( DB_SLAVE, array(), $wiki ); + } + + /** + * Get a master database connection for the specified cluster + * + * @param $cluster String: cluster name + * @return DatabaseBase object + */ + function &getMaster( $cluster ) { + $wiki = isset($this->mParams['wiki']) ? $this->mParams['wiki'] : false; + $lb =& $this->getLoadBalancer( $cluster ); + return $lb->getConnection( DB_MASTER, array(), $wiki ); + } + + /** + * Get the 'blobs' table name for this database + * + * @param $db DatabaseBase + * @return String: table name ('blobs' by default) + */ + function getTable( &$db ) { + $table = $db->getLBInfo( 'blobs table' ); + if ( is_null( $table ) ) { + $table = 'blobs'; + } + return $table; + } + + /** + * Fetch data from given URL + * @param $url String: an url of the form DB://cluster/id or DB://cluster/id/itemid for concatened storage. + * @return mixed + */ + function fetchFromURL( $url ) { + $path = explode( '/', $url ); + $cluster = $path[2]; + $id = $path[3]; + if ( isset( $path[4] ) ) { + $itemID = $path[4]; + } else { + $itemID = false; + } + + $ret =& $this->fetchBlob( $cluster, $id, $itemID ); + + if ( $itemID !== false && $ret !== false ) { + return $ret->getItem( $itemID ); + } + return $ret; + } + + /** + * Fetch a blob item out of the database; a cache of the last-loaded + * blob will be kept so that multiple loads out of a multi-item blob + * can avoid redundant database access and decompression. + * @param $cluster + * @param $id + * @param $itemID + * @return mixed + * @private + */ + function &fetchBlob( $cluster, $id, $itemID ) { + /** + * One-step cache variable to hold base blobs; operations that + * pull multiple revisions may often pull multiple times from + * the same blob. By keeping the last-used one open, we avoid + * redundant unserialization and decompression overhead. + */ + static $externalBlobCache = array(); + + $cacheID = ( $itemID === false ) ? "$cluster/$id" : "$cluster/$id/"; + if( isset( $externalBlobCache[$cacheID] ) ) { + wfDebug( "ExternalStoreDB::fetchBlob cache hit on $cacheID\n" ); + return $externalBlobCache[$cacheID]; + } + + wfDebug( "ExternalStoreDB::fetchBlob cache miss on $cacheID\n" ); + + $dbr =& $this->getSlave( $cluster ); + $ret = $dbr->selectField( $this->getTable( $dbr ), 'blob_text', array( 'blob_id' => $id ), __METHOD__ ); + if ( $ret === false ) { + wfDebugLog( 'ExternalStoreDB', "ExternalStoreDB::fetchBlob master fallback on $cacheID\n" ); + // Try the master + $dbw =& $this->getMaster( $cluster ); + $ret = $dbw->selectField( $this->getTable( $dbw ), 'blob_text', array( 'blob_id' => $id ), __METHOD__ ); + if( $ret === false) { + wfDebugLog( 'ExternalStoreDB', "ExternalStoreDB::fetchBlob master failed to find $cacheID\n" ); + } + } + if( $itemID !== false && $ret !== false ) { + // Unserialise object; caller extracts item + $ret = unserialize( $ret ); + } + + $externalBlobCache = array( $cacheID => &$ret ); + return $ret; + } + + /** + * Insert a data item into a given cluster + * + * @param $cluster String: the cluster name + * @param $data String: the data item + * @throws MWException + * @return string URL + */ + function store( $cluster, $data ) { + $dbw = $this->getMaster( $cluster ); + $id = $dbw->nextSequenceValue( 'blob_blob_id_seq' ); + $dbw->insert( $this->getTable( $dbw ), + array( 'blob_id' => $id, 'blob_text' => $data ), + __METHOD__ ); + $id = $dbw->insertId(); + if ( !$id ) { + throw new MWException( __METHOD__.': no insert ID' ); + } + if ( $dbw->getFlag( DBO_TRX ) ) { + $dbw->commit( __METHOD__ ); + } + return "DB://$cluster/$id"; + } +} diff --git a/includes/externalstore/ExternalStoreHttp.php b/includes/externalstore/ExternalStoreHttp.php new file mode 100644 index 0000000000..311e32b38c --- /dev/null +++ b/includes/externalstore/ExternalStoreHttp.php @@ -0,0 +1,45 @@ +