This avoids slave lag and makes query time account easier.
It also avoids table-level autoinc locking and slave drift
with statement-based replication in some setups.
Also refactored the use of $wgCommandLine mode in
DatabaseBase slightly, so that it can be injected.
Change-Id: I2dba6024ecf32c9ee24a3080cce3b02568c1458b
* to several groups, the most specific group defined here is used.
*
* - flags: bit field
- * - DBO_DEFAULT -- turns on DBO_TRX only if !$wgCommandLineMode (recommended)
+ * - DBO_DEFAULT -- turns on DBO_TRX only if "cliMode" is off (recommended)
* - DBO_DEBUG -- equivalent of $wgDebugDumpSql
* - DBO_TRX -- wrap entire request in a transaction
* - DBO_NOBUFFER -- turn off buffering (not useful in LocalSettings.php)
*
* - max lag: (optional) Maximum replication lag before a slave will taken out of rotation
* - is static: (optional) Set to true if the dataset is static and no replication is used.
+ * - cliMode: (optional) Connection handles will not assume that requests are short-lived
+ * nor that INSERT..SELECT can be rewritten into a buffered SELECT and INSERT.
+ * [Default: uses value of $wgCommandLineMode]
*
* These and any other user-defined properties will be assigned to the mLBInfo member
* variable of the Database object.
protected $mPassword;
/** @var string */
protected $mDBname;
+ /** @var bool */
+ protected $cliMode;
/** @var BagOStuff APC cache */
protected $srvCache;
* @param array $params Parameters passed from DatabaseBase::factory()
*/
function __construct( array $params ) {
- global $wgDBprefix, $wgDBmwschema, $wgCommandLineMode;
+ global $wgDBprefix, $wgDBmwschema;
$this->srvCache = ObjectCache::getLocalServerInstance( 'hash' );
$schema = $params['schema'];
$foreign = $params['foreign'];
+ $this->cliMode = isset( $params['cliMode'] )
+ ? $params['cliMode']
+ : ( PHP_SAPI === 'cli' );
+
$this->mFlags = $flags;
if ( $this->mFlags & DBO_DEFAULT ) {
- if ( $wgCommandLineMode ) {
+ if ( $this->cliMode ) {
$this->mFlags &= ~DBO_TRX;
} else {
$this->mFlags |= DBO_TRX;
* @return DatabaseBase|null DatabaseBase subclass or null
*/
final public static function factory( $dbType, $p = [] ) {
+ global $wgCommandLineMode;
+
$canonicalDBTypes = [
'mysql' => [ 'mysqli', 'mysql' ],
'postgres' => [],
$p['schema'] = isset( $defaultSchemas[$dbType] ) ? $defaultSchemas[$dbType] : null;
}
$p['foreign'] = isset( $p['foreign'] ) ? $p['foreign'] : false;
+ $p['cliMode'] = $wgCommandLineMode;
return new $class( $p );
} else {
* queries, like inserting a row can take a long time due to row locking. This method
* uses some simple heuristics to discount those cases.
*
- * @param string $sql
+ * @param string $sql A SQL write query
* @param float $runtime Total runtime, including RTT
*/
private function updateTrxWriteQueryTime( $sql, $runtime ) {
return $this->query( $sql, $fname );
}
- public function insertSelect( $destTable, $srcTable, $varMap, $conds,
+ public function insertSelect(
+ $destTable, $srcTable, $varMap, $conds,
+ $fname = __METHOD__, $insertOptions = [], $selectOptions = []
+ ) {
+ if ( $this->cliMode ) {
+ // For massive migrations with downtime, we don't want to select everything
+ // into memory and OOM, so do all this native on the server side if possible.
+ return $this->nativeInsertSelect(
+ $destTable,
+ $srcTable,
+ $varMap,
+ $conds,
+ $fname,
+ $insertOptions,
+ $selectOptions
+ );
+ }
+
+ // For web requests, do a locking SELECT and then INSERT. This puts the SELECT burden
+ // on only the master (without needing row-based-replication). It also makes it easy to
+ // know how big the INSERT is going to be.
+ $fields = [];
+ foreach ( $varMap as $dstColumn => $sourceColumnOrSql ) {
+ $fields[] = $this->fieldNameWithAlias( $sourceColumnOrSql, $dstColumn );
+ }
+ $selectOptions[] = 'FOR UPDATE';
+ $res = $this->select( $srcTable, implode( ',', $fields ), $conds, $fname, $selectOptions );
+ if ( !$res ) {
+ return false;
+ }
+
+ $rows = [];
+ foreach ( $res as $row ) {
+ $rows[] = (array)$row;
+ }
+
+ return $this->insert( $destTable, $rows, $fname, $insertOptions );
+ }
+
+ public function nativeInsertSelect( $destTable, $srcTable, $varMap, $conds,
$fname = __METHOD__,
$insertOptions = [], $selectOptions = []
) {
* @return null|ResultWrapper
* @throws Exception
*/
- public function insertSelect( $destTable, $srcTable, $varMap, $conds, $fname = __METHOD__,
+ public function nativeInsertSelect( $destTable, $srcTable, $varMap, $conds, $fname = __METHOD__,
$insertOptions = [], $selectOptions = []
) {
$this->mScrollableCursor = false;
try {
- $ret = parent::insertSelect(
+ $ret = parent::nativeInsertSelect(
$destTable,
$srcTable,
$varMap,
return oci_free_statement( $stmt );
}
- function insertSelect( $destTable, $srcTable, $varMap, $conds, $fname = __METHOD__,
+ function nativeInsertSelect( $destTable, $srcTable, $varMap, $conds, $fname = __METHOD__,
$insertOptions = [], $selectOptions = []
) {
$destTable = $this->tableName( $destTable );
* @param array $selectOptions
* @return bool
*/
- function insertSelect( $destTable, $srcTable, $varMap, $conds, $fname = __METHOD__,
+ function nativeInsertSelect( $destTable, $srcTable, $varMap, $conds, $fname = __METHOD__,
$insertOptions = [], $selectOptions = [] ) {
$destTable = $this->tableName( $destTable );
* This is a non DBMS depending test.
*/
class DatabaseSQLTest extends MediaWikiTestCase {
-
- /**
- * @var DatabaseTestHelper
- */
+ /** @var DatabaseTestHelper */
private $database;
protected function setUp() {
parent::setUp();
- $this->database = new DatabaseTestHelper( __CLASS__ );
+ $this->database = new DatabaseTestHelper( __CLASS__, [ 'cliMode' => true ] );
}
protected function assertLastSql( $sqlText ) {
);
}
+ protected function assertLastSqlDb( $sqlText, $db ) {
+ $this->assertEquals( $db->getLastSqls(), $sqlText );
+ }
+
/**
* @dataProvider provideSelect
* @covers DatabaseBase::select
* @dataProvider provideInsertSelect
* @covers DatabaseBase::insertSelect
*/
- public function testInsertSelect( $sql, $sqlText ) {
+ public function testInsertSelect( $sql, $sqlTextNative, $sqlSelect, $sqlInsert ) {
$this->database->insertSelect(
$sql['destTable'],
$sql['srcTable'],
isset( $sql['insertOptions'] ) ? $sql['insertOptions'] : [],
isset( $sql['selectOptions'] ) ? $sql['selectOptions'] : []
);
- $this->assertLastSql( $sqlText );
+ $this->assertLastSql( $sqlTextNative );
+
+ $dbWeb = new DatabaseTestHelper( __CLASS__, [ 'cliMode' => false ] );
+ $dbWeb->forceNextResult( [
+ array_flip( array_keys( $sql['varMap'] ) )
+ ] );
+ $dbWeb->insertSelect(
+ $sql['destTable'],
+ $sql['srcTable'],
+ $sql['varMap'],
+ $sql['conds'],
+ __METHOD__,
+ isset( $sql['insertOptions'] ) ? $sql['insertOptions'] : [],
+ isset( $sql['selectOptions'] ) ? $sql['selectOptions'] : []
+ );
+ $this->assertLastSqlDb( implode( '; ', [ $sqlSelect, $sqlInsert ] ), $dbWeb );
}
public static function provideInsertSelect() {
"INSERT INTO insert_table " .
"(field_insert,field) " .
"SELECT field_select,field2 " .
- "FROM select_table"
+ "FROM select_table",
+ "SELECT field_select AS field_insert,field2 AS field " .
+ "FROM select_table WHERE * FOR UPDATE",
+ "INSERT INTO insert_table (field_insert,field) VALUES ('0','1')"
],
[
[
"(field_insert,field) " .
"SELECT field_select,field2 " .
"FROM select_table " .
- "WHERE field = '2'"
+ "WHERE field = '2'",
+ "SELECT field_select AS field_insert,field2 AS field FROM " .
+ "select_table WHERE field = '2' FOR UPDATE",
+ "INSERT INTO insert_table (field_insert,field) VALUES ('0','1')"
],
[
[
"SELECT field_select,field2 " .
"FROM select_table " .
"WHERE field = '2' " .
- "ORDER BY field"
+ "ORDER BY field",
+ "SELECT field_select AS field_insert,field2 AS field " .
+ "FROM select_table WHERE field = '2' ORDER BY field FOR UPDATE",
+ "INSERT IGNORE INTO insert_table (field_insert,field) VALUES ('0','1')"
],
];
}
*/
protected $lastSqls = [];
+ /** @var array List of row arrays */
+ protected $nextResult = [];
+
/**
* Array of tables to be considered as existing by tableExist()
* Use setExistingTables() to alter.
*/
protected $tablesExists;
- public function __construct( $testName ) {
+ public function __construct( $testName, array $opts = [] ) {
$this->testName = $testName;
$this->profiler = new ProfilerStub( [] );
$this->trxProfiler = new TransactionProfiler();
+ $this->cliMode = isset( $opts['cliMode'] ) ? $opts['cliMode'] : true;
}
/**
$this->tablesExists = (array)$tablesExists;
}
+ /**
+ * @param mixed $res Use an array of row arrays to set row result
+ */
+ public function forceNextResult( $res ) {
+ $this->nextResult = $res;
+ }
+
protected function addSql( $sql ) {
// clean up spaces before and after some words and the whole string
$this->lastSqls[] = trim( preg_replace(
}
protected function doQuery( $sql ) {
- return [];
+ $res = $this->nextResult;
+ $this->nextResult = [];
+
+ return new FakeResultWrapper( $res );
}
}