From 65bcb9f329234aef4a5cac65a3dd9cbb60e5f2bb Mon Sep 17 00:00:00 2001 From: "Mark A. Hershberger" Date: Wed, 16 Jun 2010 18:26:47 +0000 Subject: [PATCH] * Create new DatabaseSqlite::checkForEnabledSearch() and use it from SearchSqlite::fulltextSearchSupported() instead of querying the updatelog table. This was the only use of updatelog under the non-maint code. * Add the searchindex table to the list of tables to preserve for testing. * Adapt SearchEngineTest to work with Sqlite. * Add fulltext setup for SQLite to the new installer code. * TODO: SqliteInstaller::setupSearchIndex() should not be using addHtml() --- includes/db/DatabaseSqlite.php | 27 ++++++++++++--- includes/installer/Installer.i18n.php | 3 ++ includes/installer/SqliteInstaller.php | 31 ++++++++++++++--- includes/search/SearchSqlite.php | 42 +++++++++-------------- maintenance/tests/MediaWikiParserTest.php | 1 + maintenance/tests/SearchEngineTest.php | 8 +++-- 6 files changed, 76 insertions(+), 36 deletions(-) diff --git a/includes/db/DatabaseSqlite.php b/includes/db/DatabaseSqlite.php index 59f4fb2aaa..cfd2a37534 100644 --- a/includes/db/DatabaseSqlite.php +++ b/includes/db/DatabaseSqlite.php @@ -12,6 +12,8 @@ */ class DatabaseSqlite extends DatabaseBase { + private static $fulltextEnabled = null; + var $mAffectedRows; var $mLastResult; var $mDatabaseFile; @@ -111,6 +113,22 @@ class DatabaseSqlite extends DatabaseBase { return "$dir/$dbName.sqlite"; } + /** + * Check if the searchindext table is FTS enabled. + * @returns false if not enabled. + */ + function checkForEnabledSearch() { + if ( self::$fulltextEnabled === null ) { + self::$fulltextEnabled = false; + $res = $this->query( "SELECT sql FROM sqlite_master WHERE tbl_name = 'searchindex'", __METHOD__ ); + if ( $res ) { + $row = $res->fetchRow(); + self::$fulltextEnabled = stristr($row['sql'], 'fts' ) !== false; + } + } + return self::$fulltextEnabled; + } + /** * Returns version of currently supported SQLite fulltext search module or false if none present. * @return String @@ -118,6 +136,7 @@ class DatabaseSqlite extends DatabaseBase { function getFulltextSearchModule() { $table = 'dummy_search_test'; $this->query( "DROP TABLE IF EXISTS $table", __METHOD__ ); + if ( $this->query( "CREATE VIRTUAL TABLE $table USING FTS3(dummy_field)", __METHOD__, true ) ) { $this->query( "DROP TABLE IF EXISTS $table", __METHOD__ ); return 'FTS3'; @@ -332,7 +351,7 @@ class DatabaseSqlite extends DatabaseBase { function replace( $table, $uniqueIndexes, $rows, $fname = 'DatabaseSqlite::replace' ) { if ( !count( $rows ) ) return true; - + # SQLite can't handle multi-row replaces, so divide up into multiple single-row queries if ( isset( $rows[0] ) && is_array( $rows[0] ) ) { $ret = true; @@ -498,7 +517,7 @@ class DatabaseSqlite extends DatabaseBase { if ( !$f ) { dieout( "Could not find the interwiki.sql file." ); } - + $sql = "INSERT INTO interwiki(iw_prefix,iw_url,iw_local) VALUES "; while ( !feof( $f ) ) { $line = fgets( $f, 1024 ); @@ -507,7 +526,7 @@ class DatabaseSqlite extends DatabaseBase { $this->query( "$sql $matches[1],$matches[2])" ); } } - + public function getSearchEngine() { return "SearchSqlite"; } @@ -627,7 +646,7 @@ class SQLiteField { return true; } - # isKey(), isMultipleKey() not implemented, MySQL-specific concept. + # isKey(), isMultipleKey() not implemented, MySQL-specific concept. # Suggest removal from base class [TS] function type() { diff --git a/includes/installer/Installer.i18n.php b/includes/installer/Installer.i18n.php index 4da815d612..4ea47d85e9 100644 --- a/includes/installer/Installer.i18n.php +++ b/includes/installer/Installer.i18n.php @@ -257,6 +257,9 @@ Change its permissions so that the webserver can write to it, and try again.', Check the data directory and database name below and try again.', 'config-sqlite-readonly' => 'File $1 is not writeable.', 'config-sqlite-cant-create-db' => 'Could not create database file $1.', + 'config-sqlite-fts3-downgrade' => 'PHP is missing FTS3 support, downgrading tables', + 'config-sqlite-fts3-add' => 'Adding FTS3 search capabilities', + 'config-sqlite-fts3-ok' => 'Fulltext search table appears to be in order', 'config-can-upgrade' => "There are MediaWiki tables in this database. To upgrade them to MediaWiki $1, click '''Continue'''.", 'config-upgrade-done' => "Upgrade complete. diff --git a/includes/installer/SqliteInstaller.php b/includes/installer/SqliteInstaller.php index 5dea22224e..6a54fe1a7c 100644 --- a/includes/installer/SqliteInstaller.php +++ b/includes/installer/SqliteInstaller.php @@ -16,10 +16,10 @@ class SqliteInstaller extends InstallerDBType { function getGlobalDefaults() { if ( isset( $_SERVER['DOCUMENT_ROOT'] ) ) { - $path = str_replace( - array( '/', '\\' ), - DIRECTORY_SEPARATOR, - dirname( $_SERVER['DOCUMENT_ROOT'] ) . '/data' + $path = str_replace( + array( '/', '\\' ), + DIRECTORY_SEPARATOR, + dirname( $_SERVER['DOCUMENT_ROOT'] ) . '/data' ); return array( 'wgSQLiteDataDir' => $path ); } else { @@ -159,10 +159,33 @@ class SqliteInstaller extends InstallerDBType { $this->db->reportQueryError( $err, 0, $sql, __FUNCTION__ ); } //@todo set up searchindex + $this->setupSearchIndex(); // Create default interwikis return Status::newGood(); } + function setupSearchIndex() { + global $IP; + + $module = $this->db->getFulltextSearchModule(); + $fts3tTable = $this->db->checkForEnabledSearch(); + if ( $fts3tTable && !$module ) { + $this->parent->output->addHtml + ( wfMsgHtml( 'word-separator' ) . wfMsgHtml( 'config-sqlite-fts3-downgrade' ) . wfMsgHtml( 'ellipsis' ) ); + $this->parent->output->flush(); + $this->db->sourceFile( "$IP/maintenance/sqlite/archives/searchindex-no-fts.sql" ); + } elseif ( !$fts3tTable && $module == 'FTS3' ) { + $this->parent->output->addHtml + ( wfMsgHtml( 'word-separator' ) . wfMsgHtml( 'config-sqlite-fts3-add' ) . wfMsg( 'ellipsis' ) ); + $this->parent->output->flush(); + $this->db->sourceFile( "$IP/maintenance/sqlite/archives/searchindex-fts3.sql" ); + } else { + $this->parent->output->addHtml + ( wfMsgHtml( 'word-separator' ) . wfMsgHtml( 'config-sqlite-fts3-ok' ) . wfMsgHtml( 'ellipsis' ) ); + $this->parent->output->flush(); + } + } + function doUpgrade() { global $wgDatabase; LBFactory::enableBackend(); diff --git a/includes/search/SearchSqlite.php b/includes/search/SearchSqlite.php index 54a4b5569b..6d8b3c555e 100644 --- a/includes/search/SearchSqlite.php +++ b/includes/search/SearchSqlite.php @@ -26,9 +26,6 @@ * @ingroup Search */ class SearchSqlite extends SearchEngine { - // Cached because SearchUpdate keeps recreating our class - private static $fulltextSupported = null; - /** * Creates an instance of this class * @param $db DatabaseSqlite: database object @@ -42,18 +39,11 @@ class SearchSqlite extends SearchEngine { * @return Boolean */ function fulltextSearchSupported() { - if ( self::$fulltextSupported === null ) { - self::$fulltextSupported = $this->db->selectField( - 'updatelog', - 'ul_key', - array( 'ul_key' => 'fts3' ), - __METHOD__ ) !== false; - } - return self::$fulltextSupported; + return $this->db->checkForEnabledSearch(); } - /** - * Parse the user's query and transform it into an SQL fragment which will + /** + * Parse the user's query and transform it into an SQL fragment which will * become part of a WHERE clause */ function parseQuery( $filteredText, $fulltext ) { @@ -67,7 +57,7 @@ class SearchSqlite extends SearchEngine { $filteredText, $m, PREG_SET_ORDER ) ) { foreach( $m as $bits ) { @list( /* all */, $modifier, $term, $nonQuoted, $wildcard ) = $bits; - + if( $nonQuoted != '' ) { $term = $nonQuoted; $quote = ''; @@ -86,7 +76,7 @@ class SearchSqlite extends SearchEngine { } else { $variants = array( $term ); } - + // The low-level search index does some processing on input to work // around problems with minimum lengths and encoding in MySQL's // fulltext engine. @@ -94,12 +84,12 @@ class SearchSqlite extends SearchEngine { $strippedVariants = array_map( array( $wgContLang, 'normalizeForSearch' ), $variants ); - + // Some languages such as Chinese force all variants to a canonical // form when stripping to the low-level search index, so to be sure // let's check our variants list for unique items after stripping. $strippedVariants = array_unique( $strippedVariants ); - + $searchon .= $modifier; if( count( $strippedVariants) > 1 ) $searchon .= '('; @@ -114,7 +104,7 @@ class SearchSqlite extends SearchEngine { } if( count( $strippedVariants) > 1 ) $searchon .= ')'; - + // Match individual terms or quoted phrase in result highlighting... // Note that variants will be introduced in a later stage for highlighting! $regexp = $this->regexTerm( $term, $wildcard ); @@ -129,10 +119,10 @@ class SearchSqlite extends SearchEngine { $field = $this->getIndexField( $fulltext ); return " $field MATCH '$searchon' "; } - + function regexTerm( $string, $wildcard ) { global $wgContLang; - + $regex = preg_quote( $string, '/' ); if( $wgContLang->hasWordBreaks() ) { if( $wildcard ) { @@ -172,7 +162,7 @@ class SearchSqlite extends SearchEngine { function searchTitle( $term ) { return $this->searchInternal( $term, false ); } - + protected function searchInternal( $term, $fulltext ) { global $wgCountTotalSearchHits, $wgContLang; @@ -182,7 +172,7 @@ class SearchSqlite extends SearchEngine { $filteredTerm = $this->filter( $wgContLang->lc( $term ) ); $resultSet = $this->db->query( $this->getQuery( $filteredTerm, $fulltext ) ); - + $total = null; if( $wgCountTotalSearchHits ) { $totalResult = $this->db->query( $this->getCountQuery( $filteredTerm, $fulltext ) ); @@ -192,7 +182,7 @@ class SearchSqlite extends SearchEngine { } $totalResult->free(); } - + return new SqliteSearchResultSet( $resultSet, $this->searchTerms, $total ); } @@ -226,7 +216,7 @@ class SearchSqlite extends SearchEngine { /** * Returns a query with limit for number of results set. - * @param $sql String: + * @param $sql String: * @return String */ function limitResult( $sql ) { @@ -246,7 +236,7 @@ class SearchSqlite extends SearchEngine { $this->queryNamespaces() ); } - + /** * Picks which field to index on, depending on what type of query. * @param $fulltext Boolean @@ -300,7 +290,7 @@ class SearchSqlite extends SearchEngine { $dbw = wfGetDB( DB_MASTER ); $dbw->delete( 'searchindex', array( 'rowid' => $id ), __METHOD__ ); - + $dbw->insert( 'searchindex', array( 'rowid' => $id, diff --git a/maintenance/tests/MediaWikiParserTest.php b/maintenance/tests/MediaWikiParserTest.php index f9f8782025..651637c870 100644 --- a/maintenance/tests/MediaWikiParserTest.php +++ b/maintenance/tests/MediaWikiParserTest.php @@ -28,6 +28,7 @@ class MediaWikiParserTestSuite extends PHPUnit_Framework_TestSuite { $tables[] = 'logging'; $tables[] = 'updatelog'; $tables[] = 'iwlinks'; + $tables[] = 'searchindex'; return true; } diff --git a/maintenance/tests/SearchEngineTest.php b/maintenance/tests/SearchEngineTest.php index 49a4633c5e..bb119d84cb 100644 --- a/maintenance/tests/SearchEngineTest.php +++ b/maintenance/tests/SearchEngineTest.php @@ -14,7 +14,7 @@ class SearchEngineTest extends MediaWiki_Setup { function insertSearchData() { if ( $this->pageExists( 'Not_Main_Page' ) ) { - return; + return; } $this->insertPage( "Not_Main_Page", "This is not a main page", 0 ); $this->insertPage( 'Talk:Not_Main_Page', 'This is not a talk page to the main page, see [[smithee]]', 1 ); @@ -45,7 +45,11 @@ class SearchEngineTest extends MediaWiki_Setup { } function fetchIds( $results ) { - if ( $this->db->getType() !== 'mysql' ) $this->markTestSkipped( "MySQL only" ); + $this->assertTrue( is_object( $results ) ); + + if ( $this->db->getType() !== 'mysql' && $this->db->getType() !== 'sqlite' ) { + $this->markTestSkipped( "MySQL or SQLite only" ); + } $matches = array(); while ( $row = $results->next() ) { $matches[] = $row->getTitle()->getPrefixedText(); -- 2.20.1