includes/SearchEngine.php

   1 <?php
   2 /**
   3  * @defgroup Search Search
   4  *
   5  * @file
   6  * @ingroup Search
   7  */
   8
   9 /**
  10  * Contain a class for special pages
  11  * @ingroup Search
  12  */
  13 class SearchEngine {
  14         var $limit = 10;
  15         var $offset = 0;
  16         var $searchTerms = array();
  17         var $namespaces = array( NS_MAIN );
  18         var $showRedirects = false;
  19
  20         /**
  21          * Perform a full text search query and return a result set.
  22          * If title searches are not supported or disabled, return null.
  23          *
  24          * @param string $term - Raw search term
  25          * @return SearchResultSet
  26          * @access public
  27          * @abstract
  28          */
  29         function searchText( $term ) {
  30                 return null;
  31         }
  32
  33         /**
  34          * Perform a title-only search query and return a result set.
  35          * If title searches are not supported or disabled, return null.
  36          *
  37          * @param string $term - Raw search term
  38          * @return SearchResultSet
  39          * @access public
  40          * @abstract
  41          */
  42         function searchTitle( $term ) {
  43                 return null;
  44         }
  45
  46         /**
  47          * If an exact title match can be find, or a very slightly close match,
  48          * return the title. If no match, returns NULL.
  49          *
  50          * @param string $term
  51          * @return Title
  52          */
  53         public static function getNearMatch( $searchterm ) {
  54                 global $wgContLang;
  55
  56                 $allSearchTerms = array($searchterm);
  57
  58                 if($wgContLang->hasVariants()){
  59                         $allSearchTerms = array_merge($allSearchTerms,$wgContLang->convertLinkToAllVariants($searchterm));
  60                 }
  61
  62                 foreach($allSearchTerms as $term){
  63
  64                         # Exact match? No need to look further.
  65                         $title = Title::newFromText( $term );
  66                         if (is_null($title))
  67                                 return NULL;
  68
  69                         if ( $title->getNamespace() == NS_SPECIAL || $title->isExternal()
  70                              || $title->exists() ) {
  71                                 return $title;
  72                         }
  73
  74                         # Now try all lower case (i.e. first letter capitalized)
  75                         #
  76                         $title = Title::newFromText( $wgContLang->lc( $term ) );
  77                         if ( $title && $title->exists() ) {
  78                                 return $title;
  79                         }
  80
  81                         # Now try capitalized string
  82                         #
  83                         $title = Title::newFromText( $wgContLang->ucwords( $term ) );
  84                         if ( $title && $title->exists() ) {
  85                                 return $title;
  86                         }
  87
  88                         # Now try all upper case
  89                         #
  90                         $title = Title::newFromText( $wgContLang->uc( $term ) );
  91                         if ( $title && $title->exists() ) {
  92                                 return $title;
  93                         }
  94
  95                         # Now try Word-Caps-Breaking-At-Word-Breaks, for hyphenated names etc
  96                         $title = Title::newFromText( $wgContLang->ucwordbreaks($term) );
  97                         if ( $title && $title->exists() ) {
  98                                 return $title;
  99                         }
 100
 101                         // Give hooks a chance at better match variants
 102                         $title = null;
 103                         if( !wfRunHooks( 'SearchGetNearMatch', array( $term, &$title ) ) ) {
 104                                 return $title;
 105                         }
 106                 }
 107
 108                 $title = Title::newFromText( $searchterm );
 109
 110                 # Entering an IP address goes to the contributions page
 111                 if ( ( $title->getNamespace() == NS_USER && User::isIP($title->getText() ) )
 112                         || User::isIP( trim( $searchterm ) ) ) {
 113                         return SpecialPage::getTitleFor( 'Contributions', $title->getDBkey() );
 114                 }
 115
 116
 117                 # Entering a user goes to the user page whether it's there or not
 118                 if ( $title->getNamespace() == NS_USER ) {
 119                         return $title;
 120                 }
 121
 122                 # Go to images that exist even if there's no local page.
 123                 # There may have been a funny upload, or it may be on a shared
 124                 # file repository such as Wikimedia Commons.
 125                 if( $title->getNamespace() == NS_IMAGE ) {
 126                         $image = wfFindFile( $title );
 127                         if( $image ) {
 128                                 return $title;
 129                         }
 130                 }
 131
 132                 # MediaWiki namespace? Page may be "implied" if not customized.
 133                 # Just return it, with caps forced as the message system likes it.
 134                 if( $title->getNamespace() == NS_MEDIAWIKI ) {
 135                         return Title::makeTitle( NS_MEDIAWIKI, $wgContLang->ucfirst( $title->getText() ) );
 136                 }
 137
 138                 # Quoted term? Try without the quotes...
 139                 $matches = array();
 140                 if( preg_match( '/^"([^"]+)"$/', $searchterm, $matches ) ) {
 141                         return SearchEngine::getNearMatch( $matches[1] );
 142                 }
 143
 144                 return NULL;
 145         }
 146
 147         public static function legalSearchChars() {
 148                 return "A-Za-z_'0-9\\x80-\\xFF\\-";
 149         }
 150
 151         /**
 152          * Set the maximum number of results to return
 153          * and how many to skip before returning the first.
 154          *
 155          * @param int $limit
 156          * @param int $offset
 157          * @access public
 158          */
 159         function setLimitOffset( $limit, $offset = 0 ) {
 160                 $this->limit = intval( $limit );
 161                 $this->offset = intval( $offset );
 162         }
 163
 164         /**
 165          * Set which namespaces the search should include.
 166          * Give an array of namespace index numbers.
 167          *
 168          * @param array $namespaces
 169          * @access public
 170          */
 171         function setNamespaces( $namespaces ) {
 172                 $this->namespaces = $namespaces;
 173         }
 174
 175         /**
 176          * Parse some common prefixes: all (search everything)
 177          * or namespace names
 178          *
 179          * @param string $query
 180          */
 181         function replacePrefixes( $query ){
 182                 global $wgContLang;
 183
 184                 if( strpos($query,':') === false )
 185                         return $query; // nothing to do
 186
 187                 $parsed = $query;
 188                 $allkeyword = wfMsgForContent('searchall').":";
 189                 if( strncmp($query, $allkeyword, strlen($allkeyword)) == 0 ){
 190                         $this->namespaces = null;
 191                         $parsed = substr($query,strlen($allkeyword));
 192                 } else if( strpos($query,':') !== false ) {
 193                         $prefix = substr($query,0,strpos($query,':'));
 194                         $index = $wgContLang->getNsIndex($prefix);
 195                         if($index !== false){
 196                                 $this->namespaces = array($index);
 197                                 $parsed = substr($query,strlen($prefix)+1);
 198                         }
 199                 }
 200                 if(trim($parsed) == '')
 201                         return $query; // prefix was the whole query
 202
 203                 return $parsed;
 204         }
 205
 206         /**
 207          * Make a list of searchable namespaces and their canonical names.
 208          * @return array
 209          */
 210         public static function searchableNamespaces() {
 211                 global $wgContLang;
 212                 $arr = array();
 213                 foreach( $wgContLang->getNamespaces() as $ns => $name ) {
 214                         if( $ns >= NS_MAIN ) {
 215                                 $arr[$ns] = $name;
 216                         }
 217                 }
 218                 return $arr;
 219         }
 220
 221         /**
 222          * Extract default namespaces to search from the given user's
 223          * settings, returning a list of index numbers.
 224          *
 225          * @param User $user
 226          * @return array
 227          * @static
 228          */
 229         public static function userNamespaces( &$user ) {
 230                 $arr = array();
 231                 foreach( SearchEngine::searchableNamespaces() as $ns => $name ) {
 232                         if( $user->getOption( 'searchNs' . $ns ) ) {
 233                                 $arr[] = $ns;
 234                         }
 235                 }
 236                 return $arr;
 237         }
 238
 239         /**
 240          * Find snippet highlight settings for a given user
 241          *
 242          * @param User $user
 243          * @return array contextlines, contextchars
 244          * @static
 245          */
 246         public static function userHighlightPrefs( &$user ){
 247                 //$contextlines = $user->getOption( 'contextlines',  5 );
 248                 //$contextchars = $user->getOption( 'contextchars', 50 );
 249                 $contextlines = 2; // Hardcode this. Old defaults sucked. :)
 250                 $contextchars = 75; // same as above.... :P
 251                 return array($contextlines, $contextchars);
 252         }
 253
 254         /**
 255          * An array of namespaces indexes to be searched by default
 256          *
 257          * @return array
 258          * @static
 259          */
 260         public static function defaultNamespaces(){
 261                 global $wgNamespacesToBeSearchedDefault;
 262
 263                 return array_keys($wgNamespacesToBeSearchedDefault, true);
 264         }
 265
 266         /**
 267          * Return a 'cleaned up' search string
 268          *
 269          * @return string
 270          * @access public
 271          */
 272         function filter( $text ) {
 273                 $lc = $this->legalSearchChars();
 274                 return trim( preg_replace( "/[^{$lc}]/", " ", $text ) );
 275         }
 276         /**
 277          * Load up the appropriate search engine class for the currently
 278          * active database backend, and return a configured instance.
 279          *
 280          * @fixme Ask the database class for his default search class
 281          * instead of knowing about every backend here.
 282          * @return SearchEngine
 283          */
 284         public static function create() {
 285                 global $wgSearchType;
 286                 $dbr = wfGetDB( DB_SLAVE );
 287                 if( $wgSearchType ) {
 288                         $class = $wgSearchType;
 289                 } else {
 290                         $class = $dbr->getSearchEngine();
 291                 }
 292                 $search = new $class( $dbr );
 293                 $search->setLimitOffset(0,0);
 294                 return $search;
 295         }
 296
 297         /**
 298          * Create or update the search index record for the given page.
 299          * Title and text should be pre-processed.
 300          *
 301          * @param int $id
 302          * @param string $title
 303          * @param string $text
 304          * @abstract
 305          */
 306         function update( $id, $title, $text ) {
 307                 // no-op
 308         }
 309
 310         /**
 311          * Update a search index record's title only.
 312          * Title should be pre-processed.
 313          *
 314          * @param int $id
 315          * @param string $title
 316          * @abstract
 317          */
 318         function updateTitle( $id, $title ) {
 319                 // no-op
 320         }
 321
 322         /**
 323          * Get OpenSearch suggestion template
 324          *
 325          * @return string
 326          * @static
 327          */
 328         public static function getOpenSearchTemplate() {
 329                 global $wgOpenSearchTemplate, $wgServer, $wgScriptPath;
 330                 if($wgOpenSearchTemplate)
 331                         return $wgOpenSearchTemplate;
 332                 else{
 333                         $ns = implode(',',SearchEngine::defaultNamespaces());
 334                         if(!$ns) $ns = "0";
 335                         return $wgServer . $wgScriptPath . '/api.php?action=opensearch&search={searchTerms}&namespace='.$ns;
 336                 }
 337         }
 338
 339         /**
 340          * Get internal MediaWiki Suggest template
 341          *
 342          * @return string
 343          * @static
 344          */
 345         public static function getMWSuggestTemplate() {
 346                 global $wgMWSuggestTemplate, $wgServer, $wgScriptPath;
 347                 if($wgMWSuggestTemplate)
 348                         return $wgMWSuggestTemplate;
 349                 else
 350                         return $wgServer . $wgScriptPath . '/api.php?action=opensearch&search={searchTerms}&namespace={namespaces}';
 351         }
 352 }
 353
 354 /**
 355  * @ingroup Search
 356  */
 357 class SearchResultSet {
 358         /**
 359          * Fetch an array of regular expression fragments for matching
 360          * the search terms as parsed by this engine in a text extract.
 361          *
 362          * @return array
 363          * @access public
 364          * @abstract
 365          */
 366         function termMatches() {
 367                 return array();
 368         }
 369
 370         function numRows() {
 371                 return 0;
 372         }
 373
 374         /**
 375          * Return true if results are included in this result set.
 376          * @return bool
 377          * @abstract
 378          */
 379         function hasResults() {
 380                 return false;
 381         }
 382
 383         /**
 384          * Some search modes return a total hit count for the query
 385          * in the entire article database. This may include pages
 386          * in namespaces that would not be matched on the given
 387          * settings.
 388          *
 389          * Return null if no total hits number is supported.
 390          *
 391          * @return int
 392          * @access public
 393          */
 394         function getTotalHits() {
 395                 return null;
 396         }
 397
 398         /**
 399          * Some search modes return a suggested alternate term if there are
 400          * no exact hits. Returns true if there is one on this set.
 401          *
 402          * @return bool
 403          * @access public
 404          */
 405         function hasSuggestion() {
 406                 return false;
 407         }
 408
 409         /**
 410          * @return string suggested query, null if none
 411          */
 412         function getSuggestionQuery(){
 413                 return null;
 414         }
 415
 416         /**
 417          * @return string highlighted suggested query, '' if none
 418          */
 419         function getSuggestionSnippet(){
 420                 return '';
 421         }
 422
 423         /**
 424          * Return information about how and from where the results were fetched,
 425          * should be useful for diagnostics and debugging
 426          *
 427          * @return string
 428          */
 429         function getInfo() {
 430                 return null;
 431         }
 432
 433         /**
 434          * Return a result set of hits on other (multiple) wikis associated with this one
 435          *
 436          * @return SearchResultSet
 437          */
 438         function getInterwikiResults() {
 439                 return null;
 440         }
 441
 442         /**
 443          * Check if there are results on other wikis
 444          *
 445          * @return boolean
 446          */
 447         function hasInterwikiResults() {
 448                 return $this->getInterwikiResults() != null;
 449         }
 450
 451
 452         /**
 453          * Fetches next search result, or false.
 454          * @return SearchResult
 455          * @access public
 456          * @abstract
 457          */
 458         function next() {
 459                 return false;
 460         }
 461
 462         /**
 463          * Frees the result set, if applicable.
 464          * @ access public
 465          */
 466         function free() {
 467                 // ...
 468         }
 469 }
 470
 471
 472 /**
 473  * @ingroup Search
 474  */
 475 class SearchResultTooMany {
 476         ## Some search engines may bail out if too many matches are found
 477 }
 478
 479
 480 /**
 481  * @fixme This class is horribly factored. It would probably be better to have
 482  * a useful base class to which you pass some standard information, then let
 483  * the fancy self-highlighters extend that.
 484  * @ingroup Search
 485  */
 486 class SearchResult {
 487         var $mRevision = null;
 488
 489         function SearchResult( $row ) {
 490                 $this->mTitle = Title::makeTitle( $row->page_namespace, $row->page_title );
 491                 if( !is_null($this->mTitle) )
 492                         $this->mRevision = Revision::newFromTitle( $this->mTitle );
 493         }
 494
 495         /**
 496          * Check if this is result points to an invalid title
 497          *
 498          * @return boolean
 499          * @access public
 500          */
 501         function isBrokenTitle(){
 502                 if( is_null($this->mTitle) )
 503                         return true;
 504                 return false;
 505         }
 506
 507         /**
 508          * Check if target page is missing, happens when index is out of date
 509          *
 510          * @return boolean
 511          * @access public
 512          */
 513         function isMissingRevision(){
 514                 if( !$this->mRevision )
 515                         return true;
 516                 return false;
 517         }
 518
 519         /**
 520          * @return Title
 521          * @access public
 522          */
 523         function getTitle() {
 524                 return $this->mTitle;
 525         }
 526
 527         /**
 528          * @return double or null if not supported
 529          */
 530         function getScore() {
 531                 return null;
 532         }
 533
 534         /**
 535          * Lazy initialization of article text from DB
 536          */
 537         protected function initText(){
 538                 if( !isset($this->mText) ){
 539                         $this->mText = $this->mRevision->getText();
 540                 }
 541         }
 542
 543         /**
 544          * @param array $terms terms to highlight
 545          * @return string highlighted text snippet, null (and not '') if not supported
 546          */
 547         function getTextSnippet($terms){
 548                 global $wgUser, $wgAdvancedSearchHighlighting;
 549                 $this->initText();
 550                 list($contextlines,$contextchars) = SearchEngine::userHighlightPrefs($wgUser);
 551                 $h = new SearchHighlighter();
 552                 if( $wgAdvancedSearchHighlighting )
 553                         return $h->highlightText( $this->mText, $terms, $contextlines, $contextchars );
 554                 else
 555                         return $h->highlightSimple( $this->mText, $terms, $contextlines, $contextchars );
 556         }
 557
 558         /**
 559          * @param array $terms terms to highlight
 560          * @return string highlighted title, '' if not supported
 561          */
 562         function getTitleSnippet($terms){
 563                 return '';
 564         }
 565
 566         /**
 567          * @param array $terms terms to highlight
 568          * @return string highlighted redirect name (redirect to this page), '' if none or not supported
 569          */
 570         function getRedirectSnippet($terms){
 571                 return '';
 572         }
 573
 574         /**
 575          * @return Title object for the redirect to this page, null if none or not supported
 576          */
 577         function getRedirectTitle(){
 578                 return null;
 579         }
 580
 581         /**
 582          * @return string highlighted relevant section name, null if none or not supported
 583          */
 584         function getSectionSnippet(){
 585                 return '';
 586         }
 587
 588         /**
 589          * @return Title object (pagename+fragment) for the section, null if none or not supported
 590          */
 591         function getSectionTitle(){
 592                 return null;
 593         }
 594
 595         /**
 596          * @return string timestamp
 597          */
 598         function getTimestamp(){
 599                 return $this->mRevision->getTimestamp();
 600         }
 601
 602         /**
 603          * @return int number of words
 604          */
 605         function getWordCount(){
 606                 $this->initText();
 607                 return str_word_count( $this->mText );
 608         }
 609
 610         /**
 611          * @return int size in bytes
 612          */
 613         function getByteSize(){
 614                 $this->initText();
 615                 return strlen( $this->mText );
 616         }
 617
 618         /**
 619          * @return boolean if hit has related articles
 620          */
 621         function hasRelated(){
 622                 return false;
 623         }
 624
 625         /**
 626          * @return interwiki prefix of the title (return iw even if title is broken)
 627          */
 628         function getInterwikiPrefix(){
 629                 return '';
 630         }
 631 }
 632
 633 /**
 634  * Highlight bits of wikitext
 635  *
 636  * @ingroup Search
 637  */
 638 class SearchHighlighter {
 639         var $mCleanWikitext = true;
 640
 641         function SearchHighlighter($cleanupWikitext = true){
 642                 $this->mCleanWikitext = $cleanupWikitext;
 643         }
 644
 645         /**
 646          * Default implementation of wikitext highlighting
 647          *
 648          * @param string $text
 649          * @param array $terms Terms to highlight (unescaped)
 650          * @param int $contextlines
 651          * @param int $contextchars
 652          * @return string
 653          */
 654         public function highlightText( $text, $terms, $contextlines, $contextchars ) {
 655                 global $wgLang, $wgContLang;
 656                 global $wgSearchHighlightBoundaries;
 657                 $fname = __METHOD__;
 658
 659                 if($text == '')
 660                         return '';
 661
 662                 // spli text into text + templates/links/tables
 663                 $spat = "/(\\{\\{)|(\\[\\[[^\\]:]+:)|(\n\\{\\|)";
 664                 // first capture group is for detecting nested templates/links/tables/references
 665                 $endPatterns = array(
 666                         1 => '/(\{\{)|(\}\})/', // template
 667                         2 => '/(\[\[)|(\]\])/', // image
 668                         3 => "/(\n\\{\\|)|(\n\\|\\})/"); // table
 669
 670                 // FIXME: this should prolly be a hook or something
 671                 if(function_exists('wfCite')){
 672                         $spat .= '|(<ref>)'; // references via cite extension
 673                         $endPatterns[4] = '/(<ref>)|(<\/ref>)/';
 674                 }
 675                 $spat .= '/';
 676                 $textExt = array(); // text extracts
 677                 $otherExt = array();  // other extracts
 678                 wfProfileIn( "$fname-split" );
 679                 $start = 0;
 680                 $textLen = strlen($text);
 681                 $count = 0; // sequence number to maintain ordering
 682                 while( $start < $textLen ){
 683                         // find start of template/image/table
 684                         if( preg_match( $spat, $text, $matches, PREG_OFFSET_CAPTURE, $start ) ){
 685                                 $epat = '';
 686                                 foreach($matches as $key => $val){
 687                                         if($key > 0 && $val[1] != -1){
 688                                                 if($key == 2){
 689                                                         // see if this is an image link
 690                                                         $ns = substr($val[0],2,-1);
 691                                                         if( $wgContLang->getNsIndex($ns) != NS_IMAGE )
 692                                                                 break;
 693
 694                                                 }
 695                                                 $epat = $endPatterns[$key];
 696                                                 $this->splitAndAdd( $textExt, $count, substr( $text, $start, $val[1] - $start ) );
 697                                                 $start = $val[1];
 698                                                 break;
 699                                         }
 700                                 }
 701                                 if( $epat ){
 702                                         // find end (and detect any nested elements)
 703                                         $level = 0;
 704                                         $offset = $start + 1;
 705                                         $found = false;
 706                                         while( preg_match( $epat, $text, $endMatches, PREG_OFFSET_CAPTURE, $offset ) ){
 707                                                 if( array_key_exists(2,$endMatches) ){
 708                                                         // found end
 709                                                         if($level == 0){
 710                                                                 $len = strlen($endMatches[2][0]);
 711                                                                 $off = $endMatches[2][1];
 712                                                                 $this->splitAndAdd( $otherExt, $count,
 713                                                                         substr( $text, $start, $off + $len  - $start ) );
 714                                                                 $start = $off + $len;
 715                                                                 $found = true;
 716                                                                 break;
 717                                                         } else{
 718                                                                 // end of nested element
 719                                                                 $level -= 1;
 720                                                         }
 721                                                 } else{
 722                                                         // nested
 723                                                         $level += 1;
 724                                                 }
 725                                                 $offset = $endMatches[0][1] + strlen($endMatches[0][0]);
 726                                         }
 727                                         if( ! $found ){
 728                                                 // couldn't find appropriate closing tag, skip
 729                                                 $this->splitAndAdd( $textExt, $count, substr( $text, $start, strlen($matches[0][0]) ) );
 730                                                 $start += strlen($matches[0][0]);
 731                                         }
 732                                         continue;
 733                                 }
 734                         }
 735                         // else: add as text extract
 736                         $this->splitAndAdd( $textExt, $count, substr($text,$start) );
 737                         break;
 738                 }
 739
 740                 $all = $textExt + $otherExt; // these have disjunct key sets
 741
 742                 wfProfileOut( "$fname-split" );
 743
 744                 // prepare regexps
 745                 foreach( $terms as $index => $term ) {
 746                         $terms[$index] = preg_quote( $term, '/' );
 747                         // manually do upper/lowercase stuff for utf-8 since PHP won't do it
 748                         if(preg_match('/[\x80-\xff]/', $term) ){
 749                                 $terms[$index] = preg_replace_callback('/./us',array($this,'caseCallback'),$terms[$index]);
 750                         }
 751
 752
 753                 }
 754                 $anyterm = implode( '|', $terms );
 755                 $phrase = implode("$wgSearchHighlightBoundaries+", $terms );
 756
 757                 // FIXME: a hack to scale contextchars, a correct solution
 758                 // would be to have contextchars actually be char and not byte
 759                 // length, and do proper utf-8 substrings and lengths everywhere,
 760                 // but PHP is making that very hard and unclean to implement :(
 761                 $scale = strlen($anyterm) / mb_strlen($anyterm);
 762                 $contextchars = intval( $contextchars * $scale );
 763
 764                 $patPre = "(^|$wgSearchHighlightBoundaries)";
 765                 $patPost = "($wgSearchHighlightBoundaries|$)";
 766
 767                 $pat1 = "/(".$phrase.")/ui";
 768                 $pat2 = "/$patPre(".$anyterm.")$patPost/ui";
 769
 770                 wfProfileIn( "$fname-extract" );
 771
 772                 $left = $contextlines;
 773
 774                 $snippets = array();
 775                 $offsets = array();
 776
 777                 // show beginning only if it contains all words
 778                 $first = 0;
 779                 $firstText = '';
 780                 foreach($textExt as $index => $line){
 781                         if(strlen($line)>0 && $line[0] != ';' && $line[0] != ':'){
 782                                 $firstText = $this->extract( $line, 0, $contextchars * $contextlines );
 783                                 $first = $index;
 784                                 break;
 785                         }
 786                 }
 787                 if( $firstText ){
 788                         $succ = true;
 789                         // check if first text contains all terms
 790                         foreach($terms as $term){
 791                                 if( ! preg_match("/$patPre".$term."$patPost/ui", $firstText) ){
 792                                         $succ = false;
 793                                         break;
 794                                 }
 795                         }
 796                         if( $succ ){
 797                                 $snippets[$first] = $firstText;
 798                                 $offsets[$first] = 0;
 799                         }
 800                 }
 801                 if( ! $snippets ) {
 802                         // match whole query on text
 803                         $this->process($pat1, $textExt, $left, $contextchars, $snippets, $offsets);
 804                         // match whole query on templates/tables/images
 805                         $this->process($pat1, $otherExt, $left, $contextchars, $snippets, $offsets);
 806                         // match any words on text
 807                         $this->process($pat2, $textExt, $left, $contextchars, $snippets, $offsets);
 808                         // match any words on templates/tables/images
 809                         $this->process($pat2, $otherExt, $left, $contextchars, $snippets, $offsets);
 810
 811                         ksort($snippets);
 812                 }
 813
 814                 // add extra chars to each snippet to make snippets constant size
 815                 $extended = array();
 816                 if( count( $snippets ) == 0){
 817                         // couldn't find the target words, just show beginning of article
 818                         $targetchars = $contextchars * $contextlines;
 819                         $snippets[$first] = '';
 820                         $offsets[$first] = 0;
 821                 } else{
 822                         // if begin of the article contains the whole phrase, show only that !!
 823                         if( array_key_exists($first,$snippets) && preg_match($pat1,$snippets[$first])
 824                             && $offsets[$first] < $contextchars * 2 ){
 825                                 $snippets = array ($first => $snippets[$first]);
 826                         }
 827
 828                         // calc by how much to extend existing snippets
 829                         $targetchars = intval( ($contextchars * $contextlines) / count ( $snippets ) );
 830                 }
 831
 832                 foreach($snippets as $index => $line){
 833                         $extended[$index] = $line;
 834                         $len = strlen($line);
 835                         if( $len < $targetchars - 20 ){
 836                                 // complete this line
 837                                 if($len < strlen( $all[$index] )){
 838                                         $extended[$index] = $this->extract( $all[$index], $offsets[$index], $offsets[$index]+$targetchars, $offsets[$index]);
 839                                         $len = strlen( $extended[$index] );
 840                                 }
 841
 842                                 // add more lines
 843                                 $add = $index + 1;
 844                                 while( $len < $targetchars - 20
 845                                        && array_key_exists($add,$all)
 846                                        && !array_key_exists($add,$snippets) ){
 847                                     $offsets[$add] = 0;
 848                                     $tt = "\n".$this->extract( $all[$add], 0, $targetchars - $len, $offsets[$add] );
 849                                         $extended[$add] = $tt;
 850                                         $len += strlen( $tt );
 851                                         $add++;
 852                                 }
 853                         }
 854                 }
 855
 856                 //$snippets = array_map('htmlspecialchars', $extended);
 857                 $snippets = $extended;
 858                 $last = -1;
 859                 $extract = '';
 860                 foreach($snippets as $index => $line){
 861                         if($last == -1)
 862                                 $extract .= $line; // first line
 863                         elseif($last+1 == $index && $offsets[$last]+strlen($snippets[$last]) >= strlen($all[$last]))
 864                                 $extract .= " ".$line; // continous lines
 865                         else
 866                                 $extract .= '<b> ... </b>' . $line;
 867
 868                         $last = $index;
 869                 }
 870                 if( $extract )
 871                         $extract .= '<b> ... </b>';
 872
 873                 $processed = array();
 874                 foreach($terms as $term){
 875                         if( ! isset($processed[$term]) ){
 876                                 $pat3 = "/$patPre(".$term.")$patPost/ui"; // highlight word
 877                                 $extract = preg_replace( $pat3,
 878                                         "\\1<span class='searchmatch'>\\2</span>\\3", $extract );
 879                                 $processed[$term] = true;
 880                         }
 881                 }
 882
 883                 wfProfileOut( "$fname-extract" );
 884
 885                 return $extract;
 886         }
 887
 888         /**
 889          * Split text into lines and add it to extracts array
 890          *
 891          * @param array $extracts index -> $line
 892          * @param int $count
 893          * @param string $text
 894          */
 895         function splitAndAdd(&$extracts, &$count, $text){
 896                 $split = explode( "\n", $this->mCleanWikitext? $this->removeWiki($text) : $text );
 897                 foreach($split as $line){
 898                         $tt = trim($line);
 899                         if( $tt )
 900                                 $extracts[$count++] = $tt;
 901                 }
 902         }
 903
 904         /**
 905          * Do manual case conversion for non-ascii chars
 906          *
 907          * @param unknown_type $matches
 908          */
 909         function caseCallback($matches){
 910                 global $wgContLang;
 911                 if( strlen($matches[0]) > 1 ){
 912                         return '['.$wgContLang->lc($matches[0]).$wgContLang->uc($matches[0]).']';
 913                 } else
 914                         return $matches[0];
 915         }
 916
 917         /**
 918          * Extract part of the text from start to end, but by
 919          * not chopping up words
 920          * @param string $text
 921          * @param int $start
 922          * @param int $end
 923          * @param int $posStart (out) actual start position
 924          * @param int $posEnd (out) actual end position
 925          * @return string
 926          */
 927         function extract($text, $start, $end, &$posStart = null, &$posEnd = null ){
 928                 global $wgContLang;
 929
 930                 if( $start != 0)
 931                         $start = $this->position( $text, $start, 1 );
 932                 if( $end >= strlen($text) )
 933                         $end = strlen($text);
 934                 else
 935                         $end = $this->position( $text, $end );
 936
 937                 if(!is_null($posStart))
 938                         $posStart = $start;
 939                 if(!is_null($posEnd))
 940                         $posEnd = $end;
 941
 942                 if($end > $start)
 943                         return substr($text, $start, $end-$start);
 944                 else
 945                         return '';
 946         }
 947
 948         /**
 949          * Find a nonletter near a point (index) in the text
 950          *
 951          * @param string $text
 952          * @param int $point
 953          * @param int $offset to found index
 954          * @return int nearest nonletter index, or beginning of utf8 char if none
 955          */
 956         function position($text, $point, $offset=0 ){
 957                 $tolerance = 10;
 958                 $s = max( 0, $point - $tolerance );
 959                 $l = min( strlen($text), $point + $tolerance ) - $s;
 960                 $m = array();
 961                 if( preg_match('/[ ,.!?~!@#$%^&*\(\)+=\-\\\|\[\]"\'<>]/', substr($text,$s,$l), $m, PREG_OFFSET_CAPTURE ) ){
 962                         return $m[0][1] + $s + $offset;
 963                 } else{
 964                         // check if point is on a valid first UTF8 char
 965                         $char = ord( $text[$point] );
 966                         while( $char >= 0x80 && $char < 0xc0 ) {
 967                                 // skip trailing bytes
 968                                 $point++;
 969                                 if($point >= strlen($text))
 970                                         return strlen($text);
 971                                 $char = ord( $text[$point] );
 972                         }
 973                         return $point;
 974
 975                 }
 976         }
 977
 978         /**
 979          * Search extracts for a pattern, and return snippets
 980          *
 981          * @param string $pattern regexp for matching lines
 982          * @param array $extracts extracts to search
 983          * @param int $linesleft number of extracts to make
 984          * @param int $contextchars length of snippet
 985          * @param array $out map for highlighted snippets
 986          * @param array $offsets map of starting points of snippets
 987          * @protected
 988          */
 989         function process( $pattern, $extracts, &$linesleft, &$contextchars, &$out, &$offsets ){
 990                 if($linesleft == 0)
 991                         return; // nothing to do
 992                 foreach($extracts as $index => $line){
 993                         if( array_key_exists($index,$out) )
 994                                 continue; // this line already highlighted
 995
 996                         $m = array();
 997                         if ( !preg_match( $pattern, $line, $m, PREG_OFFSET_CAPTURE ) )
 998                                 continue;
 999
1000                         $offset = $m[0][1];
1001                         $len = strlen($m[0][0]);
1002                         if($offset + $len < $contextchars)
1003                                 $begin = 0;
1004                         elseif( $len > $contextchars)
1005                                 $begin = $offset;
1006                         else
1007                                 $begin = $offset + intval( ($len - $contextchars) / 2 );
1008
1009                         $end = $begin + $contextchars;
1010
1011                         $posBegin = $begin;
1012                         // basic snippet from this line
1013                         $out[$index] = $this->extract($line,$begin,$end,$posBegin);
1014                         $offsets[$index] = $posBegin;
1015                         $linesleft--;
1016                         if($linesleft == 0)
1017                                 return;
1018                 }
1019         }
1020
1021         /**
1022          * Basic wikitext removal
1023          * @protected
1024          */
1025         function removeWiki($text) {
1026                 $fname = __METHOD__;
1027                 wfProfileIn( $fname );
1028
1029                 //$text = preg_replace("/'{2,5}/", "", $text);
1030                 //$text = preg_replace("/\[[a-z]+:\/\/[^ ]+ ([^]]+)\]/", "\\2", $text);
1031                 //$text = preg_replace("/\[\[([^]|]+)\]\]/", "\\1", $text);
1032                 //$text = preg_replace("/\[\[([^]]+\|)?([^|]]+)\]\]/", "\\2", $text);
1033                 //$text = preg_replace("/\\{\\|(.*?)\\|\\}/", "", $text);
1034                 //$text = preg_replace("/\\[\\[[A-Za-z_-]+:([^|]+?)\\]\\]/", "", $text);
1035                 $text = preg_replace("/\\{\\{([^|]+?)\\}\\}/", "", $text);
1036                 $text = preg_replace("/\\{\\{([^|]+\\|)(.*?)\\}\\}/", "\\2", $text);
1037                 $text = preg_replace("/\\[\\[([^|]+?)\\]\\]/", "\\1", $text);
1038                 $text = preg_replace_callback("/\\[\\[([^|]+\\|)(.*?)\\]\\]/", array($this,'linkReplace'), $text);
1039                 //$text = preg_replace("/\\[\\[([^|]+\\|)(.*?)\\]\\]/", "\\2", $text);
1040                 $text = preg_replace("/<\/?[^>]+>/", "", $text);
1041                 $text = preg_replace("/'''''/", "", $text);
1042                 $text = preg_replace("/('''|<\/?[iIuUbB]>)/", "", $text);
1043                 $text = preg_replace("/''/", "", $text);
1044
1045                 wfProfileOut( $fname );
1046                 return $text;
1047         }
1048
1049         /**
1050          * callback to replace [[target|caption]] kind of links, if
1051          * the target is category or image, leave it
1052          *
1053          * @param array $matches
1054          */
1055         function linkReplace($matches){
1056                 $colon = strpos( $matches[1], ':' );
1057                 if( $colon === false )
1058                         return $matches[2]; // replace with caption
1059                 global $wgContLang;
1060                 $ns = substr( $matches[1], 0, $colon );
1061                 $index = $wgContLang->getNsIndex($ns);
1062                 if( $index !== false && ($index == NS_IMAGE || $index == NS_CATEGORY) )
1063                         return $matches[0]; // return the whole thing
1064                 else
1065                         return $matches[2];
1066
1067         }
1068
1069         /**
1070      * Simple & fast snippet extraction, but gives completely unrelevant
1071      * snippets
1072      *
1073      * @param string $text
1074      * @param array $terms
1075      * @param int $contextlines
1076      * @param int $contextchars
1077      * @return string
1078      */
1079     public function highlightSimple( $text, $terms, $contextlines, $contextchars ) {
1080         global $wgLang, $wgContLang;
1081         $fname = __METHOD__;
1082
1083         $lines = explode( "\n", $text );
1084
1085         $terms = implode( '|', $terms );
1086         $terms = str_replace( '/', "\\/", $terms);
1087         $max = intval( $contextchars ) + 1;
1088         $pat1 = "/(.*)($terms)(.{0,$max})/i";
1089
1090         $lineno = 0;
1091
1092         $extract = "";
1093         wfProfileIn( "$fname-extract" );
1094         foreach ( $lines as $line ) {
1095             if ( 0 == $contextlines ) {
1096                 break;
1097             }
1098             ++$lineno;
1099             $m = array();
1100             if ( ! preg_match( $pat1, $line, $m ) ) {
1101                 continue;
1102             }
1103             --$contextlines;
1104             $pre = $wgContLang->truncate( $m[1], -$contextchars, ' ... ' );
1105
1106             if ( count( $m ) < 3 ) {
1107                 $post = '';
1108             } else {
1109                 $post = $wgContLang->truncate( $m[3], $contextchars, ' ... ' );
1110             }
1111
1112             $found = $m[2];
1113
1114             $line = htmlspecialchars( $pre . $found . $post );
1115             $pat2 = '/(' . $terms . ")/i";
1116             $line = preg_replace( $pat2,
1117               "<span class='searchmatch'>\\1</span>", $line );
1118
1119             $extract .= "${line}\n";
1120         }
1121         wfProfileOut( "$fname-extract" );
1122
1123         return $extract;
1124     }
1125
1126 }
1127
1128 /**
1129  * Dummy class to be used when non-supported Database engine is present.
1130  * @fixme Dummy class should probably try something at least mildly useful,
1131  * such as a LIKE search through titles.
1132  * @ingroup Search
1133  */
1134 class SearchEngineDummy extends SearchEngine {
1135         // no-op
1136 }