includes/search/SearchEngine.php

   1 <?php
   2 /**
   3  * @defgroup Search Search
   4  *
   5  * @file
   6  * @ingroup Search
   7  */
   8
   9 /**
  10  * Contain a class for special pages
  11  * @ingroup Search
  12  */
  13 class SearchEngine {
  14         var $limit = 10;
  15         var $offset = 0;
  16         var $prefix = '';
  17         var $searchTerms = array();
  18         var $namespaces = array( NS_MAIN );
  19         var $showRedirects = false;
  20
  21         /**
  22          * Perform a full text search query and return a result set.
  23          * If title searches are not supported or disabled, return null.
  24          * STUB
  25          *
  26          * @param $term String: raw search term
  27          * @return SearchResultSet
  28          */
  29         function searchText( $term ) {
  30                 return null;
  31         }
  32
  33         /**
  34          * Perform a title-only search query and return a result set.
  35          * If title searches are not supported or disabled, return null.
  36          * STUB
  37          *
  38          * @param $term String: raw search term
  39          * @return SearchResultSet
  40          */
  41         function searchTitle( $term ) {
  42                 return null;
  43         }
  44
  45         /** If this search backend can list/unlist redirects */
  46         function acceptListRedirects() {
  47                 return true;
  48         }
  49
  50         /**
  51          * When overridden in derived class, performs database-specific conversions
  52          * on text to be used for searching or updating search index.
  53          * Default implementation does nothing (simply returns $string).
  54          *
  55          * @param $string string: String to process
  56          * @return string
  57          */
  58         public function normalizeText( $string ) {
  59                 return $string;
  60         }
  61
  62         /**
  63          * Transform search term in cases when parts of the query came as different GET params (when supported)
  64          * e.g. for prefix queries: search=test&prefix=Main_Page/Archive -> test prefix:Main Page/Archive
  65          */
  66         function transformSearchTerm( $term ) {
  67                 return $term;
  68         }
  69
  70         /**
  71          * If an exact title match can be found, or a very slightly close match,
  72          * return the title. If no match, returns NULL.
  73          *
  74          * @param $searchterm String
  75          * @return Title
  76          */
  77         public static function getNearMatch( $searchterm ) {
  78                 $title = self::getNearMatchInternal( $searchterm );
  79
  80                 wfRunHooks( 'SearchGetNearMatchComplete', array( $searchterm, &$title ) );
  81                 return $title;
  82         }
  83
  84         /**
  85          * Really find the title match.
  86          */
  87         private static function getNearMatchInternal( $searchterm ) {
  88                 global $wgContLang;
  89
  90                 $allSearchTerms = array($searchterm);
  91
  92                 if ( $wgContLang->hasVariants() ) {
  93                         $allSearchTerms = array_merge($allSearchTerms,$wgContLang->convertLinkToAllVariants($searchterm));
  94                 }
  95
  96                 if( !wfRunHooks( 'SearchGetNearMatchBefore', array( $allSearchTerms, &$titleResult ) ) ) {
  97                         return $titleResult;
  98                 }
  99
 100                 foreach($allSearchTerms as $term) {
 101
 102                         # Exact match? No need to look further.
 103                         $title = Title::newFromText( $term );
 104                         if (is_null($title))
 105                                 return null;
 106
 107                         if ( $title->getNamespace() == NS_SPECIAL || $title->isExternal() || $title->exists() ) {
 108                                 return $title;
 109                         }
 110
 111                         # See if it still otherwise has content is some sane sense
 112                         $article = MediaWiki::articleFromTitle( $title );
 113                         if( $article->hasViewableContent() ) {
 114                                 return $title;
 115                         }
 116
 117                         # Now try all lower case (i.e. first letter capitalized)
 118                         #
 119                         $title = Title::newFromText( $wgContLang->lc( $term ) );
 120                         if ( $title && $title->exists() ) {
 121                                 return $title;
 122                         }
 123
 124                         # Now try capitalized string
 125                         #
 126                         $title = Title::newFromText( $wgContLang->ucwords( $term ) );
 127                         if ( $title && $title->exists() ) {
 128                                 return $title;
 129                         }
 130
 131                         # Now try all upper case
 132                         #
 133                         $title = Title::newFromText( $wgContLang->uc( $term ) );
 134                         if ( $title && $title->exists() ) {
 135                                 return $title;
 136                         }
 137
 138                         # Now try Word-Caps-Breaking-At-Word-Breaks, for hyphenated names etc
 139                         $title = Title::newFromText( $wgContLang->ucwordbreaks($term) );
 140                         if ( $title && $title->exists() ) {
 141                                 return $title;
 142                         }
 143
 144                         // Give hooks a chance at better match variants
 145                         $title = null;
 146                         if( !wfRunHooks( 'SearchGetNearMatch', array( $term, &$title ) ) ) {
 147                                 return $title;
 148                         }
 149                 }
 150
 151                 $title = Title::newFromText( $searchterm );
 152
 153                 # Entering an IP address goes to the contributions page
 154                 if ( ( $title->getNamespace() == NS_USER && User::isIP($title->getText() ) )
 155                         || User::isIP( trim( $searchterm ) ) ) {
 156                         return SpecialPage::getTitleFor( 'Contributions', $title->getDBkey() );
 157                 }
 158
 159
 160                 # Entering a user goes to the user page whether it's there or not
 161                 if ( $title->getNamespace() == NS_USER ) {
 162                         return $title;
 163                 }
 164
 165                 # Go to images that exist even if there's no local page.
 166                 # There may have been a funny upload, or it may be on a shared
 167                 # file repository such as Wikimedia Commons.
 168                 if( $title->getNamespace() == NS_FILE ) {
 169                         $image = wfFindFile( $title );
 170                         if( $image ) {
 171                                 return $title;
 172                         }
 173                 }
 174
 175                 # MediaWiki namespace? Page may be "implied" if not customized.
 176                 # Just return it, with caps forced as the message system likes it.
 177                 if( $title->getNamespace() == NS_MEDIAWIKI ) {
 178                         return Title::makeTitle( NS_MEDIAWIKI, $wgContLang->ucfirst( $title->getText() ) );
 179                 }
 180
 181                 # Quoted term? Try without the quotes...
 182                 $matches = array();
 183                 if( preg_match( '/^"([^"]+)"$/', $searchterm, $matches ) ) {
 184                         return SearchEngine::getNearMatch( $matches[1] );
 185                 }
 186
 187                 return null;
 188         }
 189
 190         public static function legalSearchChars() {
 191                 return "A-Za-z_'.0-9\\x80-\\xFF\\-";
 192         }
 193
 194         /**
 195          * Set the maximum number of results to return
 196          * and how many to skip before returning the first.
 197          *
 198          * @param $limit Integer
 199          * @param $offset Integer
 200          */
 201         function setLimitOffset( $limit, $offset = 0 ) {
 202                 $this->limit = intval( $limit );
 203                 $this->offset = intval( $offset );
 204         }
 205
 206         /**
 207          * Set which namespaces the search should include.
 208          * Give an array of namespace index numbers.
 209          *
 210          * @param $namespaces Array
 211          */
 212         function setNamespaces( $namespaces ) {
 213                 $this->namespaces = $namespaces;
 214         }
 215
 216         /**
 217          * Parse some common prefixes: all (search everything)
 218          * or namespace names
 219          *
 220          * @param $query String
 221          */
 222         function replacePrefixes( $query ){
 223                 global $wgContLang;
 224
 225                 $parsed = $query;
 226                 if( strpos($query,':') === false ) { // nothing to do
 227                         wfRunHooks( 'SearchEngineReplacePrefixesComplete', array( $this, $query, &$parsed ) );
 228                         return $parsed;
 229                 }
 230
 231                 $allkeyword = wfMsgForContent('searchall').":";
 232                 if( strncmp($query, $allkeyword, strlen($allkeyword)) == 0 ){
 233                         $this->namespaces = null;
 234                         $parsed = substr($query,strlen($allkeyword));
 235                 } else if( strpos($query,':') !== false ) {
 236                         $prefix = substr($query,0,strpos($query,':'));
 237                         $index = $wgContLang->getNsIndex($prefix);
 238                         if($index !== false){
 239                                 $this->namespaces = array($index);
 240                                 $parsed = substr($query,strlen($prefix)+1);
 241                         }
 242                 }
 243                 if(trim($parsed) == '')
 244                         $parsed = $query; // prefix was the whole query
 245
 246                 wfRunHooks( 'SearchEngineReplacePrefixesComplete', array( $this, $query, &$parsed ) );
 247
 248                 return $parsed;
 249         }
 250
 251         /**
 252          * Make a list of searchable namespaces and their canonical names.
 253          * @return Array
 254          */
 255         public static function searchableNamespaces() {
 256                 global $wgContLang;
 257                 $arr = array();
 258                 foreach( $wgContLang->getNamespaces() as $ns => $name ) {
 259                         if( $ns >= NS_MAIN ) {
 260                                 $arr[$ns] = $name;
 261                         }
 262                 }
 263
 264                 wfRunHooks( 'SearchableNamespaces', array( &$arr ) );
 265                 return $arr;
 266         }
 267
 268         /**
 269          * Extract default namespaces to search from the given user's
 270          * settings, returning a list of index numbers.
 271          *
 272          * @param $user User
 273          * @return Array
 274          */
 275         public static function userNamespaces( $user ) {
 276                 global $wgSearchEverythingOnlyLoggedIn;
 277
 278                 // get search everything preference, that can be set to be read for logged-in users
 279                 $searcheverything = false;
 280                 if( ( $wgSearchEverythingOnlyLoggedIn && $user->isLoggedIn() )
 281                     || !$wgSearchEverythingOnlyLoggedIn )
 282                         $searcheverything = $user->getOption('searcheverything');
 283
 284                 // searcheverything overrides other options
 285                 if( $searcheverything )
 286                         return array_keys(SearchEngine::searchableNamespaces());
 287
 288                 $arr = Preferences::loadOldSearchNs( $user );
 289                 $searchableNamespaces = SearchEngine::searchableNamespaces();
 290
 291                 $arr = array_intersect( $arr, array_keys($searchableNamespaces) ); // Filter
 292
 293                 return $arr;
 294         }
 295
 296         /**
 297          * Find snippet highlight settings for a given user
 298          *
 299          * @param $user User
 300          * @return Array contextlines, contextchars
 301          */
 302         public static function userHighlightPrefs( &$user ){
 303                 //$contextlines = $user->getOption( 'contextlines',  5 );
 304                 //$contextchars = $user->getOption( 'contextchars', 50 );
 305                 $contextlines = 2; // Hardcode this. Old defaults sucked. :)
 306                 $contextchars = 75; // same as above.... :P
 307                 return array($contextlines, $contextchars);
 308         }
 309
 310         /**
 311          * An array of namespaces indexes to be searched by default
 312          *
 313          * @return Array
 314          */
 315         public static function defaultNamespaces(){
 316                 global $wgNamespacesToBeSearchedDefault;
 317
 318                 return array_keys($wgNamespacesToBeSearchedDefault, true);
 319         }
 320
 321         /**
 322          * Get a list of namespace names useful for showing in tooltips
 323          * and preferences
 324          *
 325          * @param $namespaces Array
 326          */
 327         public static function namespacesAsText( $namespaces ){
 328                 global $wgContLang;
 329
 330                 $formatted = array_map( array($wgContLang,'getFormattedNsText'), $namespaces );
 331                 foreach( $formatted as $key => $ns ){
 332                         if ( empty($ns) )
 333                                 $formatted[$key] = wfMsg( 'blanknamespace' );
 334                 }
 335                 return $formatted;
 336         }
 337
 338         /**
 339          * Return the help namespaces to be shown on Special:Search
 340          *
 341          * @return Array
 342          */
 343         public static function helpNamespaces() {
 344                 global $wgNamespacesToBeSearchedHelp;
 345
 346                 return array_keys( $wgNamespacesToBeSearchedHelp, true );
 347         }
 348
 349         /**
 350          * Return a 'cleaned up' search string
 351          *
 352          * @param $text String
 353          * @return String
 354          */
 355         function filter( $text ) {
 356                 $lc = $this->legalSearchChars();
 357                 return trim( preg_replace( "/[^{$lc}]/", " ", $text ) );
 358         }
 359         /**
 360          * Load up the appropriate search engine class for the currently
 361          * active database backend, and return a configured instance.
 362          *
 363          * @return SearchEngine
 364          */
 365         public static function create() {
 366                 global $wgSearchType;
 367                 $dbr = wfGetDB( DB_SLAVE );
 368                 if( $wgSearchType ) {
 369                         $class = $wgSearchType;
 370                 } else {
 371                         $class = $dbr->getSearchEngine();
 372                 }
 373                 $search = new $class( $dbr );
 374                 $search->setLimitOffset(0,0);
 375                 return $search;
 376         }
 377
 378         /**
 379          * Create or update the search index record for the given page.
 380          * Title and text should be pre-processed.
 381          * STUB
 382          *
 383          * @param $id Integer
 384          * @param $title String
 385          * @param $text String
 386          */
 387         function update( $id, $title, $text ) {
 388                 // no-op
 389         }
 390
 391         /**
 392          * Update a search index record's title only.
 393          * Title should be pre-processed.
 394          * STUB
 395          *
 396          * @param $id Integer
 397          * @param $title String
 398          */
 399         function updateTitle( $id, $title ) {
 400                 // no-op
 401         }
 402
 403         /**
 404          * Get OpenSearch suggestion template
 405          *
 406          * @return String
 407          */
 408         public static function getOpenSearchTemplate() {
 409                 global $wgOpenSearchTemplate, $wgServer, $wgScriptPath;
 410                 if( $wgOpenSearchTemplate )     {
 411                         return $wgOpenSearchTemplate;
 412                 } else {
 413                         $ns = implode( '|', SearchEngine::defaultNamespaces() );
 414                         if( !$ns ) $ns = "0";
 415                         return $wgServer . $wgScriptPath . '/api.php?action=opensearch&search={searchTerms}&namespace='.$ns;
 416                 }
 417         }
 418
 419         /**
 420          * Get internal MediaWiki Suggest template
 421          *
 422          * @return String
 423          */
 424         public static function getMWSuggestTemplate() {
 425                 global $wgMWSuggestTemplate, $wgServer, $wgScriptPath;
 426                 if($wgMWSuggestTemplate)
 427                         return $wgMWSuggestTemplate;
 428                 else
 429                         return $wgServer . $wgScriptPath . '/api.php?action=opensearch&search={searchTerms}&namespace={namespaces}&suggest';
 430         }
 431 }
 432
 433 /**
 434  * @ingroup Search
 435  */
 436 class SearchResultSet {
 437         /**
 438          * Fetch an array of regular expression fragments for matching
 439          * the search terms as parsed by this engine in a text extract.
 440          * STUB
 441          *
 442          * @return Array
 443          */
 444         function termMatches() {
 445                 return array();
 446         }
 447
 448         function numRows() {
 449                 return 0;
 450         }
 451
 452         /**
 453          * Return true if results are included in this result set.
 454          * STUB
 455          *
 456          * @return Boolean
 457          */
 458         function hasResults() {
 459                 return false;
 460         }
 461
 462         /**
 463          * Some search modes return a total hit count for the query
 464          * in the entire article database. This may include pages
 465          * in namespaces that would not be matched on the given
 466          * settings.
 467          *
 468          * Return null if no total hits number is supported.
 469          *
 470          * @return Integer
 471          */
 472         function getTotalHits() {
 473                 return null;
 474         }
 475
 476         /**
 477          * Some search modes return a suggested alternate term if there are
 478          * no exact hits. Returns true if there is one on this set.
 479          *
 480          * @return Boolean
 481          */
 482         function hasSuggestion() {
 483                 return false;
 484         }
 485
 486         /**
 487          * @return String: suggested query, null if none
 488          */
 489         function getSuggestionQuery(){
 490                 return null;
 491         }
 492
 493         /**
 494          * @return String: HTML highlighted suggested query, '' if none
 495          */
 496         function getSuggestionSnippet(){
 497                 return '';
 498         }
 499
 500         /**
 501          * Return information about how and from where the results were fetched,
 502          * should be useful for diagnostics and debugging
 503          *
 504          * @return String
 505          */
 506         function getInfo() {
 507                 return null;
 508         }
 509
 510         /**
 511          * Return a result set of hits on other (multiple) wikis associated with this one
 512          *
 513          * @return SearchResultSet
 514          */
 515         function getInterwikiResults() {
 516                 return null;
 517         }
 518
 519         /**
 520          * Check if there are results on other wikis
 521          *
 522          * @return Boolean
 523          */
 524         function hasInterwikiResults() {
 525                 return $this->getInterwikiResults() != null;
 526         }
 527
 528         /**
 529          * Fetches next search result, or false.
 530          * STUB
 531          *
 532          * @return SearchResult
 533          */
 534         function next() {
 535                 return false;
 536         }
 537
 538         /**
 539          * Frees the result set, if applicable.
 540          */
 541         function free() {
 542                 // ...
 543         }
 544 }
 545
 546 /**
 547  * This class is used for different SQL-based search engines shipped with MediaWiki
 548  */
 549 class SqlSearchResultSet extends SearchResultSet {
 550         function __construct( $resultSet, $terms ) {
 551                 $this->mResultSet = $resultSet;
 552                 $this->mTerms = $terms;
 553         }
 554
 555         function termMatches() {
 556                 return $this->mTerms;
 557         }
 558
 559         function numRows() {
 560                 if ($this->mResultSet === false )
 561                         return false;
 562
 563                 return $this->mResultSet->numRows();
 564         }
 565
 566         function next() {
 567                 if ($this->mResultSet === false )
 568                         return false;
 569
 570                 $row = $this->mResultSet->fetchObject();
 571                 if ($row === false)
 572                         return false;
 573                 return new SearchResult($row);
 574         }
 575
 576         function free() {
 577                 if ($this->mResultSet === false )
 578                         return false;
 579
 580                 $this->mResultSet->free();
 581         }
 582 }
 583
 584 /**
 585  * @ingroup Search
 586  */
 587 class SearchResultTooMany {
 588         ## Some search engines may bail out if too many matches are found
 589 }
 590
 591
 592 /**
 593  * @todo Fixme: This class is horribly factored. It would probably be better to
 594  * have a useful base class to which you pass some standard information, then
 595  * let the fancy self-highlighters extend that.
 596  * @ingroup Search
 597  */
 598 class SearchResult {
 599         var $mRevision = null;
 600         var $mImage = null;
 601
 602         function __construct( $row ) {
 603                 $this->mTitle = Title::makeTitle( $row->page_namespace, $row->page_title );
 604                 if( !is_null($this->mTitle) ){
 605                         $this->mRevision = Revision::newFromTitle( $this->mTitle );
 606                         if( $this->mTitle->getNamespace() === NS_FILE )
 607                                 $this->mImage = wfFindFile( $this->mTitle );
 608                 }
 609         }
 610
 611         /**
 612          * Check if this is result points to an invalid title
 613          *
 614          * @return Boolean
 615          */
 616         function isBrokenTitle(){
 617                 if( is_null($this->mTitle) )
 618                         return true;
 619                 return false;
 620         }
 621
 622         /**
 623          * Check if target page is missing, happens when index is out of date
 624          *
 625          * @return Boolean
 626          */
 627         function isMissingRevision(){
 628                 return !$this->mRevision && !$this->mImage;
 629         }
 630
 631         /**
 632          * @return Title
 633          */
 634         function getTitle() {
 635                 return $this->mTitle;
 636         }
 637
 638         /**
 639          * @return Double or null if not supported
 640          */
 641         function getScore() {
 642                 return null;
 643         }
 644
 645         /**
 646          * Lazy initialization of article text from DB
 647          */
 648         protected function initText(){
 649                 if( !isset($this->mText) ){
 650                         if($this->mRevision != null)
 651                                 $this->mText = $this->mRevision->getText();
 652                         else // TODO: can we fetch raw wikitext for commons images?
 653                                 $this->mText = '';
 654
 655                 }
 656         }
 657
 658         /**
 659          * @param $terms Array: terms to highlight
 660          * @return String: highlighted text snippet, null (and not '') if not supported
 661          */
 662         function getTextSnippet($terms){
 663                 global $wgUser, $wgAdvancedSearchHighlighting;
 664                 $this->initText();
 665                 list($contextlines,$contextchars) = SearchEngine::userHighlightPrefs($wgUser);
 666                 $h = new SearchHighlighter();
 667                 if( $wgAdvancedSearchHighlighting )
 668                         return $h->highlightText( $this->mText, $terms, $contextlines, $contextchars );
 669                 else
 670                         return $h->highlightSimple( $this->mText, $terms, $contextlines, $contextchars );
 671         }
 672
 673         /**
 674          * @param $terms Array: terms to highlight
 675          * @return String: highlighted title, '' if not supported
 676          */
 677         function getTitleSnippet($terms){
 678                 return '';
 679         }
 680
 681         /**
 682          * @param $terms Array: terms to highlight
 683          * @return String: highlighted redirect name (redirect to this page), '' if none or not supported
 684          */
 685         function getRedirectSnippet($terms){
 686                 return '';
 687         }
 688
 689         /**
 690          * @return Title object for the redirect to this page, null if none or not supported
 691          */
 692         function getRedirectTitle(){
 693                 return null;
 694         }
 695
 696         /**
 697          * @return string highlighted relevant section name, null if none or not supported
 698          */
 699         function getSectionSnippet(){
 700                 return '';
 701         }
 702
 703         /**
 704          * @return Title object (pagename+fragment) for the section, null if none or not supported
 705          */
 706         function getSectionTitle(){
 707                 return null;
 708         }
 709
 710         /**
 711          * @return String: timestamp
 712          */
 713         function getTimestamp(){
 714                 if( $this->mRevision )
 715                         return $this->mRevision->getTimestamp();
 716                 else if( $this->mImage )
 717                         return $this->mImage->getTimestamp();
 718                 return '';
 719         }
 720
 721         /**
 722          * @return Integer: number of words
 723          */
 724         function getWordCount(){
 725                 $this->initText();
 726                 return str_word_count( $this->mText );
 727         }
 728
 729         /**
 730          * @return Integer: size in bytes
 731          */
 732         function getByteSize(){
 733                 $this->initText();
 734                 return strlen( $this->mText );
 735         }
 736
 737         /**
 738          * @return Boolean if hit has related articles
 739          */
 740         function hasRelated(){
 741                 return false;
 742         }
 743
 744         /**
 745          * @return String: interwiki prefix of the title (return iw even if title is broken)
 746          */
 747         function getInterwikiPrefix(){
 748                 return '';
 749         }
 750 }
 751
 752 /**
 753  * Highlight bits of wikitext
 754  *
 755  * @ingroup Search
 756  */
 757 class SearchHighlighter {
 758         var $mCleanWikitext = true;
 759
 760         function SearchHighlighter($cleanupWikitext = true){
 761                 $this->mCleanWikitext = $cleanupWikitext;
 762         }
 763
 764         /**
 765          * Default implementation of wikitext highlighting
 766          *
 767          * @param $text String
 768          * @param $terms Array: terms to highlight (unescaped)
 769          * @param $contextlines Integer
 770          * @param $contextchars Integer
 771          * @return String
 772          */
 773         public function highlightText( $text, $terms, $contextlines, $contextchars ) {
 774                 global $wgLang, $wgContLang;
 775                 global $wgSearchHighlightBoundaries;
 776                 $fname = __METHOD__;
 777
 778                 if($text == '')
 779                         return '';
 780
 781                 // spli text into text + templates/links/tables
 782                 $spat = "/(\\{\\{)|(\\[\\[[^\\]:]+:)|(\n\\{\\|)";
 783                 // first capture group is for detecting nested templates/links/tables/references
 784                 $endPatterns = array(
 785                         1 => '/(\{\{)|(\}\})/', // template
 786                         2 => '/(\[\[)|(\]\])/', // image
 787                         3 => "/(\n\\{\\|)|(\n\\|\\})/"); // table
 788
 789                 // FIXME: this should prolly be a hook or something
 790                 if(function_exists('wfCite')){
 791                         $spat .= '|(<ref>)'; // references via cite extension
 792                         $endPatterns[4] = '/(<ref>)|(<\/ref>)/';
 793                 }
 794                 $spat .= '/';
 795                 $textExt = array(); // text extracts
 796                 $otherExt = array();  // other extracts
 797                 wfProfileIn( "$fname-split" );
 798                 $start = 0;
 799                 $textLen = strlen($text);
 800                 $count = 0; // sequence number to maintain ordering
 801                 while( $start < $textLen ){
 802                         // find start of template/image/table
 803                         if( preg_match( $spat, $text, $matches, PREG_OFFSET_CAPTURE, $start ) ){
 804                                 $epat = '';
 805                                 foreach($matches as $key => $val){
 806                                         if($key > 0 && $val[1] != -1){
 807                                                 if($key == 2){
 808                                                         // see if this is an image link
 809                                                         $ns = substr($val[0],2,-1);
 810                                                         if( $wgContLang->getNsIndex($ns) != NS_FILE )
 811                                                                 break;
 812
 813                                                 }
 814                                                 $epat = $endPatterns[$key];
 815                                                 $this->splitAndAdd( $textExt, $count, substr( $text, $start, $val[1] - $start ) );
 816                                                 $start = $val[1];
 817                                                 break;
 818                                         }
 819                                 }
 820                                 if( $epat ){
 821                                         // find end (and detect any nested elements)
 822                                         $level = 0;
 823                                         $offset = $start + 1;
 824                                         $found = false;
 825                                         while( preg_match( $epat, $text, $endMatches, PREG_OFFSET_CAPTURE, $offset ) ){
 826                                                 if( array_key_exists(2,$endMatches) ){
 827                                                         // found end
 828                                                         if($level == 0){
 829                                                                 $len = strlen($endMatches[2][0]);
 830                                                                 $off = $endMatches[2][1];
 831                                                                 $this->splitAndAdd( $otherExt, $count,
 832                                                                         substr( $text, $start, $off + $len  - $start ) );
 833                                                                 $start = $off + $len;
 834                                                                 $found = true;
 835                                                                 break;
 836                                                         } else{
 837                                                                 // end of nested element
 838                                                                 $level -= 1;
 839                                                         }
 840                                                 } else{
 841                                                         // nested
 842                                                         $level += 1;
 843                                                 }
 844                                                 $offset = $endMatches[0][1] + strlen($endMatches[0][0]);
 845                                         }
 846                                         if( ! $found ){
 847                                                 // couldn't find appropriate closing tag, skip
 848                                                 $this->splitAndAdd( $textExt, $count, substr( $text, $start, strlen($matches[0][0]) ) );
 849                                                 $start += strlen($matches[0][0]);
 850                                         }
 851                                         continue;
 852                                 }
 853                         }
 854                         // else: add as text extract
 855                         $this->splitAndAdd( $textExt, $count, substr($text,$start) );
 856                         break;
 857                 }
 858
 859                 $all = $textExt + $otherExt; // these have disjunct key sets
 860
 861                 wfProfileOut( "$fname-split" );
 862
 863                 // prepare regexps
 864                 foreach( $terms as $index => $term ) {
 865                         // manually do upper/lowercase stuff for utf-8 since PHP won't do it
 866                         if(preg_match('/[\x80-\xff]/', $term) ){
 867                                 $terms[$index] = preg_replace_callback('/./us',array($this,'caseCallback'),$terms[$index]);
 868                         } else {
 869                                 $terms[$index] = $term;
 870                         }
 871                 }
 872                 $anyterm = implode( '|', $terms );
 873                 $phrase = implode("$wgSearchHighlightBoundaries+", $terms );
 874
 875                 // FIXME: a hack to scale contextchars, a correct solution
 876                 // would be to have contextchars actually be char and not byte
 877                 // length, and do proper utf-8 substrings and lengths everywhere,
 878                 // but PHP is making that very hard and unclean to implement :(
 879                 $scale = strlen($anyterm) / mb_strlen($anyterm);
 880                 $contextchars = intval( $contextchars * $scale );
 881
 882                 $patPre = "(^|$wgSearchHighlightBoundaries)";
 883                 $patPost = "($wgSearchHighlightBoundaries|$)";
 884
 885                 $pat1 = "/(".$phrase.")/ui";
 886                 $pat2 = "/$patPre(".$anyterm.")$patPost/ui";
 887
 888                 wfProfileIn( "$fname-extract" );
 889
 890                 $left = $contextlines;
 891
 892                 $snippets = array();
 893                 $offsets = array();
 894
 895                 // show beginning only if it contains all words
 896                 $first = 0;
 897                 $firstText = '';
 898                 foreach($textExt as $index => $line){
 899                         if(strlen($line)>0 && $line[0] != ';' && $line[0] != ':'){
 900                                 $firstText = $this->extract( $line, 0, $contextchars * $contextlines );
 901                                 $first = $index;
 902                                 break;
 903                         }
 904                 }
 905                 if( $firstText ){
 906                         $succ = true;
 907                         // check if first text contains all terms
 908                         foreach($terms as $term){
 909                                 if( ! preg_match("/$patPre".$term."$patPost/ui", $firstText) ){
 910                                         $succ = false;
 911                                         break;
 912                                 }
 913                         }
 914                         if( $succ ){
 915                                 $snippets[$first] = $firstText;
 916                                 $offsets[$first] = 0;
 917                         }
 918                 }
 919                 if( ! $snippets ) {
 920                         // match whole query on text
 921                         $this->process($pat1, $textExt, $left, $contextchars, $snippets, $offsets);
 922                         // match whole query on templates/tables/images
 923                         $this->process($pat1, $otherExt, $left, $contextchars, $snippets, $offsets);
 924                         // match any words on text
 925                         $this->process($pat2, $textExt, $left, $contextchars, $snippets, $offsets);
 926                         // match any words on templates/tables/images
 927                         $this->process($pat2, $otherExt, $left, $contextchars, $snippets, $offsets);
 928
 929                         ksort($snippets);
 930                 }
 931
 932                 // add extra chars to each snippet to make snippets constant size
 933                 $extended = array();
 934                 if( count( $snippets ) == 0){
 935                         // couldn't find the target words, just show beginning of article
 936                         $targetchars = $contextchars * $contextlines;
 937                         $snippets[$first] = '';
 938                         $offsets[$first] = 0;
 939                 } else{
 940                         // if begin of the article contains the whole phrase, show only that !!
 941                         if( array_key_exists($first,$snippets) && preg_match($pat1,$snippets[$first])
 942                             && $offsets[$first] < $contextchars * 2 ){
 943                                 $snippets = array ($first => $snippets[$first]);
 944                         }
 945
 946                         // calc by how much to extend existing snippets
 947                         $targetchars = intval( ($contextchars * $contextlines) / count ( $snippets ) );
 948                 }
 949
 950                 foreach($snippets as $index => $line){
 951                         $extended[$index] = $line;
 952                         $len = strlen($line);
 953                         if( $len < $targetchars - 20 ){
 954                                 // complete this line
 955                                 if($len < strlen( $all[$index] )){
 956                                         $extended[$index] = $this->extract( $all[$index], $offsets[$index], $offsets[$index]+$targetchars, $offsets[$index]);
 957                                         $len = strlen( $extended[$index] );
 958                                 }
 959
 960                                 // add more lines
 961                                 $add = $index + 1;
 962                                 while( $len < $targetchars - 20
 963                                        && array_key_exists($add,$all)
 964                                        && !array_key_exists($add,$snippets) ){
 965                                     $offsets[$add] = 0;
 966                                     $tt = "\n".$this->extract( $all[$add], 0, $targetchars - $len, $offsets[$add] );
 967                                         $extended[$add] = $tt;
 968                                         $len += strlen( $tt );
 969                                         $add++;
 970                                 }
 971                         }
 972                 }
 973
 974                 //$snippets = array_map('htmlspecialchars', $extended);
 975                 $snippets = $extended;
 976                 $last = -1;
 977                 $extract = '';
 978                 foreach($snippets as $index => $line){
 979                         if($last == -1)
 980                                 $extract .= $line; // first line
 981                         elseif($last+1 == $index && $offsets[$last]+strlen($snippets[$last]) >= strlen($all[$last]))
 982                                 $extract .= " ".$line; // continous lines
 983                         else
 984                                 $extract .= '<b> ... </b>' . $line;
 985
 986                         $last = $index;
 987                 }
 988                 if( $extract )
 989                         $extract .= '<b> ... </b>';
 990
 991                 $processed = array();
 992                 foreach($terms as $term){
 993                         if( ! isset($processed[$term]) ){
 994                                 $pat3 = "/$patPre(".$term.")$patPost/ui"; // highlight word
 995                                 $extract = preg_replace( $pat3,
 996                                         "\\1<span class='searchmatch'>\\2</span>\\3", $extract );
 997                                 $processed[$term] = true;
 998                         }
 999                 }
1000
1001                 wfProfileOut( "$fname-extract" );
1002
1003                 return $extract;
1004         }
1005
1006         /**
1007          * Split text into lines and add it to extracts array
1008          *
1009          * @param $extracts Array: index -> $line
1010          * @param $count Integer
1011          * @param $text String
1012          */
1013         function splitAndAdd(&$extracts, &$count, $text){
1014                 $split = explode( "\n", $this->mCleanWikitext? $this->removeWiki($text) : $text );
1015                 foreach($split as $line){
1016                         $tt = trim($line);
1017                         if( $tt )
1018                                 $extracts[$count++] = $tt;
1019                 }
1020         }
1021
1022         /**
1023          * Do manual case conversion for non-ascii chars
1024          *
1025          * @param $matches Array
1026          */
1027         function caseCallback($matches){
1028                 global $wgContLang;
1029                 if( strlen($matches[0]) > 1 ){
1030                         return '['.$wgContLang->lc($matches[0]).$wgContLang->uc($matches[0]).']';
1031                 } else
1032                         return $matches[0];
1033         }
1034
1035         /**
1036          * Extract part of the text from start to end, but by
1037          * not chopping up words
1038          * @param $text String
1039          * @param $start Integer
1040          * @param $end Integer
1041          * @param $posStart Integer: (out) actual start position
1042          * @param $posEnd Integer: (out) actual end position
1043          * @return String
1044          */
1045         function extract($text, $start, $end, &$posStart = null, &$posEnd = null ){
1046                 global $wgContLang;
1047
1048                 if( $start != 0)
1049                         $start = $this->position( $text, $start, 1 );
1050                 if( $end >= strlen($text) )
1051                         $end = strlen($text);
1052                 else
1053                         $end = $this->position( $text, $end );
1054
1055                 if(!is_null($posStart))
1056                         $posStart = $start;
1057                 if(!is_null($posEnd))
1058                         $posEnd = $end;
1059
1060                 if($end > $start)
1061                         return substr($text, $start, $end-$start);
1062                 else
1063                         return '';
1064         }
1065
1066         /**
1067          * Find a nonletter near a point (index) in the text
1068          *
1069          * @param $text String
1070          * @param $point Integer
1071          * @param $offset Integer: offset to found index
1072          * @return Integer: nearest nonletter index, or beginning of utf8 char if none
1073          */
1074         function position($text, $point, $offset=0 ){
1075                 $tolerance = 10;
1076                 $s = max( 0, $point - $tolerance );
1077                 $l = min( strlen($text), $point + $tolerance ) - $s;
1078                 $m = array();
1079                 if( preg_match('/[ ,.!?~!@#$%^&*\(\)+=\-\\\|\[\]"\'<>]/', substr($text,$s,$l), $m, PREG_OFFSET_CAPTURE ) ){
1080                         return $m[0][1] + $s + $offset;
1081                 } else{
1082                         // check if point is on a valid first UTF8 char
1083                         $char = ord( $text[$point] );
1084                         while( $char >= 0x80 && $char < 0xc0 ) {
1085                                 // skip trailing bytes
1086                                 $point++;
1087                                 if($point >= strlen($text))
1088                                         return strlen($text);
1089                                 $char = ord( $text[$point] );
1090                         }
1091                         return $point;
1092
1093                 }
1094         }
1095
1096         /**
1097          * Search extracts for a pattern, and return snippets
1098          *
1099          * @param $pattern String: regexp for matching lines
1100          * @param $extracts Array: extracts to search
1101          * @param $linesleft Integer: number of extracts to make
1102          * @param $contextchars Integer: length of snippet
1103          * @param $out Array: map for highlighted snippets
1104          * @param $offsets Array: map of starting points of snippets
1105          * @protected
1106          */
1107         function process( $pattern, $extracts, &$linesleft, &$contextchars, &$out, &$offsets ){
1108                 if($linesleft == 0)
1109                         return; // nothing to do
1110                 foreach($extracts as $index => $line){
1111                         if( array_key_exists($index,$out) )
1112                                 continue; // this line already highlighted
1113
1114                         $m = array();
1115                         if ( !preg_match( $pattern, $line, $m, PREG_OFFSET_CAPTURE ) )
1116                                 continue;
1117
1118                         $offset = $m[0][1];
1119                         $len = strlen($m[0][0]);
1120                         if($offset + $len < $contextchars)
1121                                 $begin = 0;
1122                         elseif( $len > $contextchars)
1123                                 $begin = $offset;
1124                         else
1125                                 $begin = $offset + intval( ($len - $contextchars) / 2 );
1126
1127                         $end = $begin + $contextchars;
1128
1129                         $posBegin = $begin;
1130                         // basic snippet from this line
1131                         $out[$index] = $this->extract($line,$begin,$end,$posBegin);
1132                         $offsets[$index] = $posBegin;
1133                         $linesleft--;
1134                         if($linesleft == 0)
1135                                 return;
1136                 }
1137         }
1138
1139         /**
1140          * Basic wikitext removal
1141          * @protected
1142          */
1143         function removeWiki($text) {
1144                 $fname = __METHOD__;
1145                 wfProfileIn( $fname );
1146
1147                 //$text = preg_replace("/'{2,5}/", "", $text);
1148                 //$text = preg_replace("/\[[a-z]+:\/\/[^ ]+ ([^]]+)\]/", "\\2", $text);
1149                 //$text = preg_replace("/\[\[([^]|]+)\]\]/", "\\1", $text);
1150                 //$text = preg_replace("/\[\[([^]]+\|)?([^|]]+)\]\]/", "\\2", $text);
1151                 //$text = preg_replace("/\\{\\|(.*?)\\|\\}/", "", $text);
1152                 //$text = preg_replace("/\\[\\[[A-Za-z_-]+:([^|]+?)\\]\\]/", "", $text);
1153                 $text = preg_replace("/\\{\\{([^|]+?)\\}\\}/", "", $text);
1154                 $text = preg_replace("/\\{\\{([^|]+\\|)(.*?)\\}\\}/", "\\2", $text);
1155                 $text = preg_replace("/\\[\\[([^|]+?)\\]\\]/", "\\1", $text);
1156                 $text = preg_replace_callback("/\\[\\[([^|]+\\|)(.*?)\\]\\]/", array($this,'linkReplace'), $text);
1157                 //$text = preg_replace("/\\[\\[([^|]+\\|)(.*?)\\]\\]/", "\\2", $text);
1158                 $text = preg_replace("/<\/?[^>]+>/", "", $text);
1159                 $text = preg_replace("/'''''/", "", $text);
1160                 $text = preg_replace("/('''|<\/?[iIuUbB]>)/", "", $text);
1161                 $text = preg_replace("/''/", "", $text);
1162
1163                 wfProfileOut( $fname );
1164                 return $text;
1165         }
1166
1167         /**
1168          * callback to replace [[target|caption]] kind of links, if
1169          * the target is category or image, leave it
1170          *
1171          * @param $matches Array
1172          */
1173         function linkReplace($matches){
1174                 $colon = strpos( $matches[1], ':' );
1175                 if( $colon === false )
1176                         return $matches[2]; // replace with caption
1177                 global $wgContLang;
1178                 $ns = substr( $matches[1], 0, $colon );
1179                 $index = $wgContLang->getNsIndex($ns);
1180                 if( $index !== false && ($index == NS_FILE || $index == NS_CATEGORY) )
1181                         return $matches[0]; // return the whole thing
1182                 else
1183                         return $matches[2];
1184
1185         }
1186
1187         /**
1188      * Simple & fast snippet extraction, but gives completely unrelevant
1189      * snippets
1190      *
1191      * @param $text String
1192      * @param $terms Array
1193      * @param $contextlines Integer
1194      * @param $contextchars Integer
1195      * @return String
1196      */
1197     public function highlightSimple( $text, $terms, $contextlines, $contextchars ) {
1198         global $wgLang, $wgContLang;
1199         $fname = __METHOD__;
1200
1201         $lines = explode( "\n", $text );
1202
1203         $terms = implode( '|', $terms );
1204         $max = intval( $contextchars ) + 1;
1205         $pat1 = "/(.*)($terms)(.{0,$max})/i";
1206
1207         $lineno = 0;
1208
1209         $extract = "";
1210         wfProfileIn( "$fname-extract" );
1211         foreach ( $lines as $line ) {
1212             if ( 0 == $contextlines ) {
1213                 break;
1214             }
1215             ++$lineno;
1216             $m = array();
1217             if ( ! preg_match( $pat1, $line, $m ) ) {
1218                 continue;
1219             }
1220             --$contextlines;
1221             $pre = $wgContLang->truncate( $m[1], -$contextchars );
1222
1223             if ( count( $m ) < 3 ) {
1224                 $post = '';
1225             } else {
1226                 $post = $wgContLang->truncate( $m[3], $contextchars );
1227             }
1228
1229             $found = $m[2];
1230
1231             $line = htmlspecialchars( $pre . $found . $post );
1232             $pat2 = '/(' . $terms . ")/i";
1233             $line = preg_replace( $pat2,
1234               "<span class='searchmatch'>\\1</span>", $line );
1235
1236             $extract .= "${line}\n";
1237         }
1238         wfProfileOut( "$fname-extract" );
1239
1240         return $extract;
1241     }
1242
1243 }
1244
1245 /**
1246  * Dummy class to be used when non-supported Database engine is present.
1247  * @todo Fixme: dummy class should probably try something at least mildly useful,
1248  * such as a LIKE search through titles.
1249  * @ingroup Search
1250  */
1251 class SearchEngineDummy extends SearchEngine {
1252         // no-op
1253 }