From: Robert Stojnić Date: Sun, 4 May 2008 15:31:03 +0000 (+0000) Subject: Re-commit r34072 with some modifications: X-Git-Tag: 1.31.0-rc.0~47914 X-Git-Url: https://git.cyclocoop.org/%28%28?a=commitdiff_plain;h=abf726ea02fc3bda653fc79cc992e07fec1a91bb;p=lhc%2Fweb%2Fwiklou.git Re-commit r34072 with some modifications: * turned off by default (set $wgAdvancedSearchHighlighting to turn on) * reverted r26269, \b doesn't interact very good with unicode data, so it broke highlighting of words that end/begin in nonascii chars completely * small bugfixes in unicode handling, tested in more languages * $wgSearchHighlightBoundaries need to be set to "" for CJK wikis * benchmarking: on typical simplewiki data, the code is around 4-5 slower (according to noc.wikimedia.org the old code profiles to about 0.8%), but can be up to 20 times slower on featured-size articles * update release notes (also for r33400) * fix profiling errors in SpecialSearch --- diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 554ff11b3e..c84411f10c 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -101,6 +101,8 @@ it from source control: http://www.mediawiki.org/wiki/Download_from_SVN * (bug 12542) Added hooks for expansion of Special:Listusers * Added new variable $wgSharedDBtables for altering the list of tables which are shared when $wgSharedDB is enabled. +* Drop-down AJAX search suggestions (turn on $wgEnableMWSuggest) +* More relevant search snippets (turn on $wgAdvancedSearchHighlighting) === Bug fixes in 1.13 === diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index 99f0d6ba87..976ebad417 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -1619,6 +1619,20 @@ $wgDisableCounters = false; $wgDisableTextSearch = false; $wgDisableSearchContext = false; + +/** + * Set to true to have nicer highligted text in search results, + * by default off due to execution overhead + */ +$wgAdvancedSearchHighlighting = false; + +/** + * Regexp to match word boundaries, defaults for non-CJK languages + * should be empty for CJK since the words are not separate + */ +$wgSearchHighlightBoundaries = version_compare("5.1", PHP_VERSION, "<")? '[\p{Z}\p{P}\p{C}]' + : '[ ,.;:!?~!@#$%\^&*\(\)+=\-\\|\[\]"\'<>\n\r\/{}]'; // PHP 5.0 workaround + /** * Template for OpenSearch suggestions, defaults to API action=opensearch * diff --git a/includes/SearchEngine.php b/includes/SearchEngine.php index c9294bd908..fbdd16a802 100644 --- a/includes/SearchEngine.php +++ b/includes/SearchEngine.php @@ -250,8 +250,9 @@ class SearchEngine { */ public static function userHighlightPrefs( &$user ){ //$contextlines = $user->getOption( 'contextlines', 5 ); + //$contextchars = $user->getOption( 'contextchars', 50 ); $contextlines = 2; // Hardcode this. Old defaults sucked. :) - $contextchars = $user->getOption( 'contextchars', 50 ); + $contextchars = 75; // same as above.... :P return array($contextlines, $contextchars); } @@ -358,7 +359,6 @@ class SearchEngine { } } - /** * @addtogroup Search */ @@ -544,72 +544,20 @@ class SearchResult { $this->mText = $this->mRevision->getText(); } } - + /** * @param array $terms terms to highlight * @return string highlighted text snippet, null (and not '') if not supported */ function getTextSnippet($terms){ - global $wgUser; + global $wgUser, $wgAdvancedSearchHighlighting; $this->initText(); list($contextlines,$contextchars) = SearchEngine::userHighlightPrefs($wgUser); - return $this->extractText( $this->mText, $terms, $contextlines, $contextchars); - } - - /** - * Default implementation of snippet extraction - * - * @param string $text - * @param array $terms - * @param int $contextlines - * @param int $contextchars - * @return string - */ - protected function extractText( $text, $terms, $contextlines, $contextchars ) { - global $wgLang, $wgContLang; - $fname = __METHOD__; - - $lines = explode( "\n", $text ); - - $terms = implode( '|', $terms ); - $terms = str_replace( '/', "\\/", $terms); - $max = intval( $contextchars ) + 1; - $pat1 = "/(.*)($terms)(.{0,$max})/i"; - - $lineno = 0; - - $extract = ""; - wfProfileIn( "$fname-extract" ); - foreach ( $lines as $line ) { - if ( 0 == $contextlines ) { - break; - } - ++$lineno; - $m = array(); - if ( ! preg_match( $pat1, $line, $m ) ) { - continue; - } - --$contextlines; - $pre = $wgContLang->truncate( $m[1], -$contextchars, ' ... ' ); - - if ( count( $m ) < 3 ) { - $post = ''; - } else { - $post = $wgContLang->truncate( $m[3], $contextchars, ' ... ' ); - } - - $found = $m[2]; - - $line = htmlspecialchars( $pre . $found . $post ); - $pat2 = '/(' . $terms . ")/i"; - $line = preg_replace( $pat2, - "\\1", $line ); - - $extract .= "${line}\n"; - } - wfProfileOut( "$fname-extract" ); - - return $extract; + $h = new SearchHighlighter(); + if( $wgAdvancedSearchHighlighting ) + return $h->highlightText( $this->mText, $terms, $contextlines, $contextchars ); + else + return $h->highlightSimple( $this->mText, $terms, $contextlines, $contextchars ); } /** @@ -687,6 +635,501 @@ class SearchResult { } } +/** + * Highlight bits of wikitext + * + * @addtogroup Search + */ +class SearchHighlighter { + var $mCleanWikitext = true; + + function SearchHighlighter($cleanupWikitext = true){ + $this->mCleanWikitext = $cleanupWikitext; + } + + /** + * Default implementation of wikitext highlighting + * + * @param string $text + * @param array $terms Terms to highlight (unescaped) + * @param int $contextlines + * @param int $contextchars + * @return string + */ + public function highlightText( $text, $terms, $contextlines, $contextchars ) { + global $wgLang, $wgContLang; + global $wgSearchHighlightBoundaries; + $fname = __METHOD__; + + if($text == '') + return ''; + + // spli text into text + templates/links/tables + $spat = "/(\\{\\{)|(\\[\\[[^\\]:]+:)|(\n\\{\\|)"; + // first capture group is for detecting nested templates/links/tables/references + $endPatterns = array( + 1 => '/(\{\{)|(\}\})/', // template + 2 => '/(\[\[)|(\]\])/', // image + 3 => "/(\n\\{\\|)|(\n\\|\\})/"); // table + + // FIXME: this should prolly be a hook or something + if(function_exists('wfCite')){ + $spat .= '|()'; // references via cite extension + $endPatterns[4] = '/()|(<\/ref>)/'; + } + $spat .= '/'; + $textExt = array(); // text extracts + $otherExt = array(); // other extracts + wfProfileIn( "$fname-split" ); + $start = 0; + $textLen = strlen($text); + $count = 0; // sequence number to maintain ordering + while( $start < $textLen ){ + // find start of template/image/table + if( preg_match( $spat, $text, $matches, PREG_OFFSET_CAPTURE, $start ) ){ + $epat = ''; + foreach($matches as $key => $val){ + if($key > 0 && $val[1] != -1){ + if($key == 2){ + // see if this is an image link + $ns = substr($val[0],2,-1); + if( $wgContLang->getNsIndex($ns) != NS_IMAGE ) + break; + + } + $epat = $endPatterns[$key]; + $this->splitAndAdd( $textExt, $count, substr( $text, $start, $val[1] - $start ) ); + $start = $val[1]; + break; + } + } + if( $epat ){ + // find end (and detect any nested elements) + $level = 0; + $offset = $start + 1; + $found = false; + while( preg_match( $epat, $text, $endMatches, PREG_OFFSET_CAPTURE, $offset ) ){ + if( array_key_exists(2,$endMatches) ){ + // found end + if($level == 0){ + $len = strlen($endMatches[2][0]); + $off = $endMatches[2][1]; + $this->splitAndAdd( $otherExt, $count, + substr( $text, $start, $off + $len - $start ) ); + $start = $off + $len; + $found = true; + break; + } else{ + // end of nested element + $level -= 1; + } + } else{ + // nested + $level += 1; + } + $offset = $endMatches[0][1] + strlen($endMatches[0][0]); + } + if( ! $found ){ + // couldn't find appropriate closing tag, skip + $this->splitAndAdd( $textExt, $count, substr( $text, $start, strlen($matches[0][0]) ) ); + $start += strlen($matches[0][0]); + } + continue; + } + } + // else: add as text extract + $this->splitAndAdd( $textExt, $count, substr($text,$start) ); + break; + } + + $all = $textExt + $otherExt; // these have disjunct key sets + + wfProfileOut( "$fname-split" ); + + // prepare regexps + foreach( $terms as $index => $term ) { + $terms[$index] = preg_quote( $term, '/' ); + // manually do upper/lowercase stuff for utf-8 since PHP won't do it + if(preg_match('/[\x80-\xff]/', $term) ){ + $terms[$index] = preg_replace_callback('/./us',array($this,'caseCallback'),$terms[$index]); + } + + + } + $anyterm = implode( '|', $terms ); + $phrase = implode("$wgSearchHighlightBoundaries+", $terms ); + + // FIXME: a hack to scale contextchars, a correct solution + // would be to have contextchars actually be char and not byte + // length, and do proper utf-8 substrings and lengths everywhere, + // but PHP is making that very hard and unclean to implement :( + $scale = strlen($anyterm) / mb_strlen($anyterm); + $contextchars = intval( $contextchars * $scale ); + + $patPre = "(^|$wgSearchHighlightBoundaries)"; + $patPost = "($wgSearchHighlightBoundaries|$)"; + + $pat1 = "/(".$phrase.")/ui"; + $pat2 = "/$patPre(".$anyterm.")$patPost/ui"; + + wfProfileIn( "$fname-extract" ); + + $left = $contextlines; + + $snippets = array(); + $offsets = array(); + + // show beginning only if it contains all words + $first = 0; + $firstText = ''; + foreach($textExt as $index => $line){ + if(strlen($line)>0 && $line[0] != ';' && $line[0] != ':'){ + $firstText = $this->extract( $line, 0, $contextchars * $contextlines ); + $first = $index; + break; + } + } + if( $firstText ){ + $succ = true; + // check if first text contains all terms + foreach($terms as $term){ + if( ! preg_match("/$patPre".$term."$patPost/ui", $firstText) ){ + $succ = false; + break; + } + } + if( $succ ){ + $snippets[$first] = $firstText; + $offsets[$first] = 0; + } + } + if( ! $snippets ) { + // match whole query on text + $this->process($pat1, $textExt, $left, $contextchars, $snippets, $offsets); + // match whole query on templates/tables/images + $this->process($pat1, $otherExt, $left, $contextchars, $snippets, $offsets); + // match any words on text + $this->process($pat2, $textExt, $left, $contextchars, $snippets, $offsets); + // match any words on templates/tables/images + $this->process($pat2, $otherExt, $left, $contextchars, $snippets, $offsets); + + ksort($snippets); + } + + // add extra chars to each snippet to make snippets constant size + $extended = array(); + if( count( $snippets ) == 0){ + // couldn't find the target words, just show beginning of article + $targetchars = $contextchars * $contextlines; + $snippets[$first] = ''; + $offsets[$first] = 0; + } else{ + // if begin of the article contains the whole phrase, show only that !! + if( array_key_exists($first,$snippets) && preg_match($pat1,$snippets[$first]) + && $offsets[$first] < $contextchars * 2 ){ + $snippets = array ($first => $snippets[$first]); + } + + // calc by how much to extend existing snippets + $targetchars = intval( ($contextchars * $contextlines) / count ( $snippets ) ); + } + + foreach($snippets as $index => $line){ + $extended[$index] = $line; + $len = strlen($line); + if( $len < $targetchars - 20 ){ + // complete this line + if($len < strlen( $all[$index] )){ + $extended[$index] = $this->extract( $all[$index], $offsets[$index], $offsets[$index]+$targetchars, $offsets[$index]); + $len = strlen( $extended[$index] ); + } + + // add more lines + $add = $index + 1; + while( $len < $targetchars - 20 + && array_key_exists($add,$all) + && !array_key_exists($add,$snippets) ){ + $offsets[$add] = 0; + $tt = "\n".$this->extract( $all[$add], 0, $targetchars - $len, $offsets[$add] ); + $extended[$add] = $tt; + $len += strlen( $tt ); + $add++; + } + } + } + + //$snippets = array_map('htmlspecialchars', $extended); + $snippets = $extended; + $last = -1; + $extract = ''; + foreach($snippets as $index => $line){ + if($last == -1) + $extract .= $line; // first line + elseif($last+1 == $index && $offsets[$last]+strlen($snippets[$last]) >= strlen($all[$last])) + $extract .= " ".$line; // continous lines + else + $extract .= ' ... ' . $line; + + $last = $index; + } + if( $extract ) + $extract .= ' ... '; + + $processed = array(); + foreach($terms as $term){ + if( ! isset($processed[$term]) ){ + $pat3 = "/$patPre(".$term.")$patPost/ui"; // highlight word + $extract = preg_replace( $pat3, + "\\1\\2\\3", $extract ); + $processed[$term] = true; + } + } + + wfProfileOut( "$fname-extract" ); + + return $extract; + } + + /** + * Split text into lines and add it to extracts array + * + * @param array $extracts index -> $line + * @param int $count + * @param string $text + */ + function splitAndAdd(&$extracts, &$count, $text){ + $split = explode( "\n", $this->mCleanWikitext? $this->removeWiki($text) : $text ); + foreach($split as $line){ + $tt = trim($line); + if( $tt ) + $extracts[$count++] = $tt; + } + } + + /** + * Do manual case conversion for non-ascii chars + * + * @param unknown_type $matches + */ + function caseCallback($matches){ + global $wgContLang; + if( strlen($matches[0]) > 1 ){ + return '['.$wgContLang->lc($matches[0]).$wgContLang->uc($matches[0]).']'; + } else + return $matches[0]; + } + + /** + * Extract part of the text from start to end, but by + * not chopping up words + * @param string $text + * @param int $start + * @param int $end + * @param int $posStart (out) actual start position + * @param int $posEnd (out) actual end position + * @return string + */ + function extract($text, $start, $end, &$posStart = null, &$posEnd = null ){ + global $wgContLang; + + if( $start != 0) + $start = $this->position( $text, $start, 1 ); + if( $end >= strlen($text) ) + $end = strlen($text); + else + $end = $this->position( $text, $end ); + + if(!is_null($posStart)) + $posStart = $start; + if(!is_null($posEnd)) + $posEnd = $end; + + if($end > $start) + return substr($text, $start, $end-$start); + else + return ''; + } + + /** + * Find a nonletter near a point (index) in the text + * + * @param string $text + * @param int $point + * @param int $offset to found index + * @return int nearest nonletter index, or beginning of utf8 char if none + */ + function position($text, $point, $offset=0 ){ + $tolerance = 10; + $s = max( 0, $point - $tolerance ); + $l = min( strlen($text), $point + $tolerance ) - $s; + $m = array(); + if( preg_match('/[ ,.!?~!@#$%^&*\(\)+=\-\\\|\[\]"\'<>]/', substr($text,$s,$l), $m, PREG_OFFSET_CAPTURE ) ){ + return $m[0][1] + $s + $offset; + } else{ + // check if point is on a valid first UTF8 char + $char = ord( $text[$point] ); + while( $char >= 0x80 && $char < 0xc0 ) { + // skip trailing bytes + $point++; + if($point >= strlen($text)) + return strlen($text); + $char = ord( $text[$point] ); + } + return $point; + + } + } + + /** + * Search extracts for a pattern, and return snippets + * + * @param string $pattern regexp for matching lines + * @param array $extracts extracts to search + * @param int $linesleft number of extracts to make + * @param int $contextchars length of snippet + * @param array $out map for highlighted snippets + * @param array $offsets map of starting points of snippets + * @protected + */ + function process( $pattern, $extracts, &$linesleft, &$contextchars, &$out, &$offsets ){ + if($linesleft == 0) + return; // nothing to do + foreach($extracts as $index => $line){ + if( array_key_exists($index,$out) ) + continue; // this line already highlighted + + $m = array(); + if ( !preg_match( $pattern, $line, $m, PREG_OFFSET_CAPTURE ) ) + continue; + + $offset = $m[0][1]; + $len = strlen($m[0][0]); + if($offset + $len < $contextchars) + $begin = 0; + elseif( $len > $contextchars) + $begin = $offset; + else + $begin = $offset + intval( ($len - $contextchars) / 2 ); + + $end = $begin + $contextchars; + + $posBegin = $begin; + // basic snippet from this line + $out[$index] = $this->extract($line,$begin,$end,$posBegin); + $offsets[$index] = $posBegin; + $linesleft--; + if($linesleft == 0) + return; + } + } + + /** + * Basic wikitext removal + * @protected + */ + function removeWiki($text) { + $fname = __METHOD__; + wfProfileIn( $fname ); + + //$text = preg_replace("/'{2,5}/", "", $text); + //$text = preg_replace("/\[[a-z]+:\/\/[^ ]+ ([^]]+)\]/", "\\2", $text); + //$text = preg_replace("/\[\[([^]|]+)\]\]/", "\\1", $text); + //$text = preg_replace("/\[\[([^]]+\|)?([^|]]+)\]\]/", "\\2", $text); + //$text = preg_replace("/\\{\\|(.*?)\\|\\}/", "", $text); + //$text = preg_replace("/\\[\\[[A-Za-z_-]+:([^|]+?)\\]\\]/", "", $text); + $text = preg_replace("/\\{\\{([^|]+?)\\}\\}/", "", $text); + $text = preg_replace("/\\{\\{([^|]+\\|)(.*?)\\}\\}/", "\\2", $text); + $text = preg_replace("/\\[\\[([^|]+?)\\]\\]/", "\\1", $text); + $text = preg_replace_callback("/\\[\\[([^|]+\\|)(.*?)\\]\\]/", array($this,'linkReplace'), $text); + //$text = preg_replace("/\\[\\[([^|]+\\|)(.*?)\\]\\]/", "\\2", $text); + $text = preg_replace("/<\/?[^>]+>/", "", $text); + $text = preg_replace("/'''''/", "", $text); + $text = preg_replace("/('''|<\/?[iIuUbB]>)/", "", $text); + $text = preg_replace("/''/", "", $text); + + wfProfileOut( $fname ); + return $text; + } + + /** + * callback to replace [[target|caption]] kind of links, if + * the target is category or image, leave it + * + * @param array $matches + */ + function linkReplace($matches){ + $colon = strpos( $matches[1], ':' ); + if( $colon === false ) + return $matches[2]; // replace with caption + global $wgContLang; + $ns = substr( $matches[1], 0, $colon ); + $index = $wgContLang->getNsIndex($ns); + if( $index !== false && ($index == NS_IMAGE || $index == NS_CATEGORY) ) + return $matches[0]; // return the whole thing + else + return $matches[2]; + + } + + /** + * Simple & fast snippet extraction, but gives completely unrelevant + * snippets + * + * @param string $text + * @param array $terms + * @param int $contextlines + * @param int $contextchars + * @return string + */ + public function highlightSimple( $text, $terms, $contextlines, $contextchars ) { + global $wgLang, $wgContLang; + $fname = __METHOD__; + + $lines = explode( "\n", $text ); + + $terms = implode( '|', $terms ); + $terms = str_replace( '/', "\\/", $terms); + $max = intval( $contextchars ) + 1; + $pat1 = "/(.*)($terms)(.{0,$max})/i"; + + $lineno = 0; + + $extract = ""; + wfProfileIn( "$fname-extract" ); + foreach ( $lines as $line ) { + if ( 0 == $contextlines ) { + break; + } + ++$lineno; + $m = array(); + if ( ! preg_match( $pat1, $line, $m ) ) { + continue; + } + --$contextlines; + $pre = $wgContLang->truncate( $m[1], -$contextchars, ' ... ' ); + + if ( count( $m ) < 3 ) { + $post = ''; + } else { + $post = $wgContLang->truncate( $m[3], $contextchars, ' ... ' ); + } + + $found = $m[2]; + + $line = htmlspecialchars( $pre . $found . $post ); + $pat2 = '/(' . $terms . ")/i"; + $line = preg_replace( $pat2, + "\\1", $line ); + + $extract .= "${line}\n"; + } + wfProfileOut( "$fname-extract" ); + + return $extract; + } + +} + /** * @addtogroup Search */ diff --git a/includes/SearchMySQL.php b/includes/SearchMySQL.php index f3a7e16c93..6b3f47e39f 100644 --- a/includes/SearchMySQL.php +++ b/includes/SearchMySQL.php @@ -54,10 +54,10 @@ class SearchMySQL extends SearchEngine { // Match the quoted term in result highlighting... $regexp = preg_quote( str_replace( '"', '', $terms[2] ), '/' ); } - $this->searchTerms[] = "\b$regexp\b"; + $this->searchTerms[] = $regexp; } wfDebug( "Would search with '$searchon'\n" ); - wfDebug( 'Match with /\b' . implode( '\b|\b', $this->searchTerms ) . "\b/\n" ); + wfDebug( 'Match with /' . implode( '|', $this->searchTerms ) . "/\n" ); } else { wfDebug( "Can't understand search query '{$filteredText}'\n" ); } diff --git a/includes/SpecialSearch.php b/includes/SpecialSearch.php index 0a752c7755..2c50b49a3d 100644 --- a/includes/SpecialSearch.php +++ b/includes/SpecialSearch.php @@ -374,6 +374,7 @@ class SpecialSearch { //This is not quite safe, but better than showing excerpts from non-readable pages //Note that hiding the entry entirely would screw up paging. if (!$t->userCanRead()) { + wfProfileOut( $fname ); return "
  • {$link}
    • \n"; } @@ -381,6 +382,7 @@ class SpecialSearch { // The least confusing at this point is to drop the result. // You may get less results, but... oh well. :P if( $result->isMissingRevision() ) { + wfProfileOut( $fname ); return "\n"; }