3 * Basic search engine highlighting
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
25 * Highlight bits of wikitext
29 class SearchHighlighter
{
30 protected $mCleanWikitext = true;
32 function __construct( $cleanupWikitext = true ) {
33 $this->mCleanWikitext
= $cleanupWikitext;
37 * Default implementation of wikitext highlighting
40 * @param array $terms Terms to highlight (unescaped)
41 * @param int $contextlines
42 * @param int $contextchars
45 public function highlightText( $text, $terms, $contextlines, $contextchars ) {
46 global $wgContLang, $wgSearchHighlightBoundaries;
54 // spli text into text + templates/links/tables
55 $spat = "/(\\{\\{)|(\\[\\[[^\\]:]+:)|(\n\\{\\|)";
56 // first capture group is for detecting nested templates/links/tables/references
58 1 => '/(\{\{)|(\}\})/', // template
59 2 => '/(\[\[)|(\]\])/', // image
60 3 => "/(\n\\{\\|)|(\n\\|\\})/" ); // table
62 // @todo FIXME: This should prolly be a hook or something
63 if ( function_exists( 'wfCite' ) ) {
64 $spat .= '|(<ref>)'; // references via cite extension
65 $endPatterns[4] = '/(<ref>)|(<\/ref>)/';
68 $textExt = array(); // text extracts
69 $otherExt = array(); // other extracts
71 $textLen = strlen( $text );
72 $count = 0; // sequence number to maintain ordering
73 while ( $start < $textLen ) {
74 // find start of template/image/table
75 if ( preg_match( $spat, $text, $matches, PREG_OFFSET_CAPTURE
, $start ) ) {
77 foreach ( $matches as $key => $val ) {
78 if ( $key > 0 && $val[1] != - 1 ) {
80 // see if this is an image link
81 $ns = substr( $val[0], 2, - 1 );
82 if ( $wgContLang->getNsIndex( $ns ) != NS_FILE
) {
87 $epat = $endPatterns[$key];
88 $this->splitAndAdd( $textExt, $count, substr( $text, $start, $val[1] - $start ) );
94 // find end (and detect any nested elements)
98 while ( preg_match( $epat, $text, $endMatches, PREG_OFFSET_CAPTURE
, $offset ) ) {
99 if ( array_key_exists( 2, $endMatches ) ) {
102 $len = strlen( $endMatches[2][0] );
103 $off = $endMatches[2][1];
104 $this->splitAndAdd( $otherExt, $count,
105 substr( $text, $start, $off +
$len - $start ) );
106 $start = $off +
$len;
110 // end of nested element
117 $offset = $endMatches[0][1] +
strlen( $endMatches[0][0] );
120 // couldn't find appropriate closing tag, skip
121 $this->splitAndAdd( $textExt, $count, substr( $text, $start, strlen( $matches[0][0] ) ) );
122 $start +
= strlen( $matches[0][0] );
127 // else: add as text extract
128 $this->splitAndAdd( $textExt, $count, substr( $text, $start ) );
132 $all = $textExt +
$otherExt; // these have disjunct key sets
135 foreach ( $terms as $index => $term ) {
136 // manually do upper/lowercase stuff for utf-8 since PHP won't do it
137 if ( preg_match( '/[\x80-\xff]/', $term ) ) {
138 $terms[$index] = preg_replace_callback(
140 array( $this, 'caseCallback' ),
144 $terms[$index] = $term;
147 $anyterm = implode( '|', $terms );
148 $phrase = implode( "$wgSearchHighlightBoundaries+", $terms );
150 // @todo FIXME: A hack to scale contextchars, a correct solution
151 // would be to have contextchars actually be char and not byte
152 // length, and do proper utf-8 substrings and lengths everywhere,
153 // but PHP is making that very hard and unclean to implement :(
154 $scale = strlen( $anyterm ) / mb_strlen( $anyterm );
155 $contextchars = intval( $contextchars * $scale );
157 $patPre = "(^|$wgSearchHighlightBoundaries)";
158 $patPost = "($wgSearchHighlightBoundaries|$)";
160 $pat1 = "/(" . $phrase . ")/ui";
161 $pat2 = "/$patPre(" . $anyterm . ")$patPost/ui";
163 $left = $contextlines;
168 // show beginning only if it contains all words
171 foreach ( $textExt as $index => $line ) {
172 if ( strlen( $line ) > 0 && $line[0] != ';' && $line[0] != ':' ) {
173 $firstText = $this->extract( $line, 0, $contextchars * $contextlines );
180 // check if first text contains all terms
181 foreach ( $terms as $term ) {
182 if ( !preg_match( "/$patPre" . $term . "$patPost/ui", $firstText ) ) {
188 $snippets[$first] = $firstText;
189 $offsets[$first] = 0;
193 // match whole query on text
194 $this->process( $pat1, $textExt, $left, $contextchars, $snippets, $offsets );
195 // match whole query on templates/tables/images
196 $this->process( $pat1, $otherExt, $left, $contextchars, $snippets, $offsets );
197 // match any words on text
198 $this->process( $pat2, $textExt, $left, $contextchars, $snippets, $offsets );
199 // match any words on templates/tables/images
200 $this->process( $pat2, $otherExt, $left, $contextchars, $snippets, $offsets );
205 // add extra chars to each snippet to make snippets constant size
207 if ( count( $snippets ) == 0 ) {
208 // couldn't find the target words, just show beginning of article
209 if ( array_key_exists( $first, $all ) ) {
210 $targetchars = $contextchars * $contextlines;
211 $snippets[$first] = '';
212 $offsets[$first] = 0;
215 // if begin of the article contains the whole phrase, show only that !!
216 if ( array_key_exists( $first, $snippets ) && preg_match( $pat1, $snippets[$first] )
217 && $offsets[$first] < $contextchars * 2 ) {
218 $snippets = array( $first => $snippets[$first] );
221 // calc by how much to extend existing snippets
222 $targetchars = intval( ( $contextchars * $contextlines ) / count ( $snippets ) );
225 foreach ( $snippets as $index => $line ) {
226 $extended[$index] = $line;
227 $len = strlen( $line );
228 if ( $len < $targetchars - 20 ) {
229 // complete this line
230 if ( $len < strlen( $all[$index] ) ) {
231 $extended[$index] = $this->extract(
234 $offsets[$index] +
$targetchars,
237 $len = strlen( $extended[$index] );
242 while ( $len < $targetchars - 20
243 && array_key_exists( $add, $all )
244 && !array_key_exists( $add, $snippets ) ) {
246 $tt = "\n" . $this->extract( $all[$add], 0, $targetchars - $len, $offsets[$add] );
247 $extended[$add] = $tt;
248 $len +
= strlen( $tt );
254 // $snippets = array_map( 'htmlspecialchars', $extended );
255 $snippets = $extended;
258 foreach ( $snippets as $index => $line ) {
259 if ( $last == - 1 ) {
260 $extract .= $line; // first line
261 } elseif ( $last +
1 == $index
262 && $offsets[$last] +
strlen( $snippets[$last] ) >= strlen( $all[$last] )
264 $extract .= " " . $line; // continous lines
266 $extract .= '<b> ... </b>' . $line;
272 $extract .= '<b> ... </b>';
275 $processed = array();
276 foreach ( $terms as $term ) {
277 if ( !isset( $processed[$term] ) ) {
278 $pat3 = "/$patPre(" . $term . ")$patPost/ui"; // highlight word
279 $extract = preg_replace( $pat3,
280 "\\1<span class='searchmatch'>\\2</span>\\3", $extract );
281 $processed[$term] = true;
289 * Split text into lines and add it to extracts array
291 * @param array $extracts Index -> $line
293 * @param string $text
295 function splitAndAdd( &$extracts, &$count, $text ) {
296 $split = explode( "\n", $this->mCleanWikitext ?
$this->removeWiki( $text ) : $text );
297 foreach ( $split as $line ) {
300 $extracts[$count++
] = $tt;
306 * Do manual case conversion for non-ascii chars
308 * @param array $matches
311 function caseCallback( $matches ) {
313 if ( strlen( $matches[0] ) > 1 ) {
314 return '[' . $wgContLang->lc( $matches[0] ) . $wgContLang->uc( $matches[0] ) . ']';
321 * Extract part of the text from start to end, but by
322 * not chopping up words
323 * @param string $text
326 * @param int $posStart (out) actual start position
327 * @param int $posEnd (out) actual end position
330 function extract( $text, $start, $end, &$posStart = null, &$posEnd = null ) {
332 $start = $this->position( $text, $start, 1 );
334 if ( $end >= strlen( $text ) ) {
335 $end = strlen( $text );
337 $end = $this->position( $text, $end );
340 if ( !is_null( $posStart ) ) {
343 if ( !is_null( $posEnd ) ) {
347 if ( $end > $start ) {
348 return substr( $text, $start, $end - $start );
355 * Find a nonletter near a point (index) in the text
357 * @param string $text
359 * @param int $offset Offset to found index
360 * @return int Nearest nonletter index, or beginning of utf8 char if none
362 function position( $text, $point, $offset = 0 ) {
364 $s = max( 0, $point - $tolerance );
365 $l = min( strlen( $text ), $point +
$tolerance ) - $s;
369 '/[ ,.!?~!@#$%^&*\(\)+=\-\\\|\[\]"\'<>]/',
370 substr( $text, $s, $l ),
374 return $m[0][1] +
$s +
$offset;
376 // check if point is on a valid first UTF8 char
377 $char = ord( $text[$point] );
378 while ( $char >= 0x80 && $char < 0xc0 ) {
379 // skip trailing bytes
381 if ( $point >= strlen( $text ) ) {
382 return strlen( $text );
384 $char = ord( $text[$point] );
393 * Search extracts for a pattern, and return snippets
395 * @param string $pattern Regexp for matching lines
396 * @param array $extracts Extracts to search
397 * @param int $linesleft Number of extracts to make
398 * @param int $contextchars Length of snippet
399 * @param array $out Map for highlighted snippets
400 * @param array $offsets Map of starting points of snippets
403 function process( $pattern, $extracts, &$linesleft, &$contextchars, &$out, &$offsets ) {
404 if ( $linesleft == 0 ) {
405 return; // nothing to do
407 foreach ( $extracts as $index => $line ) {
408 if ( array_key_exists( $index, $out ) ) {
409 continue; // this line already highlighted
413 if ( !preg_match( $pattern, $line, $m, PREG_OFFSET_CAPTURE
) ) {
418 $len = strlen( $m[0][0] );
419 if ( $offset +
$len < $contextchars ) {
421 } elseif ( $len > $contextchars ) {
424 $begin = $offset +
intval( ( $len - $contextchars ) / 2 );
427 $end = $begin +
$contextchars;
430 // basic snippet from this line
431 $out[$index] = $this->extract( $line, $begin, $end, $posBegin );
432 $offsets[$index] = $posBegin;
434 if ( $linesleft == 0 ) {
441 * Basic wikitext removal
443 * @param string $text
446 function removeWiki( $text ) {
449 // $text = preg_replace( "/'{2,5}/", "", $text );
450 // $text = preg_replace( "/\[[a-z]+:\/\/[^ ]+ ([^]]+)\]/", "\\2", $text );
451 // $text = preg_replace( "/\[\[([^]|]+)\]\]/", "\\1", $text );
452 // $text = preg_replace( "/\[\[([^]]+\|)?([^|]]+)\]\]/", "\\2", $text );
453 // $text = preg_replace( "/\\{\\|(.*?)\\|\\}/", "", $text );
454 // $text = preg_replace( "/\\[\\[[A-Za-z_-]+:([^|]+?)\\]\\]/", "", $text );
455 $text = preg_replace( "/\\{\\{([^|]+?)\\}\\}/", "", $text );
456 $text = preg_replace( "/\\{\\{([^|]+\\|)(.*?)\\}\\}/", "\\2", $text );
457 $text = preg_replace( "/\\[\\[([^|]+?)\\]\\]/", "\\1", $text );
458 $text = preg_replace_callback(
459 "/\\[\\[([^|]+\\|)(.*?)\\]\\]/",
460 array( $this, 'linkReplace' ),
463 // $text = preg_replace("/\\[\\[([^|]+\\|)(.*?)\\]\\]/", "\\2", $text);
464 $text = preg_replace( "/<\/?[^>]+>/", "", $text );
465 $text = preg_replace( "/'''''/", "", $text );
466 $text = preg_replace( "/('''|<\/?[iIuUbB]>)/", "", $text );
467 $text = preg_replace( "/''/", "", $text );
473 * callback to replace [[target|caption]] kind of links, if
474 * the target is category or image, leave it
476 * @param array $matches
479 function linkReplace( $matches ) {
480 $colon = strpos( $matches[1], ':' );
481 if ( $colon === false ) {
482 return $matches[2]; // replace with caption
485 $ns = substr( $matches[1], 0, $colon );
486 $index = $wgContLang->getNsIndex( $ns );
487 if ( $index !== false && ( $index == NS_FILE ||
$index == NS_CATEGORY
) ) {
488 return $matches[0]; // return the whole thing
495 * Simple & fast snippet extraction, but gives completely unrelevant
498 * @param string $text
499 * @param array $terms
500 * @param int $contextlines
501 * @param int $contextchars
504 public function highlightSimple( $text, $terms, $contextlines, $contextchars ) {
508 $lines = explode( "\n", $text );
510 $terms = implode( '|', $terms );
511 $max = intval( $contextchars ) +
1;
512 $pat1 = "/(.*)($terms)(.{0,$max})/i";
517 foreach ( $lines as $line ) {
518 if ( 0 == $contextlines ) {
523 if ( !preg_match( $pat1, $line, $m ) ) {
527 // truncate function changes ... to relevant i18n message.
528 $pre = $wgContLang->truncate( $m[1], - $contextchars, '...', false );
530 if ( count( $m ) < 3 ) {
533 $post = $wgContLang->truncate( $m[3], $contextchars, '...', false );
538 $line = htmlspecialchars( $pre . $found . $post );
539 $pat2 = '/(' . $terms . ")/i";
540 $line = preg_replace( $pat2, "<span class='searchmatch'>\\1</span>", $line );
542 $extract .= "${line}\n";
549 * Returns the first few lines of the text
551 * @param string $text
552 * @param int $contextlines Max number of returned lines
553 * @param int $contextchars Average number of characters per line
556 public function highlightNone( $text, $contextlines, $contextchars ) {
558 $text = ltrim( $text ) . "\n"; // make sure the preg_match may find the last line
559 $text = str_replace( "\n\n", "\n", $text ); // remove empty lines
560 preg_match( "/^(.*\n){0,$contextlines}/", $text, $match );
561 $text = htmlspecialchars( substr( trim( $match[0] ), 0, $contextlines * $contextchars ) ); // trim and limit to max number of chars
562 return str_replace( "\n", '<br>', $text );