3 * Basic search engine highlighting
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
25 * Highlight bits of wikitext
29 class SearchHighlighter
{
30 protected $mCleanWikitext = true;
33 * @warning If you pass false to this constructor, then
34 * the caller is responsible for HTML escaping.
36 function __construct( $cleanupWikitext = true ) {
37 $this->mCleanWikitext
= $cleanupWikitext;
41 * Wikitext highlighting when $wgAdvancedSearchHighlighting = true
44 * @param array $terms Terms to highlight (not html escaped but
45 * regex escaped via SearchDatabase::regexTerm())
46 * @param int $contextlines
47 * @param int $contextchars
50 public function highlightText( $text, $terms, $contextlines, $contextchars ) {
51 global $wgContLang, $wgSearchHighlightBoundaries;
57 // spli text into text + templates/links/tables
58 $spat = "/(\\{\\{)|(\\[\\[[^\\]:]+:)|(\n\\{\\|)";
59 // first capture group is for detecting nested templates/links/tables/references
61 1 => '/(\{\{)|(\}\})/', // template
62 2 => '/(\[\[)|(\]\])/', // image
63 3 => "/(\n\\{\\|)|(\n\\|\\})/" ]; // table
65 // @todo FIXME: This should prolly be a hook or something
66 // instead of hardcoding a class name from the Cite extension
67 if ( class_exists( 'Cite' ) ) {
68 $spat .= '|(<ref>)'; // references via cite extension
69 $endPatterns[4] = '/(<ref>)|(<\/ref>)/';
72 $textExt = []; // text extracts
73 $otherExt = []; // other extracts
75 $textLen = strlen( $text );
76 $count = 0; // sequence number to maintain ordering
77 while ( $start < $textLen ) {
78 // find start of template/image/table
79 if ( preg_match( $spat, $text, $matches, PREG_OFFSET_CAPTURE
, $start ) ) {
81 foreach ( $matches as $key => $val ) {
82 if ( $key > 0 && $val[1] != -1 ) {
84 // see if this is an image link
85 $ns = substr( $val[0], 2, -1 );
86 if ( $wgContLang->getNsIndex( $ns ) != NS_FILE
) {
91 $epat = $endPatterns[$key];
92 $this->splitAndAdd( $textExt, $count, substr( $text, $start, $val[1] - $start ) );
98 // find end (and detect any nested elements)
100 $offset = $start +
1;
102 while ( preg_match( $epat, $text, $endMatches, PREG_OFFSET_CAPTURE
, $offset ) ) {
103 if ( array_key_exists( 2, $endMatches ) ) {
106 $len = strlen( $endMatches[2][0] );
107 $off = $endMatches[2][1];
108 $this->splitAndAdd( $otherExt, $count,
109 substr( $text, $start, $off +
$len - $start ) );
110 $start = $off +
$len;
114 // end of nested element
121 $offset = $endMatches[0][1] +
strlen( $endMatches[0][0] );
124 // couldn't find appropriate closing tag, skip
125 $this->splitAndAdd( $textExt, $count, substr( $text, $start, strlen( $matches[0][0] ) ) );
126 $start +
= strlen( $matches[0][0] );
131 // else: add as text extract
132 $this->splitAndAdd( $textExt, $count, substr( $text, $start ) );
136 $all = $textExt +
$otherExt; // these have disjunct key sets
139 foreach ( $terms as $index => $term ) {
140 // manually do upper/lowercase stuff for utf-8 since PHP won't do it
141 if ( preg_match( '/[\x80-\xff]/', $term ) ) {
142 $terms[$index] = preg_replace_callback(
144 [ $this, 'caseCallback' ],
148 $terms[$index] = $term;
151 $anyterm = implode( '|', $terms );
152 $phrase = implode( "$wgSearchHighlightBoundaries+", $terms );
153 // @todo FIXME: A hack to scale contextchars, a correct solution
154 // would be to have contextchars actually be char and not byte
155 // length, and do proper utf-8 substrings and lengths everywhere,
156 // but PHP is making that very hard and unclean to implement :(
157 $scale = strlen( $anyterm ) / mb_strlen( $anyterm );
158 $contextchars = intval( $contextchars * $scale );
160 $patPre = "(^|$wgSearchHighlightBoundaries)";
161 $patPost = "($wgSearchHighlightBoundaries|$)";
163 $pat1 = "/(" . $phrase . ")/ui";
164 $pat2 = "/$patPre(" . $anyterm . ")$patPost/ui";
166 $left = $contextlines;
171 // show beginning only if it contains all words
174 foreach ( $textExt as $index => $line ) {
175 if ( strlen( $line ) > 0 && $line[0] != ';' && $line[0] != ':' ) {
176 $firstText = $this->extract( $line, 0, $contextchars * $contextlines );
183 // check if first text contains all terms
184 foreach ( $terms as $term ) {
185 if ( !preg_match( "/$patPre" . $term . "$patPost/ui", $firstText ) ) {
191 $snippets[$first] = $firstText;
192 $offsets[$first] = 0;
196 // match whole query on text
197 $this->process( $pat1, $textExt, $left, $contextchars, $snippets, $offsets );
198 // match whole query on templates/tables/images
199 $this->process( $pat1, $otherExt, $left, $contextchars, $snippets, $offsets );
200 // match any words on text
201 $this->process( $pat2, $textExt, $left, $contextchars, $snippets, $offsets );
202 // match any words on templates/tables/images
203 $this->process( $pat2, $otherExt, $left, $contextchars, $snippets, $offsets );
208 // add extra chars to each snippet to make snippets constant size
210 if ( count( $snippets ) == 0 ) {
211 // couldn't find the target words, just show beginning of article
212 if ( array_key_exists( $first, $all ) ) {
213 $targetchars = $contextchars * $contextlines;
214 $snippets[$first] = '';
215 $offsets[$first] = 0;
218 // if begin of the article contains the whole phrase, show only that !!
219 if ( array_key_exists( $first, $snippets ) && preg_match( $pat1, $snippets[$first] )
220 && $offsets[$first] < $contextchars * 2 ) {
221 $snippets = [ $first => $snippets[$first] ];
224 // calc by how much to extend existing snippets
225 $targetchars = intval( ( $contextchars * $contextlines ) / count( $snippets ) );
228 foreach ( $snippets as $index => $line ) {
229 $extended[$index] = $line;
230 $len = strlen( $line );
231 if ( $len < $targetchars - 20 ) {
232 // complete this line
233 if ( $len < strlen( $all[$index] ) ) {
234 $extended[$index] = $this->extract(
237 $offsets[$index] +
$targetchars,
240 $len = strlen( $extended[$index] );
245 while ( $len < $targetchars - 20
246 && array_key_exists( $add, $all )
247 && !array_key_exists( $add, $snippets ) ) {
249 $tt = "\n" . $this->extract( $all[$add], 0, $targetchars - $len, $offsets[$add] );
250 $extended[$add] = $tt;
251 $len +
= strlen( $tt );
257 // $snippets = array_map( 'htmlspecialchars', $extended );
258 $snippets = $extended;
261 foreach ( $snippets as $index => $line ) {
263 $extract .= $line; // first line
264 } elseif ( $last +
1 == $index
265 && $offsets[$last] +
strlen( $snippets[$last] ) >= strlen( $all[$last] )
267 $extract .= " " . $line; // continous lines
269 $extract .= '<b> ... </b>' . $line;
275 $extract .= '<b> ... </b>';
279 foreach ( $terms as $term ) {
280 if ( !isset( $processed[$term] ) ) {
281 $pat3 = "/$patPre(" . $term . ")$patPost/ui"; // highlight word
282 $extract = preg_replace( $pat3,
283 "\\1<span class='searchmatch'>\\2</span>\\3", $extract );
284 $processed[$term] = true;
292 * Split text into lines and add it to extracts array
294 * @param array $extracts Index -> $line
296 * @param string $text
298 function splitAndAdd( &$extracts, &$count, $text ) {
299 $split = explode( "\n", $this->mCleanWikitext ?
$this->removeWiki( $text ) : $text );
300 foreach ( $split as $line ) {
303 $extracts[$count++
] = $tt;
309 * Do manual case conversion for non-ascii chars
311 * @param array $matches
314 function caseCallback( $matches ) {
316 if ( strlen( $matches[0] ) > 1 ) {
317 return '[' . $wgContLang->lc( $matches[0] ) . $wgContLang->uc( $matches[0] ) . ']';
324 * Extract part of the text from start to end, but by
325 * not chopping up words
326 * @param string $text
329 * @param int $posStart (out) actual start position
330 * @param int $posEnd (out) actual end position
333 function extract( $text, $start, $end, &$posStart = null, &$posEnd = null ) {
335 $start = $this->position( $text, $start, 1 );
337 if ( $end >= strlen( $text ) ) {
338 $end = strlen( $text );
340 $end = $this->position( $text, $end );
343 if ( !is_null( $posStart ) ) {
346 if ( !is_null( $posEnd ) ) {
350 if ( $end > $start ) {
351 return substr( $text, $start, $end - $start );
358 * Find a nonletter near a point (index) in the text
360 * @param string $text
362 * @param int $offset Offset to found index
363 * @return int Nearest nonletter index, or beginning of utf8 char if none
365 function position( $text, $point, $offset = 0 ) {
367 $s = max( 0, $point - $tolerance );
368 $l = min( strlen( $text ), $point +
$tolerance ) - $s;
372 '/[ ,.!?~!@#$%^&*\(\)+=\-\\\|\[\]"\'<>]/',
373 substr( $text, $s, $l ),
377 return $m[0][1] +
$s +
$offset;
379 // check if point is on a valid first UTF8 char
380 $char = ord( $text[$point] );
381 while ( $char >= 0x80 && $char < 0xc0 ) {
382 // skip trailing bytes
384 if ( $point >= strlen( $text ) ) {
385 return strlen( $text );
387 $char = ord( $text[$point] );
396 * Search extracts for a pattern, and return snippets
398 * @param string $pattern Regexp for matching lines
399 * @param array $extracts Extracts to search
400 * @param int $linesleft Number of extracts to make
401 * @param int $contextchars Length of snippet
402 * @param array $out Map for highlighted snippets
403 * @param array $offsets Map of starting points of snippets
406 function process( $pattern, $extracts, &$linesleft, &$contextchars, &$out, &$offsets ) {
407 if ( $linesleft == 0 ) {
408 return; // nothing to do
410 foreach ( $extracts as $index => $line ) {
411 if ( array_key_exists( $index, $out ) ) {
412 continue; // this line already highlighted
416 if ( !preg_match( $pattern, $line, $m, PREG_OFFSET_CAPTURE
) ) {
421 $len = strlen( $m[0][0] );
422 if ( $offset +
$len < $contextchars ) {
424 } elseif ( $len > $contextchars ) {
427 $begin = $offset +
intval( ( $len - $contextchars ) / 2 );
430 $end = $begin +
$contextchars;
433 // basic snippet from this line
434 $out[$index] = $this->extract( $line, $begin, $end, $posBegin );
435 $offsets[$index] = $posBegin;
437 if ( $linesleft == 0 ) {
444 * Basic wikitext removal
446 * @param string $text
449 function removeWiki( $text ) {
450 $text = preg_replace( "/\\{\\{([^|]+?)\\}\\}/", "", $text );
451 $text = preg_replace( "/\\{\\{([^|]+\\|)(.*?)\\}\\}/", "\\2", $text );
452 $text = preg_replace( "/\\[\\[([^|]+?)\\]\\]/", "\\1", $text );
453 $text = preg_replace_callback(
454 "/\\[\\[([^|]+\\|)(.*?)\\]\\]/",
455 [ $this, 'linkReplace' ],
458 $text = preg_replace( "/<\/?[^>]+>/", "", $text );
459 $text = preg_replace( "/'''''/", "", $text );
460 $text = preg_replace( "/('''|<\/?[iIuUbB]>)/", "", $text );
461 $text = preg_replace( "/''/", "", $text );
463 // Note, the previous /<\/?[^>]+>/ is insufficient
464 // for XSS safety as the HTML tag can span multiple
465 // search results (T144845).
466 $text = Sanitizer
::escapeHtmlAllowEntities( $text );
471 * callback to replace [[target|caption]] kind of links, if
472 * the target is category or image, leave it
474 * @param array $matches
477 function linkReplace( $matches ) {
478 $colon = strpos( $matches[1], ':' );
479 if ( $colon === false ) {
480 return $matches[2]; // replace with caption
483 $ns = substr( $matches[1], 0, $colon );
484 $index = $wgContLang->getNsIndex( $ns );
485 if ( $index !== false && ( $index == NS_FILE ||
$index == NS_CATEGORY
) ) {
486 return $matches[0]; // return the whole thing
493 * Simple & fast snippet extraction, but gives completely unrelevant
496 * Used when $wgAdvancedSearchHighlighting is false.
498 * @param string $text
499 * @param array $terms Escaped for regex by SearchDatabase::regexTerm()
500 * @param int $contextlines
501 * @param int $contextchars
504 public function highlightSimple( $text, $terms, $contextlines, $contextchars ) {
507 $lines = explode( "\n", $text );
509 $terms = implode( '|', $terms );
510 $max = intval( $contextchars ) +
1;
511 $pat1 = "/(.*)($terms)(.{0,$max})/i";
516 foreach ( $lines as $line ) {
517 if ( 0 == $contextlines ) {
522 if ( !preg_match( $pat1, $line, $m ) ) {
526 // truncate function changes ... to relevant i18n message.
527 $pre = $wgContLang->truncate( $m[1], - $contextchars, '...', false );
529 if ( count( $m ) < 3 ) {
532 $post = $wgContLang->truncate( $m[3], $contextchars, '...', false );
537 $line = htmlspecialchars( $pre . $found . $post );
538 $pat2 = '/(' . $terms . ")/i";
539 $line = preg_replace( $pat2, "<span class='searchmatch'>\\1</span>", $line );
541 $extract .= "${line}\n";
548 * Returns the first few lines of the text
550 * @param string $text
551 * @param int $contextlines Max number of returned lines
552 * @param int $contextchars Average number of characters per line
555 public function highlightNone( $text, $contextlines, $contextchars ) {
557 $text = ltrim( $text ) . "\n"; // make sure the preg_match may find the last line
558 $text = str_replace( "\n\n", "\n", $text ); // remove empty lines
559 preg_match( "/^(.*\n){0,$contextlines}/", $text, $match );
561 // Trim and limit to max number of chars
562 $text = htmlspecialchars( substr( trim( $match[0] ), 0, $contextlines * $contextchars ) );
563 return str_replace( "\n", '<br>', $text );