New global config setting $wgMaxTocLevel: Maximum indent level of toc.
[lhc/web/wiklou.git] / includes / SearchEngine.php
1 <?php
2 # See search.doc
3
4 class SearchEngine {
5 /* private */ var $mUsertext, $mSearchterms;
6 /* private */ var $mTitlecond, $mTextcond;
7
8 var $doSearchRedirects = true;
9 var $addtoquery = array();
10 var $namespacesToSearch = array();
11 var $alternateTitle;
12 var $all_titles = false;
13
14 function SearchEngine( $text )
15 {
16 # We display the query, so let's strip it for safety
17 #
18 global $wgDBmysql4;
19 $lc = SearchEngine::legalSearchChars() . "()";
20 if( $wgDBmysql4 ) $lc .= "\"~<>*+-";
21 $this->mUsertext = trim( preg_replace( "/[^{$lc}]/", " ", $text ) );
22 $this->mSearchterms = array();
23 $this->mStrictMatching = true; # Google-style, add '+' on all terms
24 }
25
26 function queryNamespaces()
27 {
28 $namespaces = implode( ",", $this->namespacesToSearch );
29 if ($namespaces == "") {
30 $namespaces = "0";
31 }
32 return "AND cur_namespace IN (" . $namespaces . ")";
33 }
34
35 function searchRedirects()
36 {
37 if ( $this->doSearchRedirects ) {
38 return "";
39 } else {
40 return "AND cur_is_redirect=0 ";
41 }
42 }
43
44 /* private */ function initNamespaceCheckbox( $i )
45 {
46 global $wgUser, $wgNamespacesToBeSearchedDefault;
47
48 if ($wgUser->getID()) {
49 // User is logged in so we retrieve his default namespaces
50 return $wgUser->getOption( "searchNs".$i );
51 } else {
52 // User is not logged in so we give him the global default namespaces
53 return !empty($wgNamespacesToBeSearchedDefault[ $i ]);
54 }
55 }
56
57 # Display the "power search" footer. Does not actually perform the search,
58 # that is done by showResults()
59 function powersearch()
60 {
61 global $wgUser, $wgOut, $wgLang, $wgTitle, $wgRequest;
62 $sk =& $wgUser->getSkin();
63
64 $search = $wgRequest->getText( 'search' );
65 $searchx = $wgRequest->getVal( 'searchx' );
66 $listredirs = $wgRequest->getVal( 'redirs' );
67
68 $ret = wfMsg("powersearchtext"); # Text to be returned
69 $tempText = ""; # Temporary text, for substitution into $ret
70
71 if( isset( $_REQUEST["searchx"] ) ) {
72 $this->addtoquery["searchx"] = "1";
73 }
74
75 # Do namespace checkboxes
76 $namespaces = $wgLang->getNamespaces();
77 foreach ( $namespaces as $i => $namespace ) {
78 # Skip virtual namespaces
79 if ( $i < 0 ) {
80 continue;
81 }
82
83 $formVar = "ns$i";
84
85 # Initialise checkboxValues, either from defaults or from
86 # a previous invocation
87 if ( !isset( $searchx ) ) {
88 $checkboxValue = $this->initNamespaceCheckbox( $i );
89 } else {
90 $checkboxValue = $wgRequest->getVal( $formVar );
91 }
92
93 $checked = "";
94 if ( $checkboxValue == 1 ) {
95 $checked = " checked='checked'";
96 $this->addtoquery["ns{$i}"] = 1;
97 array_push( $this->namespacesToSearch, $i );
98 }
99 $name = str_replace( "_", " ", $namespaces[$i] );
100 if ( "" == $name ) {
101 $name = wfMsg( "blanknamespace" );
102 }
103
104 if ( $tempText !== "" ) {
105 $tempText .= " ";
106 }
107 $tempText .= "<input type='checkbox' value=\"1\" name=\"" .
108 "ns{$i}\"{$checked} />{$name}\n";
109 }
110 $ret = str_replace ( "$1", $tempText, $ret );
111
112 # List redirects checkbox
113
114 $checked = "";
115 if ( $listredirs == 1 ) {
116 $this->addtoquery["redirs"] = 1;
117 $checked = " checked='checked'";
118 }
119 $tempText = "<input type='checkbox' value='1' name=\"redirs\"{$checked} />\n";
120 $ret = str_replace( "$2", $tempText, $ret );
121
122 # Search field
123
124 $tempText = "<input type='text' name=\"search\" value=\"" .
125 htmlspecialchars( $search ) ."\" width='80' />\n";
126 $ret = str_replace( "$3", $tempText, $ret );
127
128 # Searchx button
129
130 $tempText = "<input type='submit' name=\"searchx\" value=\"" .
131 wfMsg("powersearch") . "\" />\n";
132 $ret = str_replace( "$9", $tempText, $ret );
133
134 $action = $sk->escapeSearchLink();
135 $ret = "<br /><br />\n<form id=\"powersearch\" method=\"get\" " .
136 "action=\"$action\">\n{$ret}\n</form>\n";
137
138 if ( isset ( $searchx ) ) {
139 if ( ! $listredirs ) {
140 $this->doSearchRedirects = false;
141 }
142 }
143 return $ret;
144 }
145
146 function setupPage() {
147 global $wgOut;
148 $wgOut->setPageTitle( wfMsg( "searchresults" ) );
149 $q = wfMsg( "searchquery", htmlspecialchars( $this->mUsertext ) );
150 $wgOut->setSubtitle( $q );
151 $wgOut->setArticleRelated( false );
152 $wgOut->setRobotpolicy( "noindex,nofollow" );
153 }
154
155 # Perform the search and construct the results page
156 function showResults()
157 {
158 global $wgUser, $wgTitle, $wgOut, $wgLang, $wgRequest;
159 global $wgDisableTextSearch, $wgInputEncoding;
160 $fname = "SearchEngine::showResults";
161
162 $search = $wgRequest->getText( 'search' );
163
164 $powersearch = $this->powersearch(); /* Need side-effects here? */
165
166 $this->setupPage();
167
168 $sk = $wgUser->getSkin();
169 $header = wfMsg( "searchresulttext", $sk->makeKnownLink(
170 wfMsg( "searchhelppage" ), wfMsg( "searchingwikipedia" ) ) );
171 $wgOut->addHTML( $header );
172
173 $this->parseQuery();
174 if ( "" == $this->mTitlecond || "" == $this->mTextcond ) {
175 $wgOut->addHTML( "<h2>" . wfMsg( "badquery" ) . "</h2>\n" .
176 "<p>" . wfMsg( "badquerytext" ) . "</p>\n" );
177 return;
178 }
179 list( $limit, $offset ) = wfCheckLimits( 20, "searchlimit" );
180
181 $searchnamespaces = $this->queryNamespaces();
182 $redircond = $this->searchRedirects();
183 $dbr =& wfGetDB( DB_SLAVE );
184 $searchindex = $dbr->tableName( 'searchindex' );
185 $cur = $dbr->tableName( 'cur' );
186
187 if ( $wgDisableTextSearch ) {
188 $wgOut->addHTML( wfMsg( "searchdisabled" ) );
189 $wgOut->addHTML( wfMsg( "googlesearch", htmlspecialchars( $search ), $GLOBALS['wgInputEncoding'] ) );
190 } else {
191 $sql = "SELECT cur_id,cur_namespace,cur_title," .
192 "cur_text FROM $cur,$searchindex " .
193 "WHERE cur_id=si_page AND {$this->mTitlecond} " .
194 "{$searchnamespaces} {$redircond}" .
195 "LIMIT {$offset}, {$limit}";
196 $res1 = $dbr->query( $sql, $fname );
197 $num = $dbr->numRows($res1);
198
199 $sk = $wgUser->getSkin();
200 $text = "";
201
202 $this->parseQuery( $dbr );
203 if ( "" == $this->mTitlecond || "" == $this->mTextcond ) {
204 $wgOut->addHTML( "<h2>" . wfMsg( "badquery" ) . "</h2>\n" .
205 "<p>" . wfMsg( "badquerytext" ) . "</p>\n" );
206 return;
207 }
208 list( $limit, $offset ) = wfCheckLimits( 20, "searchlimit" );
209
210 $searchnamespaces = $this->queryNamespaces();
211 $redircond = $this->searchRedirects();
212
213 $sql = "SELECT cur_id,cur_namespace,cur_title," .
214 "cur_text FROM $cur,$searchindex " .
215 "WHERE cur_id=si_page AND {$this->mTitlecond} " .
216 "{$searchnamespaces} {$redircond}" .
217 "LIMIT {$offset}, {$limit}";
218 $res1 = $dbr->query( $sql, $fname );
219 $num = $dbr->numRows($res1);
220
221 $sql = "SELECT cur_id,cur_namespace,cur_title," .
222 "cur_text FROM $cur,$searchindex " .
223 "WHERE cur_id=si_page AND {$this->mTextcond} " .
224 "{$searchnamespaces} {$redircond} " .
225 "LIMIT {$offset}, {$limit}";
226 $res2 = $dbr->query( $sql, $fname );
227 $num = $num + $dbr->numRows($res2);
228
229 if ( $num == $limit ) {
230 $top = wfShowingResults( $offset, $limit);
231 } else {
232 $top = wfShowingResultsNum( $offset, $limit, $num );
233 }
234 $wgOut->addHTML( "<p>{$top}</p>\n" );
235
236 # For powersearch
237
238 $a2l = "" ;
239 $akk = array_keys( $this->addtoquery ) ;
240 foreach ( $akk AS $ak ) {
241 $a2l .= "&{$ak}={$this->addtoquery[$ak]}" ;
242 }
243
244 $sl = wfViewPrevNext( $offset, $limit, "",
245 "search=" . wfUrlencode( $this->mUsertext ) . $a2l );
246 $wgOut->addHTML( "<br />{$sl}\n" );
247
248 $foundsome = false;
249
250 if ( 0 == $dbr->numRows( $res1 ) ) {
251 $wgOut->addHTML( "<h2>" . wfMsg( "notitlematches" ) .
252 "</h2>\n" );
253 } else {
254 $foundsome = true;
255 $off = $offset + 1;
256 $wgOut->addHTML( "<h2>" . wfMsg( "titlematches" ) .
257 "</h2>\n<ol start='{$off}'>" );
258
259 while ( $row = $dbr->fetchObject( $res1 ) ) {
260 $this->showHit( $row );
261 }
262 $dbr->freeResult( $res1 );
263 $wgOut->addHTML( "</ol>\n" );
264 }
265
266 if ( 0 == $dbr->numRows( $res2 ) ) {
267 $wgOut->addHTML( "<h2>" . wfMsg( "notextmatches" ) .
268 "</h2>\n" );
269 } else {
270 $foundsome = true;
271 $off = $offset + 1;
272 $wgOut->addHTML( "<h2>" . wfMsg( "textmatches" ) . "</h2>\n" .
273 "<ol start='{$off}'>" );
274 while ( $row = $dbr->fetchObject( $res2 ) ) {
275 $this->showHit( $row );
276 }
277 $dbr->freeResult( $res2 );
278 $wgOut->addHTML( "</ol>\n" );
279 }
280 if ( ! $foundsome ) {
281 $wgOut->addHTML( "<p>" . wfMsg( "nonefound" ) . "</p>\n" );
282 }
283 $wgOut->addHTML( "<p>{$sl}</p>\n" );
284 $wgOut->addHTML( $powersearch );
285 }
286 }
287
288 function legalSearchChars()
289 {
290 $lc = "A-Za-z_'0-9\\x80-\\xFF\\-";
291 return $lc;
292 }
293
294 function parseQuery( &$db )
295 {
296 global $wgDBminWordLen, $wgLang, $wgDBmysql4;
297
298 if( $wgDBmysql4 ) {
299 # Use cleaner boolean search if available
300 return $this->parseQuery4( $db );
301 }
302 # on non mysql4 database: get list of words we don't want to search for
303 require_once( "FulltextStoplist.php" );
304
305 $lc = SearchEngine::legalSearchChars() . "()";
306 $q = preg_replace( "/([()])/", " \\1 ", $this->mUsertext );
307 $q = preg_replace( "/\\s+/", " ", $q );
308 $w = explode( " ", strtolower( trim( $q ) ) );
309
310 $last = $cond = "";
311 foreach ( $w as $word ) {
312 $word = $wgLang->stripForSearch( $word );
313 if ( "and" == $word || "or" == $word || "not" == $word
314 || "(" == $word || ")" == $word ) {
315 $cond .= " " . strtoupper( $word );
316 $last = "";
317 } else if ( strlen( $word ) < $wgDBminWordLen ) {
318 continue;
319 } else if ( FulltextStoplist::inList( $word ) ) {
320 continue;
321 } else {
322 if ( "" != $last ) { $cond .= " AND"; }
323 $cond .= " (MATCH (##field##) AGAINST ('" .
324 $db->strencode( $word ). "'))";
325 $last = $word;
326 array_push( $this->mSearchterms, "\\b" . $word . "\\b" );
327 }
328 }
329 if ( 0 == count( $this->mSearchterms ) ) { return; }
330
331 $this->mTitlecond = "(" . str_replace( "##field##",
332 "si_title", $cond ) . " )";
333
334 $this->mTextcond = "(" . str_replace( "##field##",
335 "si_text", $cond ) . " AND (cur_is_redirect=0) )";
336 }
337
338 function parseQuery4( &$db )
339 {
340 global $wgLang;
341 $lc = SearchEngine::legalSearchChars();
342 $searchon = "";
343 $this->mSearchterms = array();
344
345 # FIXME: This doesn't handle parenthetical expressions.
346 if( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
347 $this->mUsertext, $m, PREG_SET_ORDER ) ) {
348 foreach( $m as $terms ) {
349 if( $searchon !== "" ) $searchon .= " ";
350 if( $this->mStrictMatching && ($terms[1] == "") ) {
351 $terms[1] = "+";
352 }
353 $searchon .= $terms[1] . $wgLang->stripForSearch( $terms[2] );
354 if( $terms[3] ) {
355 $regexp = preg_quote( $terms[3] );
356 if( $terms[4] ) $regexp .= "[0-9A-Za-z_]+";
357 } else {
358 $regexp = preg_quote( str_replace( '"', '', $terms[2] ) );
359 }
360 $this->mSearchterms[] = $regexp;
361 }
362 wfDebug( "Would search with '$searchon'\n" );
363 wfDebug( "Match with /\b" . implode( '\b|\b', $this->mSearchterms ) . "\b/\n" );
364 } else {
365 wfDebug( "Can't understand search query '$this->mUsertext'\n" );
366 }
367
368 $searchon = $db->strencode( $searchon );
369 $this->mTitlecond = " MATCH(si_title) AGAINST('$searchon' IN BOOLEAN MODE)";
370 $this->mTextcond = " (MATCH(si_text) AGAINST('$searchon' IN BOOLEAN MODE) AND cur_is_redirect=0)";
371 }
372
373 function showHit( $row )
374 {
375 global $wgUser, $wgOut;
376
377 $t = Title::makeName( $row->cur_namespace, $row->cur_title );
378 $sk = $wgUser->getSkin();
379
380 $contextlines = $wgUser->getOption( "contextlines" );
381 if ( "" == $contextlines ) { $contextlines = 5; }
382 $contextchars = $wgUser->getOption( "contextchars" );
383 if ( "" == $contextchars ) { $contextchars = 50; }
384
385 $link = $sk->makeKnownLink( $t, "" );
386 $size = wfMsg( "nbytes", strlen( $row->cur_text ) );
387 $wgOut->addHTML( "<li>{$link} ({$size})" );
388
389 $lines = explode( "\n", $row->cur_text );
390 $pat1 = "/(.*)(" . implode( "|", $this->mSearchterms ) . ")(.*)/i";
391 $lineno = 0;
392
393 foreach ( $lines as $line ) {
394 if ( 0 == $contextlines ) { break; }
395 --$contextlines;
396 ++$lineno;
397 if ( ! preg_match( $pat1, $line, $m ) ) { continue; }
398
399 $pre = $m[1];
400 if ( 0 == $contextchars ) { $pre = "..."; }
401 else {
402 if ( strlen( $pre ) > $contextchars ) {
403 $pre = "..." . substr( $pre, -$contextchars );
404 }
405 }
406 $pre = wfEscapeHTML( $pre );
407
408 if ( count( $m ) < 3 ) { $post = ""; }
409 else { $post = $m[3]; }
410
411 if ( 0 == $contextchars ) { $post = "..."; }
412 else {
413 if ( strlen( $post ) > $contextchars ) {
414 $post = substr( $post, 0, $contextchars ) . "...";
415 }
416 }
417 $post = wfEscapeHTML( $post );
418 $found = wfEscapeHTML( $m[2] );
419
420 $line = "{$pre}{$found}{$post}";
421 $pat2 = "/(" . implode( "|", $this->mSearchterms ) . ")/i";
422 $line = preg_replace( $pat2,
423 "<font color='red'>\\1</font>", $line );
424
425 $wgOut->addHTML( "<br /><small>{$lineno}: {$line}</small>\n" );
426 }
427 $wgOut->addHTML( "</li>\n" );
428 }
429
430 function goResult()
431 {
432 global $wgOut, $wgRequest, $wgGoToEdit;
433 global $wgDisableTextSearch;
434 $fname = "SearchEngine::goResult";
435
436 $search = trim( $wgRequest->getText( "search" ) );
437
438 # Try to go to page as entered.
439 #
440 $t = Title::newFromText( $search );
441
442 # If the string cannot be used to create a title
443 if( false == $t ){
444 $this->showResults();
445 return;
446 }
447
448 # Exact match? No need to look further.
449 if ( $t->getNamespace() == NS_SPECIAL || 0 != $t->getArticleID() ) {
450 $wgOut->redirect( $t->getFullURL() );
451 return;
452 }
453
454 # Now try all lower case (i.e. first letter capitalized)
455 #
456 $t = Title::newFromText( strtolower( $search ) );
457 if ( 0 != $t->getArticleID() ) {
458 $wgOut->redirect( $t->getFullURL() );
459 return;
460 }
461
462 # Now try capitalized string
463 #
464 $t = Title::newFromText( ucwords( strtolower( $search ) ) );
465 if ( 0 != $t->getArticleID() ) {
466 $wgOut->redirect( $t->getFullURL() );
467 return;
468 }
469
470 # Now try all upper case
471 #
472 $t = Title::newFromText( strtoupper( $search ) );
473 if ( 0 != $t->getArticleID() ) {
474 $wgOut->redirect( $t->getFullURL() );
475 return;
476 }
477
478 # Entering an IP address goes to the contributions page
479 if ( preg_match( '/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/', $search ) ) {
480 $title = Title::makeTitle( NS_SPECIAL, "Contributions" );
481 $wgOut->redirect( $title->getFullUrl( "target=$search" ) );
482 return;
483 }
484
485 # No match, generate an edit URL
486 $t = Title::newFromText( $this->mUsertext );
487
488 # If the feature is enabled, go straight to the edit page
489 if ( $wgGoToEdit ) {
490 $wgOut->redirect( $t->getFullURL( "action=edit" ) );
491 return;
492 }
493
494 if( $t ) {
495 $editurl = $t->escapeLocalURL( "action=edit" );
496 } else {
497 $editurl = ""; # ??
498 }
499 $wgOut->addHTML( "<p>" . wfMsg("nogomatch", $editurl ) . "</p>\n" );
500
501 # Try a fuzzy title search
502 $anyhit = false;
503 global $wgDisableFuzzySearch;
504 if(! $wgDisableFuzzySearch ){
505 foreach( array(NS_MAIN, NS_WP, NS_USER, NS_IMAGE, NS_MEDIAWIKI) as $namespace){
506 $anyhit |= SearchEngine::doFuzzyTitleSearch( $search, $namespace );
507 }
508 }
509
510 if( ! $anyhit ){
511 return $this->showResults();
512 }
513 }
514
515 /* static */ function doFuzzyTitleSearch( $search, $namespace ){
516 global $wgLang, $wgOut;
517
518 $this->setupPage();
519
520 $sstr = ucfirst($search);
521 $sstr = str_replace(" ", "_", $sstr);
522 $fuzzymatches = SearchEngine::fuzzyTitles( $sstr, $namespace );
523 $fuzzymatches = array_slice($fuzzymatches, 0, 10);
524 $slen = strlen( $search );
525 $wikitext = "";
526 foreach($fuzzymatches as $res){
527 $t = str_replace("_", " ", $res[1]);
528 $tfull = $wgLang->getNsText( $namespace ) . ":$t|$t";
529 if( $namespace == NS_MAIN )
530 $tfull = "$t";
531 $distance = $res[0];
532 $closeness = (strlen( $search ) - $distance) / strlen( $search );
533 $percent = intval( $closeness * 100 ) . "%";
534 $stars = str_repeat("*", ceil(5 * $closeness) );
535 $wikitext .= "* [[$tfull]] $percent ($stars)\n";
536 }
537 if( $wikitext ){
538 if( $namespace != NS_MAIN )
539 $wikitext = "=== " . $wgLang->getNsText( $namespace ) . " ===\n" . $wikitext;
540 $wgOut->addWikiText( $wikitext );
541 return true;
542 }
543 return false;
544 }
545
546 /* static */ function fuzzyTitles( $sstr, $namespace = NS_MAIN ){
547 $span = 0.10; // weed on title length before doing levenshtein.
548 $tolerance = 0.35; // allowed percentage of erronous characters
549 $slen = strlen($sstr);
550 $tolerance_count = ceil($tolerance * $slen);
551 $spanabs = ceil($slen * (1 + $span)) - $slen;
552 # print "Word: $sstr, len = $slen, range = [$min, $max], tolerance_count = $tolerance_count<BR>\n";
553 $result = array();
554 $cnt = 0;
555 for( $i=0; $i <= $spanabs; $i++ ){
556 $titles = SearchEngine::getTitlesByLength( $slen + $i, $namespace );
557 if( $i != 0) {
558 $titles = array_merge($titles, SearchEngine::getTitlesByLength( $slen - $i, $namespace ) );
559 }
560 foreach($titles as $t){
561 $d = levenshtein($sstr, $t);
562 if($d < $tolerance_count)
563 $result[] = array($d, $t);
564 $cnt++;
565 }
566 }
567 usort($result, "SearchEngine_pcmp");
568 return $result;
569 }
570
571 /* static */ function getTitlesByLength($aLength, $aNamespace = 0){
572 global $wgMemc, $wgDBname;
573 $fname = 'SearchEngin::getTitlesByLength';
574
575 // to avoid multiple costly SELECTs in case of no memcached
576 if( $this->all_titles ){
577 if( isset( $this->all_titles[$aLength][$aNamespace] ) ){
578 return $this->all_titles[$aLength][$aNamespace];
579 } else {
580 return array();
581 }
582 }
583
584 $mkey = "$wgDBname:titlesbylength:$aLength:$aNamespace";
585 $mkeyts = "$wgDBname:titlesbylength:createtime";
586 $ts = $wgMemc->get( $mkeyts );
587 $result = $wgMemc->get( $mkey );
588
589 if( time() - $ts < 3600 ){
590 // note: in case of insufficient memcached space, we return
591 // an empty list instead of starting to hit the DB.
592 return is_array( $result ) ? $result : array();
593 }
594
595 $wgMemc->set( $mkeyts, time() );
596
597 $dbr =& wfGetDB( DB_SLAVE );
598 $res = $dbr->select( 'cur', array( 'cur_title', 'cur_namespace' ), false, $fname );
599 $titles = array(); // length, ns, [titles]
600 while( $obj = $dbr->fetchObject( $res ) ){
601 $title = $obj->cur_title;
602 $ns = $obj->cur_namespace;
603 $len = strlen( $title );
604 $titles[$len][$ns][] = $title;
605 }
606 foreach($titles as $length => $length_arr){
607 foreach($length_arr as $ns => $title_arr){
608 $mkey = "$wgDBname:titlesbylength:$length:$ns";
609 $wgMemc->set( $mkey, $title_arr, 3600 * 24 );
610 }
611 }
612 $this->all_titles = $titles;
613 if( isset( $titles[$aLength][$aNamespace] ) )
614 return $titles[$aLength][$aNamespace];
615 else
616 return array();
617 }
618 }
619
620 /* private static */ function SearchEngine_pcmp($a, $b){ return $a[0] - $b[0]; }
621
622 ?>