slight alteration to comment
[lhc/web/wiklou.git] / includes / SearchEngine.php
1 <?php
2 # See search.doc
3
4 class SearchEngine {
5 /* private */ var $mUsertext, $mSearchterms;
6 /* private */ var $mTitlecond, $mTextcond;
7
8 var $doSearchRedirects = true;
9 var $addtoquery = array();
10 var $namespacesToSearch = array();
11 var $alternateTitle;
12 var $all_titles = false;
13
14 function SearchEngine( $text )
15 {
16 # We display the query, so let's strip it for safety
17 #
18 global $wgDBmysql4;
19 $lc = SearchEngine::legalSearchChars() . "()";
20 if( $wgDBmysql4 ) $lc .= "\"~<>*+-";
21 $this->mUsertext = trim( preg_replace( "/[^{$lc}]/", " ", $text ) );
22 $this->mSearchterms = array();
23 $this->mStrictMatching = true; # Google-style, add '+' on all terms
24 }
25
26 function queryNamespaces()
27 {
28 $namespaces = implode( ",", $this->namespacesToSearch );
29 if ($namespaces == "") {
30 $namespaces = "0";
31 }
32 return "AND cur_namespace IN (" . $namespaces . ")";
33 }
34
35 function searchRedirects()
36 {
37 if ( $this->doSearchRedirects ) {
38 return "";
39 } else {
40 return "AND cur_is_redirect=0 ";
41 }
42 }
43
44 /* private */ function initNamespaceCheckbox( $i )
45 {
46 global $wgUser, $wgNamespacesToBeSearchedDefault;
47
48 if ($wgUser->getID()) {
49 // User is logged in so we retrieve his default namespaces
50 return $wgUser->getOption( "searchNs".$i );
51 } else {
52 // User is not logged in so we give him the global default namespaces
53 return $wgNamespacesToBeSearchedDefault[ $i ];
54 }
55 }
56
57 # Display the "power search" footer. Does not actually perform the search,
58 # that is done by showResults()
59 function powersearch()
60 {
61 global $wgUser, $wgOut, $wgLang, $wgTitle, $wgRequest;
62
63 $search = $wgRequest->getText( 'search' );
64 $searchx = $wgRequest->getVal( 'searchx' );
65 $listredirs = $wgRequest->getVal( 'redirs' );
66
67 $ret = wfMsg("powersearchtext"); # Text to be returned
68 $tempText = ""; # Temporary text, for substitution into $ret
69
70 if( isset( $_REQUEST["searchx"] ) ) {
71 $this->addtoquery["searchx"] = "1";
72 }
73
74 # Do namespace checkboxes
75 $namespaces = $wgLang->getNamespaces();
76 foreach ( $namespaces as $i => $namespace ) {
77 # Skip virtual namespaces
78 if ( $i < 0 ) {
79 continue;
80 }
81
82 $formVar = "ns$i";
83
84 # Initialise checkboxValues, either from defaults or from
85 # a previous invocation
86 if ( !isset( $searchx ) ) {
87 $checkboxValue = $this->initNamespaceCheckbox( $i );
88 } else {
89 $checkboxValue = $_REQUEST[$formVar];
90 }
91
92 $checked = "";
93 if ( $checkboxValue == 1 ) {
94 $checked = " checked";
95 $this->addtoquery["ns{$i}"] = 1;
96 array_push( $this->namespacesToSearch, $i );
97 }
98 $name = str_replace( "_", " ", $namespaces[$i] );
99 if ( "" == $name ) {
100 $name = wfMsg( "blanknamespace" );
101 }
102
103 if ( $tempText !== "" ) {
104 $tempText .= " ";
105 }
106 $tempText .= "<input type=checkbox value=\"1\" name=\"" .
107 "ns{$i}\"{$checked}>{$name}\n";
108 }
109 $ret = str_replace ( "$1", $tempText, $ret );
110
111 # List redirects checkbox
112
113 $checked = "";
114 if ( $listredirs == 1 ) {
115 $this->addtoquery["redirs"] = 1;
116 $checked = " checked";
117 }
118 $tempText = "<input type=checkbox value=1 name=\"redirs\"{$checked}>\n";
119 $ret = str_replace( "$2", $tempText, $ret );
120
121 # Search field
122
123 $tempText = "<input type=text name=\"search\" value=\"" .
124 htmlspecialchars( $search ) ."\" width=80>\n";
125 $ret = str_replace( "$3", $tempText, $ret );
126
127 # Searchx button
128
129 $tempText = "<input type=submit name=\"searchx\" value=\"" .
130 wfMsg("powersearch") . "\">\n";
131 $ret = str_replace( "$9", $tempText, $ret );
132
133 global $wgScript;
134 $ret = "<br><br>\n<form id=\"powersearch\" method=\"get\" " .
135 "action=\"$wgScript\">\n{$ret}\n</form>\n";
136
137 if ( isset ( $searchx ) ) {
138 if ( ! $listredirs ) {
139 $this->doSearchRedirects = false;
140 }
141 }
142 return $ret;
143 }
144
145 # Perform the search and construct the results page
146 function showResults()
147 {
148 global $wgUser, $wgTitle, $wgOut, $wgLang, $wgDisableTextSearch;
149 global $wgInputEncoding;
150 $fname = "SearchEngine::showResults";
151
152 $search = $_REQUEST['search'];
153
154 $powersearch = $this->powersearch(); /* Need side-effects here? */
155
156 $wgOut->setPageTitle( wfMsg( "searchresults" ) );
157 $q = wfMsg( "searchquery", htmlspecialchars( $this->mUsertext ) );
158 $wgOut->setSubtitle( $q );
159 $wgOut->setArticleRelated( false );
160 $wgOut->setRobotpolicy( "noindex,nofollow" );
161
162 $sk = $wgUser->getSkin();
163 $header = wfMsg( "searchresulttext", $sk->makeKnownLink(
164 wfMsg( "searchhelppage" ), wfMsg( "searchingwikipedia" ) ) );
165 $wgOut->addHTML( $header );
166
167 $this->parseQuery();
168 if ( "" == $this->mTitlecond || "" == $this->mTextcond ) {
169 $wgOut->addHTML( "<h2>" . wfMsg( "badquery" ) . "</h2>\n" .
170 "<p>" . wfMsg( "badquerytext" ) );
171 return;
172 }
173 list( $limit, $offset ) = wfCheckLimits( 20, "searchlimit" );
174
175 $searchnamespaces = $this->queryNamespaces();
176 $redircond = $this->searchRedirects();
177
178 if ( $wgDisableTextSearch ) {
179 $wgOut->addHTML( wfMsg( "searchdisabled" ) );
180 $wgOut->addHTML( wfMsg( "googlesearch", htmlspecialchars( $search ), $GLOBALS['wgInputEncoding'] ) );
181 } else {
182 $sql = "SELECT cur_id,cur_namespace,cur_title," .
183 "cur_text FROM cur,searchindex " .
184 "WHERE cur_id=si_page AND {$this->mTitlecond} " .
185 "{$searchnamespaces} {$redircond}" .
186 "LIMIT {$offset}, {$limit}";
187 $res1 = wfQuery( $sql, DB_READ, $fname );
188 $num = wfNumRows($res1);
189
190 $sk = $wgUser->getSkin();
191 $text = "";
192
193 $this->parseQuery();
194 if ( "" == $this->mTitlecond || "" == $this->mTextcond ) {
195 $wgOut->addHTML( "<h2>" . wfMsg( "badquery" ) . "</h2>\n" .
196 "<p>" . wfMsg( "badquerytext" ) );
197 return;
198 }
199 list( $limit, $offset ) = wfCheckLimits( 20, "searchlimit" );
200
201 $searchnamespaces = $this->queryNamespaces();
202 $redircond = $this->searchRedirects();
203
204 $sql = "SELECT cur_id,cur_namespace,cur_title," .
205 "cur_text FROM cur,searchindex " .
206 "WHERE cur_id=si_page AND {$this->mTitlecond} " .
207 "{$searchnamespaces} {$redircond}" .
208 "LIMIT {$offset}, {$limit}";
209 $res1 = wfQuery( $sql, DB_READ, $fname );
210 $num = wfNumRows($res1);
211
212 $sql = "SELECT cur_id,cur_namespace,cur_title," .
213 "cur_text FROM cur,searchindex " .
214 "WHERE cur_id=si_page AND {$this->mTextcond} " .
215 "{$searchnamespaces} {$redircond} " .
216 "LIMIT {$offset}, {$limit}";
217 $res2 = wfQuery( $sql, DB_READ, $fname );
218 $num = $num + wfNumRows($res2);
219
220 if ( $num == $limit ) {
221 $top = wfShowingResults( $offset, $limit);
222 } else {
223 $top = wfShowingResultsNum( $offset, $limit, $num );
224 }
225 $wgOut->addHTML( "<p>{$top}\n" );
226
227 # For powersearch
228
229 $a2l = "" ;
230 $akk = array_keys( $this->addtoquery ) ;
231 foreach ( $akk AS $ak ) {
232 $a2l .= "&{$ak}={$this->addtoquery[$ak]}" ;
233 }
234
235 $sl = wfViewPrevNext( $offset, $limit, "",
236 "search=" . wfUrlencode( $this->mUsertext ) . $a2l );
237 $wgOut->addHTML( "<br>{$sl}\n" );
238
239 $foundsome = false;
240
241 if ( 0 == wfNumRows( $res1 ) ) {
242 $wgOut->addHTML( "<h2>" . wfMsg( "notitlematches" ) .
243 "</h2>\n" );
244 } else {
245 $foundsome = true;
246 $off = $offset + 1;
247 $wgOut->addHTML( "<h2>" . wfMsg( "titlematches" ) .
248 "</h2>\n<ol start='{$off}'>" );
249
250 while ( $row = wfFetchObject( $res1 ) ) {
251 $this->showHit( $row );
252 }
253 wfFreeResult( $res1 );
254 $wgOut->addHTML( "</ol>\n" );
255 }
256
257 if ( 0 == wfNumRows( $res2 ) ) {
258 $wgOut->addHTML( "<h2>" . wfMsg( "notextmatches" ) .
259 "</h2>\n" );
260 } else {
261 $foundsome = true;
262 $off = $offset + 1;
263 $wgOut->addHTML( "<h2>" . wfMsg( "textmatches" ) . "</h2>\n" .
264 "<ol start='{$off}'>" );
265 while ( $row = wfFetchObject( $res2 ) ) {
266 $this->showHit( $row );
267 }
268 wfFreeResult( $res2 );
269 $wgOut->addHTML( "</ol>\n" );
270 }
271 if ( ! $foundsome ) {
272 $wgOut->addHTML( "<p>" . wfMsg( "nonefound" ) . "\n" );
273 }
274 $wgOut->addHTML( "<p>{$sl}\n" );
275 $wgOut->addHTML( $powersearch );
276 }
277 }
278
279 function legalSearchChars()
280 {
281 $lc = "A-Za-z_'0-9\\x80-\\xFF\\-";
282 return $lc;
283 }
284
285 function parseQuery()
286 {
287 global $wgDBminWordLen, $wgLang, $wgDBmysql4;
288
289 if( $wgDBmysql4 ) {
290 # Use cleaner boolean search if available
291 return $this->parseQuery4();
292 }
293
294 $lc = SearchEngine::legalSearchChars() . "()";
295 $q = preg_replace( "/([()])/", " \\1 ", $this->mUsertext );
296 $q = preg_replace( "/\\s+/", " ", $q );
297 $w = explode( " ", strtolower( trim( $q ) ) );
298
299 $last = $cond = "";
300 foreach ( $w as $word ) {
301 $word = $wgLang->stripForSearch( $word );
302 if ( "and" == $word || "or" == $word || "not" == $word
303 || "(" == $word || ")" == $word ) {
304 $cond .= " " . strtoupper( $word );
305 $last = "";
306 } else if ( strlen( $word ) < $wgDBminWordLen ) {
307 continue;
308 } else if ( FulltextStoplist::inList( $word ) ) {
309 continue;
310 } else {
311 if ( "" != $last ) { $cond .= " AND"; }
312 $cond .= " (MATCH (##field##) AGAINST ('" .
313 wfStrencode( $word ). "'))";
314 $last = $word;
315 array_push( $this->mSearchterms, "\\b" . $word . "\\b" );
316 }
317 }
318 if ( 0 == count( $this->mSearchterms ) ) { return; }
319
320 $this->mTitlecond = "(" . str_replace( "##field##",
321 "si_title", $cond ) . " )";
322
323 $this->mTextcond = "(" . str_replace( "##field##",
324 "si_text", $cond ) . " AND (cur_is_redirect=0) )";
325 }
326
327 function parseQuery4()
328 {
329 global $wgLang;
330 $lc = SearchEngine::legalSearchChars();
331 $searchon = "";
332 $this->mSearchterms = array();
333
334 # FIXME: This doesn't handle parenthetical expressions.
335 if( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
336 $this->mUsertext, $m, PREG_SET_ORDER ) ) {
337 foreach( $m as $terms ) {
338 if( $searchon !== "" ) $searchon .= " ";
339 if( $this->mStrictMatching && ($terms[1] == "") ) {
340 $terms[1] = "+";
341 }
342 $searchon .= $terms[1] . $wgLang->stripForSearch( $terms[2] );
343 if( $terms[3] ) {
344 $regexp = preg_quote( $terms[3] );
345 if( $terms[4] ) $regexp .= "[0-9A-Za-z_]+";
346 } else {
347 $regexp = preg_quote( str_replace( '"', '', $terms[2] ) );
348 }
349 $this->mSearchterms[] = $regexp;
350 }
351 wfDebug( "Would search with '$searchon'\n" );
352 wfDebug( "Match with /\b" . implode( '\b|\b', $this->mSearchterms ) . "\b/\n" );
353 } else {
354 wfDebug( "Can't understand search query '$this->mUsertext'\n" );
355 }
356
357 $searchon = wfStrencode( $searchon );
358 $this->mTitlecond = " MATCH(si_title) AGAINST('$searchon' IN BOOLEAN MODE)";
359 $this->mTextcond = " (MATCH(si_text) AGAINST('$searchon' IN BOOLEAN MODE) AND cur_is_redirect=0)";
360 }
361
362 function showHit( $row )
363 {
364 global $wgUser, $wgOut;
365
366 $t = Title::makeName( $row->cur_namespace, $row->cur_title );
367 $sk = $wgUser->getSkin();
368
369 $contextlines = $wgUser->getOption( "contextlines" );
370 if ( "" == $contextlines ) { $contextlines = 5; }
371 $contextchars = $wgUser->getOption( "contextchars" );
372 if ( "" == $contextchars ) { $contextchars = 50; }
373
374 $link = $sk->makeKnownLink( $t, "" );
375 $size = wfMsg( "nbytes", strlen( $row->cur_text ) );
376 $wgOut->addHTML( "<li>{$link} ({$size})" );
377
378 $lines = explode( "\n", $row->cur_text );
379 $pat1 = "/(.*)(" . implode( "|", $this->mSearchterms ) . ")(.*)/i";
380 $lineno = 0;
381
382 foreach ( $lines as $line ) {
383 if ( 0 == $contextlines ) { break; }
384 --$contextlines;
385 ++$lineno;
386 if ( ! preg_match( $pat1, $line, $m ) ) { continue; }
387
388 $pre = $m[1];
389 if ( 0 == $contextchars ) { $pre = "..."; }
390 else {
391 if ( strlen( $pre ) > $contextchars ) {
392 $pre = "..." . substr( $pre, -$contextchars );
393 }
394 }
395 $pre = wfEscapeHTML( $pre );
396
397 if ( count( $m ) < 3 ) { $post = ""; }
398 else { $post = $m[3]; }
399
400 if ( 0 == $contextchars ) { $post = "..."; }
401 else {
402 if ( strlen( $post ) > $contextchars ) {
403 $post = substr( $post, 0, $contextchars ) . "...";
404 }
405 }
406 $post = wfEscapeHTML( $post );
407 $found = wfEscapeHTML( $m[2] );
408
409 $line = "{$pre}{$found}{$post}";
410 $pat2 = "/(" . implode( "|", $this->mSearchterms ) . ")/i";
411 $line = preg_replace( $pat2,
412 "<font color='red'>\\1</font>", $line );
413
414 $wgOut->addHTML( "<br><small>{$lineno}: {$line}</small>\n" );
415 }
416 $wgOut->addHTML( "</li>\n" );
417 }
418
419 function goResult()
420 {
421 global $wgOut, $wgDisableTextSearch;
422 $fname = "SearchEngine::goResult";
423
424 $search = $_REQUEST['search'];
425
426 # First try to go to page as entered.
427 #
428 $t = Title::newFromText( $search );
429
430 # If the string cannot be used to create a title
431 if( false == $t ){
432 $this->showResults();
433 return;
434 }
435
436 if ( 0 != $t->getArticleID() ) {
437 $wgOut->redirect( $t->getFullURL() );
438 return;
439 }
440
441 # Now try all lower case (i.e. first letter capitalized)
442 #
443 $t = Title::newFromText( strtolower( $search ) );
444 if ( 0 != $t->getArticleID() ) {
445 $wgOut->redirect( $t->getFullURL() );
446 return;
447 }
448
449 # Now try capitalized string
450 #
451 $t = Title::newFromText( ucwords( strtolower( $search ) ) );
452 if ( 0 != $t->getArticleID() ) {
453 $wgOut->redirect( $t->getFullURL() );
454 return;
455 }
456
457 # Now try all upper case
458 #
459 $t = Title::newFromText( strtoupper( $search ) );
460 if ( 0 != $t->getArticleID() ) {
461 $wgOut->redirect( $t->getFullURL() );
462 return;
463 }
464
465 # No match, generate an edit URL
466 $t = Title::newFromText( $this->mUsertext );
467 $wgOut->addHTML( wfMsg("nogomatch", $t->escapeLocalURL( "action=edit" ) ) . "\n<p>" );
468
469 # Try a fuzzy title search
470 $anyhit = false;
471 global $wgDisableFuzzySearch;
472 if(! $wgDisableFuzzySearch ){
473 foreach( array(NS_MAIN, NS_WP, NS_USER, NS_IMAGE, NS_MEDIAWIKI) as $namespace){
474 $anyhit |= SearchEngine::doFuzzyTitleSearch( $search, $namespace );
475 }
476 }
477
478 if( ! $anyhit ){
479 return $this->showResults();
480 }
481 }
482
483 /* static */ function doFuzzyTitleSearch( $search, $namespace ){
484 global $wgLang, $wgOut;
485 $sstr = ucfirst($search);
486 $sstr = str_replace(" ", "_", $sstr);
487 $fuzzymatches = SearchEngine::fuzzyTitles( $sstr, $namespace );
488 $fuzzymatches = array_slice($fuzzymatches, 0, 10);
489 $slen = strlen( $search );
490 $wikitext = "";
491 foreach($fuzzymatches as $res){
492 $t = str_replace("_", " ", $res[1]);
493 $tfull = $wgLang->getNsText( $namespace ) . ":$t|$t";
494 if( $namespace == NS_MAIN )
495 $tfull = "$t";
496 $distance = $res[0];
497 $closeness = (strlen( $search ) - $distance) / strlen( $search );
498 $percent = intval( $closeness * 100 ) . "%";
499 $stars = str_repeat("*", ceil(5 * $closeness) );
500 $wikitext .= "* [[$tfull]] $percent ($stars)\n";
501 }
502 if( $wikitext ){
503 if( $namespace != NS_MAIN )
504 $wikitext = "=== " . $wgLang->getNsText( $namespace ) . " ===\n" . $wikitext;
505 $wgOut->addWikiText( $wikitext );
506 return true;
507 }
508 return false;
509 }
510
511 /* static */ function fuzzyTitles( $sstr, $namespace = NS_MAIN ){
512 $span = 0.10; // weed on title length before doing levenshtein.
513 $tolerance = 0.35; // allowed percentage of erronous characters
514 $slen = strlen($sstr);
515 $tolerance_count = ceil($tolerance * $slen);
516 $spanabs = ceil($slen * (1 + $span)) - $slen;
517 # print "Word: $sstr, len = $slen, range = [$min, $max], tolerance_count = $tolerance_count<BR>\n";
518 $result = array();
519 $cnt = 0;
520 for( $i=0; $i <= $spanabs; $i++ ){
521 $titles = SearchEngine::getTitlesByLength( $slen + $i, $namespace );
522 if( $i != 0) {
523 $titles = array_merge($titles, SearchEngine::getTitlesByLength( $slen - $i, $namespace ) );
524 }
525 foreach($titles as $t){
526 $d = levenshtein($sstr, $t);
527 if($d < $tolerance_count)
528 $result[] = array($d, $t);
529 $cnt++;
530 }
531 }
532 usort($result, "SearchEngine_pcmp");
533 return $result;
534 }
535
536 /* static */ function getTitlesByLength($aLength, $aNamespace = 0){
537 global $wgMemc, $wgDBname;
538
539 // to avoid multiple costly SELECTs in case of no memcached
540 if( $this->all_titles ){
541 if( isset( $this->all_titles[$aLength][$aNamespace] ) ){
542 return $this->all_titles[$aLength][$aNamespace];
543 } else {
544 return array();
545 }
546 }
547
548 $mkey = "$wgDBname:titlesbylength:$aLength:$aNamespace";
549 $mkeyts = "$wgDBname:titlesbylength:createtime";
550 $ts = $wgMemc->get( $mkeyts );
551 $result = $wgMemc->get( $mkey );
552
553 if( time() - $ts < 3600 ){
554 // note: in case of insufficient memcached space, we return
555 // an empty list instead of starting to hit the DB.
556 return is_array( $result ) ? $result : array();
557 }
558
559 $wgMemc->set( $mkeyts, time() );
560
561 $res = wfQuery("SELECT cur_title, cur_namespace FROM cur", DB_READ);
562 $titles = array(); // length, ns, [titles]
563 while( $obj = wfFetchObject( $res ) ){
564 $title = $obj->cur_title;
565 $ns = $obj->cur_namespace;
566 $len = strlen( $title );
567 $titles[$len][$ns][] = $title;
568 }
569 foreach($titles as $length => $length_arr){
570 foreach($length_arr as $ns => $title_arr){
571 $mkey = "$wgDBname:titlesbylength:$length:$ns";
572 $wgMemc->set( $mkey, $title_arr, 3600 * 24 );
573 }
574 }
575 $this->all_titles = $titles;
576 if( isset( $titles[$aLength][$aNamespace] ) )
577 return $titles[$aLength][$aNamespace];
578 else
579 return array();
580 }
581 }
582
583 /* private static */ function SearchEngine_pcmp($a, $b){ return $a[0] - $b[0]; }
584
585 ?>