Starting code cleanup. Beginning to separate database logic from display output,...
[lhc/web/wiklou.git] / includes / SearchEngine.php
1 <?php
2 # See search.doc
3
4 define( "MW_SEARCH_OK", true );
5 define( "MW_SEARCH_BAD_QUERY", false );
6
7 class SearchEngine {
8 /* private */ var $mRawtext, $mUsertext, $mSearchterms;
9 /* private */ var $mTitlecond, $mTextcond;
10
11 var $doSearchRedirects = true;
12 var $addtoquery = array();
13 var $namespacesToSearch = array();
14 var $alternateTitle;
15 var $all_titles = false;
16
17 function SearchEngine( $text )
18 {
19 # We display the query, so let's strip it for safety
20 #
21 global $wgDBmysql4;
22 $lc = SearchEngine::legalSearchChars() . "()";
23 if( $wgDBmysql4 ) $lc .= "\"~<>*+-";
24 $this->mRawtext = $text;
25 $this->mUsertext = trim( preg_replace( "/[^{$lc}]/", " ", $text ) );
26 $this->mSearchterms = array();
27 $this->mStrictMatching = true; # Google-style, add '+' on all terms
28
29 $this->db =& wfGetDB( DB_SLAVE );
30 }
31
32 function queryNamespaces()
33 {
34 $namespaces = implode( ",", $this->namespacesToSearch );
35 if ($namespaces == "") {
36 $namespaces = "0";
37 }
38 return "AND cur_namespace IN (" . $namespaces . ")";
39 }
40
41 function searchRedirects()
42 {
43 if ( $this->doSearchRedirects ) {
44 return "";
45 } else {
46 return "AND cur_is_redirect=0 ";
47 }
48 }
49
50 /* private */ function initNamespaceCheckbox( $i )
51 {
52 global $wgUser, $wgNamespacesToBeSearchedDefault;
53
54 if ($wgUser->getID()) {
55 // User is logged in so we retrieve his default namespaces
56 return $wgUser->getOption( "searchNs".$i );
57 } else {
58 // User is not logged in so we give him the global default namespaces
59 return !empty($wgNamespacesToBeSearchedDefault[ $i ]);
60 }
61 }
62
63 # Display the "power search" footer. Does not actually perform the search,
64 # that is done by showResults()
65 function powersearch()
66 {
67 global $wgUser, $wgOut, $wgLang, $wgTitle, $wgRequest;
68 $sk =& $wgUser->getSkin();
69
70 $search = $this->mRawtext;
71 $searchx = $wgRequest->getVal( 'searchx' );
72 $listredirs = $wgRequest->getVal( 'redirs' );
73
74 $ret = wfMsg("powersearchtext"); # Text to be returned
75 $tempText = ""; # Temporary text, for substitution into $ret
76
77 if( isset( $_REQUEST["searchx"] ) ) {
78 $this->addtoquery["searchx"] = "1";
79 }
80
81 # Do namespace checkboxes
82 $namespaces = $wgLang->getNamespaces();
83 foreach ( $namespaces as $i => $namespace ) {
84 # Skip virtual namespaces
85 if ( $i < 0 ) {
86 continue;
87 }
88
89 $formVar = "ns$i";
90
91 # Initialise checkboxValues, either from defaults or from
92 # a previous invocation
93 if ( !isset( $searchx ) ) {
94 $checkboxValue = $this->initNamespaceCheckbox( $i );
95 } else {
96 $checkboxValue = $wgRequest->getVal( $formVar );
97 }
98
99 $checked = "";
100 if ( $checkboxValue == 1 ) {
101 $checked = " checked='checked'";
102 $this->addtoquery["ns{$i}"] = 1;
103 array_push( $this->namespacesToSearch, $i );
104 }
105 $name = str_replace( "_", " ", $namespaces[$i] );
106 if ( "" == $name ) {
107 $name = wfMsg( "blanknamespace" );
108 }
109
110 if ( $tempText !== "" ) {
111 $tempText .= " ";
112 }
113 $tempText .= "<input type='checkbox' value=\"1\" name=\"" .
114 "ns{$i}\"{$checked} />{$name}\n";
115 }
116 $ret = str_replace ( "$1", $tempText, $ret );
117
118 # List redirects checkbox
119
120 $checked = "";
121 if ( $listredirs == 1 ) {
122 $this->addtoquery["redirs"] = 1;
123 $checked = " checked='checked'";
124 }
125 $tempText = "<input type='checkbox' value='1' name=\"redirs\"{$checked} />\n";
126 $ret = str_replace( "$2", $tempText, $ret );
127
128 # Search field
129
130 $tempText = "<input type='text' name=\"search\" value=\"" .
131 htmlspecialchars( $search ) ."\" width='80' />\n";
132 $ret = str_replace( "$3", $tempText, $ret );
133
134 # Searchx button
135
136 $tempText = "<input type='submit' name=\"searchx\" value=\"" .
137 wfMsg("powersearch") . "\" />\n";
138 $ret = str_replace( "$9", $tempText, $ret );
139
140 $action = $sk->escapeSearchLink();
141 $ret = "<br /><br />\n<form id=\"powersearch\" method=\"get\" " .
142 "action=\"$action\">\n{$ret}\n</form>\n";
143
144 if ( isset ( $searchx ) ) {
145 if ( ! $listredirs ) {
146 $this->doSearchRedirects = false;
147 }
148 }
149 return $ret;
150 }
151
152 function setupPage() {
153 global $wgOut;
154 $wgOut->setPageTitle( wfMsg( "searchresults" ) );
155 $q = wfMsg( "searchquery", htmlspecialchars( $this->mRawtext ) );
156 $wgOut->setSubtitle( $q );
157 $wgOut->setArticleRelated( false );
158 $wgOut->setRobotpolicy( "noindex,nofollow" );
159 }
160
161 # Perform the search and construct the results page
162 function showResults()
163 {
164 global $wgUser, $wgTitle, $wgOut, $wgLang;
165 global $wgDisableTextSearch, $wgInputEncoding;
166 $fname = "SearchEngine::showResults";
167
168 $search = $this->mRawtext;
169
170 $powersearch = $this->powersearch(); /* Need side-effects here? */
171
172 $this->setupPage();
173
174 $sk = $wgUser->getSkin();
175 $header = wfMsg( "searchresulttext", $sk->makeKnownLink(
176 wfMsg( "searchhelppage" ), wfMsg( "searchingwikipedia" ) ) );
177 $wgOut->addHTML( $header );
178
179 $this->parseQuery();
180 if ( "" == $this->mTitlecond || "" == $this->mTextcond ) {
181 $wgOut->addHTML( "<h2>" . wfMsg( "badquery" ) . "</h2>\n" .
182 "<p>" . wfMsg( "badquerytext" ) . "</p>\n" );
183 return;
184 }
185 list( $limit, $offset ) = wfCheckLimits( 20, "searchlimit" );
186
187 $searchnamespaces = $this->queryNamespaces();
188 $redircond = $this->searchRedirects();
189
190 if ( $wgDisableTextSearch ) {
191 $wgOut->addHTML( wfMsg( "searchdisabled" ) );
192 $wgOut->addHTML( wfMsg( "googlesearch", htmlspecialchars( $search ), htmlspecialchars( $wgInputEncoding ) ) );
193 } else {
194 if( $this->parseQuery() == MW_SEARCH_BAD_QUERY ) {
195 $wgOut->addHTML( "<h2>" . wfMsg( "badquery" ) . "</h2>\n" .
196 "<p>" . wfMsg( "badquerytext" ) . "</p>\n" );
197 return;
198 }
199
200 list( $limit, $offset ) = wfCheckLimits( 20, "searchlimit" );
201 $titleMatches = $this->getMatches( $this->mTitlecond, $limit, $offset );
202 $textMatches = $this->getMatches( $this->mTextcond, $limit, $offset );
203
204 $sk = $wgUser->getSkin();
205
206 $num = count( $titleMatches ) + count( $textMatches );
207 if ( $num >= $limit ) {
208 $top = wfShowingResults( $offset, $limit );
209 } else {
210 $top = wfShowingResultsNum( $offset, $limit, $num );
211 }
212 $wgOut->addHTML( "<p>{$top}</p>\n" );
213
214 # For powersearch
215
216 $a2l = "";
217 $akk = array_keys( $this->addtoquery );
218 foreach ( $akk AS $ak ) {
219 $a2l .= "&{$ak}={$this->addtoquery[$ak]}" ;
220 }
221
222 $sl = wfViewPrevNext( $offset, $limit, "",
223 "search=" . wfUrlencode( $this->mUsertext ) . $a2l );
224 $wgOut->addHTML( "<br />{$sl}\n" );
225
226 $foundsome = $this->showMatches( $titleMatches, $offset, "notitlematches", "titlematches" )
227 || $this->showMatches( $textMatches, $offset, "notextmatches", "textmatches" );
228
229 if ( ! $foundsome ) {
230 $wgOut->addHTML( "<p>" . wfMsg( "nonefound" ) . "</p>\n" );
231 }
232 $wgOut->addHTML( "<p>{$sl}</p>\n" );
233 $wgOut->addHTML( $powersearch );
234 }
235 }
236
237 function legalSearchChars()
238 {
239 $lc = "A-Za-z_'0-9\\x80-\\xFF\\-";
240 return $lc;
241 }
242
243 function parseQuery()
244 {
245 global $wgDBminWordLen, $wgLang, $wgDBmysql4;
246
247 if( $wgDBmysql4 ) {
248 # Use cleaner boolean search if available
249 return $this->parseQuery4( $this->db );
250 }
251 # on non mysql4 database: get list of words we don't want to search for
252 require_once( "FulltextStoplist.php" );
253
254 $lc = SearchEngine::legalSearchChars() . "()";
255 $q = preg_replace( "/([()])/", " \\1 ", $this->mUsertext );
256 $q = preg_replace( "/\\s+/", " ", $q );
257 $w = explode( " ", trim( $q ) );
258
259 $last = $cond = "";
260 foreach ( $w as $word ) {
261 $word = $wgLang->stripForSearch( $word );
262 if ( "and" == $word || "or" == $word || "not" == $word
263 || "(" == $word || ")" == $word ) {
264 $cond .= " " . strtoupper( $word );
265 $last = "";
266 } else if ( strlen( $word ) < $wgDBminWordLen ) {
267 continue;
268 } else if ( FulltextStoplist::inList( $word ) ) {
269 continue;
270 } else {
271 if ( "" != $last ) { $cond .= " AND"; }
272 $cond .= " (MATCH (##field##) AGAINST ('" .
273 $this->db->strencode( $word ). "'))";
274 $last = $word;
275 array_push( $this->mSearchterms, "\\b" . $word . "\\b" );
276 }
277 }
278 if ( 0 == count( $this->mSearchterms ) ) {
279 return MW_SEARCH_BAD_QUERY;
280 }
281
282 $this->mTitlecond = "(" . str_replace( "##field##",
283 "si_title", $cond ) . " )";
284
285 $this->mTextcond = "(" . str_replace( "##field##",
286 "si_text", $cond ) . " AND (cur_is_redirect=0) )";
287
288 return MW_SEARCH_OK;
289 }
290
291 function parseQuery4()
292 {
293 global $wgLang;
294 $lc = SearchEngine::legalSearchChars();
295 $searchon = "";
296 $this->mSearchterms = array();
297
298 # FIXME: This doesn't handle parenthetical expressions.
299 if( preg_match_all( '/([-+<>~]?)(([' . $lc . ']+)(\*?)|"[^"]*")/',
300 $this->mUsertext, $m, PREG_SET_ORDER ) ) {
301 foreach( $m as $terms ) {
302 if( $searchon !== "" ) $searchon .= " ";
303 if( $this->mStrictMatching && ($terms[1] == "") ) {
304 $terms[1] = "+";
305 }
306 $searchon .= $terms[1] . $wgLang->stripForSearch( $terms[2] );
307 if( $terms[3] ) {
308 $regexp = preg_quote( $terms[3] );
309 if( $terms[4] ) $regexp .= "[0-9A-Za-z_]+";
310 } else {
311 $regexp = preg_quote( str_replace( '"', '', $terms[2] ) );
312 }
313 $this->mSearchterms[] = $regexp;
314 }
315 wfDebug( "Would search with '$searchon'\n" );
316 wfDebug( "Match with /\b" . implode( '\b|\b', $this->mSearchterms ) . "\b/\n" );
317 } else {
318 wfDebug( "Can't understand search query '$this->mUsertext'\n" );
319 }
320
321 $searchon = $this->db->strencode( $searchon );
322 $this->mTitlecond = " MATCH(si_title) AGAINST('$searchon' IN BOOLEAN MODE)";
323 $this->mTextcond = " (MATCH(si_text) AGAINST('$searchon' IN BOOLEAN MODE) AND cur_is_redirect=0)";
324 return MW_SEARCH_OK;
325 }
326
327 function &getMatches( $cond, $limit, $offset ) {
328 $searchindex = $this->db->tableName( 'searchindex' );
329 $cur = $this->db->tableName( 'cur' );
330 $searchnamespaces = $this->queryNamespaces();
331 $redircond = $this->searchRedirects();
332
333 $sql = "SELECT cur_id,cur_namespace,cur_title," .
334 "cur_text FROM $cur,$searchindex " .
335 "WHERE cur_id=si_page AND {$cond} " .
336 "{$searchnamespaces} {$redircond} " .
337 "LIMIT {$offset}, {$limit}";
338
339 $res = $this->db->query( $sql, "SearchEngine::getMatches" );
340 $matches = array();
341 while ( $row = $this->db->fetchObject( $res ) ) {
342 $matches[] = $row;
343 }
344 $this->db->freeResult( $res );
345
346 return $matches;
347 }
348
349 function showMatches( &$matches, $offset, $msgEmpty, $msgFound ) {
350 global $wgOut;
351 if ( 0 == count( $matches ) ) {
352 $wgOut->addHTML( "<h2>" . wfMsg( $msgEmpty ) .
353 "</h2>\n" );
354 return false;
355 } else {
356 $off = $offset + 1;
357 $wgOut->addHTML( "<h2>" . wfMsg( $msgFound ) .
358 "</h2>\n<ol start='{$off}'>" );
359
360 foreach( $matches as $row ) {
361 $this->showHit( $row );
362 }
363 $wgOut->addHTML( "</ol>\n" );
364 return true;
365 }
366 }
367
368 function showHit( $row )
369 {
370 global $wgUser, $wgOut, $wgLang;
371
372 $t = Title::makeName( $row->cur_namespace, $row->cur_title );
373 $sk = $wgUser->getSkin();
374
375 $contextlines = $wgUser->getOption( "contextlines" );
376 if ( "" == $contextlines ) { $contextlines = 5; }
377 $contextchars = $wgUser->getOption( "contextchars" );
378 if ( "" == $contextchars ) { $contextchars = 50; }
379
380 $link = $sk->makeKnownLink( $t, "" );
381 $size = wfMsg( "nbytes", strlen( $row->cur_text ) );
382 $wgOut->addHTML( "<li>{$link} ({$size})" );
383
384 $lines = explode( "\n", $row->cur_text );
385 $pat1 = "/(.*)(" . implode( "|", $this->mSearchterms ) . ")(.*)/i";
386 $lineno = 0;
387
388 foreach ( $lines as $line ) {
389 if ( 0 == $contextlines ) {
390 break;
391 }
392 --$contextlines;
393 ++$lineno;
394 if ( ! preg_match( $pat1, $line, $m ) ) {
395 continue;
396 }
397
398 $pre = $wgLang->truncate( $m[1], -$contextchars, "..." );
399
400 if ( count( $m ) < 3 ) {
401 $post = "";
402 } else {
403 $post = $wgLang->truncate( $m[3], $contextchars, "..." );
404 }
405
406 $found = $m[2];
407
408 $line = htmlspecialchars( $pre . $found . $post );
409 $pat2 = "/(" . implode( "|", $this->mSearchterms ) . ")/i";
410 $line = preg_replace( $pat2,
411 "<font color='red'>\\1</font>", $line );
412
413 $wgOut->addHTML( "<br /><small>{$lineno}: {$line}</small>\n" );
414 }
415 $wgOut->addHTML( "</li>\n" );
416 }
417
418 function goResult()
419 {
420 global $wgOut, $wgGoToEdit;
421 global $wgDisableTextSearch;
422 $fname = "SearchEngine::goResult";
423
424 $search = trim( $this->mRawtext );
425
426 # Try to go to page as entered.
427 #
428 $t = Title::newFromText( $search );
429
430 # If the string cannot be used to create a title
431 if( false == $t ){
432 $this->showResults();
433 return;
434 }
435
436 # Exact match? No need to look further.
437 if ( $t->getNamespace() == NS_SPECIAL || 0 != $t->getArticleID() ) {
438 $wgOut->redirect( $t->getFullURL() );
439 return;
440 }
441
442 # Now try all lower case (i.e. first letter capitalized)
443 #
444 $t = Title::newFromText( strtolower( $search ) );
445 if ( 0 != $t->getArticleID() ) {
446 $wgOut->redirect( $t->getFullURL() );
447 return;
448 }
449
450 # Now try capitalized string
451 #
452 $t = Title::newFromText( ucwords( strtolower( $search ) ) );
453 if ( 0 != $t->getArticleID() ) {
454 $wgOut->redirect( $t->getFullURL() );
455 return;
456 }
457
458 # Now try all upper case
459 #
460 $t = Title::newFromText( strtoupper( $search ) );
461 if ( 0 != $t->getArticleID() ) {
462 $wgOut->redirect( $t->getFullURL() );
463 return;
464 }
465
466 # Entering an IP address goes to the contributions page
467 if ( preg_match( '/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/', $search ) ) {
468 $title = Title::makeTitle( NS_SPECIAL, "Contributions" );
469 $wgOut->redirect( $title->getFullUrl( "target=$search" ) );
470 return;
471 }
472
473 # No match, generate an edit URL
474 $t = Title::newFromText( $this->mRawtext );
475
476 # If the feature is enabled, go straight to the edit page
477 if ( $wgGoToEdit ) {
478 $wgOut->redirect( $t->getFullURL( "action=edit" ) );
479 return;
480 }
481
482 if( $t ) {
483 $editurl = $t->escapeLocalURL( "action=edit" );
484 } else {
485 $editurl = ""; # ??
486 }
487 $wgOut->addHTML( "<p>" . wfMsg("nogomatch", $editurl ) . "</p>\n" );
488
489 # Try a fuzzy title search
490 $anyhit = false;
491 global $wgDisableFuzzySearch;
492 if(! $wgDisableFuzzySearch ){
493 foreach( array(NS_MAIN, NS_WP, NS_USER, NS_IMAGE, NS_MEDIAWIKI) as $namespace){
494 $anyhit |= SearchEngine::doFuzzyTitleSearch( $search, $namespace );
495 }
496 }
497
498 if( ! $anyhit ){
499 return $this->showResults();
500 }
501 }
502
503 /* static */ function doFuzzyTitleSearch( $search, $namespace ){
504 global $wgLang, $wgOut;
505
506 $this->setupPage();
507
508 $sstr = ucfirst($search);
509 $sstr = str_replace(" ", "_", $sstr);
510 $fuzzymatches = SearchEngine::fuzzyTitles( $sstr, $namespace );
511 $fuzzymatches = array_slice($fuzzymatches, 0, 10);
512 $slen = strlen( $search );
513 $wikitext = "";
514 foreach($fuzzymatches as $res){
515 $t = str_replace("_", " ", $res[1]);
516 $tfull = $wgLang->getNsText( $namespace ) . ":$t|$t";
517 if( $namespace == NS_MAIN )
518 $tfull = "$t";
519 $distance = $res[0];
520 $closeness = (strlen( $search ) - $distance) / strlen( $search );
521 $percent = intval( $closeness * 100 ) . "%";
522 $stars = str_repeat("*", ceil(5 * $closeness) );
523 $wikitext .= "* [[$tfull]] $percent ($stars)\n";
524 }
525 if( $wikitext ){
526 if( $namespace != NS_MAIN )
527 $wikitext = "=== " . $wgLang->getNsText( $namespace ) . " ===\n" . $wikitext;
528 $wgOut->addWikiText( $wikitext );
529 return true;
530 }
531 return false;
532 }
533
534 /* static */ function fuzzyTitles( $sstr, $namespace = NS_MAIN ){
535 $span = 0.10; // weed on title length before doing levenshtein.
536 $tolerance = 0.35; // allowed percentage of erronous characters
537 $slen = strlen($sstr);
538 $tolerance_count = ceil($tolerance * $slen);
539 $spanabs = ceil($slen * (1 + $span)) - $slen;
540 # print "Word: $sstr, len = $slen, range = [$min, $max], tolerance_count = $tolerance_count<BR>\n";
541 $result = array();
542 $cnt = 0;
543 for( $i=0; $i <= $spanabs; $i++ ){
544 $titles = SearchEngine::getTitlesByLength( $slen + $i, $namespace );
545 if( $i != 0) {
546 $titles = array_merge($titles, SearchEngine::getTitlesByLength( $slen - $i, $namespace ) );
547 }
548 foreach($titles as $t){
549 $d = levenshtein($sstr, $t);
550 if($d < $tolerance_count)
551 $result[] = array($d, $t);
552 $cnt++;
553 }
554 }
555 usort($result, "SearchEngine_pcmp");
556 return $result;
557 }
558
559 /* static */ function getTitlesByLength($aLength, $aNamespace = 0){
560 global $wgMemc, $wgDBname;
561 $fname = 'SearchEngin::getTitlesByLength';
562
563 // to avoid multiple costly SELECTs in case of no memcached
564 if( $this->all_titles ){
565 if( isset( $this->all_titles[$aLength][$aNamespace] ) ){
566 return $this->all_titles[$aLength][$aNamespace];
567 } else {
568 return array();
569 }
570 }
571
572 $mkey = "$wgDBname:titlesbylength:$aLength:$aNamespace";
573 $mkeyts = "$wgDBname:titlesbylength:createtime";
574 $ts = $wgMemc->get( $mkeyts );
575 $result = $wgMemc->get( $mkey );
576
577 if( time() - $ts < 3600 ){
578 // note: in case of insufficient memcached space, we return
579 // an empty list instead of starting to hit the DB.
580 return is_array( $result ) ? $result : array();
581 }
582
583 $wgMemc->set( $mkeyts, time() );
584
585 $res = $this->db->select( 'cur', array( 'cur_title', 'cur_namespace' ), false, $fname );
586 $titles = array(); // length, ns, [titles]
587 while( $obj = $this->db->fetchObject( $res ) ){
588 $title = $obj->cur_title;
589 $ns = $obj->cur_namespace;
590 $len = strlen( $title );
591 $titles[$len][$ns][] = $title;
592 }
593 foreach($titles as $length => $length_arr){
594 foreach($length_arr as $ns => $title_arr){
595 $mkey = "$wgDBname:titlesbylength:$length:$ns";
596 $wgMemc->set( $mkey, $title_arr, 3600 * 24 );
597 }
598 }
599 $this->all_titles = $titles;
600 if( isset( $titles[$aLength][$aNamespace] ) )
601 return $titles[$aLength][$aNamespace];
602 else
603 return array();
604 }
605 }
606
607 /* private static */ function SearchEngine_pcmp($a, $b){ return $a[0] - $b[0]; }
608
609 ?>