search.pm

   1 #!/usr/bin/perl
   2 # xapian-omega search engine plugin
   3 package IkiWiki::Plugin::search;
   4
   5 use warnings;
   6 use strict;
   7 use IkiWiki 3.00;
   8
   9 sub import {
  10         hook(type => "getsetup", id => "search", call => \&getsetup);
  11         hook(type => "checkconfig", id => "search", call => \&checkconfig);
  12         hook(type => "pagetemplate", id => "search", call => \&pagetemplate);
  13         hook(type => "indexhtml", id => "search", call => \&indexhtml);
  14         hook(type => "delete", id => "search", call => \&delete);
  15         hook(type => "cgi", id => "search", call => \&cgi);
  16         hook(type => "sessioncgi", id => "search", call => \&sessioncgi);
  17         hook(type => "disable", id => "search", call => \&disable);
  18         hook(type => "needsbuild", id => "search", call => \&needsbuild);
  19
  20         eval q{ use Search::Xapian }; # load early to work around #622591
  21 }
  22
  23 sub getsetup () {
  24         return
  25                 plugin => {
  26                         safe => 1,
  27                         rebuild => 1,
  28                         section => "web",
  29                 },
  30                 omega_cgi => {
  31                         type => "string",
  32                         example => "/usr/lib/cgi-bin/omega/omega",
  33                         description => "path to the omega cgi program",
  34                         safe => 0, # external program
  35                         rebuild => 0,
  36                 },
  37                 google_search => {
  38                         type => "booblean",
  39                         example => 1,
  40                         description => "use google site search rather than internal xapian index?",
  41                         safe => 1,
  42                         rebuild => 0,
  43                 },
  44 }
  45
  46 sub checkconfig () {
  47         foreach my $required (qw(url cgiurl)) {
  48                 if (! length $config{$required}) {
  49                         error(sprintf(gettext("Must specify %s when using the %s plugin"), $required, 'search'));
  50                 }
  51         }
  52
  53         if (! defined $config{omega_cgi}) {
  54                 $config{omega_cgi}="/usr/lib/cgi-bin/omega/omega";
  55         }
  56
  57         # Parsing "search"
  58         my @search = ();
  59         if (defined $config{search_db}
  60          and   (ref($config{search_db}) eq "ARRAY")) {
  61                 my $length = $#{$config{search_db}}+1;
  62                 my $columns = 4;
  63                 if (($length % $columns) != 0) {
  64                         error("config{search_db} length($length) must be a multiple of $columns.");
  65                  }
  66                 for(my $row = 0; $row < $length/$columns; $row++) {
  67                         my ($db, $auth, $dest, $index)
  68                          = @{$config{search_db}}[($columns*$row+0)..($columns*($row+1))];
  69                         my $auth_yesno = IkiWiki::yesno($auth);
  70                         push @search,
  71                          { db    => $db
  72                          , auth  => $auth_yesno
  73                          , dest  => $dest
  74                          , index => $index
  75                          };
  76                  }
  77          }
  78         $config{search} = \@search;
  79
  80         # This is a mass dependency, so if the search form template
  81         # changes, every page is rebuilt.
  82         add_depends("", "templates/searchform.tmpl");
  83 }
  84
  85 my $form = {};
  86 sub pagetemplate (@) {
  87         my %params=@_;
  88         my $page=$params{page};
  89         my $template=$params{template};
  90
  91         my @search = grep {
  92                 pagespec_match($page, $_->{dest})
  93          } @{$config{search}};
  94
  95         # Add search box to page header.
  96         if (@search > 0 and $template->query(name => "searchform")) {
  97                 my $form_id = join(' ', (map {$_->{db}} @search));
  98                 if (! defined $form->{$form_id}) {
  99                         my $searchform = template("searchform.tmpl", blind_cache => 1);
 100                         $searchform->param(searchaction => IkiWiki::cgiurl());
 101                         $searchform->param(searchdb => [map {{db=>$_->{db}}} @search]);
 102                         $searchform->param(html5 => $config{html5});
 103                         $form->{$form_id}=$searchform->output;
 104                  }
 105                 $template->param(searchform => $form->{$form_id});
 106          }
 107 }
 108
 109 my $scrubber;
 110 my $stemmer;
 111 sub indexhtml (@) {
 112         my %params=@_;
 113
 114         return if $config{google_search};
 115
 116         setupfiles();
 117
 118         # A unique pageterm is used to identify the document for a page.
 119         my $pageterm=pageterm($params{page});
 120         return unless defined $pageterm;
 121         my $search = (grep {
 122                 pagespec_match($params{page}, $_->{index})
 123          } @{$config{search}})[0];
 124         return unless defined $search;
 125
 126         my $db=xapiandb($search->{db});
 127         my $doc=Search::Xapian::Document->new();
 128         my $caption=pagetitle($params{page});
 129         my $title;
 130         if (exists $pagestate{$params{page}}{meta} &&
 131                 exists $pagestate{$params{page}}{meta}{title}) {
 132                 $title=$pagestate{$params{page}}{meta}{title};
 133         }
 134         else {
 135                 $title=$caption;
 136         }
 137
 138         # Remove html from text to be indexed.
 139         if (! defined $scrubber) {
 140                 eval q{use HTML::Scrubber};
 141                 if (! $@) {
 142                         $scrubber=HTML::Scrubber->new(allow => []);
 143                 }
 144         }
 145         my $toindex = defined $scrubber ? $scrubber->scrub($params{content}) : $params{content};
 146
 147         # Take 512 characters for a sample, then extend it out
 148         # if it stopped in the middle of a word.
 149         my $size=512;
 150         my ($sample)=substr($toindex, 0, $size);
 151         if (length($sample) == $size) {
 152                 my $max=length($toindex);
 153                 my $next;
 154                 while ($size < $max &&
 155                        ($next=substr($toindex, $size++, 1)) !~ /\s/) {
 156                         $sample.=$next;
 157                 }
 158         }
 159         $sample=~s/\n/ /g;
 160
 161         my $url=urlto($params{destpage}, "");
 162         if (defined $pagestate{$params{page}}{meta}{permalink}) {
 163                 $url=$pagestate{$params{page}}{meta}{permalink}
 164         }
 165
 166         # data used by omega
 167         # Decode html entities in it, since omega re-encodes them.
 168         eval q{use HTML::Entities};
 169         error $@ if $@;
 170         $doc->set_data(
 171                 "url=".$url."\n".
 172                 "sample=".decode_entities($sample)."\n".
 173                 "caption=".decode_entities($caption)."\n".
 174                 "modtime=$IkiWiki::pagemtime{$params{page}}\n".
 175                 "size=".length($params{content})."\n"
 176         );
 177
 178         # Index document and add terms for other metadata.
 179         my $tg = Search::Xapian::TermGenerator->new();
 180         if (! $stemmer) {
 181                 my $langcode=$ENV{LANG} || "en";
 182                 $langcode=~s/_.*//;
 183
 184                 # This whitelist is here to work around a xapian bug (#486138)
 185                 my @whitelist=qw{da de en es fi fr hu it no pt ru ro sv tr};
 186
 187                 if (grep { $_ eq $langcode } @whitelist) {
 188                         $stemmer=Search::Xapian::Stem->new($langcode);
 189                 }
 190                 else {
 191                         $stemmer=Search::Xapian::Stem->new("english");
 192                 }
 193         }
 194         $tg->set_stemmer($stemmer);
 195         $tg->set_document($doc);
 196         $tg->index_text($params{page}, 2);
 197         $tg->index_text($caption, 2);
 198         $tg->index_text($title, 2) if $title ne $caption;
 199         $tg->index_text($toindex);
 200         $tg->index_text(lc($title), 1, "S"); # for title:foo
 201         foreach my $link (@{$links{$params{page}}}) {
 202                 $tg->index_text(lc($link), 1, "XLINK"); # for link:bar
 203         }
 204
 205         $doc->add_term($pageterm);
 206         $db->replace_document_by_term($pageterm, $doc);
 207 }
 208
 209 sub delete (@) {
 210         return if $config{google_search};
 211
 212         foreach my $page (@_) {
 213                 my $pageterm=pageterm(pagename($page));
 214                 foreach my $search (@{$config{search}}) {
 215                         my $db=xapiandb($search->{db});
 216                         $db->delete_document_by_term($pageterm)
 217                                 if defined $pageterm;
 218                  }
 219          }
 220 }
 221
 222 sub cgi ($) {
 223         my $cgi=shift;
 224
 225         if (defined $cgi->param('P')) {
 226                 if ($config{google_search}) {
 227                         print $cgi->redirect("https://www.google.com/search?sitesearch=$config{url}&q=".$cgi->param('P'));
 228                         exit 0;
 229                  }
 230                 else {
 231                         if (defined $cgi->param('DB')) {
 232                                 my @db = map {split('/', $_)} ($cgi->param('DB'));
 233                                 my @search = map {
 234                                         my $db = $_;
 235                                         grep { $db eq $_->{db} } @{$config{search}};
 236                                  } @db;
 237                                 my $auth_needed = 0;
 238                                 foreach (@search) {
 239                                         if ($_->{auth}) {
 240                                                 $auth_needed = 1;
 241                                                 last;
 242                                          }
 243                                  }
 244                                 if (not $auth_needed) {
 245                                         # only works for GET requests
 246                                         chdir("$config{wikistatedir}/xapian") || error("chdir: $!");
 247                                         $ENV{OMEGA_CONFIG_FILE}="./omega.conf";
 248                                         $ENV{CGIURL}=IkiWiki::cgiurl();
 249                                         IkiWiki::loadindex();
 250                                         $ENV{HELPLINK}=htmllink("", "", "ikiwiki/searching",
 251                                                 noimageinline => 1, linktext => "Help");
 252                                         exec($config{omega_cgi}) || error("$config{omega_cgi} failed: $!");
 253                                  }
 254                          }
 255                  }
 256          }
 257 }
 258
 259 sub sessioncgi ($) {
 260         my $cgi=shift;
 261         my $session=shift;
 262
 263         if ($cgi->param('do') eq 'search') {
 264                 eval q{use CGI qw(-oldstyle_urls)};
 265                         # NOTE: ampersands separators are required for xapian-omega
 266                 IkiWiki::needsignin($cgi, $session);
 267                 IkiWiki::cgi_savesession($session);
 268                 my $headers;
 269                 do {
 270                         open(my $STDOUT_diverted, '>', \$headers) or die;
 271                         my $STDOUT_original = select $STDOUT_diverted;
 272                         IkiWiki::printheader($session);
 273                         select $STDOUT_original;
 274                         close $STDOUT_diverted;
 275                         local $/ = "\r\n";
 276                         chomp $headers;
 277                  };
 278                 print $headers;
 279                 chdir("$config{wikistatedir}/xapian") || error("chdir: $!");
 280                 $ENV{OMEGA_CONFIG_FILE}="./omega.conf";
 281                 $ENV{CGIURL}=IkiWiki::cgiurl();
 282                 $ENV{REQUEST_METHOD}='GET';
 283                 $ENV{QUERY_STRING}=$cgi->query_string();
 284                 IkiWiki::loadindex();
 285                 $ENV{HELPLINK}=htmllink("", "", "ikiwiki/searching",
 286                         noimageinline => 1, linktext => "Help");
 287                 exec($config{omega_cgi}) || error("$config{omega_cgi} failed: $!");
 288          }
 289 }
 290
 291 sub pageterm ($) {
 292         my $page=shift;
 293
 294         # 240 is the number used by omindex to decide when to hash an
 295         # overlong term. This does not use a compatible hash method though.
 296         eval q{use Encode};
 297         if (length encode_utf8($page) > 240) {
 298                 eval q{use Digest::SHA};
 299                 if ($@) {
 300                         debug("search: ".sprintf(gettext("need Digest::SHA to index %s"), $page)) if $@;
 301                         return undef;
 302                 }
 303
 304                 # Note no colon, therefore it's guaranteed to not overlap
 305                 # with a page with the same name as the hash..
 306                 return "U".lc(Digest::SHA::sha1_hex($page));
 307         }
 308         else {
 309                 return "U:".$page;
 310         }
 311 }
 312
 313 my $db = {};
 314 sub xapiandb ($) {
 315         my ($dbname) = @_;
 316         if (! defined $db->{$dbname}) {
 317                 eval q{
 318                         use Search::Xapian;
 319                         use Search::Xapian::WritableDatabase;
 320                 };
 321                 error($@) if $@;
 322                 $db->{$dbname}=Search::Xapian::WritableDatabase->new($config{wikistatedir}."/xapian/".$dbname,
 323                         Search::Xapian::DB_CREATE_OR_OPEN());
 324         }
 325         return $db->{$dbname};
 326 }
 327
 328 {
 329 my $setup=0;
 330 sub setupfiles () {
 331         if (! $setup and (! -e $config{wikistatedir}."/xapian" || $config{rebuild})) {
 332                 writefile("omega.conf", $config{wikistatedir}."/xapian",
 333                         "database_dir .\n".
 334                         "template_dir ./templates\n");
 335                 omega_template();
 336                 $setup=1;
 337         }
 338 }
 339 }
 340
 341 sub needsbuild {
 342         my $list=shift;
 343         if (grep {
 344                 $_ eq "templates/page.tmpl" ||
 345                 $_ eq "templates/searchquery.tmpl"
 346         } @$list) {
 347                 omega_template();
 348         }
 349 }
 350
 351 sub omega_template {
 352         # Avoid omega interpreting anything in the cgitemplate
 353         # as an omegascript command.
 354         eval q{use IkiWiki::CGI};
 355         my $template=IkiWiki::cgitemplate(undef, gettext("search"), "\0",
 356                 searchform => "", # avoid showing the small search form
 357         );
 358         eval q{use HTML::Entities};
 359         error $@ if $@;
 360         $template=encode_entities($template, '\$');
 361
 362         my $querytemplate=readfile(IkiWiki::template_file("searchquery.tmpl"));
 363         $template=~s/\0/$querytemplate/;
 364         writefile("query", $config{wikistatedir}."/xapian/templates",
 365                 $template);
 366 }
 367
 368 sub disable () {
 369         if (-d $config{wikistatedir}."/xapian") {
 370                 system("rm", "-rf", $config{wikistatedir}."/xapian");
 371         }
 372 }
 373
 374 1