From: Julien Moutinho Date: Sun, 30 Mar 2014 20:37:41 +0000 (+0200) Subject: init from IkiWiki's 9273bc5185d9633739c96570402a2c050f3fa7e2 X-Git-Url: http://git.cyclocoop.org/?p=ikiwiki%2Fsearch.git;a=commitdiff_plain;h=848109e027cc4bbac4402938d81daba2cd4c4f1b init from IkiWiki's 9273bc5185d9633739c96570402a2c050f3fa7e2 --- 848109e027cc4bbac4402938d81daba2cd4c4f1b diff --git a/search.pm b/search.pm new file mode 100644 index 0000000..24b16fe --- /dev/null +++ b/search.pm @@ -0,0 +1,289 @@ +#!/usr/bin/perl +# xapian-omega search engine plugin +package IkiWiki::Plugin::search; + +use warnings; +use strict; +use IkiWiki 3.00; + +sub import { + hook(type => "getsetup", id => "search", call => \&getsetup); + hook(type => "checkconfig", id => "search", call => \&checkconfig); + hook(type => "pagetemplate", id => "search", call => \&pagetemplate); + hook(type => "indexhtml", id => "search", call => \&indexhtml); + hook(type => "delete", id => "search", call => \&delete); + hook(type => "cgi", id => "search", call => \&cgi); + hook(type => "disable", id => "search", call => \&disable); + hook(type => "needsbuild", id => "search", call => \&needsbuild); + + eval q{ use Search::Xapian }; # load early to work around #622591 +} + +sub getsetup () { + return + plugin => { + safe => 1, + rebuild => 1, + section => "web", + }, + omega_cgi => { + type => "string", + example => "/usr/lib/cgi-bin/omega/omega", + description => "path to the omega cgi program", + safe => 0, # external program + rebuild => 0, + }, + google_search => { + type => "booblean", + example => 1, + description => "use google site search rather than internal xapian index?", + safe => 1, + rebuild => 0, + }, +} + +sub checkconfig () { + foreach my $required (qw(url cgiurl)) { + if (! length $config{$required}) { + error(sprintf(gettext("Must specify %s when using the %s plugin"), $required, 'search')); + } + } + + if (! defined $config{omega_cgi}) { + $config{omega_cgi}="/usr/lib/cgi-bin/omega/omega"; + } + + # This is a mass dependency, so if the search form template + # changes, every page is rebuilt. + add_depends("", "templates/searchform.tmpl"); +} + +my $form; +sub pagetemplate (@) { + my %params=@_; + my $page=$params{page}; + my $template=$params{template}; + + # Add search box to page header. + if ($template->query(name => "searchform")) { + if (! defined $form) { + my $searchform = template("searchform.tmpl", blind_cache => 1); + $searchform->param(searchaction => IkiWiki::cgiurl()); + $searchform->param(html5 => $config{html5}); + $form=$searchform->output; + } + + $template->param(searchform => $form); + } +} + +my $scrubber; +my $stemmer; +sub indexhtml (@) { + my %params=@_; + + return if $config{google_search}; + + setupfiles(); + + # A unique pageterm is used to identify the document for a page. + my $pageterm=pageterm($params{page}); + return unless defined $pageterm; + + my $db=xapiandb(); + my $doc=Search::Xapian::Document->new(); + my $caption=pagetitle($params{page}); + my $title; + if (exists $pagestate{$params{page}}{meta} && + exists $pagestate{$params{page}}{meta}{title}) { + $title=$pagestate{$params{page}}{meta}{title}; + } + else { + $title=$caption; + } + + # Remove html from text to be indexed. + if (! defined $scrubber) { + eval q{use HTML::Scrubber}; + if (! $@) { + $scrubber=HTML::Scrubber->new(allow => []); + } + } + my $toindex = defined $scrubber ? $scrubber->scrub($params{content}) : $params{content}; + + # Take 512 characters for a sample, then extend it out + # if it stopped in the middle of a word. + my $size=512; + my ($sample)=substr($toindex, 0, $size); + if (length($sample) == $size) { + my $max=length($toindex); + my $next; + while ($size < $max && + ($next=substr($toindex, $size++, 1)) !~ /\s/) { + $sample.=$next; + } + } + $sample=~s/\n/ /g; + + my $url=urlto($params{destpage}, ""); + if (defined $pagestate{$params{page}}{meta}{permalink}) { + $url=$pagestate{$params{page}}{meta}{permalink} + } + + # data used by omega + # Decode html entities in it, since omega re-encodes them. + eval q{use HTML::Entities}; + error $@ if $@; + $doc->set_data( + "url=".$url."\n". + "sample=".decode_entities($sample)."\n". + "caption=".decode_entities($caption)."\n". + "modtime=$IkiWiki::pagemtime{$params{page}}\n". + "size=".length($params{content})."\n" + ); + + # Index document and add terms for other metadata. + my $tg = Search::Xapian::TermGenerator->new(); + if (! $stemmer) { + my $langcode=$ENV{LANG} || "en"; + $langcode=~s/_.*//; + + # This whitelist is here to work around a xapian bug (#486138) + my @whitelist=qw{da de en es fi fr hu it no pt ru ro sv tr}; + + if (grep { $_ eq $langcode } @whitelist) { + $stemmer=Search::Xapian::Stem->new($langcode); + } + else { + $stemmer=Search::Xapian::Stem->new("english"); + } + } + $tg->set_stemmer($stemmer); + $tg->set_document($doc); + $tg->index_text($params{page}, 2); + $tg->index_text($caption, 2); + $tg->index_text($title, 2) if $title ne $caption; + $tg->index_text($toindex); + $tg->index_text(lc($title), 1, "S"); # for title:foo + foreach my $link (@{$links{$params{page}}}) { + $tg->index_text(lc($link), 1, "XLINK"); # for link:bar + } + + $doc->add_term($pageterm); + $db->replace_document_by_term($pageterm, $doc); +} + +sub delete (@) { + return if $config{google_search}; + + my $db=xapiandb(); + foreach my $page (@_) { + my $pageterm=pageterm(pagename($page)); + $db->delete_document_by_term($pageterm) if defined $pageterm; + } +} + +sub cgi ($) { + my $cgi=shift; + + if (defined $cgi->param('P')) { + if ($config{google_search}) { + print $cgi->redirect("https://www.google.com/search?sitesearch=$config{url}&q=".$cgi->param('P')); + exit 0; + } + else { + # only works for GET requests + chdir("$config{wikistatedir}/xapian") || error("chdir: $!"); + $ENV{OMEGA_CONFIG_FILE}="./omega.conf"; + $ENV{CGIURL}=IkiWiki::cgiurl(); + IkiWiki::loadindex(); + $ENV{HELPLINK}=htmllink("", "", "ikiwiki/searching", + noimageinline => 1, linktext => "Help"); + exec($config{omega_cgi}) || error("$config{omega_cgi} failed: $!"); + } + } +} + +sub pageterm ($) { + my $page=shift; + + # 240 is the number used by omindex to decide when to hash an + # overlong term. This does not use a compatible hash method though. + eval q{use Encode}; + if (length encode_utf8($page) > 240) { + eval q{use Digest::SHA}; + if ($@) { + debug("search: ".sprintf(gettext("need Digest::SHA to index %s"), $page)) if $@; + return undef; + } + + # Note no colon, therefore it's guaranteed to not overlap + # with a page with the same name as the hash.. + return "U".lc(Digest::SHA::sha1_hex($page)); + } + else { + return "U:".$page; + } +} + +my $db; +sub xapiandb () { + if (! defined $db) { + eval q{ + use Search::Xapian; + use Search::Xapian::WritableDatabase; + }; + error($@) if $@; + $db=Search::Xapian::WritableDatabase->new($config{wikistatedir}."/xapian/default", + Search::Xapian::DB_CREATE_OR_OPEN()); + } + return $db; +} + +{ +my $setup=0; +sub setupfiles () { + if (! $setup and (! -e $config{wikistatedir}."/xapian" || $config{rebuild})) { + writefile("omega.conf", $config{wikistatedir}."/xapian", + "database_dir .\n". + "template_dir ./templates\n"); + omega_template(); + $setup=1; + } +} +} + +sub needsbuild { + my $list=shift; + if (grep { + $_ eq "templates/page.tmpl" || + $_ eq "templates/searchquery.tmpl" + } @$list) { + omega_template(); + } +} + +sub omega_template { + # Avoid omega interpreting anything in the cgitemplate + # as an omegascript command. + eval q{use IkiWiki::CGI}; + my $template=IkiWiki::cgitemplate(undef, gettext("search"), "\0", + searchform => "", # avoid showing the small search form + ); + eval q{use HTML::Entities}; + error $@ if $@; + $template=encode_entities($template, '\$'); + + my $querytemplate=readfile(IkiWiki::template_file("searchquery.tmpl")); + $template=~s/\0/$querytemplate/; + writefile("query", $config{wikistatedir}."/xapian/templates", + $template); +} + +sub disable () { + if (-d $config{wikistatedir}."/xapian") { + system("rm", "-rf", $config{wikistatedir}."/xapian"); + } +} + +1 diff --git a/templates/searchform.tmpl b/templates/searchform.tmpl new file mode 100644 index 0000000..cb65d12 --- /dev/null +++ b/templates/searchform.tmpl @@ -0,0 +1,6 @@ +
+
+placeholder="search" /> +
+
diff --git a/templates/searchquery.tmpl b/templates/searchquery.tmpl new file mode 100644 index 0000000..15bc78e --- /dev/null +++ b/templates/searchquery.tmpl @@ -0,0 +1,113 @@ +$setmap{prefix,title,S} +$setmap{prefix,link,XLINK} +$set{thousand,$.}$set{decimal,.}$setmap{BN,,Any Country,uk,England,fr,France} +${ +$def{PREV, +$if{$ne{$topdoc,0},, +} +} + +$def{NEXT, +$if{$ne{$last,$msize},, +} +} + +$def{P,} +$def{PAGE,$if{$gt{$1,9},$if{$gt{$1,99},$P{$1,$div{$1,100}}}$P{$1,$mod{$div{$1,10},10}}}$P{$1,$mod{$1,10}}} + +$def{S,$1} +$def{SPAGE,$if{$gt{$1,9},$if{$gt{$1,99},$S{$1,$div{$1,100}}}$S{$1,$mod{$div{$1,10},10}}}$S{$1,$mod{$1,10}}} +} + +$def{PREV,$if{$ne{$topdoc,0},}} + +$def{PAGE,} + +$def{SPAGE,} + +$def{NEXT,$if{$ne{$last,$msize},}} + +

+ +

+
+ + +$env{HELPLINK} +
+ +$or{$html{$error}, +$if{$eq{$msize,0}, +$if{$query,No documents match your query, +
Searching $nice{$dbsize} documents +}, +$if{$not{$msizeexact}, + $nice{$add{$topdoc,1}}-$nice{$last} of about $nice{$msize} matches, + $if{$and{$eq{$last,$msize},$eq{$topdoc,0}}, + All $nice{$msize} matches, + $nice{$add{$topdoc,1}}$if{$ne{$add{$topdoc,1},$last},-$nice{$last}} of exactly $nice{$msize} matches} +} +
+
+$list{$map{$queryterms,$list{$html{$uniq{$unstem{$_}}},,/,}: $nice{$freq{$_}}},Term frequencies: ,$. ,} +
Search took $time seconds + +$hitlist{ +} +
+${$percentage%} +
+
+
+
+ +Modified:
$html{$date{$field{modtime},%Y-%m-%d}}
+$if{$field{language},Language: $html{$field{language}}
} +$if{$field{size},Size: $html{$filesize{$field{size}}}
} +
+		$html{$or{$field{caption},$field{title},$field{url},Untitled}}
+$highlight{$field{sample},$terms}$if{$field{sample},...}
+ +$percentage% relevant$. matching: +$list{$map{$terms,$html{$prettyterm{$_}}},$. , and }${for lynx:}

+ +
+ +
+${suppress next, prev, and page links if there's only one page} +$if{$ne{$lastpage,1}, +$set{a,$if{$opt{pagelink_height}, HEIGHT=$opt{pagelink_height}}$if{$opt{pagelink_width}, WIDTH=$opt{pagelink_width}}} + +${1-W ... X-(this)-Y ...} +$set{w,$min{3,$add{$thispage,-1}}} +$set{x,$max{$add{$opt{w},1},$add{$thispage,-3}}} +$set{y,$min{$lastpage,$add{$thispage,8}}} +$PREV +$map{$range{1,$opt{w}},$PAGE{$_}} +$if{$ne{$add{$opt{w},1},$opt{x}},...} +$map{$range{$opt{x},$add{$thispage,-1}},$PAGE{$_}} +$SPAGE{$thispage} +$map{$range{$add{$thispage,1},$opt{y}},$PAGE{$_}} +$if{$ne{$opt{y},$lastpage},...} +$NEXT +} +}} +

+$if{$dbname,} +$if{$ne{$topdoc,0},} +$if{$ne{$hitsperpage,10},} +$if{$fmt,} +$if{$cgi{COLLAPSE},} +$if{$queryterms,} + + +$list{$relevants,} +$if{$cgi{THRESHOLD},} +
+
$html{$version}