2 # xapian-omega search engine plugin
3 package IkiWiki
::Plugin
::search
;
10 hook
(type
=> "getsetup", id
=> "search", call
=> \
&getsetup
);
11 hook
(type
=> "checkconfig", id
=> "search", call
=> \
&checkconfig
);
12 hook
(type
=> "pagetemplate", id
=> "search", call
=> \
&pagetemplate
);
13 hook
(type
=> "indexhtml", id
=> "search", call
=> \
&indexhtml
);
14 hook
(type
=> "delete", id
=> "search", call
=> \
&delete);
15 hook
(type
=> "cgi", id
=> "search", call
=> \
&cgi
);
16 hook
(type
=> "sessioncgi", id
=> "search", call
=> \
&sessioncgi
);
17 hook
(type
=> "disable", id
=> "search", call
=> \
&disable
);
18 hook
(type
=> "needsbuild", id
=> "search", call
=> \
&needsbuild
);
20 eval q{ use Search::Xapian }; # load early to work around #622591
32 example
=> "/usr/lib/cgi-bin/omega/omega",
33 description
=> "path to the omega cgi program",
34 safe
=> 0, # external program
40 description
=> "use google site search rather than internal xapian index?",
47 foreach my $required (qw(url cgiurl)) {
48 if (! length $config{$required}) {
49 error
(sprintf(gettext
("Must specify %s when using the %s plugin"), $required, 'search'));
53 if (! defined $config{omega_cgi
}) {
54 $config{omega_cgi
}="/usr/lib/cgi-bin/omega/omega";
59 if (defined $config{search_db
}
60 and (ref($config{search_db
}) eq "ARRAY")) {
61 my $length = $#{$config{search_db}}+1;
63 if (($length % $columns) != 0) {
64 error
("config{search_db} length($length) must be a multiple of $columns.");
66 for(my $row = 0; $row < $length/$columns; $row++) {
67 my ($db, $auth, $dest, $index)
68 = @
{$config{search_db
}}[($columns*$row+0)..($columns*($row+1))];
69 my $auth_yesno = IkiWiki
::yesno
($auth);
78 $config{search
} = \
@search;
80 # This is a mass dependency, so if the search form template
81 # changes, every page is rebuilt.
82 add_depends
("", "templates/searchform.tmpl");
86 sub pagetemplate
(@
) {
88 my $page=$params{page
};
89 my $template=$params{template
};
92 pagespec_match
($page, $_->{dest
})
95 # Add search box to page header.
96 if (@search > 0 and $template->query(name
=> "searchform")) {
97 my $form_id = join(' ', (map {$_->{db
}} @search));
98 if (! defined $form->{$form_id}) {
99 my $searchform = template
("searchform.tmpl", blind_cache
=> 1);
100 $searchform->param(searchaction
=> IkiWiki
::cgiurl
());
101 $searchform->param(searchdb
=> [map {{db
=>$_->{db
}}} @search]);
102 $searchform->param(html5
=> $config{html5
});
103 $form->{$form_id}=$searchform->output;
105 $template->param(searchform
=> $form->{$form_id});
114 return if $config{google_search
};
118 # A unique pageterm is used to identify the document for a page.
119 my $pageterm=pageterm
($params{page
});
120 return unless defined $pageterm;
122 pagespec_match
($params{page
}, $_->{index})
123 } @
{$config{search
}})[0];
124 return unless defined $search;
126 my $db=xapiandb
($search->{db
});
127 my $doc=Search
::Xapian
::Document
->new();
128 my $caption=pagetitle
($params{page
});
130 if (exists $pagestate{$params{page
}}{meta
} &&
131 exists $pagestate{$params{page
}}{meta
}{title
}) {
132 $title=$pagestate{$params{page
}}{meta
}{title
};
138 # Remove html from text to be indexed.
139 if (! defined $scrubber) {
140 eval q{use HTML::Scrubber};
142 $scrubber=HTML
::Scrubber
->new(allow
=> []);
145 my $toindex = defined $scrubber ?
$scrubber->scrub($params{content
}) : $params{content
};
147 # Take 512 characters for a sample, then extend it out
148 # if it stopped in the middle of a word.
150 my ($sample)=substr($toindex, 0, $size);
151 if (length($sample) == $size) {
152 my $max=length($toindex);
154 while ($size < $max &&
155 ($next=substr($toindex, $size++, 1)) !~ /\s/) {
161 my $url=urlto
($params{destpage
}, "");
162 if (defined $pagestate{$params{page
}}{meta
}{permalink
}) {
163 $url=$pagestate{$params{page
}}{meta
}{permalink
}
167 # Decode html entities in it, since omega re-encodes them.
168 eval q{use HTML::Entities};
172 "sample=".decode_entities
($sample)."\n".
173 "caption=".decode_entities
($caption)."\n".
174 "modtime=$IkiWiki::pagemtime{$params{page}}\n".
175 "size=".length($params{content
})."\n"
178 # Index document and add terms for other metadata.
179 my $tg = Search
::Xapian
::TermGenerator
->new();
181 my $langcode=$ENV{LANG
} || "en";
184 # This whitelist is here to work around a xapian bug (#486138)
185 my @whitelist=qw{da de en es fi fr hu it
no pt ru ro sv tr
};
187 if (grep { $_ eq $langcode } @whitelist) {
188 $stemmer=Search
::Xapian
::Stem
->new($langcode);
191 $stemmer=Search
::Xapian
::Stem
->new("english");
194 $tg->set_stemmer($stemmer);
195 $tg->set_document($doc);
196 $tg->index_text($params{page
}, 2);
197 $tg->index_text($caption, 2);
198 $tg->index_text($title, 2) if $title ne $caption;
199 $tg->index_text($toindex);
200 $tg->index_text(lc($title), 1, "S"); # for title:foo
201 foreach my $link (@
{$links{$params{page
}}}) {
202 $tg->index_text(lc($link), 1, "XLINK"); # for link:bar
205 $doc->add_term($pageterm);
206 $db->replace_document_by_term($pageterm, $doc);
210 return if $config{google_search
};
212 foreach my $page (@_) {
213 my $pageterm=pageterm
(pagename
($page));
214 foreach my $search (@
{$config{search
}}) {
215 my $db=xapiandb
($search->{db
});
216 $db->delete_document_by_term($pageterm)
217 if defined $pageterm;
225 if (defined $cgi->param('P')) {
226 if ($config{google_search
}) {
227 print $cgi->redirect("https://www.google.com/search?sitesearch=$config{url}&q=".$cgi->param('P'));
231 if (defined $cgi->param('DB')) {
232 my @db = map {split('/', $_)} ($cgi->param('DB'));
235 grep { $db eq $_->{db
} } @
{$config{search
}};
244 if (not $auth_needed) {
245 # only works for GET requests
246 chdir("$config{wikistatedir}/xapian") || error
("chdir: $!");
247 $ENV{OMEGA_CONFIG_FILE
}="./omega.conf";
248 $ENV{CGIURL
}=IkiWiki
::cgiurl
();
249 IkiWiki
::loadindex
();
250 $ENV{HELPLINK
}=htmllink
("", "", "ikiwiki/searching",
251 noimageinline
=> 1, linktext
=> "Help");
252 exec($config{omega_cgi
}) || error
("$config{omega_cgi} failed: $!");
263 if ($cgi->param('do') eq 'search') {
264 eval q{use CGI qw(-oldstyle_urls)};
265 # NOTE: ampersands separators are required for xapian-omega
266 IkiWiki
::needsignin
($cgi, $session);
267 IkiWiki
::cgi_savesession
($session);
270 open(my $STDOUT_diverted, '>', \
$headers) or die;
271 my $STDOUT_original = select $STDOUT_diverted;
272 IkiWiki
::printheader
($session);
273 select $STDOUT_original;
274 close $STDOUT_diverted;
279 chdir("$config{wikistatedir}/xapian") || error
("chdir: $!");
280 $ENV{OMEGA_CONFIG_FILE
}="./omega.conf";
281 $ENV{CGIURL
}=IkiWiki
::cgiurl
();
282 $ENV{REQUEST_METHOD
}='GET';
283 $ENV{QUERY_STRING
}=$cgi->query_string();
284 IkiWiki
::loadindex
();
285 $ENV{HELPLINK
}=htmllink
("", "", "ikiwiki/searching",
286 noimageinline
=> 1, linktext
=> "Help");
287 exec($config{omega_cgi
}) || error
("$config{omega_cgi} failed: $!");
294 # 240 is the number used by omindex to decide when to hash an
295 # overlong term. This does not use a compatible hash method though.
297 if (length encode_utf8
($page) > 240) {
298 eval q{use Digest::SHA};
300 debug
("search: ".sprintf(gettext
("need Digest::SHA to index %s"), $page)) if $@
;
304 # Note no colon, therefore it's guaranteed to not overlap
305 # with a page with the same name as the hash..
306 return "U".lc(Digest
::SHA
::sha1_hex
($page));
316 if (! defined $db->{$dbname}) {
319 use Search::Xapian::WritableDatabase;
322 $db->{$dbname}=Search
::Xapian
::WritableDatabase
->new($config{wikistatedir
}."/xapian/".$dbname,
323 Search
::Xapian
::DB_CREATE_OR_OPEN
());
325 return $db->{$dbname};
331 if (! $setup and (! -e
$config{wikistatedir
}."/xapian" || $config{rebuild
})) {
332 writefile
("omega.conf", $config{wikistatedir
}."/xapian",
334 "template_dir ./templates\n");
344 $_ eq "templates/page.tmpl" ||
345 $_ eq "templates/searchquery.tmpl"
352 # Avoid omega interpreting anything in the cgitemplate
353 # as an omegascript command.
354 eval q{use IkiWiki::CGI};
355 my $template=IkiWiki
::cgitemplate
(undef, gettext
("search"), "\0",
356 searchform
=> "", # avoid showing the small search form
358 eval q{use HTML::Entities};
360 $template=encode_entities
($template, '\$');
362 my $querytemplate=readfile
(IkiWiki
::template_file
("searchquery.tmpl"));
363 $template=~s/\0/$querytemplate/;
364 writefile
("query", $config{wikistatedir
}."/xapian/templates",
369 if (-d
$config{wikistatedir
}."/xapian") {
370 system("rm", "-rf", $config{wikistatedir
}."/xapian");