diff options
Diffstat (limited to 'IkiWiki')
| -rw-r--r-- | IkiWiki/Plugin/search.pm | 215 | ||||
| -rw-r--r-- | IkiWiki/Render.pm | 4 |
2 files changed, 119 insertions, 100 deletions
diff --git a/IkiWiki/Plugin/search.pm b/IkiWiki/Plugin/search.pm index 9bf223cf0..943af1313 100644 --- a/IkiWiki/Plugin/search.pm +++ b/IkiWiki/Plugin/search.pm @@ -1,5 +1,5 @@ #!/usr/bin/perl -# hyperestraier search engine plugin +# xapian-omega search engine plugin package IkiWiki::Plugin::search; use warnings; @@ -7,33 +7,35 @@ use strict; use IkiWiki 2.00; sub import { #{{{ - hook(type => "getopt", id => "hyperestraier", - call => \&getopt); - hook(type => "checkconfig", id => "hyperestraier", - call => \&checkconfig); - hook(type => "pagetemplate", id => "hyperestraier", - call => \&pagetemplate); - hook(type => "delete", id => "hyperestraier", - call => \&delete); - hook(type => "change", id => "hyperestraier", - call => \&change); - hook(type => "cgi", id => "hyperestraier", - call => \&cgi); + hook(type => "checkconfig", id => "search", call => \&checkconfig); + hook(type => "pagetemplate", id => "search", call => \&pagetemplate); + # run last so other needsbuild hooks can modify the list + hook(type => "needsbuild", id => "search", call => \&needsbuild, + last => 1); + hook(type => "filter", id => "search", call => \&filter); + hook(type => "delete", id => "search", call => \&delete); + hook(type => "cgi", id => "search", call => \&cgi); } # }}} -sub getopt () { #{{{ - eval q{use Getopt::Long}; - error($@) if $@; - Getopt::Long::Configure('pass_through'); - GetOptions("estseek=s" => \$config{estseek}); -} #}}} - sub checkconfig () { #{{{ foreach my $required (qw(url cgiurl)) { if (! length $config{$required}) { error(sprintf(gettext("Must specify %s when using the search plugin"), $required)); } } + + if (! exists $config{omega_cgi}) { + $config{omega_cgi}="/usr/lib/cgi-bin/omega/omega"; + } + + if (! -e $config{wikistatedir}."/xapian" || $config{rebuild}) { + writefile("omega.conf", $config{wikistatedir}."/xapian", + "database_dir .\n". + "template_dir ./templates\n"); + writefile("query", $config{wikistatedir}."/xapian/templates", + IkiWiki::misctemplate(gettext("search"), + readfile(IkiWiki::template_file("searchquery.tmpl")))); + } } #}}} my $form; @@ -54,95 +56,116 @@ sub pagetemplate (@) { #{{{ } } #}}} -sub delete (@) { #{{{ - debug(gettext("cleaning hyperestraier search index")); - estcmd("purge -cl"); - estcfg(); +my %toindex; +sub needsbuild ($) { #{{{ + %toindex = map { pagename($_) => 1 } @{shift()}; } #}}} -sub change (@) { #{{{ - debug(gettext("updating hyperestraier search index")); - estcmd("gather -cm -bc -cl -sd", - map { - map { - Encode::encode_utf8($config{destdir}."/".$_) - } @{$renderedfiles{pagename($_)}}; - } @_ - ); - estcfg(); +my $scrubber; +sub filter (@) { #{{{ + my %params=@_; + + if ($params{page} eq $params{destpage} && $toindex{$params{page}}) { + # index page + my $db=xapiandb(); + my $doc=Search::Xapian::Document->new(); + my $title; + if (exists $pagestate{$params{page}}{meta} && + exists $pagestate{$params{page}}{meta}{title}) { + $title=$pagestate{$params{page}}{meta}{title}; + } + else { + $title=IkiWiki::pagetitle($params{page}); + } + + # Remove any html from text to be indexed. + # TODO: This removes html that is in eg, a markdown pre, + # which should not be removed, really. + if (! defined $scrubber) { + eval q{use HTML::Scrubber}; + if (! $@) { + $scrubber=HTML::Scrubber->new(allow => []); + } + } + my $toindex = defined $scrubber ? $scrubber->scrub($params{content}) : $params{content}; + + # Take 512 characters for a sample, then extend it out + # if it stopped in the middle of a word. + my $size=512; + my ($sample)=substr($toindex, 0, $size); + if (length($sample) == $size) { + my $max=length($toindex); + my $next; + while ($size < $max && + ($next=substr($toindex, $size++, 1)) !~ /\s/) { + $sample.=$next; + } + } + $sample=~s/\n/ /g; + + # data used by omega + $doc->set_data( + "url=".urlto($params{page}, "")."\n". + "sample=$sample\n". + "caption=$title\n". + "modtime=$IkiWiki::pagemtime{$params{page}}\n". + "size=".length($params{content})."\n" + ); + + my $tg = Search::Xapian::TermGenerator->new(); + $tg->set_stemmer(new Search::Xapian::Stem("english")); + $tg->set_document($doc); + $tg->index_text($params{page}, 2); + $tg->index_text($title, 2); + $tg->index_text($toindex); + + my $pageterm=pageterm($params{page}); + $doc->add_term($pageterm); + $db->replace_document_by_term($pageterm, $doc); + } + + return $params{content}; +} #}}} + +sub delete (@) { #{{{ + my $db=xapiandb(); + foreach my $page (@_) { + $db->delete_document_by_term(pageterm($page)); + } } #}}} sub cgi ($) { #{{{ my $cgi=shift; - if (defined $cgi->param('phrase') || defined $cgi->param("navi")) { + if (defined $cgi->param('P')) { # only works for GET requests - chdir("$config{wikistatedir}/hyperestraier") || error("chdir: $!"); - exec("./".IkiWiki::basename($config{cgiurl})) || error("estseek.cgi failed"); + chdir("$config{wikistatedir}/xapian") || error("chdir: $!"); + $ENV{OMEGA_CONFIG_FILE}="./omega.conf"; + $ENV{CGIURL}=$config{cgiurl}, + exec($config{omega_cgi}) || error("$config{omega_cgi} failed: $!"); } } #}}} -my $configured=0; -sub estcfg () { #{{{ - return if $configured; - $configured=1; - - my $estdir="$config{wikistatedir}/hyperestraier"; - my $cgi=IkiWiki::basename($config{cgiurl}); - $cgi=~s/\..*$//; - - my $newfile="$estdir/$cgi.tmpl.new"; - my $cleanup = sub { unlink($newfile) }; - open(TEMPLATE, ">:utf8", $newfile) || error("open $newfile: $!", $cleanup); - print TEMPLATE IkiWiki::misctemplate("search", - "<!--ESTFORM-->\n\n<!--ESTRESULT-->\n\n<!--ESTINFO-->\n\n", - forcebaseurl => IkiWiki::dirname($config{cgiurl})."/") || - error("write $newfile: $!", $cleanup); - close TEMPLATE || error("save $newfile: $!", $cleanup); - rename($newfile, "$estdir/$cgi.tmpl") || - error("rename $newfile: $!", $cleanup); - - $newfile="$estdir/$cgi.conf"; - open(TEMPLATE, ">$newfile") || error("open $newfile: $!", $cleanup); - my $template=template("estseek.conf"); - eval q{use Cwd 'abs_path'}; - $template->param( - index => $estdir, - tmplfile => "$estdir/$cgi.tmpl", - destdir => abs_path($config{destdir}), - url => $config{url}, - ); - print TEMPLATE $template->output || error("write $newfile: $!", $cleanup); - close TEMPLATE || error("save $newfile: $!", $cleanup); - rename($newfile, "$estdir/$cgi.conf") || - error("rename $newfile: $!", $cleanup); - - $cgi="$estdir/".IkiWiki::basename($config{cgiurl}); - unlink($cgi); - my $estseek = defined $config{estseek} ? $config{estseek} : '/usr/lib/estraier/estseek.cgi'; - symlink($estseek, $cgi) || error("symlink $estseek $cgi: $!"); -} # }}} +sub pageterm ($) { #{{{ + my $page=shift; -sub estcmd ($;@) { #{{{ - my @params=split(' ', shift); - push @params, "-cl", "$config{wikistatedir}/hyperestraier"; - if (@_) { - push @params, "-"; - } - - my $pid=open(CHILD, "|-"); - if ($pid) { - # parent - foreach (@_) { - print CHILD "$_\n"; - } - close(CHILD) || print STDERR "estcmd @params exited nonzero: $?\n"; - } - else { - # child - open(STDOUT, "/dev/null"); # shut it up (closing won't work) - exec("estcmd", @params) || error("can't run estcmd"); + # TODO: check if > 255 char page names overflow term + # length; use sha1 if so? + return "U:".$page; +} #}}} + +my $db; +sub xapiandb () { #{{{ + if (! defined $db) { + eval q{ + use Search::Xapian; + use Search::Xapian::WritableDatabase; + }; + error($@) if $@; + $db=Search::Xapian::WritableDatabase->new($config{wikistatedir}."/xapian/default", + Search::Xapian::DB_CREATE_OR_OPEN()); } + return $db; } #}}} 1 diff --git a/IkiWiki/Render.pm b/IkiWiki/Render.pm index eb01a4227..3422024d1 100644 --- a/IkiWiki/Render.pm +++ b/IkiWiki/Render.pm @@ -156,10 +156,6 @@ sub scan ($) { #{{{ my $page=pagename($file); will_render($page, htmlpage($page), 1); - # Always needs to be done, since filters might add links - # to the content. - $content=filter($page, $page, $content); - if ($config{discussion}) { # Discussion links are a special case since they're # not in the text of the page, but on its template. |