From 748aa7af777caaa32ac5ab56e707509b3739b49e Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 23 Apr 2009 14:07:28 -0400 Subject: pagespec error/failure distinction and error display by inline * Add IkiWiki::ErrorReason objects, and modify pagespecs to return them in cases where they fail to match due to a configuration or syntax error. * inline: Display a handy error message if the inline cannot display any pages due to such an error. This is perhaps somewhat incomplete, as other users of pagespecs do not display the error, and will eventually need similar modifications to inline. I should probably factor out a pagespec_match_all function and make it throw ErrorReasons. --- IkiWiki.pm | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'IkiWiki.pm') diff --git a/IkiWiki.pm b/IkiWiki.pm index 2eca82e4d..fca8da874 100644 --- a/IkiWiki.pm +++ b/IkiWiki.pm @@ -1800,7 +1800,7 @@ sub pagespec_translate ($) { $code.="IkiWiki::PageSpec::match_$1(\$page, ".safequote($2).", \@_)"; } else { - $code.="IkiWiki::FailReason->new(".safequote(qq{unknown function in pagespec "$word"}).")"; + $code.="IkiWiki::ErrorReason->new(".safequote(qq{unknown function in pagespec "$word"}).")"; } } else { @@ -1827,7 +1827,7 @@ sub pagespec_match ($$;@) { } my $sub=pagespec_translate($spec); - return IkiWiki::FailReason->new("syntax error in pagespec \"$spec\"") + return IkiWiki::ErrorReason->new("syntax error in pagespec \"$spec\"") if $@ || ! defined $sub; return $sub->($page, @params); } @@ -1861,6 +1861,10 @@ sub new { return bless \$value, $class; } +package IkiWiki::ErrorReason; + +our @ISA = 'IkiWiki::FailReason'; + package IkiWiki::SuccessReason; use overload ( @@ -2021,7 +2025,7 @@ sub match_user ($$;@) { my %params=@_; if (! exists $params{user}) { - return IkiWiki::FailReason->new("no user specified"); + return IkiWiki::ErrorReason->new("no user specified"); } if (defined $params{user} && lc $params{user} eq lc $user) { @@ -2041,7 +2045,7 @@ sub match_admin ($$;@) { my %params=@_; if (! exists $params{user}) { - return IkiWiki::FailReason->new("no user specified"); + return IkiWiki::ErrorReason->new("no user specified"); } if (defined $params{user} && IkiWiki::is_admin($params{user})) { @@ -2061,7 +2065,7 @@ sub match_ip ($$;@) { my %params=@_; if (! exists $params{ip}) { - return IkiWiki::FailReason->new("no IP specified"); + return IkiWiki::ErrorReason->new("no IP specified"); } if (defined $params{ip} && lc $params{ip} eq lc $ip) { -- cgit v1.2.3 From aa306957bac11477b914ac19b93890184ffe4062 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 23 Apr 2009 15:45:30 -0400 Subject: pagespec_match_list added and used in most appropriate places * pagespec_match_list: New API function, matches pages in a list and throws an error if the pagespec is bad. * inline, brokenlinks, calendar, linkmap, map, orphans, pagecount, pagestate, postsparkline: Display a handy error message if the pagespec is erronious. --- IkiWiki.pm | 32 +++++++++++++++++++++++---- IkiWiki/Plugin/brokenlinks.pm | 23 +++++++++---------- IkiWiki/Plugin/calendar.pm | 3 +-- IkiWiki/Plugin/external.pm | 9 +++++++- IkiWiki/Plugin/inline.pm | 17 +++----------- IkiWiki/Plugin/linkmap.pm | 7 +++--- IkiWiki/Plugin/map.pm | 49 ++++++++++++++++++++--------------------- IkiWiki/Plugin/orphans.pm | 7 +++--- IkiWiki/Plugin/pagecount.pm | 9 +++----- IkiWiki/Plugin/pagestats.pm | 11 +++++---- IkiWiki/Plugin/postsparkline.pm | 10 +++------ debian/changelog | 7 ++++-- doc/plugins/write.mdwn | 13 +++++++++++ 13 files changed, 111 insertions(+), 86 deletions(-) (limited to 'IkiWiki.pm') diff --git a/IkiWiki.pm b/IkiWiki.pm index fca8da874..e260fffea 100644 --- a/IkiWiki.pm +++ b/IkiWiki.pm @@ -18,10 +18,10 @@ use vars qw{%config %links %oldlinks %pagemtime %pagectime %pagecase use Exporter q{import}; our @EXPORT = qw(hook debug error template htmlpage add_depends pagespec_match - bestlink htmllink readfile writefile pagetype srcfile pagename - displaytime will_render gettext urlto targetpage - add_underlay pagetitle titlepage linkpage newpagefile - inject + pagespec_match_list bestlink htmllink readfile writefile + pagetype srcfile pagename displaytime will_render gettext urlto + targetpage add_underlay pagetitle titlepage linkpage + newpagefile inject %config %links %pagestate %wikistate %renderedfiles %pagesources %destsources); our $VERSION = 3.00; # plugin interface version, next is ikiwiki version @@ -1832,6 +1832,30 @@ sub pagespec_match ($$;@) { return $sub->($page, @params); } +sub pagespec_match_list ($$;@) { + my $pages=shift; + my $spec=shift; + my @params=@_; + + my $sub=pagespec_translate($spec); + error "syntax error in pagespec \"$spec\"" + if $@ || ! defined $sub; + + my @ret; + my $r; + foreach my $page (@$pages) { + $r=$sub->($page, @params); + push @ret, $page if $r; + } + + if (! @ret && defined $r && $r->isa("IkiWiki::ErrorReason")) { + error(sprintf(gettext("cannot match pages: %s"), $r)); + } + else { + return @ret; + } +} + sub pagespec_valid ($) { my $spec=shift; diff --git a/IkiWiki/Plugin/brokenlinks.pm b/IkiWiki/Plugin/brokenlinks.pm index bf0d7560d..da97dbc28 100644 --- a/IkiWiki/Plugin/brokenlinks.pm +++ b/IkiWiki/Plugin/brokenlinks.pm @@ -28,18 +28,17 @@ sub preprocess (@) { add_depends($params{page}, $params{pages}); my %broken; - foreach my $page (keys %links) { - if (pagespec_match($page, $params{pages}, location => $params{page})) { - my $discussion=gettext("discussion"); - my %seen; - foreach my $link (@{$links{$page}}) { - next if $seen{$link}; - $seen{$link}=1; - next if $link =~ /.*\/\Q$discussion\E/i && $config{discussion}; - my $bestlink=bestlink($page, $link); - next if length $bestlink; - push @{$broken{$link}}, $page; - } + foreach my $page (pagespec_match_list([keys %links], + $params{pages}, location => $params{page})) { + my $discussion=gettext("discussion"); + my %seen; + foreach my $link (@{$links{$page}}) { + next if $seen{$link}; + $seen{$link}=1; + next if $link =~ /.*\/\Q$discussion\E/i && $config{discussion}; + my $bestlink=bestlink($page, $link); + next if length $bestlink; + push @{$broken{$link}}, $page; } } diff --git a/IkiWiki/Plugin/calendar.pm b/IkiWiki/Plugin/calendar.pm index d473c8348..fe4b16072 100644 --- a/IkiWiki/Plugin/calendar.pm +++ b/IkiWiki/Plugin/calendar.pm @@ -369,8 +369,7 @@ sub preprocess (@) { my $page =$params{page}; if (! defined $cache{$pagespec}) { - foreach my $p (keys %pagesources) { - next unless pagespec_match($p, $pagespec); + foreach my $p (pagespec_match_list([keys %pagesources], $pagespec)) { my $mtime = $IkiWiki::pagectime{$p}; my $src = $pagesources{$p}; my @date = localtime($mtime); diff --git a/IkiWiki/Plugin/external.pm b/IkiWiki/Plugin/external.pm index 066f15cf1..aeee15dea 100644 --- a/IkiWiki/Plugin/external.pm +++ b/IkiWiki/Plugin/external.pm @@ -230,10 +230,17 @@ sub hook ($@) { } sub pagespec_match ($@) { - # convert pagespec_match's return object into a XML RPC boolean + # convert return object into a XML RPC boolean my $plugin=shift; return RPC::XML::boolean->new(0 + IkiWiki::pagespec_march(@_)); } +sub pagespec_match_list ($@) { + # convert return object into a XML RPC boolean + my $plugin=shift; + + return RPC::XML::boolean->new(0 + IkiWiki::pagespec_march_list(@_)); +} + 1 diff --git a/IkiWiki/Plugin/inline.pm b/IkiWiki/Plugin/inline.pm index 551c38a65..366357095 100644 --- a/IkiWiki/Plugin/inline.pm +++ b/IkiWiki/Plugin/inline.pm @@ -183,20 +183,9 @@ sub preprocess_inline (@) { $params{template} = $archive ? "archivepage" : "inlinepage"; } - my @list; - my $lastmatch; - foreach my $page (keys %pagesources) { - next if $page eq $params{page}; - $lastmatch=pagespec_match($page, $params{pages}, location => $params{page}); - if ($lastmatch) { - push @list, $page; - } - } - - if (! @list && defined $lastmatch && - $lastmatch->isa("IkiWiki::ErrorReason")) { - error(sprintf(gettext("cannot match pages: %s"), $lastmatch)); - } + my @list=pagespec_match_list( + [ grep { $_ ne $params{page}} keys %pagesources ], + $params{pages}, location => $params{page}); if (exists $params{sort} && $params{sort} eq 'title') { @list=sort { pagetitle(basename($a)) cmp pagetitle(basename($b)) } @list; diff --git a/IkiWiki/Plugin/linkmap.pm b/IkiWiki/Plugin/linkmap.pm index 941ed5f36..0137476ac 100644 --- a/IkiWiki/Plugin/linkmap.pm +++ b/IkiWiki/Plugin/linkmap.pm @@ -56,10 +56,9 @@ sub genmap ($) { # Get all the items to map. my %mapitems = (); - foreach my $item (keys %links) { - if (pagespec_match($item, $params{pages}, location => $params{page})) { - $mapitems{$item}=urlto($item, $params{destpage}); - } + foreach my $item (pagespec_match_list([keys %links], + $params{pages}, location => $params{page})) { + $mapitems{$item}=urlto($item, $params{destpage}); } my $dest=$params{page}."/linkmap.png"; diff --git a/IkiWiki/Plugin/map.pm b/IkiWiki/Plugin/map.pm index 328493116..120451b5d 100644 --- a/IkiWiki/Plugin/map.pm +++ b/IkiWiki/Plugin/map.pm @@ -32,32 +32,31 @@ sub preprocess (@) { # Get all the items to map. my %mapitems; - foreach my $page (keys %pagesources) { - if (pagespec_match($page, $params{pages}, location => $params{page})) { - if (exists $params{show} && - exists $pagestate{$page} && - exists $pagestate{$page}{meta}{$params{show}}) { - $mapitems{$page}=$pagestate{$page}{meta}{$params{show}}; - } - else { - $mapitems{$page}=''; - } - # Check for a common prefix. - if (! defined $common_prefix) { - $common_prefix=$page; - } - elsif (length $common_prefix && - $page !~ /^\Q$common_prefix\E(\/|$)/) { - my @a=split(/\//, $page); - my @b=split(/\//, $common_prefix); - $common_prefix=""; - while (@a && @b && $a[0] eq $b[0]) { - if (length $common_prefix) { - $common_prefix.="/"; - } - $common_prefix.=shift(@a); - shift @b; + foreach my $page (pagespec_match_list([keys %pagesources], + $params{pages}, location => $params{page})) { + if (exists $params{show} && + exists $pagestate{$page} && + exists $pagestate{$page}{meta}{$params{show}}) { + $mapitems{$page}=$pagestate{$page}{meta}{$params{show}}; + } + else { + $mapitems{$page}=''; + } + # Check for a common prefix. + if (! defined $common_prefix) { + $common_prefix=$page; + } + elsif (length $common_prefix && + $page !~ /^\Q$common_prefix\E(\/|$)/) { + my @a=split(/\//, $page); + my @b=split(/\//, $common_prefix); + $common_prefix=""; + while (@a && @b && $a[0] eq $b[0]) { + if (length $common_prefix) { + $common_prefix.="/"; } + $common_prefix.=shift(@a); + shift @b; } } } diff --git a/IkiWiki/Plugin/orphans.pm b/IkiWiki/Plugin/orphans.pm index 605e6e43a..cf74c9b79 100644 --- a/IkiWiki/Plugin/orphans.pm +++ b/IkiWiki/Plugin/orphans.pm @@ -35,9 +35,10 @@ sub preprocess (@) { my @orphans; my $discussion=gettext("discussion"); - foreach my $page (keys %pagesources) { - next if $linkedto{$page} || $page eq 'index'; - next unless pagespec_match($page, $params{pages}, location => $params{page}); + foreach my $page (pagespec_match_list( + [ grep { ! $linkedto{$_} && $_ ne 'index' } + keys %pagesources ], + $params{pages}, location => $params{page})) { # If the page has a link to some other page, it's # indirectly linked to a page via that page's backlinks. next if grep { diff --git a/IkiWiki/Plugin/pagecount.pm b/IkiWiki/Plugin/pagecount.pm index a143f24d0..f8881a04b 100644 --- a/IkiWiki/Plugin/pagecount.pm +++ b/IkiWiki/Plugin/pagecount.pm @@ -27,12 +27,9 @@ sub preprocess (@) { add_depends($params{page}, $params{pages}); my @pages=keys %pagesources; - return $#pages+1 if $params{pages} eq "*"; # optimisation - my $count=0; - foreach my $page (@pages) { - $count++ if pagespec_match($page, $params{pages}, location => $params{page}); - } - return $count; + @pages=pagespec_match_list(\@pages, $params{pages}, location => $params{page}) + if $params{pages} ne "*"; # optimisation; + return $#pages+1; } 1 diff --git a/IkiWiki/Plugin/pagestats.pm b/IkiWiki/Plugin/pagestats.pm index dbe69539d..8ab5d3666 100644 --- a/IkiWiki/Plugin/pagestats.pm +++ b/IkiWiki/Plugin/pagestats.pm @@ -41,12 +41,11 @@ sub preprocess (@) { my %counts; my $max = 0; - foreach my $page (keys %links) { - if (pagespec_match($page, $params{pages}, location => $params{page})) { - use IkiWiki::Render; - $counts{$page} = scalar(IkiWiki::backlinks($page)); - $max = $counts{$page} if $counts{$page} > $max; - } + foreach my $page (pagespec_match_list([keys %links], + $params{pages}, location => $params{page})) { + use IkiWiki::Render; + $counts{$page} = scalar(IkiWiki::backlinks($page)); + $max = $counts{$page} if $counts{$page} > $max; } if ($style eq 'table') { diff --git a/IkiWiki/Plugin/postsparkline.pm b/IkiWiki/Plugin/postsparkline.pm index ba43561fb..c2ebbc5eb 100644 --- a/IkiWiki/Plugin/postsparkline.pm +++ b/IkiWiki/Plugin/postsparkline.pm @@ -50,13 +50,9 @@ sub preprocess (@) { add_depends($params{page}, $params{pages}); - my @list; - foreach my $page (keys %pagesources) { - next if $page eq $params{page}; - if (pagespec_match($page, $params{pages}, location => $params{page})) { - push @list, $page; - } - } + my @list=pagespec_match_list( + [ grep { $_ ne $params{page} } keys %pagesources], + $params{pages}, location => $params{page}); @list = sort { $params{timehash}->{$b} <=> $params{timehash}->{$a} } @list; diff --git a/debian/changelog b/debian/changelog index de4ac4a8c..37b69ada0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -11,8 +11,11 @@ ikiwiki (3.11) UNRELEASED; urgency=low * Add IkiWiki::ErrorReason objects, and modify pagespecs to return them in cases where they fail to match due to a configuration or syntax error. - * inline: Display a handy error message if the inline cannot display any - pages due to such an error. + * pagespec_match_list: New API function, matches pages in a list + and throws an error if the pagespec is bad. + * inline, brokenlinks, calendar, linkmap, map, orphans, pagecount, + pagestate, postsparkline: Display a handy error message if the pagespec + is erronious. * comments: Add link to comment post form to allow user to sign in if they wish to, if the configuration makes signin optional for commenting. diff --git a/doc/plugins/write.mdwn b/doc/plugins/write.mdwn index 23df01ca7..0b358b215 100644 --- a/doc/plugins/write.mdwn +++ b/doc/plugins/write.mdwn @@ -572,6 +572,19 @@ The most often used is "location", which specifies the location the PageSpec should match against. If not passed, relative PageSpecs will match relative to the top of the wiki. +#### `pagespec_match_list($$;@)` + +Passed a reference to a list of page names, and [[ikiwiki/PageSpec]], +returns the set of pages that match the [[ikiwiki/PageSpec]]. + +Additional named parameters can be passed, to further limit the match. +The most often used is "location", which specifies the location the +PageSpec should match against. If not passed, relative PageSpecs will match +relative to the top of the wiki. + +Unlike pagespec_match, this may throw an error if there is an error in +the pagespec. + #### `bestlink($$)` Given a page and the text of a link on the page, determine which -- cgit v1.2.3 From 2f96c49bd1826ecb213ae025ad456a714aa04863 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 23 Apr 2009 16:20:18 -0400 Subject: pagespec_match_list * optimisation Add an optimisation for the semi-common case of a "*" pagespec. Can avoid doing any real processing in this case. --- IkiWiki.pm | 2 ++ IkiWiki/Plugin/pagecount.pm | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'IkiWiki.pm') diff --git a/IkiWiki.pm b/IkiWiki.pm index e260fffea..e8c0c7abc 100644 --- a/IkiWiki.pm +++ b/IkiWiki.pm @@ -1837,6 +1837,8 @@ sub pagespec_match_list ($$;@) { my $spec=shift; my @params=@_; + return @$pages if $spec eq '*'; # optimisation + my $sub=pagespec_translate($spec); error "syntax error in pagespec \"$spec\"" if $@ || ! defined $sub; diff --git a/IkiWiki/Plugin/pagecount.pm b/IkiWiki/Plugin/pagecount.pm index f8881a04b..a561e58e5 100644 --- a/IkiWiki/Plugin/pagecount.pm +++ b/IkiWiki/Plugin/pagecount.pm @@ -26,9 +26,7 @@ sub preprocess (@) { # register a dependency. add_depends($params{page}, $params{pages}); - my @pages=keys %pagesources; - @pages=pagespec_match_list(\@pages, $params{pages}, location => $params{page}) - if $params{pages} ne "*"; # optimisation; + my @pages=pagespec_match_list([keys %pagesources], $params{pages}, location => $params{page}); return $#pages+1; } -- cgit v1.2.3 From 85ae48b21eed71eeace8cd093ce92b7399cba861 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 23 Apr 2009 16:35:56 -0400 Subject: Revert "pagespec_match_list * optimisation" This reverts commit 2f96c49bd1826ecb213ae025ad456a714aa04863. I forgot about internal pages. We don't want * matching them! I left the optimisation in pagecount, where it used to live. Internal pages probably don't matter when they're just being counted. --- IkiWiki.pm | 2 -- IkiWiki/Plugin/pagecount.pm | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'IkiWiki.pm') diff --git a/IkiWiki.pm b/IkiWiki.pm index e8c0c7abc..e260fffea 100644 --- a/IkiWiki.pm +++ b/IkiWiki.pm @@ -1837,8 +1837,6 @@ sub pagespec_match_list ($$;@) { my $spec=shift; my @params=@_; - return @$pages if $spec eq '*'; # optimisation - my $sub=pagespec_translate($spec); error "syntax error in pagespec \"$spec\"" if $@ || ! defined $sub; diff --git a/IkiWiki/Plugin/pagecount.pm b/IkiWiki/Plugin/pagecount.pm index a561e58e5..f8881a04b 100644 --- a/IkiWiki/Plugin/pagecount.pm +++ b/IkiWiki/Plugin/pagecount.pm @@ -26,7 +26,9 @@ sub preprocess (@) { # register a dependency. add_depends($params{page}, $params{pages}); - my @pages=pagespec_match_list([keys %pagesources], $params{pages}, location => $params{page}); + my @pages=keys %pagesources; + @pages=pagespec_match_list(\@pages, $params{pages}, location => $params{page}) + if $params{pages} ne "*"; # optimisation; return $#pages+1; } -- cgit v1.2.3 From 2a7721febd6cac1af5e7f4b4949ffe066c62c837 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 5 May 2009 23:40:09 -0400 Subject: Avoid %links accumulating duplicates. (For TOVA) This is sorta an optimisation, and sorta a bug fix. In one test case I have available, it can speed a page build up from 3 minutes to 3 seconds. The root of the problem is that $links{$page} contains arrays of links, rather than hashes of links. And when a link is found, it is just pushed onto the array, without checking for dups. Now, the array is emptied before scanning a page, so there should not be a lot of opportunity for lots of duplicate links to pile up in it. But, in some cases, they can, and if there are hundreds of duplicate links in the array, then scanning it for matching links, as match_link and some other code does, becomes much more expensive than it needs to be. Perhaps the real right fix would be to change the data structure to a hash. But, the list of links is never accessed like that, you always want to iterate through it. I also looked at deduping the list in saveindex, but that does a lot of unnecessary work, and doesn't completly solve the problem. So, finally, I decided to add an add_link function that handles deduping, and make ikiwiki-transition remove the old dup links. --- IkiWiki.pm | 12 ++++++++++-- IkiWiki/Plugin/camelcase.pm | 2 +- IkiWiki/Plugin/img.pm | 2 +- IkiWiki/Plugin/link.pm | 2 +- IkiWiki/Plugin/meta.pm | 2 +- IkiWiki/Plugin/tag.pm | 6 +++--- debian/NEWS | 9 +++++++++ debian/changelog | 4 ++++ doc/ikiwiki-transition.mdwn | 7 +++++++ doc/plugins/write.mdwn | 22 ++++++++++++++-------- ikiwiki-transition | 19 +++++++++++++++++++ 11 files changed, 70 insertions(+), 17 deletions(-) (limited to 'IkiWiki.pm') diff --git a/IkiWiki.pm b/IkiWiki.pm index e260fffea..e6efe1889 100644 --- a/IkiWiki.pm +++ b/IkiWiki.pm @@ -21,12 +21,12 @@ our @EXPORT = qw(hook debug error template htmlpage add_depends pagespec_match pagespec_match_list bestlink htmllink readfile writefile pagetype srcfile pagename displaytime will_render gettext urlto targetpage add_underlay pagetitle titlepage linkpage - newpagefile inject + newpagefile inject add_link %config %links %pagestate %wikistate %renderedfiles %pagesources %destsources); our $VERSION = 3.00; # plugin interface version, next is ikiwiki version our $version='unknown'; # VERSION_AUTOREPLACE done by Makefile, DNE -our $installdir=''; # INSTALLDIR_AUTOREPLACE done by Makefile, DNE +our $installdir='/usr'; # INSTALLDIR_AUTOREPLACE done by Makefile, DNE # Optimisation. use Memoize; @@ -1757,6 +1757,14 @@ sub inject { use warnings; } +sub add_link ($$) { + my $page=shift; + my $link=shift; + + push @{$links{$page}}, $link + unless grep { $_ eq $link } @{$links{$page}}; +} + sub pagespec_merge ($$) { my $a=shift; my $b=shift; diff --git a/IkiWiki/Plugin/camelcase.pm b/IkiWiki/Plugin/camelcase.pm index 74a8397d7..088447d6b 100644 --- a/IkiWiki/Plugin/camelcase.pm +++ b/IkiWiki/Plugin/camelcase.pm @@ -61,7 +61,7 @@ sub scan (@) { my $content=$params{content}; while ($content =~ /$link_regexp/g) { - push @{$links{$page}}, linkpage($1) unless ignored($1) + add_link($page, linkpage($1)) unless ignored($1) } } diff --git a/IkiWiki/Plugin/img.pm b/IkiWiki/Plugin/img.pm index d295b833b..a697fea19 100644 --- a/IkiWiki/Plugin/img.pm +++ b/IkiWiki/Plugin/img.pm @@ -43,7 +43,7 @@ sub preprocess (@) { return ''; } - push @{$links{$params{page}}}, $image; + add_link($params{page}, $image); # optimisation: detect scan mode, and avoid generating the image if (! defined wantarray) { return; diff --git a/IkiWiki/Plugin/link.pm b/IkiWiki/Plugin/link.pm index b79273f96..4c1add985 100644 --- a/IkiWiki/Plugin/link.pm +++ b/IkiWiki/Plugin/link.pm @@ -86,7 +86,7 @@ sub scan (@) { my $content=$params{content}; while ($content =~ /(? pagetitle($1)); } else { my $tag=linkpage($_); $tags{$params{page}}{$tag}=1; - push @{$links{$params{page}}}, tagpage($tag); + add_link($params{page}, tagpage($tag)); return taglink($params{page}, $params{destpage}, $tag); } } diff --git a/debian/NEWS b/debian/NEWS index 22513cc4a..62e1543b3 100644 --- a/debian/NEWS +++ b/debian/NEWS @@ -1,3 +1,12 @@ +ikiwiki (3.12) UNRELEASED; urgency=low + + You may want to run `ikiwiki-transition deduplinks /path/to/srcdir` + after upgrading to this version of ikiwiki. This command will + optimise your wiki's saved state, removing duplicate information + that can slow ikiwiki down. + + -- Joey Hess Wed, 06 May 2009 00:25:06 -0400 + ikiwiki (3.01) unstable; urgency=low If your wiki uses git, and you have a `diffurl` configured in diff --git a/debian/changelog b/debian/changelog index 9244ccdb8..1e1d48d78 100644 --- a/debian/changelog +++ b/debian/changelog @@ -5,6 +5,10 @@ ikiwiki (3.12) UNRELEASED; urgency=low fails on nonexistant directories with some broken perl versions. * inline: Minor optimisation. + * add_link: New function, which plugins should use rather than + modifying %links directly, to avoid it accumulating duplicates. + * ikiwiki-transition: Add a deduplinks action, that can be used + to remove duplicate links and optimise a wiki w/o rebuilding it. -- Joey Hess Mon, 04 May 2009 19:17:39 -0400 diff --git a/doc/ikiwiki-transition.mdwn b/doc/ikiwiki-transition.mdwn index 18836d5f5..e0b853ecf 100644 --- a/doc/ikiwiki-transition.mdwn +++ b/doc/ikiwiki-transition.mdwn @@ -61,6 +61,13 @@ If this is not done explicitly, a user's plaintext password will be automatically converted to a hash when a user logs in for the first time after upgrade to ikiwiki 2.48. +# deduplinks srcdir + +In the past, bugs in ikiwiki have allowed duplicate link information +to be stored in its indexdb. This mode removes such duplicate information, +which may speed up wikis afflicted by it. Note that rebuilding the wiki +will have the same effect. + # AUTHOR Josh Triplett , Joey Hess diff --git a/doc/plugins/write.mdwn b/doc/plugins/write.mdwn index 0b358b215..28da243d5 100644 --- a/doc/plugins/write.mdwn +++ b/doc/plugins/write.mdwn @@ -107,8 +107,8 @@ adding or removing files from it. This hook is called early in the process of building the wiki, and is used as a first pass scan of the page, to collect metadata about the page. It's -mostly used to scan the page for [[WikiLinks|ikiwiki/WikiLink]], and add them to `%links`. -Present in IkiWiki 2.40 and later. +mostly used to scan the page for [[WikiLinks|ikiwiki/WikiLink]], and add +them to `%links`. Present in IkiWiki 2.40 and later. The function is passed named parameters "page" and "content". Its return value is ignored. @@ -151,11 +151,11 @@ parameter is set to a true value if the page is being previewed. If `hook` is passed an optional "scan" parameter, set to a true value, this makes the hook be called during the preliminary scan that ikiwiki makes of updated pages, before begining to render pages. This should be done if the -hook modifies data in `%links`. Note that doing so will make the hook be -run twice per page build, so avoid doing it for expensive hooks. (As an -optimisation, if your preprocessor hook is called in a void context, you -can assume it's being run in scan mode, and avoid doing expensive things at -that point.) +hook modifies data in `%links` (typically by calling `add_link`). Note that +doing so will make the hook be run twice per page build, so avoid doing it +for expensive hooks. (As an optimisation, if your preprocessor hook is +called in a void context, you can assume it's being run in scan mode, and +avoid doing expensive things at that point.) Note that if the [[htmlscrubber]] is enabled, html in preprocessor [[ikiwiki/directive]] output is sanitised, which may limit what @@ -174,7 +174,8 @@ links. The function is passed named parameters "page", "destpage", and and later. Plugins that implement linkify must also implement a scan hook, that scans -for the links on the page and adds them to `%links`. +for the links on the page and adds them to `%links` (typically by calling +`add_link`). ### htmlize @@ -753,6 +754,11 @@ Optionally, a third parameter can be passed, to specify the preferred filename of the page. For example, `targetpage("foo", "rss", "feed")` will yield something like `foo/feed.rss`. +#### `add_link($$)` + +This adds a link to `%links`, ensuring that duplicate links are not +added. Pass it the page that contains the link, and the link text. + ## Miscellaneous ### Internal use pages diff --git a/ikiwiki-transition b/ikiwiki-transition index 599261a09..f17868d73 100755 --- a/ikiwiki-transition +++ b/ikiwiki-transition @@ -220,6 +220,21 @@ sub moveprefs { IkiWiki::Setup::dump($setup); } +sub deduplinks { + my $dir=shift; + if (! defined $dir) { + usage(); + } + $config{wikistatedir}=$dir."/.ikiwiki"; + IkiWiki::loadindex(); + foreach my $page (keys %links) { + my %l; + $l{$_}=1 foreach @{$links{$page}}; + $links{$page}=[keys %l] + } + IkiWiki::saveindex(); +} + sub usage { print STDERR "Usage: ikiwiki-transition type ...\n"; print STDERR "Currently supported transition subcommands:\n"; @@ -229,6 +244,7 @@ sub usage { print STDERR "\tmoveprefs setupfile\n"; print STDERR "\thashpassword srcdir\n"; print STDERR "\tindexdb srcdir\n"; + print STDERR "\tdeduplinks srcdir\n"; exit 1; } @@ -253,6 +269,9 @@ elsif ($mode eq 'setupformat') { elsif ($mode eq 'moveprefs') { moveprefs(@ARGV); } +elsif ($mode eq 'deduplinks') { + deduplinks(@ARGV); +} else { usage(); } -- cgit v1.2.3 From 4291f2e3d7d1eb160427764e622d247145652e81 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Thu, 7 May 2009 14:02:52 -0400 Subject: Allow underlaydir to be overridden without messing up inclusion of other underlays via add_underlay. --- IkiWiki.pm | 9 ++++++++- debian/changelog | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) (limited to 'IkiWiki.pm') diff --git a/IkiWiki.pm b/IkiWiki.pm index e6efe1889..6233d2ead 100644 --- a/IkiWiki.pm +++ b/IkiWiki.pm @@ -157,6 +157,13 @@ sub getsetup () { safe => 0, # path rebuild => 0, }, + underlaydirbase => { + type => "internal", + default => "$installdir/share/ikiwiki", + description => "parent directory containing additional underlays", + safe => 0, + rebuild => 0, + }, wrappers => { type => "internal", default => [], @@ -715,7 +722,7 @@ sub add_underlay ($) { my $dir=shift; if ($dir !~ /^\//) { - $dir="$config{underlaydir}/../$dir"; + $dir="$config{underlaydirbase}/$dir"; } if (! grep { $_ eq $dir } @{$config{underlaydirs}}) { diff --git a/debian/changelog b/debian/changelog index 36f4c16fd..d8c67b5a6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,6 +2,8 @@ ikiwiki (3.13) UNRELEASED; urgency=low * ikiwiki-transition: If passed a nonexistant srcdir, or one not containing .ikiwiki, abort with an error rather than creating it. + * Allow underlaydir to be overridden without messing up inclusion + of other underlays via add_underlay. -- Joey Hess Wed, 06 May 2009 20:45:44 -0400 -- cgit v1.2.3 From 23a4ee6d15dbd9b8e8c6588a829dd30a26a8de32 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Mon, 18 May 2009 15:25:10 -0400 Subject: Allow curly braces to be used in pagespecs And avoid a whole class of potential security problems (though none that I know of actually existing..), by avoiding performing any string interpolation on user-supplied data when translating pagespecs. --- IkiWiki.pm | 16 +++++++--------- debian/changelog | 3 +++ ...spec_can__39__t_match___123__curly__125___braces.mdwn | 2 +- t/pagespec_match.t | 4 +++- 4 files changed, 14 insertions(+), 11 deletions(-) (limited to 'IkiWiki.pm') diff --git a/IkiWiki.pm b/IkiWiki.pm index 6233d2ead..061a1c6db 100644 --- a/IkiWiki.pm +++ b/IkiWiki.pm @@ -1678,12 +1678,6 @@ sub rcs_receive () { $hooks{rcs}{rcs_receive}{call}->(); } -sub safequote ($) { - my $s=shift; - $s=~s/[{}]//g; - return "q{$s}"; -} - sub add_depends ($$) { my $page=shift; my $pagespec=shift; @@ -1785,6 +1779,7 @@ sub pagespec_translate ($) { # Convert spec to perl code. my $code=""; + my @data; while ($spec=~m{ \s* # ignore whitespace ( # 1: match a single word @@ -1812,14 +1807,17 @@ sub pagespec_translate ($) { } elsif ($word =~ /^(\w+)\((.*)\)$/) { if (exists $IkiWiki::PageSpec::{"match_$1"}) { - $code.="IkiWiki::PageSpec::match_$1(\$page, ".safequote($2).", \@_)"; + push @data, $2; + $code.="IkiWiki::PageSpec::match_$1(\$page, \$data[$#data], \@_)"; } else { - $code.="IkiWiki::ErrorReason->new(".safequote(qq{unknown function in pagespec "$word"}).")"; + push @data, qq{unknown function in pagespec "$word"}; + $code.="IkiWiki::ErrorReason->new(\$data[$#data])"; } } else { - $code.=" IkiWiki::PageSpec::match_glob(\$page, ".safequote($word).", \@_)"; + push @data, $word; + $code.=" IkiWiki::PageSpec::match_glob(\$page, \$data[$#data], \@_)"; } } diff --git a/debian/changelog b/debian/changelog index c2819d0c5..7efa31cf1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -6,6 +6,9 @@ ikiwiki (3.13) UNRELEASED; urgency=low of other underlays via add_underlay. * More friendly display of markdown, textile in edit form selector (jmtd) + * Allow curly braces to be used in pagespecs, and avoid a whole class + of potential security problems, by avoiding performing any string + interpolation on user-supplied data when translating pagespecs. -- Joey Hess Wed, 06 May 2009 20:45:44 -0400 diff --git a/doc/bugs/pagespec_can__39__t_match___123__curly__125___braces.mdwn b/doc/bugs/pagespec_can__39__t_match___123__curly__125___braces.mdwn index c03f82907..e3146d92a 100644 --- a/doc/bugs/pagespec_can__39__t_match___123__curly__125___braces.mdwn +++ b/doc/bugs/pagespec_can__39__t_match___123__curly__125___braces.mdwn @@ -35,6 +35,6 @@ More tests: > * Avoid exposing user input to interpolation as a string. One > way that comes to mind is to have a local string lookup hash, > and insert each user specified string into it, then use the hash -> to lookup the specified strings at runtime. +> to lookup the specified strings at runtime. [[done]] > > --[[Joey]] diff --git a/t/pagespec_match.t b/t/pagespec_match.t index 69cf361de..4cf6fa1ff 100755 --- a/t/pagespec_match.t +++ b/t/pagespec_match.t @@ -1,7 +1,7 @@ #!/usr/bin/perl use warnings; use strict; -use Test::More tests => 51; +use Test::More tests => 53; BEGIN { use_ok("IkiWiki"); } @@ -28,6 +28,8 @@ ok(pagespec_match("a/foo", "./*", "a/b"), "relative oldstyle call"); ok(pagespec_match("foo", "./*", location => "a"), "relative toplevel"); ok(pagespec_match("foo/bar", "*", location => "baz"), "absolute"); ok(! pagespec_match("foo", "foo and bar"), "foo and bar"); +ok(pagespec_match("{f}oo", "{*}*"), "curly match"); +ok(! pagespec_match("foo", "{*}*"), "curly !match"); # The link and backlink stuff needs this. $config{userdir}=""; -- cgit v1.2.3