diff options
| -rw-r--r-- | IkiWiki.pm | 103 | ||||
| -rw-r--r-- | debian/changelog | 1 | ||||
| -rw-r--r-- | doc/ikiwiki-transition.mdwn | 12 | ||||
| -rw-r--r-- | doc/index/discussion.mdwn | 36 | ||||
| -rw-r--r-- | doc/tips/inside_dot_ikiwiki.mdwn | 65 | ||||
| -rwxr-xr-x | ikiwiki-transition | 87 |
6 files changed, 219 insertions, 85 deletions
diff --git a/IkiWiki.pm b/IkiWiki.pm index 221d1e589..684713821 100644 --- a/IkiWiki.pm +++ b/IkiWiki.pm @@ -7,6 +7,7 @@ use Encode; use HTML::Entities; use URI::Escape q{uri_escape_utf8}; use POSIX; +use Storable; use open qw{:utf8 :std}; use vars qw{%config %links %oldlinks %pagemtime %pagectime %pagecase @@ -896,39 +897,49 @@ sub loadindex () { #{{{ %oldrenderedfiles=%pagectime=(); if (! $config{rebuild}) { %pagesources=%pagemtime=%oldlinks=%links=%depends= - %destsources=%renderedfiles=%pagecase=%pagestate=(); - } - open (my $in, "<", "$config{wikistatedir}/index") || return; - while (<$in>) { - $_=possibly_foolish_untaint($_); - chomp; - my %items; - $items{link}=[]; - $items{dest}=[]; - foreach my $i (split(/ /, $_)) { - my ($item, $val)=split(/=/, $i, 2); - push @{$items{$item}}, decode_entities($val); + %destsources=%renderedfiles=%pagecase=%pagestate=(); + } + my $in; + if (! open ($in, "<", "$config{wikistatedir}/indexdb")) { + if (-e "$config{wikistatedir}/index") { + system("ikiwiki-transition", "indexdb", $config{srcdir}); + open ($in, "<", "$config{wikistatedir}/indexdb") || return; } - - next unless exists $items{src}; # skip bad lines for now - - my $page=pagename($items{src}[0]); + else { + return; + } + } + my $ret=Storable::fd_retrieve($in); + if (! defined $ret) { + return 0; + } + my %index=%$ret; + foreach my $src (keys %index) { + my %d=%{$index{$src}}; + my $page=pagename($src); + $pagectime{$page}=$d{ctime}; if (! $config{rebuild}) { - $pagesources{$page}=$items{src}[0]; - $pagemtime{$page}=$items{mtime}[0]; - $oldlinks{$page}=[@{$items{link}}]; - $links{$page}=[@{$items{link}}]; - $depends{$page}=$items{depends}[0] if exists $items{depends}; - $destsources{$_}=$page foreach @{$items{dest}}; - $renderedfiles{$page}=[@{$items{dest}}]; - $pagecase{lc $page}=$page; - foreach my $k (grep /_/, keys %items) { - my ($id, $key)=split(/_/, $k, 2); - $pagestate{$page}{decode_entities($id)}{decode_entities($key)}=$items{$k}[0]; + $pagesources{$page}=$src; + $pagemtime{$page}=$d{mtime}; + $renderedfiles{$page}=$d{dest}; + if (exists $d{links} && ref $d{links}) { + $links{$page}=$d{links}; + $oldlinks{$page}=[@{$d{links}}]; + } + if (exists $d{depends}) { + $depends{$page}=$d{depends}; + } + if (exists $d{state}) { + $pagestate{$page}=$d{state}; } } - $oldrenderedfiles{$page}=[@{$items{dest}}]; - $pagectime{$page}=$items{ctime}[0]; + $oldrenderedfiles{$page}=[@{$d{dest}}]; + } + foreach my $page (keys %pagesources) { + $pagecase{lc $page}=$page; + } + foreach my $page (keys %renderedfiles) { + $destsources{$_}=$page foreach @{$renderedfiles{$page}}; } return close($in); } #}}} @@ -938,39 +949,45 @@ sub saveindex () { #{{{ my %hookids; foreach my $type (keys %hooks) { - $hookids{encode_entities($_)}=1 foreach keys %{$hooks{$type}}; + $hookids{$_}=1 foreach keys %{$hooks{$type}}; } - my @hookids=sort keys %hookids; + my @hookids=keys %hookids; if (! -d $config{wikistatedir}) { mkdir($config{wikistatedir}); } - my $newfile="$config{wikistatedir}/index.new"; + my $newfile="$config{wikistatedir}/indexdb.new"; my $cleanup = sub { unlink($newfile) }; open (my $out, '>', $newfile) || error("cannot write to $newfile: $!", $cleanup); + my %index; foreach my $page (keys %pagemtime) { next unless $pagemtime{$page}; - my $line="mtime=$pagemtime{$page} ". - "ctime=$pagectime{$page} ". - "src=$pagesources{$page}"; - $line.=" dest=$_" foreach @{$renderedfiles{$page}}; - my %count; - $line.=" link=$_" foreach grep { ++$count{$_} == 1 } @{$links{$page}}; + my $src=$pagesources{$page}; + + $index{$src}={ + ctime => $pagectime{$page}, + mtime => $pagemtime{$page}, + dest => $renderedfiles{$page}, + links => $links{$page}, + }; + if (exists $depends{$page}) { - $line.=" depends=".encode_entities($depends{$page}, " \t\n"); + $index{$src}{depends} = $depends{$page}; } + if (exists $pagestate{$page}) { foreach my $id (@hookids) { foreach my $key (keys %{$pagestate{$page}{$id}}) { - $line.=' '.$id.'_'.encode_entities($key, " \t\n")."=".encode_entities($pagestate{$page}{$id}{$key}, " \t\n"); + $index{$src}{state}{$id}{$key}=$pagestate{$page}{$id}{$key}; } } } - print $out $line."\n" || error("failed writing to $newfile: $!", $cleanup); } + my $ret=Storable::nstore_fd(\%index, $out); + return if ! defined $ret || ! $ret; close $out || error("failed saving to $newfile: $!", $cleanup); - rename($newfile, "$config{wikistatedir}/index") || - error("failed renaming $newfile to $config{wikistatedir}/index", $cleanup); + rename($newfile, "$config{wikistatedir}/indexdb") || + error("failed renaming $newfile to $config{wikistatedir}/indexdb", $cleanup); return 1; } #}}} diff --git a/debian/changelog b/debian/changelog index 90b13ed7d..2169f310d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -58,6 +58,7 @@ ikiwiki (2.41) UNRELEASED; urgency=low * smiley: Detect smileys inside pre and code tags, and do not expand. * inline: Crazy optimisation to work around slow markdown. * Precompile pagespecs, about 10% overall speedup. + * Changed to a binary index file, written using Storable, for speed. -- martin f. krafft <madduck@debian.org> Sun, 02 Mar 2008 17:46:38 +0100 diff --git a/doc/ikiwiki-transition.mdwn b/doc/ikiwiki-transition.mdwn index 118050a6c..da3b3a8d5 100644 --- a/doc/ikiwiki-transition.mdwn +++ b/doc/ikiwiki-transition.mdwn @@ -4,7 +4,7 @@ ikiwiki-transition - transition ikiwiki pages to new syntaxes # SYNOPSIS -ikiwiki-transition prefix_directives page.mdwn... +ikiwiki-transition type ... # DESCRIPTION @@ -12,8 +12,8 @@ ikiwiki-transition prefix_directives page.mdwn... there's a major change in ikiwiki syntax. Currently only one such transition is handled, the `prefix_directives` mode -converts an ikiwiki page from the old preprocessor directive syntax, -requiring a space, to the new syntax, prefixed by '!'. +converts the specified ikiwiki page from the old preprocessor directive +syntax, requiring a space, to the new syntax, prefixed by '!'. Preprocessor directives which already use the new syntax will remain unchanged. @@ -22,6 +22,12 @@ Note that if the page contains wiki links with spaces, which some older versions of ikiwiki accepted, the prefix_directives transition will treat these as preprocessor directives and convert them. +One other transition is handled, the `indexdb` mode handles converting +a plain text `.ikiwiki/index` file to a binary `.ikiwiki/indexdb`. In this +mode, you should specify the srcdir of the wiki as the second parameter. +You do not normally need to run `ikiwiki-transition indexdb`; ikiwiki will +automatically run it as necessary. + # AUTHOR Josh Triplett <josh@freedesktop.org> diff --git a/doc/index/discussion.mdwn b/doc/index/discussion.mdwn index df4e57aba..3d17ddb2a 100644 --- a/doc/index/discussion.mdwn +++ b/doc/index/discussion.mdwn @@ -337,42 +337,6 @@ Clicking on an old "?" or going to a create link but new Markdown content exists ---- -# User database tools? - -Any tool to view user database? - -Any tool to edit the user database? - -> No, but it's fairly easy to write such tools in perl. For example, to -> list all users in the user database: - - joey@kodama:~/src/joeywiki/.ikiwiki>perl -le 'use Storable; my $userinfo=Storable::retrieve("userdb"); print $_ foreach keys %$userinfo' - http://joey.kitenet.net/ - foo - -> To list each user's email address: - - joey@kodama:~/src/joeywiki/.ikiwiki>perl -le 'use Storable; my $userinfo=Storable::retrieve("userdb"); print $userinfo->{$_}->{email} foreach keys %$userinfo' - - joey@kitenet.net - -> Editing is simply a matter of changing values and calling Storable::store(). -> I've not written actual utilities to do this yet because I've only needed -> to do it rarely, and the data I've wanted has been different each time. -> --[[Joey]] - ->> Thanks for these examples -- I have been using them. I don't know the ->> Storable yet. Can someone share an example of removing a user? (I now ->> setup account\_creation\_password and I have some spammer with different ->> login names that I have banned that I might as well remove from the ->> userdb.) - ->>> Let's see, you could do something like this: ->>> perl -le 'use Storable; my $userinfo=Storable::retrieve("userdb"); delete $$userinfo{"joey"}; Storable::lock_store($userinfo, "userdb")' ->>> I suppose I should stop being lame and create a command line tool wrapping up these operations.. --[[Joey]] - ----- - # Spaces in WikiLinks? Hello Joey, diff --git a/doc/tips/inside_dot_ikiwiki.mdwn b/doc/tips/inside_dot_ikiwiki.mdwn new file mode 100644 index 000000000..69083a9a5 --- /dev/null +++ b/doc/tips/inside_dot_ikiwiki.mdwn @@ -0,0 +1,65 @@ +[[meta title="inside .ikiwiki"]] + +The `.ikiwiki` directory contains ikiwiki's internal state. Normally, +you don't need to look in it, but here's some tips for how to do so if +you need/want to. + +## the index + +`.ikiwiki/indexdb` contains a cache of information about pages, as well +as all persisitant state about pages. It used to be a (semi) human-readable +text file, but is not anymore. + +To dump the contents of the file, enter a perl command like this. + + joey@kodama:~/src/joeywiki/.ikiwiki> perl -le 'use Storable; my $index=Storable::retrieve("indexdb"); use Data::Dumper; print Dumper $index' | head + $VAR1 = { + 'index' => { + 'ctime' => 1199739528, + 'dest' => [ + 'index.html' + ], + 'mtime' => 1199739528, + 'src' => 'index.mdwn', + 'links' => [ + 'index/discussion', + +## the user database + +`.ikiwiki/userdb` is the user database, which records preferences of all +web users. + +To list all users in the database, enter a perl command like this. +Note that the output can include both registered users, and known +openids. + + joey@kodama:~/src/joeywiki/.ikiwiki> perl -le 'use Storable; my $userinfo=Storable::retrieve("userdb"); print $_ foreach keys %$userinfo' + http://joey.kitenet.net/ + foo + +To list each user's email address: + + joey@kodama:~/src/joeywiki/.ikiwiki> perl -le 'use Storable; my $userinfo=Storable::retrieve("userdb"); print $userinfo->{$_}->{email} foreach keys %$userinfo' + + joey@kitenet.net + +To dump the entire database contents: + + joey@kodama:~/src/joeywiki/.ikiwiki> perl -le 'use Storable; my $userinfo=Storable::retrieve("userdb"); use Data::Dumper; print Dumper $userinfo' + $VAR1 = { + 'http://joey.kitenet.net/' => { + 'email' => 'joey@kitenet.net', + [...] + +Editing values is simply a matter of changing values and calling Storable::nstore(). +So to change a user's password: + + joey@kodama:~/src/joeywiki/.ikiwiki> perl -le 'use Storable; my $userinfo=Storable::retrieve("userdb"); $userinfo->{"foo"}->{email}=q{foo@bar}; Storable::lock_nstore($userinfo, "underdb")' + +To remove that user: + + joey@kodama:~/src/joeywiki/.ikiwiki> perl -le 'use Storable; my $userinfo=Storable::retrieve("userdb"); delete $userinfo->{"foo"}; Storable::lock_nstore($userinfo, "underdb")' + +I've not written actual utilities to do this yet because I've only needed +to do it rarely, and the data I've wanted has been different each time. +--[[Joey]] diff --git a/ikiwiki-transition b/ikiwiki-transition index 1fd23cec5..0177f98a9 100755 --- a/ikiwiki-transition +++ b/ikiwiki-transition @@ -1,6 +1,8 @@ #!/usr/bin/perl -i use warnings; use strict; +use IkiWiki; +use HTML::Entities; my $regex = qr{ (\\?) # 1: escape? @@ -48,10 +50,32 @@ sub prefix_directives { } } +sub indexdb { + $config{wikistatedir}=shift()."/.ikiwiki"; + + if (! defined $config{wikistatedir}) { + usage(); + } + + if (! IkiWiki::oldloadindex()) { + die "failed to load index\n"; + } + if (! IkiWiki::saveindex()) { + die "failed to save indexdb\n" + } + if (! IkiWiki::loadindex()) { + die "transition failed, cannot load new indexdb\n"; + } + if (! unlink("$config{wikistatedir}/index")) { + die "unlink failed: $!\n"; + } +} + sub usage { - print STDERR "Usage: ikiwiki-transition type file ...\n"; - print STDERR "Currently supported transition types:\n"; - print STDERR " prefix_directives\n"; + print STDERR "Usage: ikiwiki-transition type ...\n"; + print STDERR "Currently supported transition subcommands:\n"; + print STDERR " prefix_directives file\n"; + print STDERR " indexdb srcdir\n"; exit 1; } @@ -61,6 +85,63 @@ my $mode=shift; if ($mode eq 'prefix_directives') { prefix_directives(@ARGV); } +elsif ($mode eq 'indexdb') { + indexdb(@ARGV); +} else { usage(); } + +package IkiWiki; + +# A slightly modified version of the old loadindex function. +sub oldloadindex { + %oldrenderedfiles=%pagectime=(); + if (! $config{rebuild}) { + %pagesources=%pagemtime=%oldlinks=%links=%depends= + %destsources=%renderedfiles=%pagecase=%pagestate=(); + } + open (my $in, "<", "$config{wikistatedir}/index") || return; + while (<$in>) { + chomp; + my %items; + $items{link}=[]; + $items{dest}=[]; + foreach my $i (split(/ /, $_)) { + my ($item, $val)=split(/=/, $i, 2); + push @{$items{$item}}, decode_entities($val); + } + + next unless exists $items{src}; # skip bad lines for now + + my $page=pagename($items{src}[0]); + if (! $config{rebuild}) { + $pagesources{$page}=$items{src}[0]; + $pagemtime{$page}=$items{mtime}[0]; + $oldlinks{$page}=[@{$items{link}}]; + $links{$page}=[@{$items{link}}]; + $depends{$page}=$items{depends}[0] if exists $items{depends}; + $destsources{$_}=$page foreach @{$items{dest}}; + $renderedfiles{$page}=[@{$items{dest}}]; + $pagecase{lc $page}=$page; + foreach my $k (grep /_/, keys %items) { + my ($id, $key)=split(/_/, $k, 2); + $pagestate{$page}{decode_entities($id)}{decode_entities($key)}=$items{$k}[0]; + } + } + $oldrenderedfiles{$page}=[@{$items{dest}}]; + $pagectime{$page}=$items{ctime}[0]; + } + + # saveindex relies on %hooks being populated, else it won't save + # the page state owned by a given hook. But no plugins are loaded + # by this program, so populate %hooks with all hook ids that + # currently have page state. + foreach my $page (keys %pagemtime) { + foreach my $id (keys %{$pagestate{$page}}) { + $hooks{_dummy}{$id}=1; + } + } + + return close($in); +} |