From 7a40bcab9a223d29189632cb05d26bc558927520 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kodama.kitenet.net>
Date: Tue, 29 Jan 2008 17:36:25 -0500
Subject: add missing test to avoid uninitialised value when a page with
 metadata is removed

---
 IkiWiki/Plugin/aggregate.pm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'IkiWiki/Plugin/aggregate.pm')

diff --git a/IkiWiki/Plugin/aggregate.pm b/IkiWiki/Plugin/aggregate.pm
index 71368e254..c3cbbae05 100644
--- a/IkiWiki/Plugin/aggregate.pm
+++ b/IkiWiki/Plugin/aggregate.pm
@@ -66,7 +66,8 @@ sub needsbuild (@) { #{{{
 	loadstate(); # if not already loaded
 
 	foreach my $feed (values %feeds) {
-		if (grep { $_ eq $pagesources{$feed->{sourcepage}} } @$needsbuild) {
+		if (exists $pagesources{$page} && 
+		    grep { $_ eq $pagesources{$feed->{sourcepage}} } @$needsbuild) {
 			# Mark all feeds originating on this page as removable;
 			# preprocess will unmark those that still exist.
 			remove_feeds($feed->{sourcepage});
-- 
cgit v1.2.3


From 3803266b8fd129b71390c11a212cf52c83c3506a Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kodama.kitenet.net>
Date: Tue, 29 Jan 2008 17:50:11 -0500
Subject: merged the recentchanges branch

misc fixes
---
 IkiWiki/Plugin/aggregate.pm |  2 +-
 debian/NEWS                 |  7 +++++++
 debian/changelog            |  2 +-
 debian/postinst             |  2 +-
 po/ikiwiki.pot              | 44 ++++++++++++++++++++++----------------------
 t/pagespec_match.t          |  2 +-
 6 files changed, 33 insertions(+), 26 deletions(-)

(limited to 'IkiWiki/Plugin/aggregate.pm')

diff --git a/IkiWiki/Plugin/aggregate.pm b/IkiWiki/Plugin/aggregate.pm
index c3cbbae05..2a4d10411 100644
--- a/IkiWiki/Plugin/aggregate.pm
+++ b/IkiWiki/Plugin/aggregate.pm
@@ -66,7 +66,7 @@ sub needsbuild (@) { #{{{
 	loadstate(); # if not already loaded
 
 	foreach my $feed (values %feeds) {
-		if (exists $pagesources{$page} && 
+		if (exists $pagesources{$feed->{sourcepage}} && 
 		    grep { $_ eq $pagesources{$feed->{sourcepage}} } @$needsbuild) {
 			# Mark all feeds originating on this page as removable;
 			# preprocess will unmark those that still exist.
diff --git a/debian/NEWS b/debian/NEWS
index 43c8b3ca2..c3ed55c03 100644
--- a/debian/NEWS
+++ b/debian/NEWS
@@ -5,6 +5,10 @@ ikiwiki (2.30) unstable; urgency=low
   now a static page, not a CGI. Users can subscribe to its rss/atom feeds.
   Custom RecentChanges pages can be easily set up that display only changes
   to a subset of pages, or only changes by a subset of users.
+  
+  All wikis need to be rebuilt on upgrade to this version. If you listed your
+  wiki in /etc/ikiwiki/wikilist this will be done automatically when the
+  Debian package is upgraded. Or use ikiwiki-mass-rebuild to force a rebuild.
 
   With this excellent new RecentChanges support, the mail notification system
   is showing its age (and known to be variously buggy and underimplemented for
@@ -12,6 +16,9 @@ ikiwiki (2.30) unstable; urgency=low
   from this version. If you were subscribed to commit mails, you should be
   able to accomplish the same thing by subscribing to a RecentChanges feed.
 
+  The "svnrepo" and "notify" fields in setup files are no longer used, and
+  silently ignored. You may want to remove them from your setup file.
+
  -- Joey Hess <joeyh@debian.org>  Tue, 29 Jan 2008 17:18:31 -0500
 
 ikiwiki (2.20) unstable; urgency=low
diff --git a/debian/changelog b/debian/changelog
index 77202ed1f..ec8f7f1b1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-ikiwiki (2.21) UNRELEASED; urgency=low
+ikiwiki (2.30) UNRELEASED; urgency=low
 
   [ Joey Hess ]
   * Old versions of git-init don't support --git-dir or GIT_DIR with
diff --git a/debian/postinst b/debian/postinst
index 018e04f78..26c44a88b 100755
--- a/debian/postinst
+++ b/debian/postinst
@@ -4,7 +4,7 @@ set -e
 
 # Change this when some incompatible change is made that requires
 # rebuilding all wikis.
-firstcompat=2.1
+firstcompat=2.30
 
 if [ "$1" = configure ] && \
    dpkg --compare-versions "$2" lt "$firstcompat"; then
diff --git a/po/ikiwiki.pot b/po/ikiwiki.pot
index 027e10e8a..cb25d0475 100644
--- a/po/ikiwiki.pot
+++ b/po/ikiwiki.pot
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2008-01-29 17:15-0500\n"
+"POT-Creation-Date: 2008-01-29 17:42-0500\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -67,67 +67,67 @@ msgstr ""
 msgid "You are banned."
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:82
+#: ../IkiWiki/Plugin/aggregate.pm:83
 #, perl-format
 msgid "missing %s parameter"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:110
+#: ../IkiWiki/Plugin/aggregate.pm:111
 msgid "new feed"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:124
+#: ../IkiWiki/Plugin/aggregate.pm:125
 msgid "posts"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:126
+#: ../IkiWiki/Plugin/aggregate.pm:127
 msgid "new"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:236
+#: ../IkiWiki/Plugin/aggregate.pm:237
 #, perl-format
 msgid "expiring %s (%s days old)"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:243
+#: ../IkiWiki/Plugin/aggregate.pm:244
 #, perl-format
 msgid "expiring %s"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:269
+#: ../IkiWiki/Plugin/aggregate.pm:270
 #, perl-format
 msgid "processed ok at %s"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:274
+#: ../IkiWiki/Plugin/aggregate.pm:275
 #, perl-format
 msgid "checking feed %s ..."
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:279
+#: ../IkiWiki/Plugin/aggregate.pm:280
 #, perl-format
 msgid "could not find feed at %s"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:294
+#: ../IkiWiki/Plugin/aggregate.pm:295
 msgid "feed not found"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:305
+#: ../IkiWiki/Plugin/aggregate.pm:306
 #, perl-format
 msgid "(invalid UTF-8 stripped from feed)"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:311
+#: ../IkiWiki/Plugin/aggregate.pm:312
 #, perl-format
 msgid "(feed entities escaped)"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:317
+#: ../IkiWiki/Plugin/aggregate.pm:318
 msgid "feed crashed XML::Feed!"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:391
+#: ../IkiWiki/Plugin/aggregate.pm:392
 #, perl-format
 msgid "creating new page %s"
 msgstr ""
@@ -146,20 +146,20 @@ msgstr ""
 msgid "%s parameter is required"
 msgstr ""
 
-#: ../IkiWiki/Plugin/edittemplate.pm:40
+#: ../IkiWiki/Plugin/edittemplate.pm:41
 msgid "template not specified"
 msgstr ""
 
-#: ../IkiWiki/Plugin/edittemplate.pm:43
+#: ../IkiWiki/Plugin/edittemplate.pm:44
 msgid "match not specified"
 msgstr ""
 
-#: ../IkiWiki/Plugin/edittemplate.pm:48
+#: ../IkiWiki/Plugin/edittemplate.pm:49
 #, perl-format
 msgid "edittemplate %s registered for %s"
 msgstr ""
 
-#: ../IkiWiki/Plugin/edittemplate.pm:110
+#: ../IkiWiki/Plugin/edittemplate.pm:111
 msgid "failed to process"
 msgstr ""
 
@@ -240,15 +240,15 @@ msgstr ""
 msgid "failed to load Markdown.pm perl module (%s) or /usr/bin/markdown (%s)"
 msgstr ""
 
-#: ../IkiWiki/Plugin/meta.pm:118
+#: ../IkiWiki/Plugin/meta.pm:119
 msgid "stylesheet not found"
 msgstr ""
 
-#: ../IkiWiki/Plugin/meta.pm:142
+#: ../IkiWiki/Plugin/meta.pm:143
 msgid "redir page not found"
 msgstr ""
 
-#: ../IkiWiki/Plugin/meta.pm:155
+#: ../IkiWiki/Plugin/meta.pm:156
 msgid "redir cycle is not allowed"
 msgstr ""
 
diff --git a/t/pagespec_match.t b/t/pagespec_match.t
index cb98ab149..038472967 100755
--- a/t/pagespec_match.t
+++ b/t/pagespec_match.t
@@ -1,7 +1,7 @@
 #!/usr/bin/perl
 use warnings;
 use strict;
-use Test::More tests => 54;
+use Test::More tests => 52;
 
 BEGIN { use_ok("IkiWiki"); }
 
-- 
cgit v1.2.3


From 077901368358f2c0f6958f44a783c50074fe7405 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kodama.kitenet.net>
Date: Sat, 2 Feb 2008 23:56:13 -0500
Subject: * aggregate: Forking a child broke the one state that mattered:
 Forcing   the aggregating page to be rebuilt. Fix this.

---
 IkiWiki/Plugin/aggregate.pm | 29 ++++++++++++++++++-----------
 debian/changelog            |  2 ++
 2 files changed, 20 insertions(+), 11 deletions(-)

(limited to 'IkiWiki/Plugin/aggregate.pm')

diff --git a/IkiWiki/Plugin/aggregate.pm b/IkiWiki/Plugin/aggregate.pm
index 2a4d10411..736b0e0d5 100644
--- a/IkiWiki/Plugin/aggregate.pm
+++ b/IkiWiki/Plugin/aggregate.pm
@@ -37,16 +37,19 @@ sub checkconfig () { #{{{
 			debug("wiki is locked by another process, not aggregating");
 			exit 1;
 		}
-		
+
+		loadstate();
+		my @feeds=needsaggregate();
+		return unless @feeds;
+
 		# Fork a child process to handle the aggregation.
-		# The parent process will then handle building the result.
-		# This avoids messy code to clear state accumulated while
-		# aggregating.
+		# The parent process will then handle building the
+		# result. This avoids messy code to clear state
+		# accumulated while aggregating.
 		defined(my $pid = fork) or error("Can't fork: $!");
 		if (! $pid) {
-			loadstate();
 			IkiWiki::loadindex();
-			aggregate();
+			aggregate(@feeds);
 			expire();
 			savestate();
 			exit 0;
@@ -55,6 +58,8 @@ sub checkconfig () { #{{{
 		if ($?) {
 			error "aggregation failed with code $?";
 		}
+		$IkiWiki::forcerebuild{$_->{sourcepage}}=1
+			foreach @feeds;
 		
 		IkiWiki::unlockwiki();
 	}
@@ -254,7 +259,12 @@ sub expire () { #{{{
 	}
 } #}}}
 
-sub aggregate () { #{{{
+sub needsaggregate () { #{{{
+	return values %feeds if $config{rebuild};
+	return grep { time - $_->{lastupdate} >= $_->{updateinterval} } values %feeds;
+} #}}}
+
+sub aggregate (@) { #{{{
 	eval q{use XML::Feed};
 	error($@) if $@;
 	eval q{use URI::Fetch};
@@ -262,15 +272,12 @@ sub aggregate () { #{{{
 	eval q{use HTML::Entities};
 	error($@) if $@;
 
-	foreach my $feed (values %feeds) {
-		next unless $config{rebuild} || 
-			time - $feed->{lastupdate} >= $feed->{updateinterval};
+	foreach my $feed (@_) {
 		$feed->{lastupdate}=time;
 		$feed->{newposts}=0;
 		$feed->{message}=sprintf(gettext("processed ok at %s"),
 			displaytime($feed->{lastupdate}));
 		$feed->{error}=0;
-		$IkiWiki::forcerebuild{$feed->{sourcepage}}=1;
 
 		debug(sprintf(gettext("checking feed %s ..."), $feed->{name}));
 
diff --git a/debian/changelog b/debian/changelog
index b7096aec4..df7b8b424 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,6 +6,8 @@ ikiwiki (2.31) UNRELEASED; urgency=low
     that contributes to a page's content and using the youngest of them all,
     as well as special cases for things like the version plugin, and it's just
     too complex to do.
+  * aggregate: Forking a child broke the one state that mattered: Forcing
+    the aggregating page to be rebuilt. Fix this.
 
  -- Joey Hess <joeyh@debian.org>  Sat, 02 Feb 2008 23:36:31 -0500
 
-- 
cgit v1.2.3


From 1f6591f0a61415777a662d979c5c142c7f4ad7cd Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kodama.kitenet.net>
Date: Sun, 3 Feb 2008 03:04:19 -0500
Subject: * aggregate: Revert use of forking to not save state, that was not
 the right   approach.

---
 IkiWiki/Plugin/aggregate.pm | 52 +++++++++++++++++----------------------------
 debian/changelog            |  4 ++--
 2 files changed, 22 insertions(+), 34 deletions(-)

(limited to 'IkiWiki/Plugin/aggregate.pm')

diff --git a/IkiWiki/Plugin/aggregate.pm b/IkiWiki/Plugin/aggregate.pm
index 736b0e0d5..0f50fab06 100644
--- a/IkiWiki/Plugin/aggregate.pm
+++ b/IkiWiki/Plugin/aggregate.pm
@@ -37,30 +37,14 @@ sub checkconfig () { #{{{
 			debug("wiki is locked by another process, not aggregating");
 			exit 1;
 		}
-
+	
 		loadstate();
-		my @feeds=needsaggregate();
-		return unless @feeds;
-
-		# Fork a child process to handle the aggregation.
-		# The parent process will then handle building the
-		# result. This avoids messy code to clear state
-		# accumulated while aggregating.
-		defined(my $pid = fork) or error("Can't fork: $!");
-		if (! $pid) {
-			IkiWiki::loadindex();
-			aggregate(@feeds);
-			expire();
-			savestate();
-			exit 0;
-		}
-		waitpid($pid,0);
-		if ($?) {
-			error "aggregation failed with code $?";
-		}
-		$IkiWiki::forcerebuild{$_->{sourcepage}}=1
-			foreach @feeds;
-		
+		IkiWiki::loadindex();
+		aggregate();
+		expire();
+		savestate();
+		clearstate();
+
 		IkiWiki::unlockwiki();
 	}
 } #}}}
@@ -148,7 +132,7 @@ sub loadstate () { #{{{
 	return if $state_loaded;
 	$state_loaded=1;
 	if (-e "$config{wikistatedir}/aggregate") {
-		open(IN, "$config{wikistatedir}/aggregate") ||
+		open(IN, "<", "$config{wikistatedir}/aggregate") ||
 			die "$config{wikistatedir}/aggregate: $!";
 		while (<IN>) {
 			$_=IkiWiki::possibly_foolish_untaint($_);
@@ -186,7 +170,7 @@ sub savestate () { #{{{
 	error($@) if $@;
 	my $newfile="$config{wikistatedir}/aggregate.new";
 	my $cleanup = sub { unlink($newfile) };
-	open (OUT, ">$newfile") || error("open $newfile: $!", $cleanup);
+	open (OUT, ">", $newfile) || error("open $newfile: $!", $cleanup);
 	foreach my $data (values %feeds, values %guids) {
 		if ($data->{remove}) {
 			if ($data->{name}) {
@@ -228,6 +212,12 @@ sub savestate () { #{{{
 		error("rename $newfile: $!", $cleanup);
 } #}}}
 
+sub clearstate () { #{{{
+	%feeds=();
+	%guids=();
+	$state_loaded=0;
+} #}}}
+
 sub expire () { #{{{
 	foreach my $feed (values %feeds) {
 		next unless $feed->{expireage} || $feed->{expirecount};
@@ -259,12 +249,7 @@ sub expire () { #{{{
 	}
 } #}}}
 
-sub needsaggregate () { #{{{
-	return values %feeds if $config{rebuild};
-	return grep { time - $_->{lastupdate} >= $_->{updateinterval} } values %feeds;
-} #}}}
-
-sub aggregate (@) { #{{{
+sub aggregate () { #{{{
 	eval q{use XML::Feed};
 	error($@) if $@;
 	eval q{use URI::Fetch};
@@ -272,12 +257,15 @@ sub aggregate (@) { #{{{
 	eval q{use HTML::Entities};
 	error($@) if $@;
 
-	foreach my $feed (@_) {
+	foreach my $feed (values %feeds) {
+		next unless $config{rebuild} || 
+			time - $feed->{lastupdate} >= $feed->{updateinterval};
 		$feed->{lastupdate}=time;
 		$feed->{newposts}=0;
 		$feed->{message}=sprintf(gettext("processed ok at %s"),
 			displaytime($feed->{lastupdate}));
 		$feed->{error}=0;
+		$IkiWiki::forcerebuild{$feed->{sourcepage}}=1;
 
 		debug(sprintf(gettext("checking feed %s ..."), $feed->{name}));
 
diff --git a/debian/changelog b/debian/changelog
index b4ff54d9f..590c1a84e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,14 +6,14 @@ ikiwiki (2.31) UNRELEASED; urgency=low
     that contributes to a page's content and using the youngest of them all,
     as well as special cases for things like the version plugin, and it's just
     too complex to do.
-  * aggregate: Forking a child broke the one state that mattered: Forcing
-    the aggregating page to be rebuilt. Fix this.
   * cgi hooks are now run before ikiwiki state is loaded.
   * This allows locking the wiki before loading state, which avoids some
     tricky locking code when saving a web edit.
   * poll: This plugin turns out to have edited pages w/o doing any locking.
     Oops. Convert it from a cgi to a sessioncgi hook, which will work
     much better.
+  * aggregate: Revert use of forking to not save state, that was not the right
+    approach.
 
  -- Joey Hess <joeyh@debian.org>  Sat, 02 Feb 2008 23:36:31 -0500
 
-- 
cgit v1.2.3


From 38affb0c1c4e2b89beb63d6f8dc3f172eee7bd02 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kodama.kitenet.net>
Date: Sun, 3 Feb 2008 15:17:15 -0500
Subject: add aggregate locking functions

---
 IkiWiki/Plugin/aggregate.pm | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

(limited to 'IkiWiki/Plugin/aggregate.pm')

diff --git a/IkiWiki/Plugin/aggregate.pm b/IkiWiki/Plugin/aggregate.pm
index 0f50fab06..cfc4ec955 100644
--- a/IkiWiki/Plugin/aggregate.pm
+++ b/IkiWiki/Plugin/aggregate.pm
@@ -495,4 +495,26 @@ sub htmlfn ($) { #{{{
 	return shift().".".$config{htmlext};
 } #}}}
 
+my $aggregatelock;
+
+sub lockaggregate () { #{{{
+	# Take an exclusive lock to prevent multiple concurrent aggregators.
+	# Returns true if the lock was aquired.
+	if (! -d $config{wikistatedir}) {
+		mkdir($config{wikistatedir});
+	}
+	open($aggregatelock, '>', "$config{wikistatedir}/aggregatelock") ||
+		error ("cannot open to $config{wikistatedir}/aggregatelock: $!");
+	if (! flock($aggregatelock, 2 | 4)) { # LOCK_EX | LOCK_NB
+		close($aggregatelock) || error("failed closing aggregatelock: $!");
+		return 0;
+	}
+	return 1;
+} #}}}
+
+sub unlockaggregate () { #{{{
+	return close($aggregatelock) if $aggregatelock;
+	return;
+} #}}}
+
 1
-- 
cgit v1.2.3


From 9d54cc4659248f9820f47a021b694405d75404a8 Mon Sep 17 00:00:00 2001
From: Joey Hess <joey@kodama.kitenet.net>
Date: Sun, 3 Feb 2008 16:48:26 -0500
Subject: implement aggregate_locking design

Now aggregation will not lock the wiki. Any changes made during aggregaton are
merged in with the changed state accumulated while aggregating. A separate
lock file prevents multiple concurrent aggregators. Garbage collection
of orphaned guids is much improved. loadstate() is only called once
per process, so tricky support for reloading wiki state is not needed.

(Tested fairly thuroughly.)
---
 IkiWiki/Plugin/aggregate.pm     | 181 +++++++++++++++++++++++++++-------------
 debian/changelog                |   7 +-
 doc/todo/aggregate_locking.mdwn |   6 +-
 po/ikiwiki.pot                  |  30 +++----
 4 files changed, 146 insertions(+), 78 deletions(-)

(limited to 'IkiWiki/Plugin/aggregate.pm')

diff --git a/IkiWiki/Plugin/aggregate.pm b/IkiWiki/Plugin/aggregate.pm
index cfc4ec955..ba40ee6bc 100644
--- a/IkiWiki/Plugin/aggregate.pm
+++ b/IkiWiki/Plugin/aggregate.pm
@@ -33,33 +33,62 @@ sub getopt () { #{{{
 sub checkconfig () { #{{{
 	if ($config{aggregate} && ! ($config{post_commit} && 
 	                             IkiWiki::commit_hook_enabled())) {
-		if (! IkiWiki::lockwiki(0)) {
-			debug("wiki is locked by another process, not aggregating");
-			exit 1;
-		}
-	
+		# See if any feeds need aggregation.
 		loadstate();
-		IkiWiki::loadindex();
-		aggregate();
-		expire();
-		savestate();
-		clearstate();
+		my @feeds=needsaggregate();
+		return unless @feeds;
+		if (! lockaggregate()) {
+			debug("an aggregation process is already running");
+			return;
+		}
+		# force a later rebuild of source pages
+		$IkiWiki::forcerebuild{$_->{sourcepage}}=1
+			foreach @feeds;
+
+		# Fork a child process to handle the aggregation.
+		# The parent process will then handle building the
+		# result. This avoids messy code to clear state
+		# accumulated while aggregating.
+		defined(my $pid = fork) or error("Can't fork: $!");
+		if (! $pid) {
+			IkiWiki::loadindex();
+
+			# Aggregation happens without the main wiki lock
+			# being held. This allows editing pages etc while
+			# aggregation is running.
+			aggregate(@feeds);
+
+			IkiWiki::lockwiki;
+			# Merge changes, since aggregation state may have
+			# changed on disk while the aggregation was happening.
+			mergestate();
+			expire();
+			savestate();
+			IkiWiki::unlockwiki;
+			exit 0;
+		}
+		waitpid($pid,0);
+		if ($?) {
+			error "aggregation failed with code $?";
+		}
 
-		IkiWiki::unlockwiki();
+		clearstate();
+		unlockaggregate();
 	}
 } #}}}
 
 sub needsbuild (@) { #{{{
 	my $needsbuild=shift;
 	
-	loadstate(); # if not already loaded
+	loadstate();
 
 	foreach my $feed (values %feeds) {
 		if (exists $pagesources{$feed->{sourcepage}} && 
 		    grep { $_ eq $pagesources{$feed->{sourcepage}} } @$needsbuild) {
-			# Mark all feeds originating on this page as removable;
-			# preprocess will unmark those that still exist.
-			remove_feeds($feed->{sourcepage});
+			# Mark all feeds originating on this page as 
+			# not yet seen; preprocess will unmark those that
+			# still exist.
+			markunseen($feed->{sourcepage});
 		}
 	}
 } # }}}
@@ -92,8 +121,7 @@ sub preprocess (@) { #{{{
 	$feed->{updateinterval}=defined $params{updateinterval} ? $params{updateinterval} * 60 : 15 * 60;
 	$feed->{expireage}=defined $params{expireage} ? $params{expireage} : 0;
 	$feed->{expirecount}=defined $params{expirecount} ? $params{expirecount} : 0;
-	delete $feed->{remove};
-	delete $feed->{expired};
+	delete $feed->{unseen};
 	$feed->{lastupdate}=0 unless defined $feed->{lastupdate};
 	$feed->{numposts}=0 unless defined $feed->{numposts};
 	$feed->{newposts}=0 unless defined $feed->{newposts};
@@ -123,16 +151,27 @@ sub delete (@) { #{{{
 	# Remove feed data for removed pages.
 	foreach my $file (@files) {
 		my $page=pagename($file);
-		remove_feeds($page);
+		markunseen($page);
+	}
+} #}}}
+
+sub markunseen ($) { #{{{
+	my $page=shift;
+
+	foreach my $id (keys %feeds) {
+		if ($feeds{$id}->{sourcepage} eq $page) {
+			$feeds{$id}->{unseen}=1;
+		}
 	}
 } #}}}
 
 my $state_loaded=0;
+
 sub loadstate () { #{{{
 	return if $state_loaded;
 	$state_loaded=1;
 	if (-e "$config{wikistatedir}/aggregate") {
-		open(IN, "<", "$config{wikistatedir}/aggregate") ||
+		open(IN, "$config{wikistatedir}/aggregate") ||
 			die "$config{wikistatedir}/aggregate: $!";
 		while (<IN>) {
 			$_=IkiWiki::possibly_foolish_untaint($_);
@@ -166,32 +205,13 @@ sub loadstate () { #{{{
 
 sub savestate () { #{{{
 	return unless $state_loaded;
+	garbage_collect();
 	eval q{use HTML::Entities};
 	error($@) if $@;
 	my $newfile="$config{wikistatedir}/aggregate.new";
 	my $cleanup = sub { unlink($newfile) };
-	open (OUT, ">", $newfile) || error("open $newfile: $!", $cleanup);
+	open (OUT, ">$newfile") || error("open $newfile: $!", $cleanup);
 	foreach my $data (values %feeds, values %guids) {
-		if ($data->{remove}) {
-			if ($data->{name}) {
-				foreach my $guid (values %guids) {
-					if ($guid->{feed} eq $data->{name}) {
-						$guid->{remove}=1;
-					}
-				}
-			}
-			else {
-				unlink pagefile($data->{page})
-					if exists $data->{page};
-			}
-			next;
-		}
-		elsif ($data->{expired} && exists $data->{page}) {
-			unlink pagefile($data->{page});
-			delete $data->{page};
-			delete $data->{md5};
-		}
-
 		my @line;
 		foreach my $field (keys %$data) {
 			if ($field eq "name" || $field eq "feed" ||
@@ -212,6 +232,63 @@ sub savestate () { #{{{
 		error("rename $newfile: $!", $cleanup);
 } #}}}
 
+sub garbage_collect () { #{{{
+	foreach my $name (keys %feeds) {
+		# remove any feeds that were not seen while building the pages
+		# that used to contain them
+		if ($feeds{$name}->{unseen}) {
+			delete $feeds{$name};
+		}
+	}
+
+	foreach my $guid (values %guids) {
+		# any guid whose feed is gone should be removed
+		if (! exists $feeds{$guid->{feed}}) {
+			unlink pagefile($guid->{page})
+				if exists $guid->{page};
+			delete $guids{$guid->{guid}};
+		}
+		# handle expired guids
+		elsif ($guid->{expired} && exists $guid->{page}) {
+			unlink pagefile($guid->{page});
+			delete $guid->{page};
+			delete $guid->{md5};
+		}
+	}
+} #}}}
+
+sub mergestate () { #{{{
+	# Load the current state in from disk, and merge into it
+	# values from the state in memory that might have changed
+	# during aggregation.
+	my %myfeeds=%feeds;
+	my %myguids=%guids;
+	clearstate();
+	loadstate();
+
+	# All that can change in feed state during aggregation is a few
+	# fields.
+	foreach my $name (keys %myfeeds) {
+		if (exists $feeds{$name}) {
+			foreach my $field (qw{message lastupdate numposts
+			                      newposts error}) {
+				$feeds{$name}->{$field}=$myfeeds{$name}->{$field};
+			}
+		}
+	}
+
+	# New guids can be created during aggregation.
+	# It's also possible that guids were removed from the on-disk state
+	# while the aggregation was in process. That would only happen if
+	# their feed was also removed, so any removed guids added back here
+	# will be garbage collected later.
+	foreach my $guid (keys %myguids) {
+		if (! exists $guids{$guid}) {
+			$guids{$guid}=$myguids{$guid};
+		}
+	}
+} #}}}
+
 sub clearstate () { #{{{
 	%feeds=();
 	%guids=();
@@ -249,7 +326,12 @@ sub expire () { #{{{
 	}
 } #}}}
 
-sub aggregate () { #{{{
+sub needsaggregate () { #{{{
+	return values %feeds if $config{rebuild};
+	return grep { time - $_->{lastupdate} >= $_->{updateinterval} } values %feeds;
+} #}}}
+
+sub aggregate (@) { #{{{
 	eval q{use XML::Feed};
 	error($@) if $@;
 	eval q{use URI::Fetch};
@@ -257,15 +339,12 @@ sub aggregate () { #{{{
 	eval q{use HTML::Entities};
 	error($@) if $@;
 
-	foreach my $feed (values %feeds) {
-		next unless $config{rebuild} || 
-			time - $feed->{lastupdate} >= $feed->{updateinterval};
+	foreach my $feed (@_) {
 		$feed->{lastupdate}=time;
 		$feed->{newposts}=0;
 		$feed->{message}=sprintf(gettext("processed ok at %s"),
 			displaytime($feed->{lastupdate}));
 		$feed->{error}=0;
-		$IkiWiki::forcerebuild{$feed->{sourcepage}}=1;
 
 		debug(sprintf(gettext("checking feed %s ..."), $feed->{name}));
 
@@ -473,18 +552,6 @@ sub htmlabs ($$) { #{{{
 	return $ret;
 } #}}}
 
-sub remove_feeds () { #{{{
-	my $page=shift;
-
-	my %removed;
-	foreach my $id (keys %feeds) {
-		if ($feeds{$id}->{sourcepage} eq $page) {
-			$feeds{$id}->{remove}=1;
-			$removed{$id}=1;
-		}
-	}
-} #}}}
-
 sub pagefile ($) { #{{{
 	my $page=shift;
 
diff --git a/debian/changelog b/debian/changelog
index 99ee5cdc2..1266666e3 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -6,19 +6,22 @@ ikiwiki (2.31) UNRELEASED; urgency=low
     that contributes to a page's content and using the youngest of them all,
     as well as special cases for things like the version plugin, and it's just
     too complex to do.
+  * aggregate: Forking a child broke the one state that mattered: Forcing
+    the aggregating page to be rebuilt. Fix this.
   * cgi hooks are now run before ikiwiki state is loaded.
   * This allows locking the wiki before loading state, which avoids some
     tricky locking code when saving a web edit.
   * poll: This plugin turns out to have edited pages w/o doing any locking.
     Oops. Convert it from a cgi to a sessioncgi hook, which will work
     much better.
-  * aggregate: Revert use of forking to not save state, that was not the right
-    approach.
   * recentchanges: Improve handling of links on the very static changes pages
     by thunking to the CGI, which can redirect to the page, or allow it to be
     created if it doesn't exist.
   * recentchanges: Exipre all *._change pages, even if the directory
     they're in has changed.
+  * aggregate: Lots of changes; aggregation can now run without locking the
+    wiki, and there is a separate aggregatelock to prevent multiple concurrent
+    aggregation runs.
 
  -- Joey Hess <joeyh@debian.org>  Sat, 02 Feb 2008 23:36:31 -0500
 
diff --git a/doc/todo/aggregate_locking.mdwn b/doc/todo/aggregate_locking.mdwn
index 91df662a7..b6c82e923 100644
--- a/doc/todo/aggregate_locking.mdwn
+++ b/doc/todo/aggregate_locking.mdwn
@@ -46,16 +46,12 @@ would be loaded, and there would be no reason to worry about aggregating.
 
 Or aggregation could be kept in checkconfig, like so:
 
-* lock wiki
 * load aggregation state
-* unlock wiki
 * get list of feeds needing aggregation
 * exit if none
 * attempt to take aggregation lock, exit if another aggregation is happening
 * fork a child process to do the aggregation
-  * lock wiki
   * load wiki state (needed for aggregation to run)
-  * unlock wiki
   * aggregate
   * lock wiki
   * reload aggregation state
@@ -64,3 +60,5 @@ Or aggregation could be kept in checkconfig, like so:
 * drop aggregation lock
 * force rebuild of sourcepages of feeds that were aggregated
 * exit checkconfig and continue with usual refresh process
+
+[[done]]
diff --git a/po/ikiwiki.pot b/po/ikiwiki.pot
index 69e231ada..62810a687 100644
--- a/po/ikiwiki.pot
+++ b/po/ikiwiki.pot
@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2008-02-03 14:52-0500\n"
+"POT-Creation-Date: 2008-02-03 16:05-0500\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -67,67 +67,67 @@ msgstr ""
 msgid "You are banned."
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:72
+#: ../IkiWiki/Plugin/aggregate.pm:100
 #, perl-format
 msgid "missing %s parameter"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:100
+#: ../IkiWiki/Plugin/aggregate.pm:127
 msgid "new feed"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:114
+#: ../IkiWiki/Plugin/aggregate.pm:141
 msgid "posts"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:116
+#: ../IkiWiki/Plugin/aggregate.pm:143
 msgid "new"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:232
+#: ../IkiWiki/Plugin/aggregate.pm:307
 #, perl-format
 msgid "expiring %s (%s days old)"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:239
+#: ../IkiWiki/Plugin/aggregate.pm:314
 #, perl-format
 msgid "expiring %s"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:265
+#: ../IkiWiki/Plugin/aggregate.pm:343
 #, perl-format
 msgid "processed ok at %s"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:270
+#: ../IkiWiki/Plugin/aggregate.pm:347
 #, perl-format
 msgid "checking feed %s ..."
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:275
+#: ../IkiWiki/Plugin/aggregate.pm:352
 #, perl-format
 msgid "could not find feed at %s"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:290
+#: ../IkiWiki/Plugin/aggregate.pm:367
 msgid "feed not found"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:301
+#: ../IkiWiki/Plugin/aggregate.pm:378
 #, perl-format
 msgid "(invalid UTF-8 stripped from feed)"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:307
+#: ../IkiWiki/Plugin/aggregate.pm:384
 #, perl-format
 msgid "(feed entities escaped)"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:313
+#: ../IkiWiki/Plugin/aggregate.pm:390
 msgid "feed crashed XML::Feed!"
 msgstr ""
 
-#: ../IkiWiki/Plugin/aggregate.pm:387
+#: ../IkiWiki/Plugin/aggregate.pm:464
 #, perl-format
 msgid "creating new page %s"
 msgstr ""
-- 
cgit v1.2.3