diff options
Diffstat (limited to 'IkiWiki/Plugin/aggregate.pm')
| -rw-r--r-- | IkiWiki/Plugin/aggregate.pm | 171 |
1 files changed, 97 insertions, 74 deletions
diff --git a/IkiWiki/Plugin/aggregate.pm b/IkiWiki/Plugin/aggregate.pm index 26c5cc9ae..e1baae666 100644 --- a/IkiWiki/Plugin/aggregate.pm +++ b/IkiWiki/Plugin/aggregate.pm @@ -4,7 +4,7 @@ package IkiWiki::Plugin::aggregate; use warnings; use strict; -use IkiWiki 2.00; +use IkiWiki 3.00; use HTML::Parser; use HTML::Tagset; use HTML::Entities; @@ -14,7 +14,7 @@ use open qw{:utf8 :std}; my %feeds; my %guids; -sub import { #{{{ +sub import { hook(type => "getopt", id => "aggregate", call => \&getopt); hook(type => "getsetup", id => "aggregate", call => \&getsetup); hook(type => "checkconfig", id => "aggregate", call => \&checkconfig); @@ -26,9 +26,9 @@ sub import { #{{{ if (exists $config{aggregate_webtrigger} && $config{aggregate_webtrigger}) { hook(type => "cgi", id => "aggregate", call => \&cgi); } -} # }}} +} -sub getopt () { #{{{ +sub getopt () { eval q{use Getopt::Long}; error($@) if $@; Getopt::Long::Configure('pass_through'); @@ -36,9 +36,9 @@ sub getopt () { #{{{ "aggregate" => \$config{aggregate}, "aggregateinternal!" => \$config{aggregateinternal}, ); -} #}}} +} -sub getsetup () { #{{{ +sub getsetup () { return plugin => { safe => 1, @@ -46,7 +46,7 @@ sub getsetup () { #{{{ }, aggregateinternal => { type => "boolean", - example => 0, + example => 1, description => "enable aggregation to internal pages?", safe => 0, # enabling needs manual transition rebuild => 0, @@ -58,16 +58,20 @@ sub getsetup () { #{{{ safe => 1, rebuild => 0, }, -} #}}} +} + +sub checkconfig () { + if (! defined $config{aggregateinternal}) { + $config{aggregateinternal}=1; + } -sub checkconfig () { #{{{ if ($config{aggregate} && ! ($config{post_commit} && IkiWiki::commit_hook_enabled())) { launchaggregation(); } -} #}}} +} -sub cgi ($) { #{{{ +sub cgi ($) { my $cgi=shift; if (defined $cgi->param('do') && @@ -90,9 +94,9 @@ sub cgi ($) { #{{{ } exit 0; } -} #}}} +} -sub launchaggregation () { #{{{ +sub launchaggregation () { # See if any feeds need aggregation. loadstate(); my @feeds=needsaggregate(); @@ -135,16 +139,16 @@ sub launchaggregation () { #{{{ unlockaggregate(); return 1; -} #}}} +} # Pages with extension _aggregated have plain html markup, pass through. -sub htmlize (@) { #{{{ +sub htmlize (@) { my %params=@_; return $params{content}; -} #}}} +} # Used by ikiwiki-transition aggregateinternal. -sub migrate_to_internal { #{{{ +sub migrate_to_internal { if (! lockaggregate()) { error("an aggregation process is currently running"); } @@ -190,9 +194,9 @@ sub migrate_to_internal { #{{{ IkiWiki::unlockwiki; unlockaggregate(); -} #}}} +} -sub needsbuild (@) { #{{{ +sub needsbuild (@) { my $needsbuild=shift; loadstate(); @@ -206,9 +210,9 @@ sub needsbuild (@) { #{{{ markunseen($feed->{sourcepage}); } } -} # }}} +} -sub preprocess (@) { #{{{ +sub preprocess (@) { my %params=@_; foreach my $required (qw{name url}) { @@ -245,6 +249,7 @@ sub preprocess (@) { #{{{ $feed->{template}=$params{template} . ".tmpl"; delete $feed->{unseen}; $feed->{lastupdate}=0 unless defined $feed->{lastupdate}; + $feed->{lasttry}=$feed->{lastupdate} unless defined $feed->{lasttry}; $feed->{numposts}=0 unless defined $feed->{numposts}; $feed->{newposts}=0 unless defined $feed->{newposts}; $feed->{message}=gettext("new feed") unless defined $feed->{message}; @@ -265,9 +270,9 @@ sub preprocess (@) { #{{{ ($feed->{newposts} ? "; ".$feed->{newposts}. " ".gettext("new") : ""). ")"; -} # }}} +} -sub delete (@) { #{{{ +sub delete (@) { my @files=@_; # Remove feed data for removed pages. @@ -275,9 +280,9 @@ sub delete (@) { #{{{ my $page=pagename($file); markunseen($page); } -} #}}} +} -sub markunseen ($) { #{{{ +sub markunseen ($) { my $page=shift; foreach my $id (keys %feeds) { @@ -285,11 +290,11 @@ sub markunseen ($) { #{{{ $feeds{$id}->{unseen}=1; } } -} #}}} +} my $state_loaded=0; -sub loadstate () { #{{{ +sub loadstate () { return if $state_loaded; $state_loaded=1; if (-e "$config{wikistatedir}/aggregate") { @@ -323,9 +328,9 @@ sub loadstate () { #{{{ close IN; } -} #}}} +} -sub savestate () { #{{{ +sub savestate () { return unless $state_loaded; garbage_collect(); my $newfile="$config{wikistatedir}/aggregate.new"; @@ -342,7 +347,8 @@ sub savestate () { #{{{ push @line, "tag=$_" foreach @{$data->{tags}}; } else { - push @line, "$field=".$data->{$field}; + push @line, "$field=".$data->{$field} + if defined $data->{$field}; } } print OUT join(" ", @line)."\n" || error("write $newfile: $!", $cleanup); @@ -350,9 +356,9 @@ sub savestate () { #{{{ close OUT || error("save $newfile: $!", $cleanup); rename($newfile, "$config{wikistatedir}/aggregate") || error("rename $newfile: $!", $cleanup); -} #}}} +} -sub garbage_collect () { #{{{ +sub garbage_collect () { foreach my $name (keys %feeds) { # remove any feeds that were not seen while building the pages # that used to contain them @@ -375,9 +381,9 @@ sub garbage_collect () { #{{{ delete $guid->{md5}; } } -} #}}} +} -sub mergestate () { #{{{ +sub mergestate () { # Load the current state in from disk, and merge into it # values from the state in memory that might have changed # during aggregation. @@ -390,8 +396,8 @@ sub mergestate () { #{{{ # fields. foreach my $name (keys %myfeeds) { if (exists $feeds{$name}) { - foreach my $field (qw{message lastupdate numposts - newposts error}) { + foreach my $field (qw{message lastupdate lasttry + numposts newposts error}) { $feeds{$name}->{$field}=$myfeeds{$name}->{$field}; } } @@ -407,15 +413,15 @@ sub mergestate () { #{{{ $guids{$guid}=$myguids{$guid}; } } -} #}}} +} -sub clearstate () { #{{{ +sub clearstate () { %feeds=(); %guids=(); $state_loaded=0; -} #}}} +} -sub expire () { #{{{ +sub expire () { foreach my $feed (values %feeds) { next unless $feed->{expireage} || $feed->{expirecount}; my $count=0; @@ -444,24 +450,24 @@ sub expire () { #{{{ } } } -} #}}} +} -sub needsaggregate () { #{{{ +sub needsaggregate () { return values %feeds if $config{rebuild}; return grep { time - $_->{lastupdate} >= $_->{updateinterval} } values %feeds; -} #}}} +} -sub aggregate (@) { #{{{ +sub aggregate (@) { eval q{use XML::Feed}; error($@) if $@; eval q{use URI::Fetch}; error($@) if $@; foreach my $feed (@_) { - $feed->{lastupdate}=time; + $feed->{lasttry}=time; $feed->{newposts}=0; - $feed->{message}=sprintf(gettext("processed ok at %s"), - displaytime($feed->{lastupdate})); + $feed->{message}=sprintf(gettext("last checked %s"), + displaytime($feed->{lasttry})); $feed->{error}=0; debug(sprintf(gettext("checking feed %s ..."), $feed->{name})); @@ -483,6 +489,10 @@ sub aggregate (@) { #{{{ debug($feed->{message}); next; } + + # lastupdate is only set if we were able to contact the server + $feed->{lastupdate}=$feed->{lasttry}; + if ($res->status == URI::Fetch::URI_GONE()) { $feed->{message}=gettext("feed not found"); $feed->{error}=1; @@ -496,15 +506,19 @@ sub aggregate (@) { #{{{ # that contains invalid UTF-8 sequences. Convert # feed to ascii to try to work around. $feed->{message}.=" ".sprintf(gettext("(invalid UTF-8 stripped from feed)")); - $content=Encode::decode_utf8($content, 0); - $f=eval{XML::Feed->parse(\$content)}; + $f=eval { + $content=Encode::decode_utf8($content, 0); + XML::Feed->parse(\$content) + }; } if ($@) { # Another possibility is badly escaped entities. $feed->{message}.=" ".sprintf(gettext("(feed entities escaped)")); $content=~s/\&(?!amp)(\w+);/&$1;/g; - $content=Encode::decode_utf8($content, 0); - $f=eval{XML::Feed->parse(\$content)}; + $f=eval { + $content=Encode::decode_utf8($content, 0); + XML::Feed->parse(\$content) + }; } if ($@) { $feed->{message}=gettext("feed crashed XML::Feed!")." ($@)"; @@ -520,10 +534,15 @@ sub aggregate (@) { #{{{ } foreach my $entry ($f->entries) { - my $content=$content=$entry->content->body; + # XML::Feed doesn't work around XML::Atom's bizarre + # API, so we will. Real unicode strings? Yes please. + # See [[bugs/Aggregated_Atom_feeds_are_double-encoded]] + local $XML::Atom::ForceUnicode = 1; + + my $c=$entry->content; # atom feeds may have no content, only a summary - if (! defined $content && ref $entry->summary) { - $content=$entry->summary->body; + if (! defined $c && ref $entry->summary) { + $c=$entry->summary; } add_page( @@ -531,15 +550,16 @@ sub aggregate (@) { #{{{ copyright => $f->copyright, title => defined $entry->title ? decode_entities($entry->title) : "untitled", link => $entry->link, - content => defined $content ? $content : "", + content => (defined $c && defined $c->body) ? $c->body : "", guid => defined $entry->id ? $entry->id : time."_".$feed->{name}, ctime => $entry->issued ? ($entry->issued->epoch || time) : time, + base => (defined $c && $c->can("base")) ? $c->base : undef, ); } } -} #}}} +} -sub add_page (@) { #{{{ +sub add_page (@) { my %params=@_; my $feed=$params{feed}; @@ -605,7 +625,8 @@ sub add_page (@) { #{{{ my $template=template($feed->{template}, blind_cache => 1); $template->param(title => $params{title}) if defined $params{title} && length($params{title}); - $template->param(content => htmlescape(htmlabs($params{content}, $feed->{feedurl}))); + $template->param(content => wikiescape(htmlabs($params{content}, + defined $params{base} ? $params{base} : $feed->{feedurl}))); $template->param(name => $feed->{name}); $template->param(url => $feed->{url}); $template->param(copyright => $params{copyright}) @@ -625,23 +646,25 @@ sub add_page (@) { #{{{ # Store it in pagectime for expiry code to use also. $IkiWiki::pagectime{$guid->{page}}=$mtime; } -} #}}} + else { + # Dummy value for expiry code. + $IkiWiki::pagectime{$guid->{page}}=time; + } +} -sub htmlescape ($) { #{{{ +sub wikiescape ($) { # escape accidental wikilinks and preprocessor stuff - my $html=shift; - $html=~s/(?<!\\)\[\[/\\\[\[/g; - return $html; -} #}}} + return encode_entities(shift, '\[\]'); +} -sub urlabs ($$) { #{{{ +sub urlabs ($$) { my $url=shift; my $urlbase=shift; URI->new_abs($url, $urlbase)->as_string; -} #}}} +} -sub htmlabs ($$) { #{{{ +sub htmlabs ($$) { # Convert links in html from relative to absolute. # Note that this is a heuristic, which is not specified by the rss # spec and may not be right for all feeds. Also, see Debian @@ -677,15 +700,15 @@ sub htmlabs ($$) { #{{{ $p->eof; return $ret; -} #}}} +} -sub htmlfn ($) { #{{{ +sub htmlfn ($) { return shift().".".($config{aggregateinternal} ? "_aggregated" : $config{htmlext}); -} #}}} +} my $aggregatelock; -sub lockaggregate () { #{{{ +sub lockaggregate () { # Take an exclusive lock to prevent multiple concurrent aggregators. # Returns true if the lock was aquired. if (! -d $config{wikistatedir}) { @@ -698,11 +721,11 @@ sub lockaggregate () { #{{{ return 0; } return 1; -} #}}} +} -sub unlockaggregate () { #{{{ +sub unlockaggregate () { return close($aggregatelock) if $aggregatelock; return; -} #}}} +} 1 |