From 50b95a0742d74a4f73f5a8ed9ffc50733080de77 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Nov 2008 18:34:18 -0500 Subject: response --- doc/plugins/contrib/po.mdwn | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/plugins/contrib/po.mdwn b/doc/plugins/contrib/po.mdwn index 3077b4858..0fd06cb81 100644 --- a/doc/plugins/contrib/po.mdwn +++ b/doc/plugins/contrib/po.mdwn @@ -158,3 +158,5 @@ Any thoughts on this? >>>>> Joey, please have a look at my branch, your help would be really >>>>> welcome for the security research, as I'm almost done with what >>>>> I am able to do myself in this area. --[[intrigeri]] +>>>>>> +>>>>>> I came up with a patch for the WrapI18N issue --[[Joey]] -- cgit v1.2.3 From 521d76e5082b5d32bf1fe85b55d85f54f2030090 Mon Sep 17 00:00:00 2001 From: intrigeri Date: Wed, 12 Nov 2008 01:06:17 +0100 Subject: pagetitle vs meta: announcing a working prototype Signed-off-by: intrigeri --- doc/bugs/pagetitle_function_does_not_respect_meta_titles.mdwn | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/bugs/pagetitle_function_does_not_respect_meta_titles.mdwn b/doc/bugs/pagetitle_function_does_not_respect_meta_titles.mdwn index 77c86eba1..158656a13 100644 --- a/doc/bugs/pagetitle_function_does_not_respect_meta_titles.mdwn +++ b/doc/bugs/pagetitle_function_does_not_respect_meta_titles.mdwn @@ -8,5 +8,9 @@ The `IkiWiki::pagetitle` function does not respect title changes via `meta.title > - Using inline would avoid the redefinition + code duplication. > - A few plugins would need to be upgraded. > - It may be necessary to adapt the testsuite in `t/pagetitle.t`, as well. - +> > --[[intrigeri]] +> +>> It was actually more complicated than expected. A working prototype is +>> now in my `meta` branch, see my userpage for the up-to-date url. +>> Thus tagging [[patch]]. --[[intrigeri]] -- cgit v1.2.3 From ecd4f0ee553cc3e11491f236763d6cc7cd6dd8a1 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Nov 2008 20:48:02 -0500 Subject: make unlockwiki drop the cgilock This is necessary so that things that fork to the background, like pinger, and inline ping, don't block other cgis from running. Note that websetup also calls unlockwiki, before refreshing / rebuilding the wiki. It makes perfect sense for that not to block other cgis. --- IkiWiki.pm | 1 + IkiWiki/Wrapper.pm | 10 +++++++--- debian/changelog | 20 ++++++++++---------- po/ikiwiki.pot | 10 +++++----- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/IkiWiki.pm b/IkiWiki.pm index d949566d8..5e21e7090 100644 --- a/IkiWiki.pm +++ b/IkiWiki.pm @@ -1295,6 +1295,7 @@ sub lockwiki () { #{{{ } #}}} sub unlockwiki () { #{{{ + POSIX::close($ENV{IKIWIKI_CGILOCK_FD}) if exists $ENV{IKIWIKI_CGILOCK_FD}; return close($wikilock) if $wikilock; return; } #}}} diff --git a/IkiWiki/Wrapper.pm b/IkiWiki/Wrapper.pm index 7a2d4381a..31e30ad2e 100644 --- a/IkiWiki/Wrapper.pm +++ b/IkiWiki/Wrapper.pm @@ -72,12 +72,16 @@ EOF # Avoid more than one ikiwiki cgi running at a time by # taking a cgi lock. Since ikiwiki uses several MB of # memory, a pile up of processes could cause thrashing - # otherwise. + # otherwise. The fd of the lock is stored in + # IKIWIKI_CGILOCK_FD so unlockwiki can close it. $pre_exec=<<"EOF"; { int fd=open("$config{wikistatedir}/cgilock", O_CREAT | O_RDWR, 0666); - if (fd != -1) - flock(fd, LOCK_EX); + if (fd != -1 && flock(fd, LOCK_EX) == 0) { + char *fd_s; + asprintf(&fd_s, "%i", fd); + setenv("IKIWIKI_CGILOCK_FD", fd_s, 1); + } } EOF } diff --git a/debian/changelog b/debian/changelog index 2088b07ec..99f35482e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,13 @@ -ikiwiki (2.69) UNRELEASED; urgency=low +ikiwiki (2.69) unstable; urgency=low + * Avoid multiple ikiwiki cgi processes piling up, eating all memory, + and thrashing, by making the cgi wrapper wait on a cgilock. + If you had to set apache's MaxClients low to avoid ikiwiki thrashing your + server, you can now turn it up to a high value. + * Stop busy-waiting in lockwiki, as this could delay ikiwiki from waking up + for up to one second. The bailout code is no longer needed after above + change. + * Remove support for unused optional wait parameter from lockwiki. * aggregate: Try to query XML::Feed for the base url when derelevatising links. Since this needs the just released XML::Feed 0.3, as well as a not yet released XML::RSS, it will fall back to the old method @@ -14,16 +22,8 @@ ikiwiki (2.69) UNRELEASED; urgency=low * tag: Normalize tagbase so leading/trailing slashes in it don't break things. * bzr: Fix dates for recentchanges. - * Avoid multiple ikiwiki cgi processes piling up, eating all memory, - and thrashing, by making the cgi wrapper wait on a cgilock. - If you had to set apache's MaxClients low to avoid ikiwiki thrashing your - server, you can now turn it up to a high value. - * Stop busy-waiting in lockwiki, as this could delay ikiwiki from waking up - for up to one second. The bailout code is no longer needed after above - change. - * Remove support for unused optional wait parameter from lockwiki. - -- Joey Hess Thu, 06 Nov 2008 16:01:00 -0500 + -- Joey Hess Tue, 11 Nov 2008 20:35:55 -0500 ikiwiki (2.68) unstable; urgency=low diff --git a/po/ikiwiki.pot b/po/ikiwiki.pot index 8cf3853e0..feb36c742 100644 --- a/po/ikiwiki.pot +++ b/po/ikiwiki.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2008-11-11 15:36-0500\n" +"POT-Creation-Date: 2008-11-11 20:48-0500\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -910,19 +910,19 @@ msgstr "" #. translators: The first parameter is a filename, and the second is #. translators: a (probably not translated) error message. -#: ../IkiWiki/Wrapper.pm:93 +#: ../IkiWiki/Wrapper.pm:97 #, perl-format msgid "failed to write %s: %s" msgstr "" #. translators: The parameter is a C filename. -#: ../IkiWiki/Wrapper.pm:150 +#: ../IkiWiki/Wrapper.pm:154 #, perl-format msgid "failed to compile %s" msgstr "" #. translators: The parameter is a filename. -#: ../IkiWiki/Wrapper.pm:170 +#: ../IkiWiki/Wrapper.pm:174 #, perl-format msgid "successfully generated %s" msgstr "" @@ -969,7 +969,7 @@ msgstr "" msgid "preprocessing loop detected on %s at depth %i" msgstr "" -#: ../IkiWiki.pm:1672 +#: ../IkiWiki.pm:1673 msgid "yes" msgstr "" -- cgit v1.2.3 From 2ab3132d3b88afa4b23dec3bf2a38f185e7f2fed Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Nov 2008 21:01:16 -0500 Subject: document other files in .ikiwiki --- doc/tips/inside_dot_ikiwiki.mdwn | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/tips/inside_dot_ikiwiki.mdwn b/doc/tips/inside_dot_ikiwiki.mdwn index b649636dc..1f76ce4bd 100644 --- a/doc/tips/inside_dot_ikiwiki.mdwn +++ b/doc/tips/inside_dot_ikiwiki.mdwn @@ -63,3 +63,28 @@ To remove that user: I've not written actual utilities to do this yet because I've only needed to do it rarely, and the data I've wanted has been different each time. --[[Joey]] + +## the session database + +`.ikiwiki/sessions.db` is the session database. See the [[cpan CGI::Session]] +documentation for more details. + +## lockfiles + +In case you're curious, here's what the various lock files do. + +* `.ikiwiki/lockfile` is the master ikiwiki lock file. Ikiwiki takes this + lock before reading/writing state. +* `.ikiwiki/commitlock` is locked as a semophore, to disable the commit hook + from doing anything. +* `.ikiwiki/cgilock` is locked by the cgi wrapper, to ensure that only + one ikiwiki process is run at a time to handle cgi requests. + +## plugin state files + +Some plugins create other files to store their state. + +* `.ikiwiki/aggregate` is a plain text database used by the aggregate plugin + to record feeds and known posts. +* `.ikiwiki/xapian/` is created by the search plugin, and contains xapian-omega + configuration and the xapian database. -- cgit v1.2.3 From 2c858c9c95e287ebe3740a94f983f6ae9d6fb080 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Tue, 11 Nov 2008 21:07:06 -0500 Subject: add news item for ikiwiki 2.69 --- doc/news/version_2.64.mdwn | 25 ------------------------- doc/news/version_2.69.mdwn | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 25 deletions(-) delete mode 100644 doc/news/version_2.64.mdwn create mode 100644 doc/news/version_2.69.mdwn diff --git a/doc/news/version_2.64.mdwn b/doc/news/version_2.64.mdwn deleted file mode 100644 index 137ca1a5c..000000000 --- a/doc/news/version_2.64.mdwn +++ /dev/null @@ -1,25 +0,0 @@ -ikiwiki 2.64 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * Avoid uninitialised value when --dumpsetup is used and no srcdir/destdir - specified. - * ddate: Stop clobbering timeformat when not enabled. - * progress: New plugin to generate progress bars (willu) - * Add allow\_symlinks\_before\_srcdir to config so websetup doesn't eat it. - * img: Support sizes like 200x. Closes: #[475149](http://bugs.debian.org/475149) - * goodstuff: Remove otl plugin from the bundle since it needs a significant - external dependency and is not commonly used. If you use otl, make sure - you explicitly enable it now. - * goodstuff: Add more, progress, and table plugins to the bundle. - * Improve error message if external plugin fails to load. Closes: #[498458](http://bugs.debian.org/498458) - * Directive documentation broken out of the plugin documentation and into - pages suitable to be used as an underlay. Thanks to Willu for doing most - of the tedious work. - * Move the directive documentation into its own underlay, separate from - basewiki, since it's sorta large compared to the rest of basewiki. - * listdirectives: Enable use of the directives underlay. - * Removed the obsolete blog page from the basewiki. ikiwiki/blog still - remains, but is now deprecated too. - * Removed old redirecton pages from basewiki (helponformatting, - markdown, openid, pagespec, preprocessordirective, subpage, wikilink). - * inline: Treat rootpage as a link, so that it can refer to a subpage - without hardcoding the path."""]] \ No newline at end of file diff --git a/doc/news/version_2.69.mdwn b/doc/news/version_2.69.mdwn new file mode 100644 index 000000000..a277541fe --- /dev/null +++ b/doc/news/version_2.69.mdwn @@ -0,0 +1,24 @@ +ikiwiki 2.69 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Avoid multiple ikiwiki cgi processes piling up, eating all memory, + and thrashing, by making the cgi wrapper wait on a cgilock. + If you had to set apache's MaxClients low to avoid ikiwiki thrashing your + server, you can now turn it up to a high value. + * Stop busy-waiting in lockwiki, as this could delay ikiwiki from waking up + for up to one second. The bailout code is no longer needed after above + change. + * Remove support for unused optional wait parameter from lockwiki. + * aggregate: Try to query XML::Feed for the base url when derelevatising + links. Since this needs the just released XML::Feed 0.3, as well + as a not yet released XML::RSS, it will fall back to the old method + if no xml:base info is available. + * meta: Plugin is now enabled by default since the basewiki uses it. + * txt: Do not encode quotes when filtering the txt, as that broke + later parsing of any directives on the page. + * Fix the link() pagespec to match links that are internally recorded as + absolute. + * Add rel=nofollow to recentchanges\_links for the same (weak) reasons it + was earlier added to edit links. + * tag: Normalize tagbase so leading/trailing slashes in it don't break + things. + * bzr: Fix dates for recentchanges."""]] \ No newline at end of file -- cgit v1.2.3 From 716560b7f15b6e15b246c39c11eb8181d91c8662 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 12 Nov 2008 17:19:41 -0500 Subject: check for invalid utf-8, and toss it back to avoid crashes Since ikiwiki uses open :utf8, perl assumes that files contain valid utf-8. If it turns out to be malformed it may later crash while processing strings read from them, with 'Malformed UTF-8 character (fatal)'. As at least a quick fix, use utf8::valid as soon as data is read, and if it's not valid, call encode_utf8 on the string, thus clearing the utf-8 flag. This may cause follow-on encoding problems, but will avoid this crash, and the input file was broken anyway, so GIGO is a reasonable response. (I looked at calling decode_utf8 after, but it seemed to cause more trouble than it was worth. BTW, use open ':encoding(utf8)' avaoids this problem, but the corrupted data later causes Storable to crash when writing the index.) This is a quick fix, clearly imperfect: - It might be better to explicitly call decode_utf8 when reading files, rather than using the IO layer. - Data read other than by readfile() can still sneak in bad utf-8. While ikiwiki does very little file input not using it, stdin for the CGI would be one way. --- IkiWiki.pm | 4 ++++ debian/changelog | 6 ++++++ doc/security.mdwn | 9 +++++++++ 3 files changed, 19 insertions(+) diff --git a/IkiWiki.pm b/IkiWiki.pm index 5e21e7090..735dc97b1 100644 --- a/IkiWiki.pm +++ b/IkiWiki.pm @@ -721,6 +721,10 @@ sub readfile ($;$$) { #{{{ binmode($in) if ($binary); return \*$in if $wantfd; my $ret=<$in>; + # check for invalid utf-8, and toss it back to avoid crashes + if (! utf8::valid($ret)) { + $ret=encode_utf8($ret); + } close $in || error("failed to read $file: $!"); return $ret; } #}}} diff --git a/debian/changelog b/debian/changelog index 99f35482e..3838a3e90 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +ikiwiki (2.70) UNRELEASED; urgency=low + + * Avoid crash on malformed utf-8 discovered by intrigeri. + + -- Joey Hess Wed, 12 Nov 2008 17:30:33 -0500 + ikiwiki (2.69) unstable; urgency=low * Avoid multiple ikiwiki cgi processes piling up, eating all memory, diff --git a/doc/security.mdwn b/doc/security.mdwn index 0841abf49..1bc7b9e60 100644 --- a/doc/security.mdwn +++ b/doc/security.mdwn @@ -407,3 +407,12 @@ discovered on 30 May 2008 and fixed the same day. ([[!cve CVE-2008-0169]]) I recommend upgrading to 2.48 immediatly if your wiki allows both password and openid logins. + +## Malformed UTF-8 DOS + +Feeding ikiwiki page sources containing certian forms of malformed UTF-8 +can cause it to crash. This can potentially be used for a denial of service +attack. + +intrigeri discovered this problem on 12 Nov 2008 and a patch put in place +later that day. -- cgit v1.2.3 From 97cb03681172400a11824007f2fc61a519f68717 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 12 Nov 2008 17:35:21 -0500 Subject: note fix versions --- doc/security.mdwn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/security.mdwn b/doc/security.mdwn index 1bc7b9e60..b067a8a16 100644 --- a/doc/security.mdwn +++ b/doc/security.mdwn @@ -415,4 +415,5 @@ can cause it to crash. This can potentially be used for a denial of service attack. intrigeri discovered this problem on 12 Nov 2008 and a patch put in place -later that day. +later that day, in version 2.70. The fix was backported to testing as version +2.53.2, and to stable as version 1.33.7. -- cgit v1.2.3 From 31a1a3215b715b2356c67378605c955af48ef1e5 Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 12 Nov 2008 18:01:46 -0500 Subject: releasing version 2.70 --- debian/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 3838a3e90..dd19c1463 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,8 @@ -ikiwiki (2.70) UNRELEASED; urgency=low +ikiwiki (2.70) unstable; urgency=low * Avoid crash on malformed utf-8 discovered by intrigeri. - -- Joey Hess Wed, 12 Nov 2008 17:30:33 -0500 + -- Joey Hess Wed, 12 Nov 2008 17:45:58 -0500 ikiwiki (2.69) unstable; urgency=low -- cgit v1.2.3 From 699aa79c2e2dbec02a38ba6ff32e9c861ee9f7be Mon Sep 17 00:00:00 2001 From: Joey Hess Date: Wed, 12 Nov 2008 18:08:10 -0500 Subject: add news item for ikiwiki 2.70 --- doc/news/version_2.65.mdwn | 25 ------------------------- doc/news/version_2.70.mdwn | 3 +++ 2 files changed, 3 insertions(+), 25 deletions(-) delete mode 100644 doc/news/version_2.65.mdwn create mode 100644 doc/news/version_2.70.mdwn diff --git a/doc/news/version_2.65.mdwn b/doc/news/version_2.65.mdwn deleted file mode 100644 index db6afd988..000000000 --- a/doc/news/version_2.65.mdwn +++ /dev/null @@ -1,25 +0,0 @@ -ikiwiki 2.65 released with [[!toggle text="these changes"]] -[[!toggleable text=""" - * aggregate: Expire excess or old items on the same pass that adds them, - not only on subsequent passes. - * editdiff: Broken since 2.62 due to wrong syntax, now fixed. - * aggregate: Support atom feeds with only a summary element, and no content - elements. - * progress: Display an error if the progress cannot be parsed, and allow - the percent parameter to only optionally end with "%". - * Fix reversion in use of ikiwiki -verbose -setup with a setup file that - enables syslog. Setup output is once again output to stdout in this - case. - * edittemplate: Default new page file type to the same type as the template. - (willu) - * edittemplate: Add "silent" parameter. (Willu) - * edittemplate: Link to template, to allow creating it. (Willu) - * editpage: Add a missing check that the page name contains only legal - characters, in addition to the existing check for pruned filenames. - * Print a debug message if a page has multiple source files. - * Add keepextension parameter to htmlize hook. (Willu) - * rename, remove: Don't rely on a form parameter to tell whether the page - should be treated as an attachment. - * rename: Add support for moving SubPages of a page when renaming it. - (Sponsored by The TOVA Company.) - * rename: Hide type field from rename form when renaming attachments."""]] \ No newline at end of file diff --git a/doc/news/version_2.70.mdwn b/doc/news/version_2.70.mdwn new file mode 100644 index 000000000..f0830efa1 --- /dev/null +++ b/doc/news/version_2.70.mdwn @@ -0,0 +1,3 @@ +ikiwiki 2.70 released with [[!toggle text="these changes"]] +[[!toggleable text=""" + * Avoid crash on malformed utf-8 discovered by intrigeri."""]] \ No newline at end of file -- cgit v1.2.3