* -CSD does not affect modules, so readfile() was not using the utf-8 input

  layer, which led to lots of problems; make it force read files as utf-8.
  Closes: #373203
* writefile() likewise needs to use the utf8 output layer.
* Remove the -CSD from ikiwiki's hashbang since it's useless to have it
  there.
* Revert some of the decode_utf8 changes in CGI.pm that seem unnecessary
  given the readfile fix.
* Add utf-8 testcases for readfile and htmlize.

1 files changed, 12 insertions, 2 deletions

diff --git a/IkiWiki.pm b/IkiWiki.pm
index 357c1cd2d..76472302e 100644
--- a/IkiWiki.pm
+++ b/IkiWiki.pm
@@ -163,7 +163,12 @@ sub readfile ($;$) { #{{{
 	
 	local $/=undef;
 	open (IN, $file) || error("failed to read $file: $!");
-	binmode(IN) if $binary;
+	if (! $binary) {
+		binmode(IN, ":utf8");
+	}
+	else {
+		binmode(IN);
+	}
 	my $ret=<IN>;
 	close IN;
 	return $ret;
@@ -195,7 +200,12 @@ sub writefile ($$$;$) { #{{{
 	}
 	
 	open (OUT, ">$destdir/$file") || error("failed to write $destdir/$file: $!");
-	binmode(OUT) if $binary;
+	if (! $binary) {
+		binmode(OUT, ":utf8");
+	}
+	else {
+		binmode(OUT);
+	}
 	print OUT $content;
 	close OUT;
 } #}}}