summaryrefslogtreecommitdiff
path: root/IkiWiki/Plugin/amazon_s3.pm
blob: 10bf358e44a69148954fc36a0557330016fa4024 (plain)
  1. #!/usr/bin/perl
  2. package IkiWiki::Plugin::amazon_s3;
  3. use warnings;
  4. no warnings 'redefine';
  5. use strict;
  6. use IkiWiki 3.00;
  7. use IkiWiki::Render;
  8. use Net::Amazon::S3;
  9. # Store references to real subs before overriding them.
  10. our %subs;
  11. BEGIN {
  12. foreach my $sub (qw{IkiWiki::writefile IkiWiki::prune}) {
  13. $subs{$sub}=\&$sub;
  14. }
  15. };
  16. sub import {
  17. hook(type => "getopt", id => "amazon_s3", call => \&getopt);
  18. hook(type => "getsetup", id => "amazon_s3", call => \&getsetup);
  19. hook(type => "checkconfig", id => "amazon_s3", call => \&checkconfig);
  20. }
  21. sub getopt () {
  22. eval q{use Getopt::Long};
  23. error($@) if $@;
  24. Getopt::Long::Configure('pass_through');
  25. GetOptions("delete-bucket" => sub {
  26. my $bucket=getbucket();
  27. debug(gettext("deleting bucket.."));
  28. my $resp = $bucket->list_all or die $bucket->err . ": " . $bucket->errstr;
  29. foreach my $key (@{$resp->{keys}}) {
  30. debug("\t".$key->{key});
  31. $bucket->delete_key($key->{key}) or die $bucket->err . ": " . $bucket->errstr;
  32. }
  33. $bucket->delete_bucket or die $bucket->err . ": " . $bucket->errstr;
  34. debug(gettext("done"));
  35. exit(0);
  36. });
  37. }
  38. sub getsetup () {
  39. return
  40. plugin => {
  41. safe => 0,
  42. rebuild => 0,
  43. },
  44. amazon_s3_key_id => {
  45. type => "string",
  46. example => "XXXXXXXXXXXXXXXXXXXX",
  47. description => "public access key id",
  48. safe => 1,
  49. rebuild => 0,
  50. },
  51. amazon_s3_key_id => {
  52. type => "string",
  53. example => "$ENV{HOME}/.s3_key",
  54. description => "file holding secret key (must not be readable by others!)",
  55. safe => 0, # ikiwiki reads this file
  56. rebuild => 0,
  57. },
  58. amazon_s3_bucket => {
  59. type => "string",
  60. example => "mywiki",
  61. description => "globally unique name of bucket to store wiki in",
  62. safe => 1,
  63. rebuild => 1,
  64. },
  65. amazon_s3_prefix => {
  66. type => "string",
  67. example => "wiki/",
  68. description => "a prefix to prepend to each page name",
  69. safe => 1,
  70. rebuild => 1,
  71. },
  72. amazon_s3_location => {
  73. type => "string",
  74. example => "EU",
  75. description => "which S3 datacenter to use (leave blank for default)",
  76. safe => 1,
  77. rebuild => 1,
  78. },
  79. amazon_s3_dupindex => {
  80. type => "boolean",
  81. example => 0,
  82. description => "store each index file twice? (allows urls ending in \"/index.html\" and \"/\")",
  83. safe => 1,
  84. rebuild => 1,
  85. },
  86. }
  87. sub checkconfig {
  88. foreach my $field (qw{amazon_s3_key_id amazon_s3_key_file
  89. amazon_s3_bucket}) {
  90. if (! exists $config{$field} || ! defined $config{$field}) {
  91. error(sprintf(gettext("Must specify %s"), $field));
  92. }
  93. }
  94. if (! exists $config{amazon_s3_prefix} ||
  95. ! defined $config{amazon_s3_prefix}) {
  96. $config{amazon_s3_prefix}="wiki/";
  97. }
  98. }
  99. {
  100. my $bucket;
  101. sub getbucket {
  102. return $bucket if defined $bucket;
  103. open(IN, "<", $config{amazon_s3_key_file}) || error($config{amazon_s3_key_file}.": ".$!);
  104. my $key=<IN>;
  105. chomp $key;
  106. close IN;
  107. my $s3=Net::Amazon::S3->new({
  108. aws_access_key_id => $config{amazon_s3_key_id},
  109. aws_secret_access_key => $key,
  110. retry => 1,
  111. });
  112. # make sure the bucket exists
  113. if (exists $config{amazon_s3_location}) {
  114. $bucket=$s3->add_bucket({
  115. bucket => $config{amazon_s3_bucket},
  116. location_constraint => $config{amazon_s3_location},
  117. });
  118. }
  119. else {
  120. $bucket=$s3->add_bucket({
  121. bucket => $config{amazon_s3_bucket},
  122. });
  123. }
  124. if (! $bucket) {
  125. error(gettext("Failed to create bucket in S3: ").
  126. $s3->err.": ".$s3->errstr."\n");
  127. }
  128. return $bucket;
  129. }
  130. }
  131. # Given a file, return any S3 keys associated with it.
  132. sub file2keys ($) {
  133. my $file=shift;
  134. my @keys;
  135. if ($file =~ /^\Q$config{destdir}\/\E(.*)/) {
  136. push @keys, $config{amazon_s3_prefix}.$1;
  137. # Munge foo/index.html to foo/
  138. if ($keys[0]=~/(^|.*\/)index.$config{htmlext}$/) {
  139. # A duplicate might need to be stored under the
  140. # unmunged name too.
  141. if (!$config{usedirs} || $config{amazon_s3_dupindex}) {
  142. push @keys, $1;
  143. }
  144. else {
  145. @keys=($1);
  146. }
  147. }
  148. }
  149. return @keys;
  150. }
  151. package IkiWiki;
  152. use File::MimeInfo;
  153. use Encode;
  154. # This is a wrapper around the real writefile.
  155. sub writefile ($$$;$$) {
  156. my $file=shift;
  157. my $destdir=shift;
  158. my $content=shift;
  159. my $binary=shift;
  160. my $writer=shift;
  161. # First, write the file to disk.
  162. my $ret=$IkiWiki::Plugin::amazon_s3::subs{'IkiWiki::writefile'}->($file, $destdir, $content, $binary, $writer);
  163. my @keys=IkiWiki::Plugin::amazon_s3::file2keys("$destdir/$file");
  164. # Store the data in S3.
  165. if (@keys) {
  166. my $bucket=IkiWiki::Plugin::amazon_s3::getbucket();
  167. # The http layer tries to downgrade utf-8
  168. # content, but that can fail (see
  169. # http://rt.cpan.org/Ticket/Display.html?id=35710),
  170. # so force convert it to bytes.
  171. $content=encode_utf8($content) if defined $content;
  172. my %opts=(
  173. acl_short => 'public-read',
  174. content_type => mimetype("$destdir/$file"),
  175. );
  176. # If there are multiple keys to write, data is sent
  177. # multiple times.
  178. # TODO: investigate using the new copy operation.
  179. # (It may not be robust enough.)
  180. foreach my $key (@keys) {
  181. my $res;
  182. if (! $writer) {
  183. $res=$bucket->add_key($key, $content, \%opts);
  184. }
  185. else {
  186. # This test for empty files is a workaround
  187. # for this bug:
  188. # http://rt.cpan.org//Ticket/Display.html?id=35731
  189. if (-z "$destdir/$file") {
  190. $res=$bucket->add_key($key, "", \%opts);
  191. }
  192. else {
  193. # read back in the file that the writer emitted
  194. $res=$bucket->add_key_filename($key, "$destdir/$file", \%opts);
  195. }
  196. }
  197. if (! $res) {
  198. error(gettext("Failed to save file to S3: ").
  199. $bucket->err.": ".$bucket->errstr."\n");
  200. }
  201. }
  202. }
  203. return $ret;
  204. }
  205. # This is a wrapper around the real prune.
  206. sub prune ($) {
  207. my $file=shift;
  208. my @keys=IkiWiki::Plugin::amazon_s3::file2keys($file);
  209. # Prune files out of S3 too.
  210. if (@keys) {
  211. my $bucket=IkiWiki::Plugin::amazon_s3::getbucket();
  212. foreach my $key (@keys) {
  213. my $res=$bucket->delete_key($key);
  214. if (! $res) {
  215. error(gettext("Failed to delete file from S3: ").
  216. $bucket->err.": ".$bucket->errstr."\n");
  217. }
  218. }
  219. }
  220. return $IkiWiki::Plugin::amazon_s3::subs{'IkiWiki::prune'}->($file);
  221. }
  222. 1