summaryrefslogtreecommitdiff
path: root/localwebstats
blob: 0e17e5bfbcb8ee90bd9ac4eda9d857ac68f3d539 (plain)
  1. #!/bin/bash
  2. #
  3. # /usr/local/sbin/localwebstats
  4. # Copyright 2001-2002 Jonas Smedegaard <dr@jones.dk>
  5. #
  6. # $Id: localwebstats,v 1.18 2002-03-18 20:12:07 jonas Exp $
  7. #
  8. # Webstats maintenance script
  9. #
  10. # halt on errors (NB! this is a bashism...)
  11. set -e
  12. function usage() {
  13. echo "Usage: $(basename $0) init|update|prelogrotate|postlogrotate|ignore <website> [<website>...]"
  14. echo " If no website is given, all are attempted"
  15. echo " Tip: Automagically runs when symlinked to /etc/cron.{daily,weekly,monthly}/"
  16. exit 1
  17. }
  18. function exit1() {
  19. echo "Error: $1"
  20. echo "Exiting..."
  21. exit 1
  22. }
  23. # automagically configure when run from cron dirs
  24. case $(dirname $0) in
  25. /etc/cron.daily)
  26. stamp=update
  27. ;;
  28. /etc/cron.weekly)
  29. stamp=ignore
  30. ;;
  31. /etc/cron.monthly)
  32. stamp=ignore
  33. ;;
  34. *)
  35. stamp=$1
  36. shift || usage
  37. ;;
  38. esac
  39. case "$stamp" in
  40. init|update|prelogrotate|postlogrotate|ignore)
  41. ;;
  42. *)
  43. usage
  44. ;;
  45. esac
  46. if [ "$stamp" = "ignore" ]; then
  47. [ $DEBUG ] && echo "Asked to ignore - exiting silently..."
  48. exit 0
  49. fi
  50. ROOT=1
  51. user=www-data
  52. group=www-data
  53. LOGROOT=/var/log/apache
  54. WEBROOT=/var/www
  55. # Options: $1=LOGROOT, $2=WEBROOT, $3=WEBSITE
  56. function statsdir() { echo $2/VIRTUAL/stats.$(dnsdomainname)/www/$3; }
  57. function webdirs() { find $1 -type d -mindepth 1 -maxdepth 1 | grep '\.*\.' | sed 's!$1!!'; }
  58. #function logfiles() { $(ls -r $LOGDIR/*-access*.gz) $(ls -r $LOGDIR/access*.??.gz) $(ls -r $LOGDIR/access*.?.gz) $(ls -r $LOGDIR/access*.?); }
  59. function logcontentresolved() { for file in $(find $1/$3 -name '????.??.00.gz' -type f -mindepth 1 -maxdepth 1 -follow | sort); do zcat $file; done; for file in $(find $1/$3 -name '????.??.00' -type f -mindepth 1 -maxdepth 1 -follow | sort); do cat $file; done; }
  60. function logcontent() { for file in $(find $1/$3 -name '????.??.??.gz' ! -name '*00.gz' -type f -mindepth 1 -maxdepth 1 -follow | sort); do zcat $file; done; for file in $(find $1/$3 -name '????.??.??' ! -name '*00' -type f -mindepth 1 -maxdepth 1 -follow | sort); do cat $file; done; }
  61. function host() { cat $2/VIRTUAL/$3/hostname || exit1 "Unable to get hostname for virtual host."; }
  62. function domain() { cat $2/VIRTUAL/$3/domainname || exit1 "Unable to get domainname for virtual host."; }
  63. function analog_cfg() { echo /etc/analog_$3.conf; }
  64. function rmagic_cfg() { echo /etc/rmagic/rmagic_$3.conf; }
  65. function webalizer_cfg() { echo /etc/webalizer_$3.conf; }
  66. function modlogan_cfg() { echo /etc/modlogan/modlogan_$3.conf; }
  67. function pre_init() { true; }
  68. function post_init() { true; }
  69. function pre_update() { true; }
  70. function post_update() { true; }
  71. # The above can be overridden
  72. LOCALCONFIG=/etc/local/webstats.conf
  73. . $LOCALCONFIG || exit1 "Unable to read local config file $LOCALCONFIG"
  74. # variables and functions too boring to be configurable
  75. JDRESOLVE_BIN="/usr/bin/jdresolve"
  76. JDRESOLVE_DB="/var/cache/jdresolve/hosts.db"
  77. #JDRESOLVE_OPTIONS="-r -t 5 --database=$JDRESOLVE_DB --dbfirst --expiredb=48"
  78. # DB access doesn't work currently (no output...)
  79. #JDRESOLVE_OPTIONS="-r --database=$JDRESOLVE_DB --dbfirst --expiredb=48"
  80. JDRESOLVE_OPTIONS="-r"
  81. JDRESOLVE_MERGE_OPTIONS="--mergedb --database=$JDRESOLVE_DB"
  82. LOGRESOLVE_BIN="/usr/sbin/logresolve"
  83. ANALOG_BIN="/usr/bin/analog"
  84. RMAGIC_BIN="/usr/bin/rmagic"
  85. WEBALIZER_BIN="/usr/bin/webalizer"
  86. MODLOGAN_BIN="/usr/bin/modlogan"
  87. AWSTATS_BIN="/usr/lib/cgi-bin/awstats.pl"
  88. function awstats_setlog() { sed -e "s!^\(LogFile=\).*\$!\\1$2!" $1 > $1.tmp; mv $1.tmp $1; }
  89. # Webiste/independent checks
  90. test -d $WEBROOT || exit1 "Webroot \"$WEBROOT\" doesn't exist"
  91. test -d $LOGROOT || exit1 "Logroot \"$LOGROOT\" doesn't exist"
  92. if [ -x $JDRESOLVE_BIN ]; then
  93. touch $JDRESOLVE_DB || exit1 "Couldn't touch JDRESOLVE_DB"
  94. fi
  95. # Generate stats for websites from stdin or all default sites
  96. WEBSITES=$@
  97. if [ "$WEBSITES" = "" ]; then
  98. WEBSITES=$(webdirs $LOGROOT $WEBROOT $WEBSITE)
  99. fi
  100. for WEBSITE in $WEBSITES; do
  101. STATSDIR=$(statsdir $LOGROOT $WEBROOT $WEBSITE) || exit1 "Unable to resolve STATSDIR."
  102. # FIXME test -d $STATSDIR/.. || exit1 "Directory above STATSDIR doesn't exist."
  103. ANALOG_CFG=$(analog_cfg $LOGROOT $WEBROOT $WEBSITE) || exit1 "Unable to resolve ANALOG_CFG."
  104. RMAGIC_CFG=$(rmagic_cfg $LOGROOT $WEBROOT $WEBSITE) || exit1 "Unable to resolve RMAGIC_CFG."
  105. WEBALIZER_CFG=$(webalizer_cfg $LOGROOT $WEBROOT $WEBSITE) || exit1 "Unable to resolve WEBALIZER_CFG."
  106. MODLOGAN_CFG=$(modlogan_cfg $LOGROOT $WEBROOT $WEBSITE) || exit1 "Unable to resolve MODLOGAN_CFG."
  107. if [ $stamp = "init" ]; then
  108. [ $DEBUG ] && echo "Execute $WEBSITE PRE_INIT"
  109. pre_init $LOGROOT $WEBROOT $WEBSITE || exit1 "Error executing PRE_INIT."
  110. fi
  111. if [ $stamp = "update" ]; then
  112. [ $DEBUG ] && echo "Execute $WEBSITE PRE_UPDATE"
  113. pre_update $LOGROOT $WEBROOT $WEBSITE || exit1 "Error executing PRE_UPDATE."
  114. fi
  115. HOST=$(host $LOGROOT $WEBROOT $WEBSITE) || exit1 "Unable to get hostname for virtual host."
  116. DOMAIN=$(domain $LOGROOT $WEBROOT $WEBSITE) || exit1 "Unable to get domainname for virtual host."
  117. ANALOG_OPTIONS="-G +g/etc/analog.conf +g$ANALOG_CFG +A -a"
  118. RMAGIC_OPTIONS=""
  119. WEBALIZER_OPTIONS="-c /etc/webalizer.conf -c $WEBALIZER_CFG -o $STATSDIR/webalizer -f"
  120. MODLOGAN_OPTIONS="-c $MODLOGAN_CFG"
  121. AWSTATS_OPTIONS="-update"
  122. FQDN_ESC=`echo "$HOST.$DOMAIN" | sed -e 's/\./\\\./g'` # needed for awstats config
  123. if [ $DEBUG ]; then
  124. echo "Making stats for $WEBSITE in $STATSDIR:"
  125. JDRESOLVE_OPTIONS="$JDRESOLVE_OPTIONS -p"
  126. ANALOG_OPTIONS="$ANALOG_OPTIONS +q"
  127. RMAGIC_OPTIONS="$RMAGIC_OPTIONS"
  128. WEBALIZER_OPTIONS="$WEBALIZER_OPTIONS -T"
  129. MODLOGAN_OPTIONS="$MODLOGAN_OPTIONS"
  130. AWSTATS_OPTIONS="$AWSTATS_OPTIONS -showsteps"
  131. else
  132. JDRESOLVE_OPTIONS="$JDRESOLVE_DB -n"
  133. ANALOG_OPTIONS="$ANALOG_OPTIONS -q"
  134. RMAGIC_OPTIONS="$RMAGIC_OPTIONS -statistics_Verbose=NONE"
  135. WEBALIZER_OPTIONS="$WEBALIZER_OPTIONS -Q"
  136. MODLOGAN_OPTIONS="$MODLOGAN_OPTIONS"
  137. AWSTATS_OPTIONS="$AWSTATS_OPTIONS"
  138. fi
  139. if [ $stamp = "init" ]; then
  140. [ $DEBUG ] && echo "$WEBSITE: Purge STATSDIR"
  141. rm -rf $STATSDIR
  142. mkdir $STATSDIR
  143. fi
  144. LOGDATARESOLVED="$STATSDIR/rawlog_old.txt"
  145. LOGDATA="$STATSDIR/rawlog_new.txt"
  146. LOGDATATMP="$STATSDIR/rawlog_incoming.txt"
  147. touch $LOGDATARESOLVED $LOGDATA $LOGDATATMP || exit1 "Couldn't touch LOGDATA files."
  148. #FIXME run init if folders doesn't exist
  149. if [ $stamp = "init" ]; then
  150. logcontentresolved $LOGROOT $WEBROOT $WEBSITE >> $LOGDATARESOLVED
  151. if [ -x $ANALOG_BIN ]; then
  152. [ $DEBUG ] && echo "$WEBSITE: Create/update analog config"
  153. echo "\
  154. # NB! This file is automatically generated. Do not edit directly!
  155. # Instead, put additions/overrides in $ANALOG_CFG.local
  156. HOSTNAME $HOST.$DOMAIN
  157. HOSTURL http://$HOST.$DOMAIN/
  158. BASEURL http://$HOST.$DOMAIN
  159. LANGUAGE DANISH
  160. "\
  161. > $ANALOG_CFG
  162. [ -s $ANALOG_CFG.local ] && cat $ANALOG_CFG.local >> $ANALOG_CFG
  163. [ -d $STATSDIR/analog ] || mkdir $STATSDIR/analog
  164. [ $DEBUG ] && echo "$WEBSITE: Create initial analog stats"
  165. rm -f $STATSDIR/analog/cache.data
  166. $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED" -C"CACHEOUTFILE $STATSDIR/analog/cache.data" -C"OUTFILE $STATSDIR/analog/index.html"
  167. fi
  168. if [ -x $RMAGIC_BIN -a -x $ANALOG_BIN ]; then
  169. [ $DEBUG ] && echo "$WEBSITE: Create/update Report Magic config"
  170. echo "\
  171. # NB! This file is automatically generated. Do not edit directly!
  172. # Instead, put additions/overrides in $RMAGIC_CFG.local
  173. [statistics]
  174. File_In = $STATSDIR/rmagic/report.dat
  175. Frame_File_Out = $STATSDIR/rmagic/index.html
  176. Language = en
  177. [reports]
  178. File_Out = $STATSDIR/rmagic/
  179. [QUICK]
  180. Rows = ALL
  181. [navigation]
  182. File_Out = navfile.html
  183. "\
  184. > $RMAGIC_CFG
  185. [ -s $RMAGIC_CFG.local ] && $RMAGIC_OPTIONS="$RMAGIC_OPTIONS -statistics_Include=$RMAGIC_CFG.local"
  186. [ -d $STATSDIR/rmagic ] || mkdir $STATSDIR/rmagic
  187. [ $DEBUG ] && echo "$WEBSITE: Create initial Report Magic stats"
  188. $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"CACHEFILE $STATSDIR/analog/cache.data" -C"LANGUAGE ENGLISH" -C"OUTPUT COMPUTER" -C"OUTFILE $STATSDIR/rmagic/report.dat"
  189. $RMAGIC_BIN $RMAGIC_OPTIONS $RMAGIC_CFG
  190. fi
  191. if [ -x $WEBALIZER_BIN ]; then
  192. [ $DEBUG ] && echo "$WEBSITE: Create/update Webalizer config"
  193. echo "\
  194. # NB! This file is automatically generated. Do not edit directly!
  195. # Instead, put additions/overrides in $WEBALIZER_CFG.local
  196. HostName $HOST.$DOMAIN
  197. HideSite *$DOMAIN
  198. HideReferrer $DOMAIN/
  199. "\
  200. > $WEBALIZER_CFG
  201. [ -s $WEBALIZER_CFG.local ] && cat $WEBALIZER_CFG.local >> $WEBALIZER_CFG
  202. [ -d $STATSDIR/webalizer ] || mkdir $STATSDIR/webalizer
  203. [ $DEBUG ] && echo "$WEBSITE: Create initial Webalizer stats"
  204. cat $LOGDATARESOLVED | $WEBALIZER_BIN $WEBALIZER_OPTIONS -N 0 - || true
  205. fi
  206. if [ -x $MODLOGAN_BIN ]; then
  207. [ $DEBUG ] && echo "$WEBSITE: Create/update ModLogAn config"
  208. echo "\
  209. # NB! This file is automatically generated. Do not edit directly!
  210. # Instead, put additions/overrides in $MODLOGAN_CFG.local
  211. [global]
  212. includepath = /etc/modlogan
  213. include = modlogan.def.conf,global
  214. loadplugin = input_clf
  215. loadplugin = output_modlogan
  216. loadplugin = processor_web
  217. outputdir=$STATSDIR/modlogan
  218. incremental = 0
  219. debug_level = 0
  220. [processor_web]
  221. include = modlogan.def.conf,processor_web
  222. searchengines = match.searchengines.conf
  223. debug_searchengines = 0
  224. hidereferrer = \"^http://$HOST.$DOMAIN\"
  225. [output_modlogan]
  226. include = modlogan.def.conf,output_modlogan
  227. hostname = http://$HOST.$DOMAIN
  228. [input_clf]
  229. include = match.os.conf,matchos
  230. include = match.ua.conf,matchua
  231. inputfile = -
  232. "\
  233. > $MODLOGAN_CFG
  234. [ -s $MODLOGAN_CFG.local ] && cat $MODLOGAN_CFG.local >> $MODLOGAN_CFG
  235. [ -d $STATSDIR/modlogan ] || mkdir $STATSDIR/modlogan
  236. [ $ROOT ] && chown $user:$group $STATSDIR/modlogan
  237. [ $DEBUG ] && echo "$WEBSITE: Create initial ModLogAn stats"
  238. if [ $ROOT ]; then
  239. cat $LOGDATARESOLVED | su $user -c "$MODLOGAN_BIN $MODLOGAN_OPTIONS" || true
  240. else
  241. cat $LOGDATARESOLVED | $MODLOGAN_BIN $MODLOGAN_OPTIONS || true
  242. fi
  243. fi
  244. if [ -x $AWSTATS_BIN ]; then
  245. [ $DEBUG ] && echo "$WEBSITE: Create/update AWStats config"
  246. echo "\
  247. # NB! This file is automatically generated. Do not edit directly!
  248. # Instead, put additions/overrides in $AWSTATS_CFG.local
  249. LogFile=\"$LOGDATATMP\"
  250. LogFormat=4
  251. DNSLookup=0
  252. DirData=\"$STATSDIR/awstats\"
  253. AllowToUpdateStatsFromBrowser=0
  254. DirCgi=\"http://cgi.jones.dk/cgi-bin\"
  255. DirIcons=\"http://stats.jones.dk/awstats-icon\"
  256. SiteDomain=\"$FQDN_ESC\"
  257. HostAliases=\"$FQDN_ESC\"
  258. Lang=\"dk\"
  259. DirLang=\"/usr/share/awstats/lang\"
  260. DefaultFile=\"index.html\"
  261. SkipHosts=\"\"
  262. SkipFiles=\"\"
  263. ShowLinksOnUrl=1
  264. ShowFlagLinks=0
  265. "\
  266. >/etc/awstats/awstats.$WEBSITE.conf
  267. [ -d $STATSDIR/awstats ] || mkdir $STATSDIR/awstats
  268. [ $DEBUG ] && echo "$WEBSITE: Create initial AWStats stats"
  269. awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATARESOLVED
  270. $AWSTATS_BIN -config=$WEBSITE $AWSTATS_OPTIONS -output > $STATSDIR/awstats/index.html
  271. awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATATMP
  272. fi
  273. [ $DEBUG ] && echo "$WEBSITE: Compress DNS-resolved logdata"
  274. gzip -9 $LOGDATARESOLVED
  275. fi
  276. if [ -x $JDRESOLVE_BIN ]; then
  277. [ $DEBUG ] && echo "$WEBSITE: DNS-resolve new logdata using jdresolve"
  278. # DB access doesn't work currently (no output...)
  279. # logcontent $LOGROOT $WEBROOT $WEBSITE | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA
  280. # jdresolve $JDRESOLVE_MERGE_OPTIONS $LOGDATA
  281. logcontent $LOGROOT $WEBROOT $WEBSITE | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA
  282. elif [ -x $LOGRESOLVE_BIN ]; then
  283. [ $DEBUG ] && echo "$WEBSITE: DNS-resolve new logdata using logresolve"
  284. logcontent $LOGROOT $WEBROOT $WEBSITE | $LOGRESOLVE_BIN >> $LOGDATA
  285. else
  286. [ $DEBUG ] && echo "$WEBSITE: Merging new logdata without DNS-resolving (no resolver found)"
  287. logcontent $LOGROOT $WEBROOT $WEBSITE >> $LOGDATA
  288. fi
  289. if [ -s $LOGDATA ]; then
  290. if [ -x $ANALOG_BIN ]; then
  291. [ $DEBUG ] && echo "$WEBSITE: Update analog stats"
  292. $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"CACHEFILE $STATSDIR/analog/cache.data" -C"LOGFILE $LOGDATA" -C"OUTFILE $STATSDIR/analog/index.html"
  293. fi
  294. if [ -x $RMAGIC_BIN -a -x $ANALOG_BIN ]; then
  295. [ $DEBUG ] && echo "$WEBSITE: Update Report Magic stats"
  296. $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"CACHEFILE $STATSDIR/analog/cache.data" -C"LOGFILE $LOGDATA" -C"LANGUAGE ENGLISH" -C"OUTPUT COMPUTER" -C"OUTFILE $STATSDIR/rmagic/report.dat"
  297. $RMAGIC_BIN $RMAGIC_OPTIONS $RMAGIC_CFG
  298. fi
  299. if [ -x $WEBALIZER_BIN ]; then
  300. [ $DEBUG ] && echo "$WEBSITE: Update Webalog stats"
  301. cat $LOGDATA | $WEBALIZER_BIN $WEBALIZER_OPTIONS -N 0 -
  302. fi
  303. if [ -x $MODLOGAN_BIN ]; then
  304. [ $DEBUG ] && echo "$WEBSITE: Update ModLogAn stats"
  305. if [ $ROOT ]; then
  306. cat $LOGDATA | su $user -c "$MODLOGAN_BIN $MODLOGAN_OPTIONS" >/dev/null
  307. else
  308. cat $LOGDATA | $MODLOGAN_BIN $MODLOGAN_OPTIONS >/dev/null
  309. fi
  310. fi
  311. if [ -x $AWSTATS_BIN ]; then
  312. [ $DEBUG ] && echo "$WEBSITE: Update AWStats stats"
  313. awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATA
  314. $AWSTATS_BIN -config=$WEBSITE $AWSTATS_OPTIONS -output > $STATSDIR/awstats/index.html
  315. awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATATMP
  316. fi
  317. gzip -f9 $LOGDATA
  318. fi
  319. if [ $stamp = "init" ]; then
  320. [ $DEBUG ] && echo "Execute $WEBSITE POST_INIT"
  321. post_init $LOGROOT $WEBROOT $WEBSITE || exit1 "Error executing POST_INIT."
  322. fi
  323. if [ $stamp = "update" ]; then
  324. [ $DEBUG ] && echo "Execute $WEBSITE POST_UPDATE"
  325. post_update $LOGROOT $WEBROOT $WEBSITE || exit1 "Error executing POST_UPDATE."
  326. fi
  327. done