Major overhaul of localwebstats:

  * Correctly split reset and update runs.
  * Improve awstats defaults.
  * Make sure all LOGDATA files are (at least) touched before use.
  * Remove double -Q Webalizer option.
  * Added option "init" that does a reset and overwrites config files to
    default values.
  * Replace inline `cmd` commands with the (as I understand it) newer
    $(cmd) variant.
  * Replace ${var} with the simpler $var.
  * Replace == tests with sh-compliant =.
  * Add variable STATSDIR and remove WEBDIR.
  * Put all binaries as variables in top of script (but below configurable
    ones).
  * No longer make symlinks into website (doesn't work well, and I have
    switched to using a separate stats website instead).

1 files changed, 141 insertions, 96 deletions

diff --git a/localwebstats b/localwebstats
index 0f331f8..3f27ada 100755
--- a/localwebstats
+++ b/localwebstats
@@ -6,7 +6,7 @@
 #set -e
 
 function usage()	{
-	echo "Usage: `basename $0` update|prelogrotate|postlogrotate|reset <website> [<website>...]"
+	echo "Usage: $(basename $0) init|reset|update|prelogrotate|postlogrotate|ignore <website> [<website>...]"
 	echo "  If no website is given, all are attempted"
 	echo "  Tip: Automagically runs when symlinked to /etc/cron.{daily,weekly,monthly}/"
 	exit 1
@@ -19,12 +19,12 @@ function exit1()	{
 }
 
 # automagically configure when run from cron dirs
-case `dirname $0` in
+case $(dirname $0) in
     /etc/cron.daily)
-	stamp=ignore
+	stamp=update
 	;;
     /etc/cron.weekly)
-	stamp=update
+	stamp=ignore
 	;;
     /etc/cron.monthly)
 	stamp=ignore
@@ -36,14 +36,14 @@ case `dirname $0` in
 esac
 
 case "$stamp" in
-    update|prelogrotate|postlogrotate|reset|ignore)
+    init|reset|update|prelogrotate|postlogrotate|ignore)
 	;;
     *)
 	usage
 	;;
 esac
 
-if [ "$stamp" == "ignore" ]; then
+if [ "$stamp" = "ignore" ]; then
 	if [ $DEBUG ]; then
 		echo "Asked to ignore - exiting silently..."
 	fi
@@ -52,13 +52,11 @@ fi
 
 WEBROOT=/var/www
 LOGROOT=/var/log/apache
-WEBALIZER_OPTIONS="-q"
-WEBALIZER_OPTIONS="-Q"
 
-function webdir()		{ echo $1/VIRTUAL/$2; }
+function statsdir()		{ echo $1/VIRTUAL/stats.$(dnsdomainname)/www/$2; }
 function webdirs()		{ echo `find /var/log/apache -type d -print`; }
-#function webdirs()		{ for dir in $(find ${LOGROOT} -type d -print); do echo $dir; done; }
-#function logdir()		{ echo ${LOGROOT}/${WEBSITE}; }
+#function webdirs()		{ for dir in $(find $LOGROOT -type d -print); do echo $dir; done; }
+#function logdir()		{ echo $LOGROOT/$WEBSITE; }
 #function logfiles()		{ $(ls -r $LOGDIR/*-access*.gz) $(ls -r $LOGDIR/access*.??.gz) $(ls -r $LOGDIR/access*.?.gz) $(ls -r $LOGDIR/access*.?); }
 function logfiles()		{ ls $1/$2/????.??.?? | grep -v '\.00$'; }
 function logfilesgz()		{ ls $1/$2/????.??.??.gz | grep -v '\.00\.gz$'; }
@@ -66,131 +64,178 @@ function logfilesresolved()	{ ls $1/$2/????.??.00; }
 function logfilesresolvedgz()	{ ls $1/$2/????.??.00.gz; }
 function host()			{ cat $1/VIRTUAL/$2/hostname || exit1 "Unable to get hostname for virtual host."; }
 function domain()		{ cat $1/VIRTUAL/$2/domainname || exit1 "Unable to get domainname for virtual host."; }
+function analog_cfg()		{ echo /etc/analog_$2.conf; }
+function webalizer_cfg()	{ echo /etc/webalizer_$2.conf; }
 
 # The above can be overridden in /etc/local/www
 
 . /etc/local/www || exit1 "Unable to read local preferences"
 
-# variables too boring to be configurable
+# variables and functions too boring to be configurable
+JDRESOLVE_BIN="/usr/bin/jdresolve"
 JDRESOLVE_DB="/var/cache/jdresolve/hosts.db"
-#JDRESOLVE_OPTIONS="-r -t 5 --database=${JDRESOLVE_DB} --dbfirst --expiredb=48"
-JDRESOLVE_OPTIONS="-r --database=${JDRESOLVE_DB} --dbfirst --expiredb=48"
-JDRESOLVE_MERGE_OPTIONS="--mergedb --database=${JDRESOLVE_DB}"
-
-test -d ${WEBROOT} || exit1 "Webroot \"${WEBROOT}\" doesn't exist"
-test -d ${LOGROOT} || exit1 "Logroot \"${LOGROOT}\" doesn't exist"
+#JDRESOLVE_OPTIONS="-r -t 5 --database=$JDRESOLVE_DB --dbfirst --expiredb=48"
+JDRESOLVE_OPTIONS="-r --database=$JDRESOLVE_DB --dbfirst --expiredb=48"
+JDRESOLVE_MERGE_OPTIONS="--mergedb --database=$JDRESOLVE_DB"
+LOGRESOLVE_BIN="/usr/sbin/logresolve"
+ANALOG_BIN="/usr/bin/analog"
+WEBALIZER_BIN="/usr/bin/webalizer"
+AWSTATS_BIN="/usr/lib/cgi-bin/awstats.pl"
+
+function awstats_setlog()	{ sed -e "s!^\(LogFile=\).*\$!\\1$2!" $1 > $1.tmp; mv $1.tmp $1; }
+
+# Webiste/independent checks
+test -d $WEBROOT || exit1 "Webroot \"$WEBROOT\" doesn't exist"
+test -d $LOGROOT || exit1 "Logroot \"$LOGROOT\" doesn't exist"
+if [ -x $JDRESOLVE_BIN ]; then
+	touch $JDRESOLVE_DB || exit1 "Couldn't touch JDRESOLVE_DB"
+fi
 
+# Generate stats for websites from stdin or all default sites
 WEBSITES=$@
-if [ "$WEBSITES" == "" ]; then
+if [ "$WEBSITES" = "" ]; then
 	WEBSITES=$(webdirs)
 fi
 
 for WEBSITE in $WEBSITES; do
-	WEBDIR=$(webdir $WEBROOT ${WEBSITE}) || exit1 "Unable to resolve WEBDIR."
-
-	test -d ${WEBDIR} || exit1 "Webdir \"${WEBDIR}\" doesn't exist"
+	STATSDIR=$(statsdir $WEBROOT $WEBSITE) || exit1 "Unable to resolve STATSDIR."
+# FIXME	test -d $STATSDIR/.. || exit1 "Directory above STATSDIR doesn't exist."
 
-	HOST=$(host ${WEBROOT} ${WEBSITE}) || exit1 "Unable to get hostname for virtual host."
-	DOMAIN=$(domain ${WEBROOT} ${WEBSITE}) || exit1 "Unable to get domainname for virtual host."
-	LOGFILES=$(logfiles ${LOGROOT} ${WEBSITE})
-	LOGFILESGZ=$(logfilesgz ${LOGROOT} ${WEBSITE})
-	LOGFILESRESOLVED=$(logfilesresolved ${LOGROOT} ${WEBSITE})
-	LOGFILESRESOLVEDGZ=$(logfilesresolvedgz ${LOGROOT} ${WEBSITE})
+	ANALOG_CFG=$(analog_cfg $WEBROOT $WEBSITE) || exit1 "Unable to resolve ANALOG_CFG."
+	WEBALIZER_CFG=$(webalizer_cfg $WEBROOT $WEBSITE) || exit1 "Unable to resolve WEBALIZER_CFG."
 
-	if [ -x /usr/bin/jdresolve ]; then
-		[ ${DEBUG} ] && echo "Checking access to ${JDRESOLVE_DB}"
-		touch ${JDRESOLVE_DB} || exit1 "Couldn't touch JDRESOLVE_DB"
-	fi
+	HOST=$(host $WEBROOT $WEBSITE) || exit1 "Unable to get hostname for virtual host."
+	DOMAIN=$(domain $WEBROOT $WEBSITE) || exit1 "Unable to get domainname for virtual host."
+	LOGFILES=$(logfiles $LOGROOT $WEBSITE)
+	LOGFILESGZ=$(logfilesgz $LOGROOT $WEBSITE)
+	LOGFILESRESOLVED=$(logfilesresolved $LOGROOT $WEBSITE)
+	LOGFILESRESOLVEDGZ=$(logfilesresolvedgz $LOGROOT $WEBSITE)
 
-	WEBALIZER_OPTIONS="-c /etc/webalizer.conf -c $WEBDIR/webalizer.conf -o $WEBDIR/stats/webalizer -f"
-	ANALOG_OPTIONS="-G +g/etc/analog.conf +g$WEBDIR/analog.conf -a"
+	ANALOG_OPTIONS="-G +g/etc/analog.conf +g$ANALOG_CFG +A -a"
+	WEBALIZER_OPTIONS="-c /etc/webalizer.conf -c $WEBALIZER_CFG -o $STATSDIR/webalizer -f"
 	AWSTATS_OPTIONS="-update"
 	FQDN_ESC=`echo "$HOST.$DOMAIN" | sed -e 's/\./\\\./g'` # needed for awstats config
 
-	if [ ${DEBUG} ]; then
-		echo "Making stats for ${WEBSITE} in ${WEBDIR}:"
-		JDRESOLVE_OPTIONS="${JDRESOLVE_OPTIONS} -p"
-		ANALOG_OPTIONS="${ANALOG_OPTIONS} +q"
-		WEBALIZER_OPTIONS="${WEBALIZER_OPTIONS} -T"
-		AWSTATS_OPTIONS="${AWSTATS_OPTIONS} -showsteps"
+	if [ $DEBUG ]; then
+		echo "Making stats for $WEBSITE in $STATSDIR:"
+		JDRESOLVE_OPTIONS="$JDRESOLVE_OPTIONS -p"
+		ANALOG_OPTIONS="$ANALOG_OPTIONS +q"
+		WEBALIZER_OPTIONS="$WEBALIZER_OPTIONS -T"
+		AWSTATS_OPTIONS="$AWSTATS_OPTIONS -showsteps"
 	else
-		JDRESOLVE_OPTIONS="${JDRESOLVE_DB} -n"
-		ANALOG_OPTIONS="${ANALOG_OPTIONS} -q"
-		WEBALIZER_OPTIONS="${WEBALIZER_OPTIONS} -Q"
-		AWSTATS_OPTIONS="${AWSTATS_OPTIONS}"
+		JDRESOLVE_OPTIONS="$JDRESOLVE_DB -n"
+		ANALOG_OPTIONS="$ANALOG_OPTIONS -q"
+		WEBALIZER_OPTIONS="$WEBALIZER_OPTIONS -Q"
+		AWSTATS_OPTIONS="$AWSTATS_OPTIONS"
 	fi
 
-	if [ $stamp == "reset" ]; then
-		rm -rf $WEBDIR/stats
-		mkdir $WEBDIR/stats
+	if [ $stamp = "init" -o $stamp = "reset" ]; then
+		[ $DEBUG ] && echo "$WEBSITE: Purge STATSDIR"
+		rm -rf $STATSDIR
+		mkdir $STATSDIR
 	fi
 
-	LOGDATA="${WEBDIR}/stats/rawlog.txt"
-	LOGDATATMP="${WEBDIR}/stats/rawlog_tmp.txt"
-	zcat ${LOGFILESRESOLVEDGZ} > ${LOGDATA}
-	cat ${LOGFILESRESOLVED} >> ${LOGDATA}
-	if [ -x /usr/bin/jdresolve ]; then
-		zcat ${LOGFILESGZ} | jdresolve ${JDRESOLVE_OPTIONS} - >> ${LOGDATA}
-		cat ${LOGFILES} | jdresolve ${JDRESOLVE_OPTIONS} - >> ${LOGDATA}
-		jdresolve ${JDRESOLVE_MERGE_OPTIONS} ${LOGDATA}
-	elif [ -x /usr/sbin/logresolve ]; then
-		zcat ${LOGFILESGZ} | logresolve >> ${LOGDATA}
-		cat ${LOGFILES} | logresolve >> ${LOGDATA}
-	else
-		zcat ${LOGFILESGZ} >> ${LOGDATA}
-		cat ${LOGFILES} >> ${LOGDATA}
-	fi
-
-	if [ -x /usr/bin/analog ]; then
-		if [ $stamp == "reset" ]; then
-			test ! -f $WEBDIR/analog.conf && echo "\
+	LOGDATARESOLVED="$STATSDIR/rawlog_resolved.txt"
+	LOGDATA="$STATSDIR/rawlog_unresolved.txt"
+	LOGDATATMP="$STATSDIR/rawlog_new.txt"
+	touch $LOGDATARESOLVED $LOGDATA $LOGDATATMP || exit1 "Couldn't touch LOGDATA files."
+
+	if [ $stamp = "init" -o $stamp = "reset" ]; then
+		for gzlog in $LOGFILESRESOLVEDGZ; do zcat $gzlog >> $LOGDATARESOLVED; done
+		for log in $LOGFILESRESOLVED; do cat $log >> $LOGDATARESOLVED; done
+		if [ -x $ANALOG_BIN ]; then
+			if [ ! -f $ANALOG_CFG -o $stamp = "init" ]; then
+				[ $DEBUG ] && echo "$WEBSITE: Create analog config"
+				echo "\
 HOSTNAME	$HOST.$DOMAIN
+HOSTURL		http://$HOST.$DOMAIN/
+BASEURL		http://$HOST.$DOMAIN
+LANGUAGE	DANISH
+JPEGCHARTS	ON
 "\
-			>$WEBDIR/analog.conf
-			mkdir ${WEBDIR}/stats/analog
-			cat ${LOGDATA} | analog ${ANALOG_OPTIONS} -C"LOGFILE none" -C"LOGFILE stdin" > $WEBDIR/stats/analog/index.html
+				> $ANALOG_CFG
+			fi
+			mkdir $STATSDIR/analog
+			[ $DEBUG ] && echo "$WEBSITE: Create initial analog stats"
+#			$ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED" -C"OUTFILE $STATSDIR/analog/index.html"
+			$ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED" -C"CACHEOUTFILE $STATSDIR/analog/cache.data" -C"OUTPUT NONE"
 		fi
-#		cat ${LOGDATA} | analog ${ANALOG_OPTIONS} -C"LOGFILE none" -C"LOGFILE stdin" > $WEBDIR/stats/analog/index.html
-	fi
-	if [ -x /usr/bin/webalizer ]; then
-
-		if [ $stamp == "reset" ]; then
-			test ! -f $WEBDIR/webalizer.conf && echo "\
+		if [ -x $WEBALIZER_BIN ]; then
+			if [ ! -f $WEBALIZER_CFG -o $stamp = "init" ]; then
+				[ $DEBUG ] && echo "$WEBSITE: Create Webalizer config"
+				echo "\
 HostName	$HOST.$DOMAIN
 HideSite	*$DOMAIN
 HideReferrer	$DOMAIN/
 "\
-			> $WEBDIR/webalizer.conf
-			mkdir $WEBDIR/stats/webalizer
-			cat ${LOGDATA} | webalizer $WEBALIZER_OPTIONS -N 0 -
+				> $WEBALIZER_CFG
+			fi
+			mkdir $STATSDIR/webalizer
+			[ $DEBUG ] && echo "$WEBSITE: Create initial Webalizer stats"
+			cat $LOGDATARESOLVED | $WEBALIZER_BIN $WEBALIZER_OPTIONS -N 0 -
 		fi
-#		cat ${LOGDATA} | webalizer $WEBALIZER_OPTIONS -N 0 -
-	fi
-	if [ -x /usr/lib/cgi-bin/awstats.pl ]; then
-		if [ $stamp == "reset" ]; then
-			test ! -f /etc/awstats/awstats.$WEBSITE.conf && echo "\
-LogFile=\"/var/www/VIRTUAL/$WEBSITE/stats/rawlog.txt\"
+		if [ -x $AWSTATS_BIN ]; then
+			if [ ! -f /etc/awstats/awstats.$WEBSITE.conf -o $stamp = "init" ]; then
+				[ $DEBUG ] && echo "$WEBSITE: Create AWStats config"
+				echo "\
+LogFile=\"$LOGDATARESOLVED\"
 LogFormat=4
 DNSLookup=0
 DirData=\"/var/cache/awstats\"
-DirIcons=\"/awstats-icon\"
+AllowToUpdateStatsFromBrowser=0
+DirCgi=\"http://cgi.jones.dk/cgi-bin\"
+DirIcons=\"http://stats.jones.dk/awstats-icon\"
 SiteDomain=\"$FQDN_ESC\"
-HostAliases=\"$FQDN_ESC localhost 127\.0\.0\.1\"
+HostAliases=\"$FQDN_ESC\"
 Lang=\"dk\"
 DirLang=\"/usr/share/awstats/lang\"
 DefaultFile=\"index.html\"
+SkipHosts=\"\"
+SkipFiles=\"\"
+ShowLinksOnUrl=1
 ShowFlagLinks=0
-AllowToUpdateStatsFromBrowser=0
-ShowLinksOnUrl=0
-DirCgi=\"http://cgi.jones.dk/cgi-bin\"
-DirIcons=\"http://stats.jones.dk/awstats-icon\"
 "\
-			>/etc/awstats/awstats.$WEBSITE.conf
-			mkdir $WEBDIR/stats/awstats
-			/usr/lib/cgi-bin/awstats.pl -config=${WEBSITE} $AWSTATS_OPTIONS -output > $WEBDIR/stats/awstats/index.html
+				>/etc/awstats/awstats.$WEBSITE.conf
+			fi
+			mkdir $STATSDIR/awstats
+			[ $DEBUG ] && echo "$WEBSITE: Create initial AWStats stats"
+			awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATARESOLVED
+			$AWSTATS_BIN -config=$WEBSITE $AWSTATS_OPTIONS -output > $STATSDIR/awstats/index.html
+			awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATA
 		fi
-		/usr/lib/cgi-bin/awstats.pl -config=${WEBSITE} $AWSTATS_OPTIONS -output > $WEBDIR/stats/awstats/index.html
+		[ $DEBUG ] && echo "$WEBSITE: Compress DNS-resolved logdata"
+		gzip -9 $LOGDATARESOLVED
+	fi
+
+	if [ -x $JDRESOLVE_BIN ]; then
+		[ $DEBUG ] && echo "$WEBSITE: DNS-resolve new logdata using jdresolve"
+		for gzlog in $LOGFILESGZ; do zcat $gzlog | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA; done
+		for log in $LOGFILES; do cat $log | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA; done
+		jdresolve $JDRESOLVE_MERGE_OPTIONS $LOGDATA
+	elif [ -x $LOGRESOLVE_BIN ]; then
+		[ $DEBUG ] && echo "$WEBSITE: DNS-resolve new logdata using logresolve"
+		for gzlog in $LOGFILESGZ; do zcat $gzlog | $LOGRESOLVE_BIN >> $LOGDATA; done
+		for log in $LOGFILES; do cat $log | $LOGRESOLVE_BIN >> $LOGDATA; done
+	else
+		[ $DEBUG ] && echo "$WEBSITE: Merging new logdata without DNS-resolving (no resolver found)"
+		for gzlog in $LOGFILESGZ; do zcat $gzlog >> $LOGDATA; done
+		for log in $LOGFILES; do cat $log >> $LOGDATA}; done
+	fi
+
+	if [ -x $ANALOG_BIN ]; then
+		[ $DEBUG ] && echo "$WEBSITE: Update analog stats"
+		# TODO: use incremental update
+		$ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED.gz" -C"LOGFILE $LOGDATA" -C"OUTFILE $STATSDIR/analog/index.html"
+		$ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATA" -C"CACHEFILE $STATSDIR/analog/cache.data" -C"OUTFILE $STATSDIR/analog/index.html"
+	fi
+	if [ -x $WEBALIZER_BIN ]; then
+		[ $DEBUG ] && echo "$WEBSITE: Update Webalog stats"
+		zcat -f $LOGDATA | $WEBALIZER_BIN $WEBALIZER_OPTIONS -N 0 -
+	fi
+	if [ -x $AWSTATS_BIN ]; then
+		[ $DEBUG ] && echo "$WEBSITE: Update AWStats stats"
+		awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATA
+		$AWSTATS_BIN -config=$WEBSITE $AWSTATS_OPTIONS -output > $STATSDIR/awstats/index.html
+		awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATATMP
 	fi
-	test -L $WEBDIR/www/stats || ln -s $WEBDIR/stats $WEBDIR/www/stats
-	test -L $WEBDIR/stats/stats && rm $WEBDIR/stats/stats
 done