Several updates to localwebstats:

  - Include *.local config files on init. Remove option reset.
  - Improve webdirs() function.
  - Add logcontent* functions and reorder logfile* functions to reflect the order they are used.
  - Rename LOGDATA* output filenames to be more obvious.
  - Make sure output is generated even if last pool of data is empty.
  - Avoid using DB features of jdresolve - they are broken currently, it seems.

1 files changed, 62 insertions, 60 deletions

diff --git a/localwebstats b/localwebstats
index 3f27ada..30a9020 100755
--- a/localwebstats
+++ b/localwebstats
@@ -6,7 +6,7 @@
 #set -e
 
 function usage()	{
-	echo "Usage: $(basename $0) init|reset|update|prelogrotate|postlogrotate|ignore <website> [<website>...]"
+	echo "Usage: $(basename $0) init|update|prelogrotate|postlogrotate|ignore <website> [<website>...]"
 	echo "  If no website is given, all are attempted"
 	echo "  Tip: Automagically runs when symlinked to /etc/cron.{daily,weekly,monthly}/"
 	exit 1
@@ -36,7 +36,7 @@ case $(dirname $0) in
 esac
 
 case "$stamp" in
-    init|reset|update|prelogrotate|postlogrotate|ignore)
+    init|update|prelogrotate|postlogrotate|ignore)
 	;;
     *)
 	usage
@@ -54,14 +54,15 @@ WEBROOT=/var/www
 LOGROOT=/var/log/apache
 
 function statsdir()		{ echo $1/VIRTUAL/stats.$(dnsdomainname)/www/$2; }
-function webdirs()		{ echo `find /var/log/apache -type d -print`; }
-#function webdirs()		{ for dir in $(find $LOGROOT -type d -print); do echo $dir; done; }
+function webdirs()		{ find $LOGROOT -type d -mindepth 1 -maxdepth 1 | grep '\.*\.' | sed 's!$LOGROOT!!'; }
 #function logdir()		{ echo $LOGROOT/$WEBSITE; }
 #function logfiles()		{ $(ls -r $LOGDIR/*-access*.gz) $(ls -r $LOGDIR/access*.??.gz) $(ls -r $LOGDIR/access*.?.gz) $(ls -r $LOGDIR/access*.?); }
-function logfiles()		{ ls $1/$2/????.??.?? | grep -v '\.00$'; }
-function logfilesgz()		{ ls $1/$2/????.??.??.gz | grep -v '\.00\.gz$'; }
-function logfilesresolved()	{ ls $1/$2/????.??.00; }
 function logfilesresolvedgz()	{ ls $1/$2/????.??.00.gz; }
+function logfilesresolved()	{ ls $1/$2/????.??.00; }
+function logfilesgz()		{ ls $1/$2/????.??.??.gz | grep -v '\.00\.gz$'; }
+function logfiles()		{ ls $1/$2/????.??.?? | grep -v '\.00$'; }
+function logcontentresolved()	{ for file in `logfilesresolvedgz $1 $2`; do zcat $file; done; for file in `logfilesresolved $1 $2`; do cat $file; done; }
+function logcontent()		{ for file in `logfilesgz $1 $2`; do zcat $file; done; for file in `logfiles $1 $2`; do cat $file; done; }
 function host()			{ cat $1/VIRTUAL/$2/hostname || exit1 "Unable to get hostname for virtual host."; }
 function domain()		{ cat $1/VIRTUAL/$2/domainname || exit1 "Unable to get domainname for virtual host."; }
 function analog_cfg()		{ echo /etc/analog_$2.conf; }
@@ -69,13 +70,15 @@ function webalizer_cfg()	{ echo /etc/webalizer_$2.conf; }
 
 # The above can be overridden in /etc/local/www
 
-. /etc/local/www || exit1 "Unable to read local preferences"
+. /etc/local/www || exit1 "Unable to read prefs file"
 
 # variables and functions too boring to be configurable
 JDRESOLVE_BIN="/usr/bin/jdresolve"
 JDRESOLVE_DB="/var/cache/jdresolve/hosts.db"
 #JDRESOLVE_OPTIONS="-r -t 5 --database=$JDRESOLVE_DB --dbfirst --expiredb=48"
-JDRESOLVE_OPTIONS="-r --database=$JDRESOLVE_DB --dbfirst --expiredb=48"
+# DB access doesn't work currently (no output...)
+#JDRESOLVE_OPTIONS="-r --database=$JDRESOLVE_DB --dbfirst --expiredb=48"
+JDRESOLVE_OPTIONS="-r"
 JDRESOLVE_MERGE_OPTIONS="--mergedb --database=$JDRESOLVE_DB"
 LOGRESOLVE_BIN="/usr/sbin/logresolve"
 ANALOG_BIN="/usr/bin/analog"
@@ -106,9 +109,6 @@ for WEBSITE in $WEBSITES; do
 
 	HOST=$(host $WEBROOT $WEBSITE) || exit1 "Unable to get hostname for virtual host."
 	DOMAIN=$(domain $WEBROOT $WEBSITE) || exit1 "Unable to get domainname for virtual host."
-	LOGFILES=$(logfiles $LOGROOT $WEBSITE)
-	LOGFILESGZ=$(logfilesgz $LOGROOT $WEBSITE)
-	LOGFILESRESOLVED=$(logfilesresolved $LOGROOT $WEBSITE)
 	LOGFILESRESOLVEDGZ=$(logfilesresolvedgz $LOGROOT $WEBSITE)
 
 	ANALOG_OPTIONS="-G +g/etc/analog.conf +g$ANALOG_CFG +A -a"
@@ -129,55 +129,58 @@ for WEBSITE in $WEBSITES; do
 		AWSTATS_OPTIONS="$AWSTATS_OPTIONS"
 	fi
 
-	if [ $stamp = "init" -o $stamp = "reset" ]; then
+	if [ $stamp = "init" -o $stamp = "init" ]; then
 		[ $DEBUG ] && echo "$WEBSITE: Purge STATSDIR"
 		rm -rf $STATSDIR
 		mkdir $STATSDIR
 	fi
 
-	LOGDATARESOLVED="$STATSDIR/rawlog_resolved.txt"
-	LOGDATA="$STATSDIR/rawlog_unresolved.txt"
-	LOGDATATMP="$STATSDIR/rawlog_new.txt"
+	LOGDATARESOLVED="$STATSDIR/rawlog_old.txt"
+	LOGDATA="$STATSDIR/rawlog_new.txt"
+	LOGDATATMP="$STATSDIR/rawlog_incoming.txt"
 	touch $LOGDATARESOLVED $LOGDATA $LOGDATATMP || exit1 "Couldn't touch LOGDATA files."
 
-	if [ $stamp = "init" -o $stamp = "reset" ]; then
-		for gzlog in $LOGFILESRESOLVEDGZ; do zcat $gzlog >> $LOGDATARESOLVED; done
-		for log in $LOGFILESRESOLVED; do cat $log >> $LOGDATARESOLVED; done
+	if [ $stamp = "init" ]; then
+		logcontentresolved $LOGROOT $WEBSITE >> $LOGDATARESOLVED
 		if [ -x $ANALOG_BIN ]; then
-			if [ ! -f $ANALOG_CFG -o $stamp = "init" ]; then
-				[ $DEBUG ] && echo "$WEBSITE: Create analog config"
-				echo "\
+			[ $DEBUG ] && echo "$WEBSITE: Create/update analog config"
+			echo "\
+# NB! This file is automatically generated. Do not edit directly!
+# Instead, put additions/overrides in $ANALOG_CFG.local
+
 HOSTNAME	$HOST.$DOMAIN
 HOSTURL		http://$HOST.$DOMAIN/
 BASEURL		http://$HOST.$DOMAIN
 LANGUAGE	DANISH
-JPEGCHARTS	ON
 "\
-				> $ANALOG_CFG
-			fi
+			> $ANALOG_CFG
+			[ -s $ANALOG_CFG.local ] && cat $ANALOG_CFG.local >> $ANALOG_CFG
 			mkdir $STATSDIR/analog
 			[ $DEBUG ] && echo "$WEBSITE: Create initial analog stats"
-#			$ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED" -C"OUTFILE $STATSDIR/analog/index.html"
-			$ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED" -C"CACHEOUTFILE $STATSDIR/analog/cache.data" -C"OUTPUT NONE"
+			$ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED" -C"CACHEOUTFILE $STATSDIR/analog/cache.data" -C"OUTFILE $STATSDIR/analog/index.html"
 		fi
 		if [ -x $WEBALIZER_BIN ]; then
-			if [ ! -f $WEBALIZER_CFG -o $stamp = "init" ]; then
-				[ $DEBUG ] && echo "$WEBSITE: Create Webalizer config"
-				echo "\
+			[ $DEBUG ] && echo "$WEBSITE: Create/update Webalizer config"
+			echo "\
+# NB! This file is automatically generated. Do not edit directly!
+# Instead, put additions/overrides in $WEBALIZER_CFG.local
+
 HostName	$HOST.$DOMAIN
 HideSite	*$DOMAIN
 HideReferrer	$DOMAIN/
 "\
-				> $WEBALIZER_CFG
-			fi
+			> $WEBALIZER_CFG
+			[ -s $WEBALIZER_CFG.local ] && cat $WEBALIZER_CFG.local >> $WEBALIZER_CFG
 			mkdir $STATSDIR/webalizer
 			[ $DEBUG ] && echo "$WEBSITE: Create initial Webalizer stats"
 			cat $LOGDATARESOLVED | $WEBALIZER_BIN $WEBALIZER_OPTIONS -N 0 -
 		fi
 		if [ -x $AWSTATS_BIN ]; then
-			if [ ! -f /etc/awstats/awstats.$WEBSITE.conf -o $stamp = "init" ]; then
-				[ $DEBUG ] && echo "$WEBSITE: Create AWStats config"
-				echo "\
+			[ $DEBUG ] && echo "$WEBSITE: Create/update AWStats config"
+			echo "\
+# NB! This file is automatically generated. Do not edit directly!
+# Instead, put additions/overrides in $AWSTATS_CFG.local
+
 LogFile=\"$LOGDATARESOLVED\"
 LogFormat=4
 DNSLookup=0
@@ -195,13 +198,13 @@ SkipFiles=\"\"
 ShowLinksOnUrl=1
 ShowFlagLinks=0
 "\
-				>/etc/awstats/awstats.$WEBSITE.conf
-			fi
+			>/etc/awstats/awstats.$WEBSITE.conf
+			[ -s $WEBALIZER_CFG.local ] && cat $WEBALIZER_CFG.local >> $WEBALIZER_CFG
 			mkdir $STATSDIR/awstats
 			[ $DEBUG ] && echo "$WEBSITE: Create initial AWStats stats"
 			awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATARESOLVED
 			$AWSTATS_BIN -config=$WEBSITE $AWSTATS_OPTIONS -output > $STATSDIR/awstats/index.html
-			awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATA
+			awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATATMP
 		fi
 		[ $DEBUG ] && echo "$WEBSITE: Compress DNS-resolved logdata"
 		gzip -9 $LOGDATARESOLVED
@@ -209,33 +212,32 @@ ShowFlagLinks=0
 
 	if [ -x $JDRESOLVE_BIN ]; then
 		[ $DEBUG ] && echo "$WEBSITE: DNS-resolve new logdata using jdresolve"
-		for gzlog in $LOGFILESGZ; do zcat $gzlog | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA; done
-		for log in $LOGFILES; do cat $log | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA; done
-		jdresolve $JDRESOLVE_MERGE_OPTIONS $LOGDATA
+# DB access doesn't work currently (no output...)
+#		logcontent $LOGROOT $WEBSITE | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA
+#		jdresolve $JDRESOLVE_MERGE_OPTIONS $LOGDATA
+		logcontent $LOGROOT $WEBSITE | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA
 	elif [ -x $LOGRESOLVE_BIN ]; then
 		[ $DEBUG ] && echo "$WEBSITE: DNS-resolve new logdata using logresolve"
-		for gzlog in $LOGFILESGZ; do zcat $gzlog | $LOGRESOLVE_BIN >> $LOGDATA; done
-		for log in $LOGFILES; do cat $log | $LOGRESOLVE_BIN >> $LOGDATA; done
+		logcontent $LOGROOT $WEBSITE | $LOGRESOLVE_BIN >> $LOGDATA
 	else
 		[ $DEBUG ] && echo "$WEBSITE: Merging new logdata without DNS-resolving (no resolver found)"
-		for gzlog in $LOGFILESGZ; do zcat $gzlog >> $LOGDATA; done
-		for log in $LOGFILES; do cat $log >> $LOGDATA}; done
+		logcontent $LOGROOT $WEBSITE >> $LOGDATA
 	fi
 
-	if [ -x $ANALOG_BIN ]; then
-		[ $DEBUG ] && echo "$WEBSITE: Update analog stats"
-		# TODO: use incremental update
-		$ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED.gz" -C"LOGFILE $LOGDATA" -C"OUTFILE $STATSDIR/analog/index.html"
-		$ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATA" -C"CACHEFILE $STATSDIR/analog/cache.data" -C"OUTFILE $STATSDIR/analog/index.html"
-	fi
-	if [ -x $WEBALIZER_BIN ]; then
-		[ $DEBUG ] && echo "$WEBSITE: Update Webalog stats"
-		zcat -f $LOGDATA | $WEBALIZER_BIN $WEBALIZER_OPTIONS -N 0 -
-	fi
-	if [ -x $AWSTATS_BIN ]; then
-		[ $DEBUG ] && echo "$WEBSITE: Update AWStats stats"
-		awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATA
-		$AWSTATS_BIN -config=$WEBSITE $AWSTATS_OPTIONS -output > $STATSDIR/awstats/index.html
-		awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATATMP
+	if [ -s $LOGDATA ]; then
+		if [ -x $ANALOG_BIN ]; then
+			[ $DEBUG ] && echo "$WEBSITE: Update analog stats"
+			$ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATA" -C"CACHEFILE $STATSDIR/analog/cache.data" -C"OUTFILE $STATSDIR/analog/index.html"
+		fi
+		if [ -x $WEBALIZER_BIN ]; then
+			[ $DEBUG ] && echo "$WEBSITE: Update Webalog stats"
+			zcat -f $LOGDATA | $WEBALIZER_BIN $WEBALIZER_OPTIONS -N 0 -
+		fi
+		if [ -x $AWSTATS_BIN ]; then
+			[ $DEBUG ] && echo "$WEBSITE: Update AWStats stats"
+			awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATA
+			$AWSTATS_BIN -config=$WEBSITE $AWSTATS_OPTIONS -output > $STATSDIR/awstats/index.html
+			awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATATMP
+		fi
 	fi
 done