From 234e8dee7d1e13635cba17737d0d178fe6208b06 Mon Sep 17 00:00:00 2001 From: Jonas Smedegaard Date: Wed, 9 Jan 2002 01:40:42 +0000 Subject: Several updates to localwebstats: - Include *.local config files on init. Remove option reset. - Improve webdirs() function. - Add logcontent* functions and reorder logfile* functions to reflect the order they are used. - Rename LOGDATA* output filenames to be more obvious. - Make sure output is generated even if last pool of data is empty. - Avoid using DB features of jdresolve - they are broken currently, it seems. --- localwebstats | 122 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 62 insertions(+), 60 deletions(-) (limited to 'localwebstats') diff --git a/localwebstats b/localwebstats index 3f27ada..30a9020 100755 --- a/localwebstats +++ b/localwebstats @@ -6,7 +6,7 @@ #set -e function usage() { - echo "Usage: $(basename $0) init|reset|update|prelogrotate|postlogrotate|ignore [...]" + echo "Usage: $(basename $0) init|update|prelogrotate|postlogrotate|ignore [...]" echo " If no website is given, all are attempted" echo " Tip: Automagically runs when symlinked to /etc/cron.{daily,weekly,monthly}/" exit 1 @@ -36,7 +36,7 @@ case $(dirname $0) in esac case "$stamp" in - init|reset|update|prelogrotate|postlogrotate|ignore) + init|update|prelogrotate|postlogrotate|ignore) ;; *) usage @@ -54,14 +54,15 @@ WEBROOT=/var/www LOGROOT=/var/log/apache function statsdir() { echo $1/VIRTUAL/stats.$(dnsdomainname)/www/$2; } -function webdirs() { echo `find /var/log/apache -type d -print`; } -#function webdirs() { for dir in $(find $LOGROOT -type d -print); do echo $dir; done; } +function webdirs() { find $LOGROOT -type d -mindepth 1 -maxdepth 1 | grep '\.*\.' | sed 's!$LOGROOT!!'; } #function logdir() { echo $LOGROOT/$WEBSITE; } #function logfiles() { $(ls -r $LOGDIR/*-access*.gz) $(ls -r $LOGDIR/access*.??.gz) $(ls -r $LOGDIR/access*.?.gz) $(ls -r $LOGDIR/access*.?); } -function logfiles() { ls $1/$2/????.??.?? | grep -v '\.00$'; } -function logfilesgz() { ls $1/$2/????.??.??.gz | grep -v '\.00\.gz$'; } -function logfilesresolved() { ls $1/$2/????.??.00; } function logfilesresolvedgz() { ls $1/$2/????.??.00.gz; } +function logfilesresolved() { ls $1/$2/????.??.00; } +function logfilesgz() { ls $1/$2/????.??.??.gz | grep -v '\.00\.gz$'; } +function logfiles() { ls $1/$2/????.??.?? | grep -v '\.00$'; } +function logcontentresolved() { for file in `logfilesresolvedgz $1 $2`; do zcat $file; done; for file in `logfilesresolved $1 $2`; do cat $file; done; } +function logcontent() { for file in `logfilesgz $1 $2`; do zcat $file; done; for file in `logfiles $1 $2`; do cat $file; done; } function host() { cat $1/VIRTUAL/$2/hostname || exit1 "Unable to get hostname for virtual host."; } function domain() { cat $1/VIRTUAL/$2/domainname || exit1 "Unable to get domainname for virtual host."; } function analog_cfg() { echo /etc/analog_$2.conf; } @@ -69,13 +70,15 @@ function webalizer_cfg() { echo /etc/webalizer_$2.conf; } # The above can be overridden in /etc/local/www -. /etc/local/www || exit1 "Unable to read local preferences" +. /etc/local/www || exit1 "Unable to read prefs file" # variables and functions too boring to be configurable JDRESOLVE_BIN="/usr/bin/jdresolve" JDRESOLVE_DB="/var/cache/jdresolve/hosts.db" #JDRESOLVE_OPTIONS="-r -t 5 --database=$JDRESOLVE_DB --dbfirst --expiredb=48" -JDRESOLVE_OPTIONS="-r --database=$JDRESOLVE_DB --dbfirst --expiredb=48" +# DB access doesn't work currently (no output...) +#JDRESOLVE_OPTIONS="-r --database=$JDRESOLVE_DB --dbfirst --expiredb=48" +JDRESOLVE_OPTIONS="-r" JDRESOLVE_MERGE_OPTIONS="--mergedb --database=$JDRESOLVE_DB" LOGRESOLVE_BIN="/usr/sbin/logresolve" ANALOG_BIN="/usr/bin/analog" @@ -106,9 +109,6 @@ for WEBSITE in $WEBSITES; do HOST=$(host $WEBROOT $WEBSITE) || exit1 "Unable to get hostname for virtual host." DOMAIN=$(domain $WEBROOT $WEBSITE) || exit1 "Unable to get domainname for virtual host." - LOGFILES=$(logfiles $LOGROOT $WEBSITE) - LOGFILESGZ=$(logfilesgz $LOGROOT $WEBSITE) - LOGFILESRESOLVED=$(logfilesresolved $LOGROOT $WEBSITE) LOGFILESRESOLVEDGZ=$(logfilesresolvedgz $LOGROOT $WEBSITE) ANALOG_OPTIONS="-G +g/etc/analog.conf +g$ANALOG_CFG +A -a" @@ -129,55 +129,58 @@ for WEBSITE in $WEBSITES; do AWSTATS_OPTIONS="$AWSTATS_OPTIONS" fi - if [ $stamp = "init" -o $stamp = "reset" ]; then + if [ $stamp = "init" -o $stamp = "init" ]; then [ $DEBUG ] && echo "$WEBSITE: Purge STATSDIR" rm -rf $STATSDIR mkdir $STATSDIR fi - LOGDATARESOLVED="$STATSDIR/rawlog_resolved.txt" - LOGDATA="$STATSDIR/rawlog_unresolved.txt" - LOGDATATMP="$STATSDIR/rawlog_new.txt" + LOGDATARESOLVED="$STATSDIR/rawlog_old.txt" + LOGDATA="$STATSDIR/rawlog_new.txt" + LOGDATATMP="$STATSDIR/rawlog_incoming.txt" touch $LOGDATARESOLVED $LOGDATA $LOGDATATMP || exit1 "Couldn't touch LOGDATA files." - if [ $stamp = "init" -o $stamp = "reset" ]; then - for gzlog in $LOGFILESRESOLVEDGZ; do zcat $gzlog >> $LOGDATARESOLVED; done - for log in $LOGFILESRESOLVED; do cat $log >> $LOGDATARESOLVED; done + if [ $stamp = "init" ]; then + logcontentresolved $LOGROOT $WEBSITE >> $LOGDATARESOLVED if [ -x $ANALOG_BIN ]; then - if [ ! -f $ANALOG_CFG -o $stamp = "init" ]; then - [ $DEBUG ] && echo "$WEBSITE: Create analog config" - echo "\ + [ $DEBUG ] && echo "$WEBSITE: Create/update analog config" + echo "\ +# NB! This file is automatically generated. Do not edit directly! +# Instead, put additions/overrides in $ANALOG_CFG.local + HOSTNAME $HOST.$DOMAIN HOSTURL http://$HOST.$DOMAIN/ BASEURL http://$HOST.$DOMAIN LANGUAGE DANISH -JPEGCHARTS ON "\ - > $ANALOG_CFG - fi + > $ANALOG_CFG + [ -s $ANALOG_CFG.local ] && cat $ANALOG_CFG.local >> $ANALOG_CFG mkdir $STATSDIR/analog [ $DEBUG ] && echo "$WEBSITE: Create initial analog stats" -# $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED" -C"OUTFILE $STATSDIR/analog/index.html" - $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED" -C"CACHEOUTFILE $STATSDIR/analog/cache.data" -C"OUTPUT NONE" + $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED" -C"CACHEOUTFILE $STATSDIR/analog/cache.data" -C"OUTFILE $STATSDIR/analog/index.html" fi if [ -x $WEBALIZER_BIN ]; then - if [ ! -f $WEBALIZER_CFG -o $stamp = "init" ]; then - [ $DEBUG ] && echo "$WEBSITE: Create Webalizer config" - echo "\ + [ $DEBUG ] && echo "$WEBSITE: Create/update Webalizer config" + echo "\ +# NB! This file is automatically generated. Do not edit directly! +# Instead, put additions/overrides in $WEBALIZER_CFG.local + HostName $HOST.$DOMAIN HideSite *$DOMAIN HideReferrer $DOMAIN/ "\ - > $WEBALIZER_CFG - fi + > $WEBALIZER_CFG + [ -s $WEBALIZER_CFG.local ] && cat $WEBALIZER_CFG.local >> $WEBALIZER_CFG mkdir $STATSDIR/webalizer [ $DEBUG ] && echo "$WEBSITE: Create initial Webalizer stats" cat $LOGDATARESOLVED | $WEBALIZER_BIN $WEBALIZER_OPTIONS -N 0 - fi if [ -x $AWSTATS_BIN ]; then - if [ ! -f /etc/awstats/awstats.$WEBSITE.conf -o $stamp = "init" ]; then - [ $DEBUG ] && echo "$WEBSITE: Create AWStats config" - echo "\ + [ $DEBUG ] && echo "$WEBSITE: Create/update AWStats config" + echo "\ +# NB! This file is automatically generated. Do not edit directly! +# Instead, put additions/overrides in $AWSTATS_CFG.local + LogFile=\"$LOGDATARESOLVED\" LogFormat=4 DNSLookup=0 @@ -195,13 +198,13 @@ SkipFiles=\"\" ShowLinksOnUrl=1 ShowFlagLinks=0 "\ - >/etc/awstats/awstats.$WEBSITE.conf - fi + >/etc/awstats/awstats.$WEBSITE.conf + [ -s $WEBALIZER_CFG.local ] && cat $WEBALIZER_CFG.local >> $WEBALIZER_CFG mkdir $STATSDIR/awstats [ $DEBUG ] && echo "$WEBSITE: Create initial AWStats stats" awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATARESOLVED $AWSTATS_BIN -config=$WEBSITE $AWSTATS_OPTIONS -output > $STATSDIR/awstats/index.html - awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATA + awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATATMP fi [ $DEBUG ] && echo "$WEBSITE: Compress DNS-resolved logdata" gzip -9 $LOGDATARESOLVED @@ -209,33 +212,32 @@ ShowFlagLinks=0 if [ -x $JDRESOLVE_BIN ]; then [ $DEBUG ] && echo "$WEBSITE: DNS-resolve new logdata using jdresolve" - for gzlog in $LOGFILESGZ; do zcat $gzlog | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA; done - for log in $LOGFILES; do cat $log | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA; done - jdresolve $JDRESOLVE_MERGE_OPTIONS $LOGDATA +# DB access doesn't work currently (no output...) +# logcontent $LOGROOT $WEBSITE | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA +# jdresolve $JDRESOLVE_MERGE_OPTIONS $LOGDATA + logcontent $LOGROOT $WEBSITE | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA elif [ -x $LOGRESOLVE_BIN ]; then [ $DEBUG ] && echo "$WEBSITE: DNS-resolve new logdata using logresolve" - for gzlog in $LOGFILESGZ; do zcat $gzlog | $LOGRESOLVE_BIN >> $LOGDATA; done - for log in $LOGFILES; do cat $log | $LOGRESOLVE_BIN >> $LOGDATA; done + logcontent $LOGROOT $WEBSITE | $LOGRESOLVE_BIN >> $LOGDATA else [ $DEBUG ] && echo "$WEBSITE: Merging new logdata without DNS-resolving (no resolver found)" - for gzlog in $LOGFILESGZ; do zcat $gzlog >> $LOGDATA; done - for log in $LOGFILES; do cat $log >> $LOGDATA}; done + logcontent $LOGROOT $WEBSITE >> $LOGDATA fi - if [ -x $ANALOG_BIN ]; then - [ $DEBUG ] && echo "$WEBSITE: Update analog stats" - # TODO: use incremental update - $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED.gz" -C"LOGFILE $LOGDATA" -C"OUTFILE $STATSDIR/analog/index.html" - $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATA" -C"CACHEFILE $STATSDIR/analog/cache.data" -C"OUTFILE $STATSDIR/analog/index.html" - fi - if [ -x $WEBALIZER_BIN ]; then - [ $DEBUG ] && echo "$WEBSITE: Update Webalog stats" - zcat -f $LOGDATA | $WEBALIZER_BIN $WEBALIZER_OPTIONS -N 0 - - fi - if [ -x $AWSTATS_BIN ]; then - [ $DEBUG ] && echo "$WEBSITE: Update AWStats stats" - awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATA - $AWSTATS_BIN -config=$WEBSITE $AWSTATS_OPTIONS -output > $STATSDIR/awstats/index.html - awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATATMP + if [ -s $LOGDATA ]; then + if [ -x $ANALOG_BIN ]; then + [ $DEBUG ] && echo "$WEBSITE: Update analog stats" + $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATA" -C"CACHEFILE $STATSDIR/analog/cache.data" -C"OUTFILE $STATSDIR/analog/index.html" + fi + if [ -x $WEBALIZER_BIN ]; then + [ $DEBUG ] && echo "$WEBSITE: Update Webalog stats" + zcat -f $LOGDATA | $WEBALIZER_BIN $WEBALIZER_OPTIONS -N 0 - + fi + if [ -x $AWSTATS_BIN ]; then + [ $DEBUG ] && echo "$WEBSITE: Update AWStats stats" + awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATA + $AWSTATS_BIN -config=$WEBSITE $AWSTATS_OPTIONS -output > $STATSDIR/awstats/index.html + awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATATMP + fi fi done -- cgit v1.2.3