From 4862fb660c680e846624412c13f96f03a0e61da8 Mon Sep 17 00:00:00 2001 From: Jonas Smedegaard Date: Fri, 4 Jan 2002 23:57:01 +0000 Subject: Major overhaul of localwebstats: * Correctly split reset and update runs. * Improve awstats defaults. * Make sure all LOGDATA files are (at least) touched before use. * Remove double -Q Webalizer option. * Added option "init" that does a reset and overwrites config files to default values. * Replace inline `cmd` commands with the (as I understand it) newer $(cmd) variant. * Replace ${var} with the simpler $var. * Replace == tests with sh-compliant =. * Add variable STATSDIR and remove WEBDIR. * Put all binaries as variables in top of script (but below configurable ones). * No longer make symlinks into website (doesn't work well, and I have switched to using a separate stats website instead). --- localwebstats | 237 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 141 insertions(+), 96 deletions(-) (limited to 'localwebstats') diff --git a/localwebstats b/localwebstats index 0f331f8..3f27ada 100755 --- a/localwebstats +++ b/localwebstats @@ -6,7 +6,7 @@ #set -e function usage() { - echo "Usage: `basename $0` update|prelogrotate|postlogrotate|reset [...]" + echo "Usage: $(basename $0) init|reset|update|prelogrotate|postlogrotate|ignore [...]" echo " If no website is given, all are attempted" echo " Tip: Automagically runs when symlinked to /etc/cron.{daily,weekly,monthly}/" exit 1 @@ -19,12 +19,12 @@ function exit1() { } # automagically configure when run from cron dirs -case `dirname $0` in +case $(dirname $0) in /etc/cron.daily) - stamp=ignore + stamp=update ;; /etc/cron.weekly) - stamp=update + stamp=ignore ;; /etc/cron.monthly) stamp=ignore @@ -36,14 +36,14 @@ case `dirname $0` in esac case "$stamp" in - update|prelogrotate|postlogrotate|reset|ignore) + init|reset|update|prelogrotate|postlogrotate|ignore) ;; *) usage ;; esac -if [ "$stamp" == "ignore" ]; then +if [ "$stamp" = "ignore" ]; then if [ $DEBUG ]; then echo "Asked to ignore - exiting silently..." fi @@ -52,13 +52,11 @@ fi WEBROOT=/var/www LOGROOT=/var/log/apache -WEBALIZER_OPTIONS="-q" -WEBALIZER_OPTIONS="-Q" -function webdir() { echo $1/VIRTUAL/$2; } +function statsdir() { echo $1/VIRTUAL/stats.$(dnsdomainname)/www/$2; } function webdirs() { echo `find /var/log/apache -type d -print`; } -#function webdirs() { for dir in $(find ${LOGROOT} -type d -print); do echo $dir; done; } -#function logdir() { echo ${LOGROOT}/${WEBSITE}; } +#function webdirs() { for dir in $(find $LOGROOT -type d -print); do echo $dir; done; } +#function logdir() { echo $LOGROOT/$WEBSITE; } #function logfiles() { $(ls -r $LOGDIR/*-access*.gz) $(ls -r $LOGDIR/access*.??.gz) $(ls -r $LOGDIR/access*.?.gz) $(ls -r $LOGDIR/access*.?); } function logfiles() { ls $1/$2/????.??.?? | grep -v '\.00$'; } function logfilesgz() { ls $1/$2/????.??.??.gz | grep -v '\.00\.gz$'; } @@ -66,131 +64,178 @@ function logfilesresolved() { ls $1/$2/????.??.00; } function logfilesresolvedgz() { ls $1/$2/????.??.00.gz; } function host() { cat $1/VIRTUAL/$2/hostname || exit1 "Unable to get hostname for virtual host."; } function domain() { cat $1/VIRTUAL/$2/domainname || exit1 "Unable to get domainname for virtual host."; } +function analog_cfg() { echo /etc/analog_$2.conf; } +function webalizer_cfg() { echo /etc/webalizer_$2.conf; } # The above can be overridden in /etc/local/www . /etc/local/www || exit1 "Unable to read local preferences" -# variables too boring to be configurable +# variables and functions too boring to be configurable +JDRESOLVE_BIN="/usr/bin/jdresolve" JDRESOLVE_DB="/var/cache/jdresolve/hosts.db" -#JDRESOLVE_OPTIONS="-r -t 5 --database=${JDRESOLVE_DB} --dbfirst --expiredb=48" -JDRESOLVE_OPTIONS="-r --database=${JDRESOLVE_DB} --dbfirst --expiredb=48" -JDRESOLVE_MERGE_OPTIONS="--mergedb --database=${JDRESOLVE_DB}" - -test -d ${WEBROOT} || exit1 "Webroot \"${WEBROOT}\" doesn't exist" -test -d ${LOGROOT} || exit1 "Logroot \"${LOGROOT}\" doesn't exist" +#JDRESOLVE_OPTIONS="-r -t 5 --database=$JDRESOLVE_DB --dbfirst --expiredb=48" +JDRESOLVE_OPTIONS="-r --database=$JDRESOLVE_DB --dbfirst --expiredb=48" +JDRESOLVE_MERGE_OPTIONS="--mergedb --database=$JDRESOLVE_DB" +LOGRESOLVE_BIN="/usr/sbin/logresolve" +ANALOG_BIN="/usr/bin/analog" +WEBALIZER_BIN="/usr/bin/webalizer" +AWSTATS_BIN="/usr/lib/cgi-bin/awstats.pl" + +function awstats_setlog() { sed -e "s!^\(LogFile=\).*\$!\\1$2!" $1 > $1.tmp; mv $1.tmp $1; } + +# Webiste/independent checks +test -d $WEBROOT || exit1 "Webroot \"$WEBROOT\" doesn't exist" +test -d $LOGROOT || exit1 "Logroot \"$LOGROOT\" doesn't exist" +if [ -x $JDRESOLVE_BIN ]; then + touch $JDRESOLVE_DB || exit1 "Couldn't touch JDRESOLVE_DB" +fi +# Generate stats for websites from stdin or all default sites WEBSITES=$@ -if [ "$WEBSITES" == "" ]; then +if [ "$WEBSITES" = "" ]; then WEBSITES=$(webdirs) fi for WEBSITE in $WEBSITES; do - WEBDIR=$(webdir $WEBROOT ${WEBSITE}) || exit1 "Unable to resolve WEBDIR." - - test -d ${WEBDIR} || exit1 "Webdir \"${WEBDIR}\" doesn't exist" + STATSDIR=$(statsdir $WEBROOT $WEBSITE) || exit1 "Unable to resolve STATSDIR." +# FIXME test -d $STATSDIR/.. || exit1 "Directory above STATSDIR doesn't exist." - HOST=$(host ${WEBROOT} ${WEBSITE}) || exit1 "Unable to get hostname for virtual host." - DOMAIN=$(domain ${WEBROOT} ${WEBSITE}) || exit1 "Unable to get domainname for virtual host." - LOGFILES=$(logfiles ${LOGROOT} ${WEBSITE}) - LOGFILESGZ=$(logfilesgz ${LOGROOT} ${WEBSITE}) - LOGFILESRESOLVED=$(logfilesresolved ${LOGROOT} ${WEBSITE}) - LOGFILESRESOLVEDGZ=$(logfilesresolvedgz ${LOGROOT} ${WEBSITE}) + ANALOG_CFG=$(analog_cfg $WEBROOT $WEBSITE) || exit1 "Unable to resolve ANALOG_CFG." + WEBALIZER_CFG=$(webalizer_cfg $WEBROOT $WEBSITE) || exit1 "Unable to resolve WEBALIZER_CFG." - if [ -x /usr/bin/jdresolve ]; then - [ ${DEBUG} ] && echo "Checking access to ${JDRESOLVE_DB}" - touch ${JDRESOLVE_DB} || exit1 "Couldn't touch JDRESOLVE_DB" - fi + HOST=$(host $WEBROOT $WEBSITE) || exit1 "Unable to get hostname for virtual host." + DOMAIN=$(domain $WEBROOT $WEBSITE) || exit1 "Unable to get domainname for virtual host." + LOGFILES=$(logfiles $LOGROOT $WEBSITE) + LOGFILESGZ=$(logfilesgz $LOGROOT $WEBSITE) + LOGFILESRESOLVED=$(logfilesresolved $LOGROOT $WEBSITE) + LOGFILESRESOLVEDGZ=$(logfilesresolvedgz $LOGROOT $WEBSITE) - WEBALIZER_OPTIONS="-c /etc/webalizer.conf -c $WEBDIR/webalizer.conf -o $WEBDIR/stats/webalizer -f" - ANALOG_OPTIONS="-G +g/etc/analog.conf +g$WEBDIR/analog.conf -a" + ANALOG_OPTIONS="-G +g/etc/analog.conf +g$ANALOG_CFG +A -a" + WEBALIZER_OPTIONS="-c /etc/webalizer.conf -c $WEBALIZER_CFG -o $STATSDIR/webalizer -f" AWSTATS_OPTIONS="-update" FQDN_ESC=`echo "$HOST.$DOMAIN" | sed -e 's/\./\\\./g'` # needed for awstats config - if [ ${DEBUG} ]; then - echo "Making stats for ${WEBSITE} in ${WEBDIR}:" - JDRESOLVE_OPTIONS="${JDRESOLVE_OPTIONS} -p" - ANALOG_OPTIONS="${ANALOG_OPTIONS} +q" - WEBALIZER_OPTIONS="${WEBALIZER_OPTIONS} -T" - AWSTATS_OPTIONS="${AWSTATS_OPTIONS} -showsteps" + if [ $DEBUG ]; then + echo "Making stats for $WEBSITE in $STATSDIR:" + JDRESOLVE_OPTIONS="$JDRESOLVE_OPTIONS -p" + ANALOG_OPTIONS="$ANALOG_OPTIONS +q" + WEBALIZER_OPTIONS="$WEBALIZER_OPTIONS -T" + AWSTATS_OPTIONS="$AWSTATS_OPTIONS -showsteps" else - JDRESOLVE_OPTIONS="${JDRESOLVE_DB} -n" - ANALOG_OPTIONS="${ANALOG_OPTIONS} -q" - WEBALIZER_OPTIONS="${WEBALIZER_OPTIONS} -Q" - AWSTATS_OPTIONS="${AWSTATS_OPTIONS}" + JDRESOLVE_OPTIONS="$JDRESOLVE_DB -n" + ANALOG_OPTIONS="$ANALOG_OPTIONS -q" + WEBALIZER_OPTIONS="$WEBALIZER_OPTIONS -Q" + AWSTATS_OPTIONS="$AWSTATS_OPTIONS" fi - if [ $stamp == "reset" ]; then - rm -rf $WEBDIR/stats - mkdir $WEBDIR/stats + if [ $stamp = "init" -o $stamp = "reset" ]; then + [ $DEBUG ] && echo "$WEBSITE: Purge STATSDIR" + rm -rf $STATSDIR + mkdir $STATSDIR fi - LOGDATA="${WEBDIR}/stats/rawlog.txt" - LOGDATATMP="${WEBDIR}/stats/rawlog_tmp.txt" - zcat ${LOGFILESRESOLVEDGZ} > ${LOGDATA} - cat ${LOGFILESRESOLVED} >> ${LOGDATA} - if [ -x /usr/bin/jdresolve ]; then - zcat ${LOGFILESGZ} | jdresolve ${JDRESOLVE_OPTIONS} - >> ${LOGDATA} - cat ${LOGFILES} | jdresolve ${JDRESOLVE_OPTIONS} - >> ${LOGDATA} - jdresolve ${JDRESOLVE_MERGE_OPTIONS} ${LOGDATA} - elif [ -x /usr/sbin/logresolve ]; then - zcat ${LOGFILESGZ} | logresolve >> ${LOGDATA} - cat ${LOGFILES} | logresolve >> ${LOGDATA} - else - zcat ${LOGFILESGZ} >> ${LOGDATA} - cat ${LOGFILES} >> ${LOGDATA} - fi - - if [ -x /usr/bin/analog ]; then - if [ $stamp == "reset" ]; then - test ! -f $WEBDIR/analog.conf && echo "\ + LOGDATARESOLVED="$STATSDIR/rawlog_resolved.txt" + LOGDATA="$STATSDIR/rawlog_unresolved.txt" + LOGDATATMP="$STATSDIR/rawlog_new.txt" + touch $LOGDATARESOLVED $LOGDATA $LOGDATATMP || exit1 "Couldn't touch LOGDATA files." + + if [ $stamp = "init" -o $stamp = "reset" ]; then + for gzlog in $LOGFILESRESOLVEDGZ; do zcat $gzlog >> $LOGDATARESOLVED; done + for log in $LOGFILESRESOLVED; do cat $log >> $LOGDATARESOLVED; done + if [ -x $ANALOG_BIN ]; then + if [ ! -f $ANALOG_CFG -o $stamp = "init" ]; then + [ $DEBUG ] && echo "$WEBSITE: Create analog config" + echo "\ HOSTNAME $HOST.$DOMAIN +HOSTURL http://$HOST.$DOMAIN/ +BASEURL http://$HOST.$DOMAIN +LANGUAGE DANISH +JPEGCHARTS ON "\ - >$WEBDIR/analog.conf - mkdir ${WEBDIR}/stats/analog - cat ${LOGDATA} | analog ${ANALOG_OPTIONS} -C"LOGFILE none" -C"LOGFILE stdin" > $WEBDIR/stats/analog/index.html + > $ANALOG_CFG + fi + mkdir $STATSDIR/analog + [ $DEBUG ] && echo "$WEBSITE: Create initial analog stats" +# $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED" -C"OUTFILE $STATSDIR/analog/index.html" + $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED" -C"CACHEOUTFILE $STATSDIR/analog/cache.data" -C"OUTPUT NONE" fi -# cat ${LOGDATA} | analog ${ANALOG_OPTIONS} -C"LOGFILE none" -C"LOGFILE stdin" > $WEBDIR/stats/analog/index.html - fi - if [ -x /usr/bin/webalizer ]; then - - if [ $stamp == "reset" ]; then - test ! -f $WEBDIR/webalizer.conf && echo "\ + if [ -x $WEBALIZER_BIN ]; then + if [ ! -f $WEBALIZER_CFG -o $stamp = "init" ]; then + [ $DEBUG ] && echo "$WEBSITE: Create Webalizer config" + echo "\ HostName $HOST.$DOMAIN HideSite *$DOMAIN HideReferrer $DOMAIN/ "\ - > $WEBDIR/webalizer.conf - mkdir $WEBDIR/stats/webalizer - cat ${LOGDATA} | webalizer $WEBALIZER_OPTIONS -N 0 - + > $WEBALIZER_CFG + fi + mkdir $STATSDIR/webalizer + [ $DEBUG ] && echo "$WEBSITE: Create initial Webalizer stats" + cat $LOGDATARESOLVED | $WEBALIZER_BIN $WEBALIZER_OPTIONS -N 0 - fi -# cat ${LOGDATA} | webalizer $WEBALIZER_OPTIONS -N 0 - - fi - if [ -x /usr/lib/cgi-bin/awstats.pl ]; then - if [ $stamp == "reset" ]; then - test ! -f /etc/awstats/awstats.$WEBSITE.conf && echo "\ -LogFile=\"/var/www/VIRTUAL/$WEBSITE/stats/rawlog.txt\" + if [ -x $AWSTATS_BIN ]; then + if [ ! -f /etc/awstats/awstats.$WEBSITE.conf -o $stamp = "init" ]; then + [ $DEBUG ] && echo "$WEBSITE: Create AWStats config" + echo "\ +LogFile=\"$LOGDATARESOLVED\" LogFormat=4 DNSLookup=0 DirData=\"/var/cache/awstats\" -DirIcons=\"/awstats-icon\" +AllowToUpdateStatsFromBrowser=0 +DirCgi=\"http://cgi.jones.dk/cgi-bin\" +DirIcons=\"http://stats.jones.dk/awstats-icon\" SiteDomain=\"$FQDN_ESC\" -HostAliases=\"$FQDN_ESC localhost 127\.0\.0\.1\" +HostAliases=\"$FQDN_ESC\" Lang=\"dk\" DirLang=\"/usr/share/awstats/lang\" DefaultFile=\"index.html\" +SkipHosts=\"\" +SkipFiles=\"\" +ShowLinksOnUrl=1 ShowFlagLinks=0 -AllowToUpdateStatsFromBrowser=0 -ShowLinksOnUrl=0 -DirCgi=\"http://cgi.jones.dk/cgi-bin\" -DirIcons=\"http://stats.jones.dk/awstats-icon\" "\ - >/etc/awstats/awstats.$WEBSITE.conf - mkdir $WEBDIR/stats/awstats - /usr/lib/cgi-bin/awstats.pl -config=${WEBSITE} $AWSTATS_OPTIONS -output > $WEBDIR/stats/awstats/index.html + >/etc/awstats/awstats.$WEBSITE.conf + fi + mkdir $STATSDIR/awstats + [ $DEBUG ] && echo "$WEBSITE: Create initial AWStats stats" + awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATARESOLVED + $AWSTATS_BIN -config=$WEBSITE $AWSTATS_OPTIONS -output > $STATSDIR/awstats/index.html + awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATA fi - /usr/lib/cgi-bin/awstats.pl -config=${WEBSITE} $AWSTATS_OPTIONS -output > $WEBDIR/stats/awstats/index.html + [ $DEBUG ] && echo "$WEBSITE: Compress DNS-resolved logdata" + gzip -9 $LOGDATARESOLVED + fi + + if [ -x $JDRESOLVE_BIN ]; then + [ $DEBUG ] && echo "$WEBSITE: DNS-resolve new logdata using jdresolve" + for gzlog in $LOGFILESGZ; do zcat $gzlog | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA; done + for log in $LOGFILES; do cat $log | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA; done + jdresolve $JDRESOLVE_MERGE_OPTIONS $LOGDATA + elif [ -x $LOGRESOLVE_BIN ]; then + [ $DEBUG ] && echo "$WEBSITE: DNS-resolve new logdata using logresolve" + for gzlog in $LOGFILESGZ; do zcat $gzlog | $LOGRESOLVE_BIN >> $LOGDATA; done + for log in $LOGFILES; do cat $log | $LOGRESOLVE_BIN >> $LOGDATA; done + else + [ $DEBUG ] && echo "$WEBSITE: Merging new logdata without DNS-resolving (no resolver found)" + for gzlog in $LOGFILESGZ; do zcat $gzlog >> $LOGDATA; done + for log in $LOGFILES; do cat $log >> $LOGDATA}; done + fi + + if [ -x $ANALOG_BIN ]; then + [ $DEBUG ] && echo "$WEBSITE: Update analog stats" + # TODO: use incremental update + $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATARESOLVED.gz" -C"LOGFILE $LOGDATA" -C"OUTFILE $STATSDIR/analog/index.html" + $ANALOG_BIN $ANALOG_OPTIONS -C"LOGFILE none" -C"LOGFILE $LOGDATA" -C"CACHEFILE $STATSDIR/analog/cache.data" -C"OUTFILE $STATSDIR/analog/index.html" + fi + if [ -x $WEBALIZER_BIN ]; then + [ $DEBUG ] && echo "$WEBSITE: Update Webalog stats" + zcat -f $LOGDATA | $WEBALIZER_BIN $WEBALIZER_OPTIONS -N 0 - + fi + if [ -x $AWSTATS_BIN ]; then + [ $DEBUG ] && echo "$WEBSITE: Update AWStats stats" + awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATA + $AWSTATS_BIN -config=$WEBSITE $AWSTATS_OPTIONS -output > $STATSDIR/awstats/index.html + awstats_setlog /etc/awstats/awstats.$WEBSITE.conf $LOGDATATMP fi - test -L $WEBDIR/www/stats || ln -s $WEBDIR/stats $WEBDIR/www/stats - test -L $WEBDIR/stats/stats && rm $WEBDIR/stats/stats done -- cgit v1.2.3