summaryrefslogtreecommitdiff
path: root/localwebresolve
blob: a8252a9fc1558ea793d15273db38968c644dd28d (plain)
  1. #!/bin/sh
  2. #
  3. # /usr/local/bin/localwebresolve
  4. # Copyright 2002-2006 Jonas Smedegaard <dr@jones.dk>
  5. #
  6. # $Id: localwebresolve,v 1.14 2006-07-16 10:32:35 jonas Exp $
  7. #
  8. # Webstats maintenance script
  9. #
  10. # Hint: The following works within Midnight Commander (mc):
  11. # for y in 2003 2004; do for m in $(seq -w 1 12); do localwebresolve $y.$m $(basename $(pwd)); done; done
  12. #
  13. set -e
  14. function usage() {
  15. echo "Usage: $(basename $0) <YYYY.MM> [<website> [<website>...]]"
  16. echo " If no website is given, all are attempted"
  17. exit 1
  18. }
  19. function exit1() {
  20. echo "Error: $1"
  21. echo "Exiting..."
  22. exit 1
  23. }
  24. case $1 in
  25. ????.??)
  26. YEARDOTMONTH=$1
  27. ;;
  28. *)
  29. usage
  30. ;;
  31. esac
  32. shift
  33. ROOT=1
  34. user=www-data
  35. group=www-data
  36. LOGROOT=/var/log/apache-vhosts
  37. WEBROOT=/var/www
  38. # Options: $1=LOGROOT, $2=WEBROOT, $3=WEBSITE, $4=YEARDOTMONTH
  39. function resolveddir() { echo $1/$3; }
  40. function webdirs() { find $1 -type d -mindepth 1 -maxdepth 1 | grep '\.*\.' | sed 's!$1!!'; }
  41. #function logfiles() { $(ls -r $LOGDIR/*-access*.gz) $(ls -r $LOGDIR/access*.??.gz) $(ls -r $LOGDIR/access*.?.gz) $(ls -r $LOGDIR/access*.?); }
  42. function logcontent() { for file in $(find $1/$3 -name "$YEARDOTMONTH.??.gz" ! -name '*00.gz' -type f -mindepth 1 -maxdepth 1 -follow | sort); do zcat $file; done; for file in $(find $1/$3 -name "$YEARDOTMONTH.??" ! -name '*00' -type f -mindepth 1 -maxdepth 1 -follow | sort); do cat $file; done; }
  43. function logremove() { for file in $(find $1/$3 -name "$YEARDOTMONTH.??.gz" ! -name '*00.gz' -type f -mindepth 1 -maxdepth 1 -follow | sort); do rm -f $file; done; for file in $(find $1/$3 -name "$YEARDOTMONTH.??" ! -name '*00' -type f -mindepth 1 -maxdepth 1 -follow | sort); do rm -f $file; done; }
  44. # The above can be overridden
  45. LOCALCONFIG=/etc/local/webresolve.conf
  46. . $LOCALCONFIG || exit1 "Unable to read local config file $LOCALCONFIG"
  47. # variables and functions too boring to be configurable
  48. JDRESOLVE_BIN="/usr/bin/jdresolve"
  49. JDRESOLVE_OPTIONS="-r -t 5"
  50. JDRESOLVE_DB="/var/cache/jdresolve/hosts.db"
  51. JDRESOLVE_EXPIRY="48"
  52. LOGRESOLVE_BIN="/usr/sbin/logresolve"
  53. # Website/independent checks
  54. test -d $WEBROOT || exit1 "Webroot \"$WEBROOT\" doesn't exist"
  55. test -d $LOGROOT || exit1 "Logroot \"$LOGROOT\" doesn't exist"
  56. if [ -x "$JDRESOLVE_BIN" -a -n "$JDRESOLVE_DB" ]; then
  57. JDRESOLVE_OPTIONS="$JDRESOLVE_OPTIONS --database=$JDRESOLVE_DB --dbfirst"
  58. test -d $(dirname "$JDRESOLVE_DB") || exit1 "Cache dir for jdresolve doesn't exist"
  59. if [ -n "$JDRESOLVE_EXPIRY" ]; then
  60. if [ $DEBUG ]; then
  61. find "$JDRESOLVE_DB" -mtime +1 -exec jdresolve $JDRESOLVE_OPTIONS --expiredb="$JDRESOLVE_EXPIRY" ';'
  62. else
  63. find "$JDRESOLVE_DB" -mtime +1 -exec jdresolve $JDRESOLVE_OPTIONS --expiredb="$JDRESOLVE_EXPIRY" ';' > /dev/null 2>&1
  64. fi
  65. fi
  66. fi
  67. # Resolve logs for websites from stdin or all default sites
  68. WEBSITES=$@
  69. if [ "$WEBSITES" = "" ]; then
  70. WEBSITES=$(webdirs $LOGROOT $WEBROOT $WEBSITE)
  71. fi
  72. for WEBSITE in $WEBSITES; do
  73. RESOLVEDDIR=$(resolveddir $LOGROOT $WEBROOT $WEBSITE) || exit1 "Unable to resolve RESOLVEDDIR."
  74. # FIXME test -d $RESOLVEDDIR/.. || exit1 "Directory above RESOLVEDDIR doesn't exist."
  75. if [ $DEBUG ]; then
  76. echo "Resolving logfiles $YEARDOTMONTH for $WEBSITE in $RESOLVEDDIR:"
  77. JDRESOLVE_OPTIONS="$JDRESOLVE_OPTIONS -p"
  78. else
  79. JDRESOLVE_OPTIONS="$JDRESOLVE_OPTIONS -n"
  80. fi
  81. LOGDATA="$RESOLVEDDIR/$YEARDOTMONTH.00"
  82. [ -e $LOGDATA ] && exit1 "File $LOGDATA exists already."
  83. [ -e $LOGDATA.gz ] && exit1 "File $LOGDATA exists (compressed) already."
  84. touch $LOGDATA || exit1 "Couldn't touch LOGDATA files."
  85. if [ -x $JDRESOLVE_BIN ]; then
  86. [ $DEBUG ] && echo "$WEBSITE: DNS-resolve new logdata using jdresolve"
  87. logcontent $LOGROOT $WEBROOT $WEBSITE | $JDRESOLVE_BIN $JDRESOLVE_OPTIONS - >> $LOGDATA
  88. elif [ -x $LOGRESOLVE_BIN ]; then
  89. [ $DEBUG ] && echo "$WEBSITE: DNS-resolve new logdata using logresolve"
  90. logcontent $LOGROOT $WEBROOT $WEBSITE | $LOGRESOLVE_BIN >> $LOGDATA
  91. else
  92. [ $DEBUG ] && echo "$WEBSITE: Merging new logdata without DNS-resolving (no resolver found)"
  93. logcontent $LOGROOT $WEBROOT $WEBSITE >> $LOGDATA
  94. fi
  95. if [ -s $LOGDATA ]; then
  96. gzip -f9 $LOGDATA
  97. fi
  98. logremove $LOGROOT $WEBROOT $WEBSITE
  99. done