Infrastructure/Awstats

Awstats

In production -- https://admin.fedoraproject.org/awstats/

Proxy Servers

• Save logs in files that are rotated hourly:
• On proxy1:

LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %v proxy1.fedoraproject.org" awstats

• On proxy2:

LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %v proxy2.fedoraproject.org" awstats

• On both:

CustomLog "|/usr/sbin/rotatelogs /var/log/httpd/access_log.%Y-%m-%d-%H-%M-%S 3600" awstats
ErrorLog "|/usr/sbin/rotatelogs /var/log/httpd/error_log.%Y-%m-%d-%H-%M-%S 3600"

• Clean up old log files:

#!/bin/sh

/usr/sbin/tmpwatch --mtime 672 /var/log/httpd

• Rsync is then used to copy log files to server that will run awstats

Awstats Server

• In /etc/awstats/awstats.fedoraproject.org.conf:

LogFile="/usr/share/awstats/tools/logresolvemerge.pl /var/log/proxies/*/access_log.* |"
LogType=W
LogFormat="%host %other %logname %time1 %methodurl %code %bytesd %refererquot %uaquot %virtualname %cluster"
LogSeparator=" "
SiteDomain="fedoraproject.org"
HostAliases="REGEX[.*\.fedoraproject\.org] "
DNSLookup=0
DirData="/var/lib/awstats"
DirCgi="/awstats"
DirIcons="/awstatsicons"
AllowToUpdateStatsFromBrowser=0
AllowFullYearView=2
EnableLockForUpdate=1
DNSStaticCacheFile="dnscache.txt"
DNSLastUpdateCacheFile="dnscachelastupdate.txt"
SkipDNSLookupFor=""
AllowAccessFromWebToAuthenticatedUsersOnly=0
AllowAccessFromWebToFollowingAuthenticatedUsers=""
AllowAccessFromWebToFollowingIPAddresses=""
CreateDirDataIfNotExists=0
BuildHistoryFormat=text
BuildReportFormat=html
SaveDatabaseFilesWithPermissionsForEveryone=0
PurgeLogFile=0
ArchiveLogRecords=0
KeepBackupOfHistoricFiles=1
DefaultFile="index.html"
SkipHosts="127.0.0.1"
SkipUserAgents=""
SkipFiles=""
SkipReferrersBlackList=""
OnlyHosts=""
OnlyUserAgents=""
OnlyFiles=""
NotPageList="css js class gif jpg jpeg png bmp ico swf"
ValidHTTPCodes="200 304"
ValidSMTPCodes="1 250"
AuthenticatedUsersNotCaseSensitive=0
URLNotCaseSensitive=0
URLWithAnchor=0
URLQuerySeparators="?;"
URLWithQuery=0
URLWithQueryWithOnlyFollowingParameters=""
URLWithQueryWithoutFollowingParameters=""
URLReferrerWithQuery=0
WarningMessages=1
ErrorMessages=""
DebugMessages=0
NbOfLinesForCorruptedLog=50
WrapperScript=""
DecodeUA=0
MiscTrackerUrl=""
LevelForBrowsersDetection=allphones
LevelForOSDetection=2
LevelForRefererAnalyze=2
LevelForRobotsDetection=2
LevelForSearchEnginesDetection=2
LevelForKeywordsDetection=2
LevelForFileTypesDetection=2
LevelForWormsDetection=2
UseFramesWhenCGI=1
DetailedReportsOnNewWindows=0
Expires=3600
MaxRowsInHTMLOutput=1000
Lang="auto"
DirLang="/usr/share/awstats/lang"
ShowMenu=1
ShowSummary=UVPHB
ShowMonthStats=UVPHB
ShowDaysOfMonthStats=VPHB
ShowDaysOfWeekStats=PHB
ShowHoursStats=PHB
ShowDomainsStats=PHB
ShowHostsStats=PHBL
ShowAuthenticatedUsers=0
ShowRobotsStats=HBL
ShowWormsStats=HBL
ShowEMailSenders=0
ShowEMailReceivers=0
ShowSessionsStats=1
ShowPagesStats=PBEX
ShowFileTypesStats=HBC
ShowFileSizesStats=0
ShowOSStats=1
ShowBrowsersStats=1
ShowScreenSizeStats=0
ShowOriginStats=PH
ShowKeyphrasesStats=1
ShowKeywordsStats=1
ShowMiscStats=1
ShowHTTPErrorsStats=1
ShowSMTPErrorsStats=0
ShowClusterStats=PHB
AddDataArrayMonthStats=1
AddDataArrayShowDaysOfMonthStats=1
AddDataArrayShowDaysOfWeekStats=1
AddDataArrayShowHoursStats=1
IncludeInternalLinksInOriginSection=0
MaxNbOfDomain = 10
MinHitDomain  = 1
MaxNbOfHostsShown = 10
MinHitHost    = 1
MaxNbOfLoginShown = 10
MinHitLogin   = 1
MaxNbOfRobotShown = 10
MinHitRobot   = 1
MaxNbOfPageShown = 10
MinHitFile    = 1
MaxNbOfOsShown = 10
MinHitOs      = 1
MaxNbOfBrowsersShown = 10
MinHitBrowser = 1
MaxNbOfScreenSizesShown = 5
MinHitScreenSize = 1
MaxNbOfWindowSizesShown = 5
MinHitWindowSize = 1
MaxNbOfRefererShown = 10
MinHitRefer   = 1
MaxNbOfKeyphrasesShown = 10
MinHitKeyphrase = 1
MaxNbOfKeywordsShown = 10
MinHitKeyword = 1
MaxNbOfEMailsShown = 20
MinHitEMail   = 1
FirstDayOfWeek=0
ShowFlagLinks=""
ShowLinksOnUrl=1
UseHTTPSLinkForUrl=""
MaxLengthOfShownURL=64
HTMLHeadSection=""
HTMLEndSection=""
Logo="awstats_logo6.png"
LogoLink="http://awstats.sourceforge.net"
BarWidth   = 260
BarHeight  = 90
StyleSheet=""
LoadPlugin="tooltips"
#LoadPlugin="geoipfree"

• In /etc/cron.hourly/awstats-fedoraproject.org:

#!/bin/sh

REPORT_DIR=/var/lib/awstats

mkdir -p $REPORT_DIR/current

/usr/share/awstats/tools/awstats_buildstaticpages.pl -config=fedoraproject.org -dir=$REPORT_DIR/current

ln -sf awstats.fedoraproject.org.html $REPORT_DIR/current/index.html

rsync -acH $REPORT_DIR/ <wherever>

• In /etc/cron.monthly/awstats-fedoraproject.org:

#!/bin/sh

REPORT_DIR=/var/lib/awstats

YEAR=<code>date --date="last month" +"%Y"</code>
MONTH=<code>date --date="last month" +"%m"</code>

mkdir -p $REPORT_DIR/$YEAR-$MONTH

/usr/share/awstats/tools/awstats_buildstaticpages.pl -config=fedoraproject.org -month=$MONTH -year=$YEAR -dir=$REPORT_DIR/$YEAR-$MONTH

ln -sf awstats.fedoraproject.org.html $REPORT_DIR/$YEAR-$MONTH/index.html