1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
|
#! /usr/bin/env python
# -*- coding: utf-8 -*-
# * Copyright (C) 2013 Adiscon GmbH.
# * This file is part of RSyslog
# *
# * This script processes impstats logfiles and searches for abnormalities
# *
import sys
import datetime
import time
import os
# Include regex definitions
import statslog_regex
from statslog_regex import *
# Set default variables
szInput = "rsyslog-stats.log"
bHelpOutput = False
nEvictedAlarm = 5 # In Percent, when this specific amount of requests raises the evicted counter, a problem is reported.
nFailedAlarm = 1 # Number of fails to occur until a problem is reported.
nDiscardedAlarm = 1 # Number of discards to occur until a problem is reported.
# Helper variables
nLogLineNum = 0
nLogFileCount = 0
outputData = {}
# ArrayID's in Logdata
LN_DATE = 0
LN_HOST = 1
LN_LINENUM = 2
LN_DATA = 3
# Open Errorlog on init
errorlog = open("statslog-analyzer.corrupted.log", 'w')
# Process Arguments
for arg in sys.argv: # [-4:]:
if arg.find("--input=") != -1:
szInput = arg[8:]
elif arg.find("--evictedalarm=") != -1:
nEvictedAlarm = int(arg[15:])
if nEvictedAlarm < 0 or nEvictedAlarm > 100:
nEvictedAlarm = 5 # Resett to default if value is invalid
elif arg.find("--failedalarm=") != -1:
nFailedAlarm = int(arg[14:])
elif arg.find("--discardedalarm=") != -1:
nDiscardedAlarm = int(arg[17:])
elif arg.find("--h") != -1 or arg.find("-h") != -1 or arg.find("--help") != -1:
bHelpOutput = True
if bHelpOutput:
print "\n\nStatslog-analyzer command line options:"
print "======================================="
print " --input=<filename> Contains the path and filename of your impstats logfile. "
print " Default is 'rsyslog-stats.log' \n"
print " --evictedalarm=<number> Default is 5 which is in percent."
print " So the value can be between 0 and 100.\n"
print " --failedalarm=<number> Number of fails that have to occur before raising an alarm.\n"
print " --discardedalarm=<number> Number of discards that have to occur before raising an alarm.\n"
print " --h / -h / --help Displays this help message. \n"
print "\n Sampleline: ./statslog-analyzer.py --input=rsyslog-stats.log --evictedalarm=5 --failedalarm=1 --discardedalarm=1"
else:
print " Start sorting impstats file '" + szInput+ "'"
# Open inout file
inputfile = open(szInput, 'r')
for line in inputfile.readlines():
if line.find("rsyslogd-pstats") != -1:
# Init variables
aData = {}
iLogRegExIndex = 0
bLogProcessed = False
# Loop through Regex parsers
for loglineregex in loglineregexes:
# Parse IMPStats Line!
result = loglineregex.split(line)
# Found valid logline, save into file!
if len(result) >= loglineindexes[iLogRegExIndex]["LN_LOGDATA"] and result[ loglineindexes[iLogRegExIndex]["LN_SYSLOGTAG"] ] == "rsyslogd-pstats":
# Convert Datetime!
try:
iMonth = int( result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ] )
filedate = datetime.datetime.strptime(result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ] + " " + str(datetime.datetime.now().year) + " " + result[ loglineindexes[iLogRegExIndex]["LN_DAY"] ] + " " + result[ loglineindexes[iLogRegExIndex]["LN_TIME"] ] ,"%m %Y %d %H:%M:%S")
except ValueError:
filedate = datetime.datetime.strptime(result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ] + " " + str(datetime.datetime.now().year) + " " + result[ loglineindexes[iLogRegExIndex]["LN_DAY"] ] + " " + result[ loglineindexes[iLogRegExIndex]["LN_TIME"] ] ,"%b %Y %d %H:%M:%S")
# Split logdata into Array
aProperties = result[ loglineindexes[iLogRegExIndex]["LN_LOGDATA"] ].split(" ")
for szProperty in aProperties:
aProperty = szProperty.split("=")
if len(aProperty) > 1:
# aData.append(aProperty[1]) # Append FieldData
aData[aProperty[0]] = aProperty[1]
else:
errorlog.write("Corrupted Logline at line " + str(nLogLineNum) + " failed to parse: " + line)
break
# Remove invalid characters from StatsID!
# szStatsID = re.sub("[^a-zA-Z0-9]", "_", result[ loglineindexes[iLogRegExIndex]["LN_LOGOBJECT"] ])
szStatsID = result[ loglineindexes[iLogRegExIndex]["LN_LOGOBJECT"] ]
# Open Statsdata per ID
if szStatsID not in outputData:
outputData[szStatsID] = []
# Add data into array
outputData[szStatsID].append( [ filedate.strftime("%Y/%b/%d %H:%M:%S"),
result[ loglineindexes[iLogRegExIndex]["LN_HOST"] ],
nLogLineNum,
aData ] )
# Set Log as processed, abort Loop!
bLogProcessed = True
# Increment Logreged Counter
iLogRegExIndex += 1
# Abort Loop if processed
if bLogProcessed:
break
# Debug output if format not detected
if bLogProcessed == False:
print "Fail parsing logline: "
print result
# Increment helper counter
nLogLineNum += 1
# Close input file
inputfile.close()
# Data sorted, now analyze data!
print " Sorting finished with '" + str(nLogLineNum) + "' loglines processed."
print " Start analyzing logdata now"
#print outputData
#sys.exit(0)
# Init variables
aPossibleProblems = {}
# Loop through data
for szSingleStatsID in outputData:
# Init variables
aProblemFound = {}
bEvictedMsgs = False
bFailedMsgs = False
aPossibleProblems[szSingleStatsID] = []
iLineNum = 0
szLineDate = ""
szHost = ""
# Loop through loglines
for singleStatLine in outputData[szSingleStatsID]:
iLineNum = singleStatLine[LN_LINENUM]
szLineDate = singleStatLine[LN_DATE]
szHost = singleStatLine[LN_HOST]
# Init helper values
iLogEvictedData = 0
iLogRequestsData = 0
# Loop through data
for logDataID in singleStatLine[LN_DATA]:
if logDataID.find("discarded") != -1 or logDataID.find("failed") != -1:
iLogData = int(singleStatLine[LN_DATA][logDataID])
if iLogData > 0:
if logDataID not in aProblemFound:
# Init Value
aProblemFound[logDataID] = {}
aProblemFound[logDataID]['type'] = logDataID
aProblemFound[logDataID]['value'] = iLogData
aProblemFound[logDataID]['startline'] = iLineNum
aProblemFound[logDataID]['startdate'] = szLineDate
elif aProblemFound[logDataID]['value'] > iLogData:
# Logdata was resetted
aProblemFound[logDataID]['endline'] = iLineNum
aProblemFound[logDataID]['enddate'] = szLineDate
# Add to possible Problems Array
if logDataID.find("discarded") != -1:
if aProblemFound[logDataID]['value'] >= nDiscardedAlarm:
aPossibleProblems[szSingleStatsID].append( aProblemFound[logDataID] )
elif logDataID.find("failed") != -1:
if aProblemFound[logDataID]['value'] >= nFailedAlarm:
aPossibleProblems[szSingleStatsID].append( aProblemFound[logDataID] )
# Reinit
del aProblemFound[logDataID]
else:
aProblemFound[logDataID]['value'] = int(singleStatLine[LN_DATA][logDataID])
aProblemFound[logDataID]['endline'] = iLineNum
aProblemFound[logDataID]['enddate'] = szLineDate
elif logDataID.find("evicted") != -1 or logDataID.find("requests") != -1:
iLogData = int(singleStatLine[LN_DATA][logDataID])
if "omfile" not in aProblemFound:
# Init Value
aProblemFound['omfile'] = {}
aProblemFound['omfile']['type'] = "omfile"
aProblemFound['omfile']['evicted'] = 0 # Init value
aProblemFound['omfile']['requests'] = 0 # Init value
aProblemFound['omfile'][logDataID] = iLogData # Set value
aProblemFound['omfile']['startline'] = iLineNum
aProblemFound['omfile']['startdate'] = szLineDate
elif aProblemFound['omfile'][logDataID] > iLogData:
# Logdata was resetted
aProblemFound['omfile']['endline'] = iLineNum
aProblemFound['omfile']['enddate'] = szLineDate
# Add to possible Problems Array if exceeds alarm level
if (aProblemFound['omfile']['evicted'] / (aProblemFound['omfile']['requests']/100)) >= nEvictedAlarm:
aPossibleProblems[szSingleStatsID].append( aProblemFound['omfile'] )
# Reinit
del aProblemFound['omfile']
else:
aProblemFound['omfile'][logDataID] = int(singleStatLine[LN_DATA][logDataID])
aProblemFound['omfile']['endline'] = iLineNum
aProblemFound['omfile']['enddate'] = szLineDate
#print logDataID + "=" + singleStatLine[LN_DATA][logDataID]
# Finish possible problems array
if len(aProblemFound) > 0:
for szProblemID in aProblemFound:
if aProblemFound[szProblemID]['type'].find('omfile') != -1:
# Only add if exceeds percentage
if (aProblemFound[szProblemID]['evicted'] / (aProblemFound[szProblemID]['requests']/100)) >= nEvictedAlarm:
aPossibleProblems[szSingleStatsID].append( aProblemFound[szProblemID] )
elif aProblemFound[szProblemID]['type'].find('failed') != -1:
# Only add if exceeds Alarm Level
if aProblemFound[szProblemID]['value'] >= nFailedAlarm:
aPossibleProblems[szSingleStatsID].append( aProblemFound[szProblemID] )
elif aProblemFound[szProblemID]['type'].find('discarded') != -1:
# Only add if exceeds Alarm Level
if aProblemFound[szProblemID]['value'] >= nDiscardedAlarm:
aPossibleProblems[szSingleStatsID].append( aProblemFound[szProblemID] )
else:
# Add to problem array!
aPossibleProblems[szSingleStatsID].append( aProblemFound[szProblemID] )
# Debug break
# sys.exit(0)
print " End analyzing logdata, showing results now:"
# Output all found problems!
for szProblemID in aPossibleProblems:
if len(aPossibleProblems[szProblemID]) > 0:
print "\n Problems found for Counter '" + szProblemID + "': "
for aProblem in aPossibleProblems[szProblemID]:
if aProblem['type'].find("discarded") != -1:
print " - " + str(aProblem['value']) + " Messages were discarded (" + aProblem['type'] + ") between line " + str(aProblem['startline']) + " and " + str(aProblem['endline']) + " (Startdate: '" + str(aProblem['startdate']) + "' Enddate: '" + str(aProblem['enddate']) + "') "
elif aProblem['type'].find("failed") != -1:
print " - " + str(aProblem['value']) + " Messages failed (" + aProblem['type'] + ") between line " + str(aProblem['startline']) + " and " + str(aProblem['endline']) + " (Startdate: '" + str(aProblem['startdate']) + "' Enddate: '" + str(aProblem['enddate']) + "') "
elif aProblem['type'].find("omfile") != -1:
nEvictedInPercent = aProblem['evicted'] / (aProblem['requests']/100)
print " - " + str(nEvictedInPercent) + "% File Requests were evicted ('" + str(aProblem['requests']) + "' requests, '" + str(aProblem['evicted']) + "' evicted) between line " + str(aProblem['startline']) + " and " + str(aProblem['endline']) + " (Startdate: '" + str(aProblem['startdate']) + "' Enddate: '" + str(aProblem['enddate']) + "') "
# Finished
print "\n\n"
|
