1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
|
#! /usr/bin/env python
# -*- coding: utf-8 -*-
# * Copyright (C) 2013 Adiscon GmbH.
# * This file is part of RSyslog
# *
# * This script processes impstats logfiles and splits them into csv files
# *
import sys
import datetime
import time
import os
# Include regex definitions
import statslog_regex
from statslog_regex import *
# Set default variables
szInput = "rsyslog-stats.log"
szOutputDir = "./"
bSingleObjectOutput = True
bHelpOutput = False
bEnableCharts = False
bLogarithmicChart = False
bLineChart = True
bBarChart = False
bFilledLineChart = False
bChartCalcDelta = False
szChartsFormat = "svg"
# Helper variables
nLogLineNum = 0
nLogFileCount = 0
szChartAddArgs = ""
# Init result with file handles
outputFiles = {}
# Open Errorlog on init
errorlog = open("statslog-splitter.corrupted.log", 'w')
# Process Arguments
for arg in sys.argv: # [-4:]:
if arg.find("--input=") != -1:
szInput = arg[8:]
elif arg.find("--outputdir=") != -1:
szOutputDir = arg[12:]
elif arg.find("--singlefile") != -1:
bSingleObjectOutput = True
elif arg.find("--enablecharts") != -1:
bEnableCharts = True
elif arg.find("--chartsformat=") != -1:
szChartsFormat = arg[15:]
elif arg.find("--logarithmic") != -1:
bLogarithmicChart = True
elif arg.find("--linechart") != -1:
bLineChart = True
bBarChart = False
elif arg.find("--barchart") != -1:
bLineChart = False
bBarChart = True
elif arg.find("--filledlinechart") != -1:
bFilledLineChart = True
elif arg.find("--chartscalcdelta") != -1:
bChartCalcDelta = True
elif arg.find("--h") != -1 or arg.find("-h") != -1 or arg.find("--help") != -1:
bHelpOutput = True
if bHelpOutput:
print "\n\nStatslog-splitter command line options:"
print "======================================="
print " --input=<filename> Contains the path and filename of your impstats logfile. "
print " Default is 'rsyslog-stats.log' \n"
print " --outputdir=<dir> Output directory to be used. "
print " Default is current directory. "
print " --h / -h / --help Displays this help message. \n"
print " --singlefile Splits the stats logfile into single CSV Files"
print " Default is enabled."
print " --enablecharts Generate Charts for each exported CSV File."
print " Default is disabled."
print " --chartsformat=<svg|png> Format which should be used for Charts."
print " Default is svg format"
print " --logarithmic Uses Logarithmic to scale the Y Axis, maybe useful in some cases. Default is OFF."
print " --linechart If set, line charts will be generated (Default)."
print " --barchart If set, bar charts will be generated."
print " --filledlinechart Use filled lines on Linechart, maybe useful in some cases. Default is OFF."
print " --chartscalcdelta If set, charts will use calculated delta values instead of cumulative values."
print "\n Sampleline: ./statslog-splitter.py singlefile --input=rsyslog-stats.log --outputdir=/home/user/csvlogs/ --enablecharts --chartsformat=png"
elif bSingleObjectOutput:
inputfile = open(szInput, 'r')
for line in inputfile.readlines():
if line.find("rsyslogd-pstats") != -1:
# Init variables
aFields = []
aData = []
iLogRegExIndex = 0
bLogProcessed = False
# Loop through Regex parsers
for loglineregex in loglineregexes:
# Parse IMPStats Line!
result = loglineregex.split(line)
# Found valid logline, save into file!
if len(result) >= loglineindexes[iLogRegExIndex]["LN_LOGDATA"] and result[ loglineindexes[iLogRegExIndex]["LN_SYSLOGTAG"] ] == "rsyslogd-pstats":
# Convert Datetime!
try:
iMonth = int( result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ] )
filedate = datetime.datetime.strptime(result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ] + " " + str(datetime.datetime.now().year) + " " + result[ loglineindexes[iLogRegExIndex]["LN_DAY"] ] + " " + result[ loglineindexes[iLogRegExIndex]["LN_TIME"] ] ,"%m %Y %d %H:%M:%S")
except ValueError:
filedate = datetime.datetime.strptime(result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ] + " " + str(datetime.datetime.now().year) + " " + result[ loglineindexes[iLogRegExIndex]["LN_DAY"] ] + " " + result[ loglineindexes[iLogRegExIndex]["LN_TIME"] ] ,"%b %Y %d %H:%M:%S")
# Split logdata into Array
aProperties = result[ loglineindexes[iLogRegExIndex]["LN_LOGDATA"] ].split(" ")
for szProperty in aProperties:
aProperty = szProperty.split("=")
aFields.append(aProperty[0]) # Append FieldID
if len(aProperty) > 1:
aData.append(aProperty[1]) # Append FieldData
else:
errorlog.write("Corrupted Logline at line " + str(nLogLineNum) + " failed to parse: " + line)
break
# Remove invalid characters for filename!
szFileName = re.sub("[^a-zA-Z0-9]", "_", result[ loglineindexes[iLogRegExIndex]["LN_LOGOBJECT"] ]) + ".csv"
# Open file for writing!
if szFileName not in outputFiles:
print "Creating file : " + szOutputDir + "/" + szFileName
outputFiles[szFileName] = open(szOutputDir + "/" + szFileName, 'w')
nLogFileCount += 1
# Output CSV Header
outputFiles[szFileName].write("Date;")
outputFiles[szFileName].write("Host;")
outputFiles[szFileName].write("Object;")
for szField in aFields:
outputFiles[szFileName].write(szField + ";")
outputFiles[szFileName].write("\n")
#else:
# print "already open: " + szFileName
# Output CSV Data
outputFiles[szFileName].write(filedate.strftime("%Y/%b/%d %H:%M:%S") + ";")
outputFiles[szFileName].write(result[ loglineindexes[iLogRegExIndex]["LN_HOST"] ] + ";")
outputFiles[szFileName].write(result[ loglineindexes[iLogRegExIndex]["LN_LOGOBJECT"] ] + ";")
for szData in aData:
outputFiles[szFileName].write(szData + ";")
outputFiles[szFileName].write("\n")
# Set Log as processed, abort Loop!
bLogProcessed = True
#print result[LN_LOGOBJECT]
#print result[LN_LOGDATA]
# Increment Logreged Counter
iLogRegExIndex += 1
# Abort Loop if processed
if bLogProcessed:
break
# Debug output if format not detected
if bLogProcessed == False:
print "Fail parsing logline: "
print result
# Increment helper counter
nLogLineNum += 1
#print result
#break
#print "IMPStats Line found: " + line
# Close outfiles
for outFileName in outputFiles:
outputFiles[outFileName].close()
# Close input file
inputfile.close()
print "\n File " + szInput + " has been processed"
print " " + str(nLogFileCount) + " Logfiles have been exported to " + szOutputDir
if bEnableCharts:
# Open HTML Code
szHtmlCode = "<!DOCTYPE html><html><head></head><body><center>"
# Add mandetory args
if bLineChart:
szChartAddArgs += " --linechart"
elif bBarChart:
szChartAddArgs += " --barchart"
# Add optional args
if bLogarithmicChart:
szChartAddArgs += " --logarithmic"
if bFilledLineChart:
szChartAddArgs += " --filledlinechart"
if bChartCalcDelta:
szChartAddArgs += " --chartscalcdelta"
# Default SVG Format!
if szChartsFormat.find("svg") != -1:
for outFileName in outputFiles:
iReturn = os.system("./statslog-graph.py " + szChartAddArgs + " --input=" + szOutputDir + "/" + outFileName + "")
print "Chart SVG generated for '" + outFileName + "': " + str(iReturn)
szHtmlCode += "<figure><embed type=\"image/svg+xml\" src=\"" + outFileName[:-4] + ".svg" + "\" />" + "</figure><br/><br/>"
# Otherwise PNG Output!
else:
for outFileName in outputFiles:
iReturn = os.system("./statslog-graph.py " + szChartAddArgs + " --input=" + szOutputDir + "/" + outFileName + " --convertpng")
print "Chart PNG generated for '" + outFileName + "': " + str(iReturn)
szHtmlCode += "<img src=\"" + outFileName[:-4] + ".png" + "\" width=\"800\" height=\"600\"/>" + "<br/><br/>"
print " " + str(nLogFileCount) + " Charts have been written " + szOutputDir
# Close HTML Code
szHtmlCode += "</center></body></html>"
# Write HTML Index site!
outHtmlFile = open(szOutputDir + "/index.html", 'w')
outHtmlFile.write(szHtmlCode)
outHtmlFile.close()
print " HTML Index with all charts has been written to " + szOutputDir + "/index.html"
print "\n\n"
# Close Error log on exit
errorlog.close()
|
