From bc194a9b76ec858fdffa622dbc5186f484681b75 Mon Sep 17 00:00:00 2001
From: Andre Lorbach <alorbach@adiscon.com>
Date: Tue, 2 Jul 2013 15:09:30 +0200
Subject: Added script to split impstats logfiles into processable csv files

Python 2.6 or higher is needed to run the script
---
 plugins/impstats/statslog-splitter.py | 133 ++++++++++++++++++++++++++++++++++
 1 file changed, 133 insertions(+)
 create mode 100755 plugins/impstats/statslog-splitter.py

(limited to 'plugins/impstats/statslog-splitter.py')

diff --git a/plugins/impstats/statslog-splitter.py b/plugins/impstats/statslog-splitter.py
new file mode 100755
index 00000000..eddb5914
--- /dev/null
+++ b/plugins/impstats/statslog-splitter.py
@@ -0,0 +1,133 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# 	* Copyright (C) 2013 Adiscon GmbH.
+#	* This file is part of RSyslog
+#	* 
+#	* This script processes impstats logfiles and splits them into csv files
+#	*
+
+import sys
+import datetime
+import time
+import re
+
+# Set default variables
+szInput = "rsyslog-stats.log"
+szOutputDir = "./"
+nSingleObjectOutput = 1
+nHelpOutput = 0
+nLogLineNum = 0
+nLogFileCount = 0
+
+# Create regex for logline
+loglineregex = re.compile(r"(...)(?:.|..)([0-9]{1,2}) ([0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}) ([a-zA-Z0-9_\-\.]{1,256}) ([A-Za-z0-9_\-\/\.]{1,32}): (.*?): (.*?) \n")
+# Array Indexes
+LN_MONTH = 1;LN_DAY = 2;LN_TIME = 3;LN_HOST = 4;LN_SYSLOGTAG = 5;LN_LOGOBJECT = 6;LN_LOGDATA = 7
+
+# Init result with file handles
+outputFiles = {}
+
+# Open Errorlog on init
+errorlog = open("statslog-splitter.corrupted.log", 'w')
+
+# Process Arguments
+for arg in sys.argv[-4:]:
+	if arg.find("--input=") != -1:
+		szInput = arg[8:]
+	elif arg.find("--outputdir=") != -1:
+		szOutputDir = arg[12:]
+	elif arg.find("--h") != -1 or arg.find("-h") != -1 or arg.find("--help") != -1:
+		nHelpOutput = 1
+
+#sys.exit(0)
+
+if nHelpOutput == 1: 
+	print "\n\nStatslog-splitter command line options:"
+	print "======================================="
+	print "	--input=<filename>	Contains the path and filename of your impstats logfile. "
+	print "				Default is 'rsyslog-stats.log' \n"
+	print "	--outputdir=<dir>		Output directory to be used. "
+	print "				Default is current directory. "
+	print "	--h / -h / --help	Displays this help message. \n"
+	print "	singlefile		Splits the stats logfile into single CSV Files"
+	print "				(Default)"
+	print "\n	Sampleline: ./statslog-splitter.py singlefile --input=rsyslog-stats.log --outputdir=/home/user/csvlogs/"
+elif nSingleObjectOutput == 1:
+	inputfile = open(szInput, 'r')
+	for line in inputfile.readlines():
+		if line.find("rsyslogd-pstats") != -1:
+			# Init variables
+			aFields = []
+			aData = []
+
+			# Parse IMPStats Line!
+			result = loglineregex.split(line)
+			# Found valid logline, save into file! 
+			if len(result) >= LN_LOGDATA and result[LN_SYSLOGTAG] == "rsyslogd-pstats":
+				# Convert Datetime!
+				filedate = datetime.datetime.strptime(result[LN_MONTH] + " " + str(datetime.datetime.now().year) + " " + result[LN_DAY] + " " + result[LN_TIME] ,"%b %Y %d %H:%M:%S")
+
+				# Split logdata into Array
+				aProperties = result[LN_LOGDATA].split(" ")
+				for szProperty in aProperties:
+					aProperty = szProperty.split("=")
+					aFields.append(aProperty[0])	# Append FieldID
+					if len(aProperty) > 1:
+						aData.append(aProperty[1])	# Append FieldData
+					else: 
+						errorlog.write("Corrupted Logline at line " + str(nLogLineNum) + " failed to parse: " + line)
+						break
+
+				# Remove invalid characters for filename!
+				szFileName = re.sub("[^a-zA-Z0-9]", "_", result[LN_LOGOBJECT]) + ".csv"
+
+				# Open file for writing!
+				if szFileName not in outputFiles:
+       					print "Creating file : " + szOutputDir + "/" + szFileName
+					outputFiles[szFileName] = open(szOutputDir + "/" + szFileName, 'w')
+					nLogFileCount += 1
+					
+					# Output CSV Header
+					outputFiles[szFileName].write("Date;")
+					outputFiles[szFileName].write("Host;")
+					outputFiles[szFileName].write("Object;")
+					for szField in aFields:
+						outputFiles[szFileName].write(szField + ";")
+					outputFiles[szFileName].write("\n")
+				#else:
+				#	print "already open: " + szFileName
+				
+				# Output CSV Data
+				outputFiles[szFileName].write(filedate.strftime("%Y/%b/%d %H:%M:%S") + ";")
+				outputFiles[szFileName].write(result[LN_HOST] + ";")
+				outputFiles[szFileName].write(result[LN_LOGOBJECT] + ";")
+				for szData in aData:
+					outputFiles[szFileName].write(szData + ";")
+				outputFiles[szFileName].write("\n")
+
+				#print result[LN_LOGOBJECT]
+				#print result[LN_LOGDATA]
+			else:
+				print "Fail parsing logline: "
+				print result
+				break
+
+
+			# Increment helper counter
+			nLogLineNum += 1 
+
+			#print result
+			#break
+
+			#print "IMPStats Line found: " + line
+
+	# Close input file
+	inputfile.close()
+
+	print "\n	File " + szInput + " has been processed"
+	print "	" + str(nLogFileCount) + " Logfiles have been exported to " + szOutputDir
+	print "\n\n"
+
+# Close Error log on exit
+errorlog.close()
-- 
cgit v1.2.3


From 1d22bd58d4a19d63294fb89f4f7a054e7220074a Mon Sep 17 00:00:00 2001
From: Andre Lorbach <alorbach@adiscon.com>
Date: Wed, 3 Jul 2013 12:16:40 +0200
Subject: Finsihed implementing chart gemerator script for impstats.

CSV Data generated by statslog-splitter is needed.
To run the chart script, the following packages need to be installed:
	- python pip
	- python cairosvg
With pip install these packages:
	- pip install CairoSVG tinycss cssselect pygal
---
 plugins/impstats/statslog-splitter.py | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

(limited to 'plugins/impstats/statslog-splitter.py')

diff --git a/plugins/impstats/statslog-splitter.py b/plugins/impstats/statslog-splitter.py
index eddb5914..82cc01cc 100755
--- a/plugins/impstats/statslog-splitter.py
+++ b/plugins/impstats/statslog-splitter.py
@@ -15,8 +15,8 @@ import re
 # Set default variables
 szInput = "rsyslog-stats.log"
 szOutputDir = "./"
-nSingleObjectOutput = 1
-nHelpOutput = 0
+bSingleObjectOutput = True
+bHelpOutput = True
 nLogLineNum = 0
 nLogFileCount = 0
 
@@ -37,23 +37,29 @@ for arg in sys.argv[-4:]:
 		szInput = arg[8:]
 	elif arg.find("--outputdir=") != -1:
 		szOutputDir = arg[12:]
+	elif arg.find("--singlefile=") != -1:
+		bSingleObjectOutput = True
 	elif arg.find("--h") != -1 or arg.find("-h") != -1 or arg.find("--help") != -1:
-		nHelpOutput = 1
+		bHelpOutput = True
 
 #sys.exit(0)
 
-if nHelpOutput == 1: 
+if bHelpOutput == 1: 
 	print "\n\nStatslog-splitter command line options:"
 	print "======================================="
-	print "	--input=<filename>	Contains the path and filename of your impstats logfile. "
-	print "				Default is 'rsyslog-stats.log' \n"
+	print "	--input=<filename>		Contains the path and filename of your impstats logfile. "
+	print "					Default is 'rsyslog-stats.log' \n"
 	print "	--outputdir=<dir>		Output directory to be used. "
-	print "				Default is current directory. "
-	print "	--h / -h / --help	Displays this help message. \n"
-	print "	singlefile		Splits the stats logfile into single CSV Files"
-	print "				(Default)"
-	print "\n	Sampleline: ./statslog-splitter.py singlefile --input=rsyslog-stats.log --outputdir=/home/user/csvlogs/"
-elif nSingleObjectOutput == 1:
+	print "					Default is current directory. "
+	print "	--h / -h / --help		Displays this help message. \n"
+	print "	--singlefile			Splits the stats logfile into single CSV Files"
+	print "					Default is enabled."
+	print " --enablecharts			Generate Charts for each exported CSV File.
+	print "					Default is disabled."
+	print " --chartsformat=<svg|png>	Format which should be used for Charts.
+	print "					Default is svg format
+	print "\n	Sampleline: ./statslog-splitter.py singlefile --input=rsyslog-stats.log --outputdir=/home/user/csvlogs/ --enablecharts --chartsformat=png"
+elif bSingleObjectOutput:
 	inputfile = open(szInput, 'r')
 	for line in inputfile.readlines():
 		if line.find("rsyslogd-pstats") != -1:
-- 
cgit v1.2.3


From 7b16f5c49766fb8d08835e2f2839d4514dc29f9a Mon Sep 17 00:00:00 2001
From: Andre Lorbach <alorbach@adiscon.com>
Date: Wed, 3 Jul 2013 13:21:46 +0200
Subject: Finished statslog graph and splitter script.

Also added needed javascript files, so they can be included into the svg files.
This removes dependencies on external sources.

The statslog-splitter can now automatically generated SVG or PNG charts together
with an index.html in the output directory. This helps taking a quick view on all
generated stats data.
---
 plugins/impstats/statslog-splitter.py | 56 ++++++++++++++++++++++++++++++-----
 1 file changed, 48 insertions(+), 8 deletions(-)

(limited to 'plugins/impstats/statslog-splitter.py')

diff --git a/plugins/impstats/statslog-splitter.py b/plugins/impstats/statslog-splitter.py
index 82cc01cc..b342f94c 100755
--- a/plugins/impstats/statslog-splitter.py
+++ b/plugins/impstats/statslog-splitter.py
@@ -11,12 +11,17 @@ import sys
 import datetime
 import time
 import re
+import os
 
 # Set default variables
 szInput = "rsyslog-stats.log"
 szOutputDir = "./"
 bSingleObjectOutput = True
-bHelpOutput = True
+bHelpOutput = False
+bEnableCharts = False
+szChartsFormat = "svg"
+
+# Helper variables
 nLogLineNum = 0
 nLogFileCount = 0
 
@@ -37,14 +42,16 @@ for arg in sys.argv[-4:]:
 		szInput = arg[8:]
 	elif arg.find("--outputdir=") != -1:
 		szOutputDir = arg[12:]
-	elif arg.find("--singlefile=") != -1:
+	elif arg.find("--singlefile") != -1:
 		bSingleObjectOutput = True
+	elif arg.find("--enablecharts") != -1:
+		bEnableCharts = True
+	elif arg.find("--chartsformat=") != -1:
+		szChartsFormat = arg[15:]
 	elif arg.find("--h") != -1 or arg.find("-h") != -1 or arg.find("--help") != -1:
 		bHelpOutput = True
 
-#sys.exit(0)
-
-if bHelpOutput == 1: 
+if bHelpOutput: 
 	print "\n\nStatslog-splitter command line options:"
 	print "======================================="
 	print "	--input=<filename>		Contains the path and filename of your impstats logfile. "
@@ -54,10 +61,10 @@ if bHelpOutput == 1:
 	print "	--h / -h / --help		Displays this help message. \n"
 	print "	--singlefile			Splits the stats logfile into single CSV Files"
 	print "					Default is enabled."
-	print " --enablecharts			Generate Charts for each exported CSV File.
+	print " --enablecharts			Generate Charts for each exported CSV File."
 	print "					Default is disabled."
-	print " --chartsformat=<svg|png>	Format which should be used for Charts.
-	print "					Default is svg format
+	print " --chartsformat=<svg|png>	Format which should be used for Charts."
+	print "					Default is svg format"
 	print "\n	Sampleline: ./statslog-splitter.py singlefile --input=rsyslog-stats.log --outputdir=/home/user/csvlogs/ --enablecharts --chartsformat=png"
 elif bSingleObjectOutput:
 	inputfile = open(szInput, 'r')
@@ -128,11 +135,44 @@ elif bSingleObjectOutput:
 
 			#print "IMPStats Line found: " + line
 
+	# Close outfiles
+	for outFileName in outputFiles:
+		outputFiles[outFileName].close()
+
 	# Close input file
 	inputfile.close()
 
 	print "\n	File " + szInput + " has been processed"
 	print "	" + str(nLogFileCount) + " Logfiles have been exported to " + szOutputDir
+	
+	if bEnableCharts: 
+		# Open HTML Code
+		szHtmlCode =	"<!DOCTYPE html><html><head></head><body><center>"
+
+		# Default SVG Format!
+		if szChartsFormat.find("svg") != -1:
+			for outFileName in outputFiles:
+				iReturn = os.system("./statslog-graph.py --input=" + szOutputDir + "/" + outFileName + "")
+				print "Chart SVG generated for '" + outFileName + "': " + str(iReturn)
+				szHtmlCode +=	"<figure><embed type=\"image/svg+xml\" src=\"" + outFileName[:-4] + ".svg" + "\" />" + "</figure><br/><br/>"
+		# Otherwise PNG Output!
+		else: 
+			for outFileName in outputFiles:
+				iReturn = os.system("./statslog-graph.py --input=" + szOutputDir + "/" + outFileName + " --convertpng")
+				print "Chart PNG generated for '" + outFileName + "': " + str(iReturn)
+				szHtmlCode +=	"<img src=\"" + outFileName[:-4] + ".png" + "\" width=\"800\" height=\"600\"/>" + "<br/><br/>"
+
+		print "	" + str(nLogFileCount) + " Charts have been written " + szOutputDir
+		
+		# Close HTML Code
+		szHtmlCode +=	"</center></body></html>"
+
+		# Write HTML Index site!
+		outHtmlFile = open(szOutputDir + "/index.html", 'w')
+		outHtmlFile.write(szHtmlCode)
+		outHtmlFile.close()
+		print "	HTML Index with all charts has been written to " + szOutputDir + "/index.html"
+
 	print "\n\n"
 
 # Close Error log on exit
-- 
cgit v1.2.3


From 036096c1b4d7cb67b35d9da25f2cc031b1a6bbff Mon Sep 17 00:00:00 2001
From: Andre Lorbach <alorbach@adiscon.com>
Date: Wed, 3 Jul 2013 16:50:18 +0200
Subject: Added more command line options to customize the charts and their
 output

---
 plugins/impstats/statslog-splitter.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

(limited to 'plugins/impstats/statslog-splitter.py')

diff --git a/plugins/impstats/statslog-splitter.py b/plugins/impstats/statslog-splitter.py
index b342f94c..e7102c51 100755
--- a/plugins/impstats/statslog-splitter.py
+++ b/plugins/impstats/statslog-splitter.py
@@ -19,11 +19,14 @@ szOutputDir = "./"
 bSingleObjectOutput = True
 bHelpOutput = False
 bEnableCharts = False
+bLogarithmicChart = False
+bFilledLineChart = False
 szChartsFormat = "svg"
 
 # Helper variables
 nLogLineNum = 0
 nLogFileCount = 0
+szChartOptionalArgs = ""
 
 # Create regex for logline
 loglineregex = re.compile(r"(...)(?:.|..)([0-9]{1,2}) ([0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}) ([a-zA-Z0-9_\-\.]{1,256}) ([A-Za-z0-9_\-\/\.]{1,32}): (.*?): (.*?) \n")
@@ -48,6 +51,10 @@ for arg in sys.argv[-4:]:
 		bEnableCharts = True
 	elif arg.find("--chartsformat=") != -1:
 		szChartsFormat = arg[15:]
+	elif arg.find("--logarithmic") != -1:
+		bLogarithmicChart = True
+	elif arg.find("--filledlinechart") != -1:
+		bFilledLineChart = True
 	elif arg.find("--h") != -1 or arg.find("-h") != -1 or arg.find("--help") != -1:
 		bHelpOutput = True
 
@@ -65,6 +72,8 @@ if bHelpOutput:
 	print "					Default is disabled."
 	print " --chartsformat=<svg|png>	Format which should be used for Charts."
 	print "					Default is svg format"
+	print " --logarithmic			Uses Logarithmic to scale the Y Axis, maybe useful in some cases. Default is OFF"
+	print " --filledlinechart		Use filled lines on Linechart, maybe useful in some cases. Default is OFF"
 	print "\n	Sampleline: ./statslog-splitter.py singlefile --input=rsyslog-stats.log --outputdir=/home/user/csvlogs/ --enablecharts --chartsformat=png"
 elif bSingleObjectOutput:
 	inputfile = open(szInput, 'r')
@@ -149,16 +158,22 @@ elif bSingleObjectOutput:
 		# Open HTML Code
 		szHtmlCode =	"<!DOCTYPE html><html><head></head><body><center>"
 
+		# Add optional args 
+		if bLogarithmicChart: 
+			szChartOptionalArgs += " --logarithmic"
+		if bFilledLineChart: 
+			szChartOptionalArgs += " --filledlinechart"
+
 		# Default SVG Format!
 		if szChartsFormat.find("svg") != -1:
 			for outFileName in outputFiles:
-				iReturn = os.system("./statslog-graph.py --input=" + szOutputDir + "/" + outFileName + "")
+				iReturn = os.system("./statslog-graph.py " + szChartOptionalArgs + " --input=" + szOutputDir + "/" + outFileName + "")
 				print "Chart SVG generated for '" + outFileName + "': " + str(iReturn)
 				szHtmlCode +=	"<figure><embed type=\"image/svg+xml\" src=\"" + outFileName[:-4] + ".svg" + "\" />" + "</figure><br/><br/>"
 		# Otherwise PNG Output!
 		else: 
 			for outFileName in outputFiles:
-				iReturn = os.system("./statslog-graph.py --input=" + szOutputDir + "/" + outFileName + " --convertpng")
+				iReturn = os.system("./statslog-graph.py " + szChartOptionalArgs + " --input=" + szOutputDir + "/" + outFileName + " --convertpng")
 				print "Chart PNG generated for '" + outFileName + "': " + str(iReturn)
 				szHtmlCode +=	"<img src=\"" + outFileName[:-4] + ".png" + "\" width=\"800\" height=\"600\"/>" + "<br/><br/>"
 
-- 
cgit v1.2.3


From ebe5266336f1421f20edf844872ad2e2ab80c4b3 Mon Sep 17 00:00:00 2001
From: Andre Lorbach <alorbach@adiscon.com>
Date: Wed, 3 Jul 2013 18:07:04 +0200
Subject: Added support for multiple regex commands, useful for parsing
 multiple rsyslog formats

---
 plugins/impstats/statslog-splitter.py | 132 ++++++++++++++++++++--------------
 1 file changed, 79 insertions(+), 53 deletions(-)

(limited to 'plugins/impstats/statslog-splitter.py')

diff --git a/plugins/impstats/statslog-splitter.py b/plugins/impstats/statslog-splitter.py
index e7102c51..6a6870bf 100755
--- a/plugins/impstats/statslog-splitter.py
+++ b/plugins/impstats/statslog-splitter.py
@@ -28,10 +28,19 @@ nLogLineNum = 0
 nLogFileCount = 0
 szChartOptionalArgs = ""
 
-# Create regex for logline
-loglineregex = re.compile(r"(...)(?:.|..)([0-9]{1,2}) ([0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}) ([a-zA-Z0-9_\-\.]{1,256}) ([A-Za-z0-9_\-\/\.]{1,32}): (.*?): (.*?) \n")
-# Array Indexes
-LN_MONTH = 1;LN_DAY = 2;LN_TIME = 3;LN_HOST = 4;LN_SYSLOGTAG = 5;LN_LOGOBJECT = 6;LN_LOGDATA = 7
+# Create regex for loglines
+loglineregexes = []
+loglineindexes = []
+
+# Traditional Format
+# Sample Line: Jun 26 14:21:44 nhpljt084 rsyslogd-pstats: main Q[DA]: size=0 enqueued=0 full=0 discarded.full=0 discarded.nf=0 maxqsize=0 
+loglineregexes.append( re.compile(r"(...)(?:.|..)([0-9]{1,2}) ([0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}) ([a-zA-Z0-9_\-\.]{1,256}) ([A-Za-z0-9_\-\/\.]{1,32}): (.*?): (.*?) \n") )
+loglineindexes.append( {"LN_YEAR":-1, "LN_MONTH":1, "LN_DAY":2, "LN_TIME":3, "LN_HOST":4, "LN_SYSLOGTAG":5, "LN_LOGOBJECT":6, "LN_LOGDATA":7} )
+
+# Newer Format
+# Sample format: 2013-07-03T17:22:55.680078+02:00 devdebian6 rsyslogd-pstats: main Q: size=358 enqueued=358 full=0 discarded.full=0 discarded.nf=0 maxqsize=358 
+loglineregexes.append( re.compile(r"([0-9]{4,4})-([0-9]{1,2})-([0-9]{1,2})T([0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2})\.[0-9]{1,6}.[0-9]{1,2}:[0-9]{1,2} ([a-zA-Z0-9_\-\.]{1,256}) ([A-Za-z0-9_\-\/\.]{1,32}): (.*?): (.*?) \n") )
+loglineindexes.append( {"LN_YEAR":1, "LN_MONTH":2, "LN_DAY":3, "LN_TIME":4, "LN_HOST":5, "LN_SYSLOGTAG":6, "LN_LOGOBJECT":7, "LN_LOGDATA":8} )
 
 # Init result with file handles
 outputFiles = {}
@@ -82,59 +91,76 @@ elif bSingleObjectOutput:
 			# Init variables
 			aFields = []
 			aData = []
-
-			# Parse IMPStats Line!
-			result = loglineregex.split(line)
-			# Found valid logline, save into file! 
-			if len(result) >= LN_LOGDATA and result[LN_SYSLOGTAG] == "rsyslogd-pstats":
-				# Convert Datetime!
-				filedate = datetime.datetime.strptime(result[LN_MONTH] + " " + str(datetime.datetime.now().year) + " " + result[LN_DAY] + " " + result[LN_TIME] ,"%b %Y %d %H:%M:%S")
-
-				# Split logdata into Array
-				aProperties = result[LN_LOGDATA].split(" ")
-				for szProperty in aProperties:
-					aProperty = szProperty.split("=")
-					aFields.append(aProperty[0])	# Append FieldID
-					if len(aProperty) > 1:
-						aData.append(aProperty[1])	# Append FieldData
-					else: 
-						errorlog.write("Corrupted Logline at line " + str(nLogLineNum) + " failed to parse: " + line)
-						break
-
-				# Remove invalid characters for filename!
-				szFileName = re.sub("[^a-zA-Z0-9]", "_", result[LN_LOGOBJECT]) + ".csv"
-
-				# Open file for writing!
-				if szFileName not in outputFiles:
-       					print "Creating file : " + szOutputDir + "/" + szFileName
-					outputFiles[szFileName] = open(szOutputDir + "/" + szFileName, 'w')
-					nLogFileCount += 1
+			iLogRegExIndex = 0
+			bLogProcessed = False
+
+			# Loop through Regex parsers
+			for loglineregex in loglineregexes:
+				# Parse IMPStats Line!
+				result = loglineregex.split(line)
+				# Found valid logline, save into file! 
+				if len(result) >= loglineindexes[iLogRegExIndex]["LN_LOGDATA"] and result[ loglineindexes[iLogRegExIndex]["LN_SYSLOGTAG"] ] == "rsyslogd-pstats":
+					# Convert Datetime!
+					if isinstance( result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ], int ):
+						filedate = datetime.datetime.strptime(result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ] + " " + str(datetime.datetime.now().year) + " " + result[ loglineindexes[iLogRegExIndex]["LN_DAY"] ] + " " + result[ loglineindexes[iLogRegExIndex]["LN_TIME"] ] ,"%b %Y %d %H:%M:%S")
+					else:
+						filedate = datetime.datetime.strptime(result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ] + " " + str(datetime.datetime.now().year) + " " + result[ loglineindexes[iLogRegExIndex]["LN_DAY"] ] + " " + result[ loglineindexes[iLogRegExIndex]["LN_TIME"] ] ,"%m %Y %d %H:%M:%S")
+
+					# Split logdata into Array
+					aProperties = result[ loglineindexes[iLogRegExIndex]["LN_LOGDATA"] ].split(" ")
+					for szProperty in aProperties:
+						aProperty = szProperty.split("=")
+						aFields.append(aProperty[0])	# Append FieldID
+						if len(aProperty) > 1:
+							aData.append(aProperty[1])	# Append FieldData
+						else: 
+							errorlog.write("Corrupted Logline at line " + str(nLogLineNum) + " failed to parse: " + line)
+							break
+
+					# Remove invalid characters for filename!
+					szFileName = re.sub("[^a-zA-Z0-9]", "_", result[ loglineindexes[iLogRegExIndex]["LN_LOGOBJECT"] ]) + ".csv"
+
+					# Open file for writing!
+					if szFileName not in outputFiles:
+						print "Creating file : " + szOutputDir + "/" + szFileName
+						outputFiles[szFileName] = open(szOutputDir + "/" + szFileName, 'w')
+						nLogFileCount += 1
+						
+						# Output CSV Header
+						outputFiles[szFileName].write("Date;")
+						outputFiles[szFileName].write("Host;")
+						outputFiles[szFileName].write("Object;")
+						for szField in aFields:
+							outputFiles[szFileName].write(szField + ";")
+						outputFiles[szFileName].write("\n")
+					#else:
+					#	print "already open: " + szFileName
 					
-					# Output CSV Header
-					outputFiles[szFileName].write("Date;")
-					outputFiles[szFileName].write("Host;")
-					outputFiles[szFileName].write("Object;")
-					for szField in aFields:
-						outputFiles[szFileName].write(szField + ";")
+					# Output CSV Data
+					outputFiles[szFileName].write(filedate.strftime("%Y/%b/%d %H:%M:%S") + ";")
+					outputFiles[szFileName].write(result[ loglineindexes[iLogRegExIndex]["LN_HOST"] ] + ";")
+					outputFiles[szFileName].write(result[ loglineindexes[iLogRegExIndex]["LN_LOGOBJECT"] ] + ";")
+					for szData in aData:
+						outputFiles[szFileName].write(szData + ";")
 					outputFiles[szFileName].write("\n")
-				#else:
-				#	print "already open: " + szFileName
-				
-				# Output CSV Data
-				outputFiles[szFileName].write(filedate.strftime("%Y/%b/%d %H:%M:%S") + ";")
-				outputFiles[szFileName].write(result[LN_HOST] + ";")
-				outputFiles[szFileName].write(result[LN_LOGOBJECT] + ";")
-				for szData in aData:
-					outputFiles[szFileName].write(szData + ";")
-				outputFiles[szFileName].write("\n")
-
-				#print result[LN_LOGOBJECT]
-				#print result[LN_LOGDATA]
-			else:
+
+					# Set Log as processed, abort Loop!
+					bLogProcessed = True
+
+					#print result[LN_LOGOBJECT]
+					#print result[LN_LOGDATA]
+
+				# Increment Logreged Counter
+				iLogRegExIndex += 1
+
+				# Abort Loop if processed
+				if bLogProcessed: 
+					break
+
+			# Debug output if format not detected
+			if bLogProcessed == False: 
 				print "Fail parsing logline: "
 				print result
-				break
-
 
 			# Increment helper counter
 			nLogLineNum += 1 
-- 
cgit v1.2.3


From 2953e6dc81a0c32c8c45e3db12cdb9b65a4a0137 Mon Sep 17 00:00:00 2001
From: Andre Lorbach <alorbach@adiscon.com>
Date: Wed, 3 Jul 2013 18:18:59 +0200
Subject: Fixed Datetime detection

---
 plugins/impstats/statslog-splitter.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'plugins/impstats/statslog-splitter.py')

diff --git a/plugins/impstats/statslog-splitter.py b/plugins/impstats/statslog-splitter.py
index 6a6870bf..3452e3b3 100755
--- a/plugins/impstats/statslog-splitter.py
+++ b/plugins/impstats/statslog-splitter.py
@@ -101,10 +101,11 @@ elif bSingleObjectOutput:
 				# Found valid logline, save into file! 
 				if len(result) >= loglineindexes[iLogRegExIndex]["LN_LOGDATA"] and result[ loglineindexes[iLogRegExIndex]["LN_SYSLOGTAG"] ] == "rsyslogd-pstats":
 					# Convert Datetime!
-					if isinstance( result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ], int ):
-						filedate = datetime.datetime.strptime(result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ] + " " + str(datetime.datetime.now().year) + " " + result[ loglineindexes[iLogRegExIndex]["LN_DAY"] ] + " " + result[ loglineindexes[iLogRegExIndex]["LN_TIME"] ] ,"%b %Y %d %H:%M:%S")
-					else:
+					try:
+						iMonth = int( result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ] )
 						filedate = datetime.datetime.strptime(result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ] + " " + str(datetime.datetime.now().year) + " " + result[ loglineindexes[iLogRegExIndex]["LN_DAY"] ] + " " + result[ loglineindexes[iLogRegExIndex]["LN_TIME"] ] ,"%m %Y %d %H:%M:%S")
+					except ValueError:
+						filedate = datetime.datetime.strptime(result[ loglineindexes[iLogRegExIndex]["LN_MONTH"] ] + " " + str(datetime.datetime.now().year) + " " + result[ loglineindexes[iLogRegExIndex]["LN_DAY"] ] + " " + result[ loglineindexes[iLogRegExIndex]["LN_TIME"] ] ,"%b %Y %d %H:%M:%S")
 
 					# Split logdata into Array
 					aProperties = result[ loglineindexes[iLogRegExIndex]["LN_LOGDATA"] ].split(" ")
-- 
cgit v1.2.3