From 32db6291bc310062cffd3266fc297c7b13c571a7 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 16 Mar 2011 16:59:49 +0100
Subject: backporting patch for clean make distcheck

---
 plugins/imdiag/imdiag.c | 3 +++
 plugins/imfile/imfile.c | 8 ++++++++
 runtime/cfsysline.c     | 3 +++
 3 files changed, 14 insertions(+)

diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c
index 0a69ee43..404cebc7 100644
--- a/plugins/imdiag/imdiag.c
+++ b/plugins/imdiag/imdiag.c
@@ -433,6 +433,9 @@ CODESTARTmodExit
 		net.DestructPermittedPeers(&pPermPeersRoot);
 	}
 
+	/* free some globals to keep valgrind happy */
+	free(pszInputName);
+
 	/* release objects we used */
 	objRelease(net, LM_NET_FILENAME);
 	objRelease(netstrm, LM_NETSTRMS_FILENAME);
diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c
index acb58dad..cac3a55d 100644
--- a/plugins/imfile/imfile.c
+++ b/plugins/imfile/imfile.c
@@ -331,6 +331,11 @@ ENDrunInput
  */
 BEGINwillRun
 CODESTARTwillRun
+	/* free config variables we do no longer needed */
+	free(pszFileName);
+	free(pszFileTag);
+	free(pszStateFile);
+
 	if(iFilPtr == 0) {
 		errmsg.LogError(0, RS_RET_NO_RUN, "No files configured to be monitored");
 		ABORT_FINALIZE(RS_RET_NO_RUN);
@@ -398,6 +403,9 @@ CODESTARTafterRun
 			persistStrmState(&files[i]);
 			strm.Destruct(&(files[i].pStrm));
 		}
+		free(files[i].pszFileName);
+		free(files[i].pszTag);
+		free(files[i].pszStateFile);
 	}
 
 	if(pInputName != NULL)
diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c
index 037e9f84..646ab39b 100644
--- a/runtime/cfsysline.c
+++ b/runtime/cfsysline.c
@@ -41,6 +41,7 @@
 #include "obj.h"
 #include "errmsg.h"
 #include "srUtils.h"
+#include "unicode-helper.h"
 
 
 /* static data */
@@ -511,6 +512,8 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void
 	CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pNewVal, 0));
 	pStrB = NULL;
 
+	DBGPRINTF("doGetWord: get newval '%s' (len %d), hdlr %p\n",
+		  pNewVal, (int) ustrlen(pNewVal), pSetHdlr);
 	/* we got the word, now set it */
 	if(pSetHdlr == NULL) {
 		/* we should set value directly to var */
-- 
cgit v1.2.3


From 709ce7647ca3a44c7c6d47e0445e70a9254a2fdb Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 16 Mar 2011 17:17:34 +0100
Subject: preparing for 5.7.9

---
 ChangeLog       | 5 ++++-
 configure.ac    | 2 +-
 doc/manual.html | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 595a4585..13f6608d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,10 +1,13 @@
 ---------------------------------------------------------------------------
-Version 5.7.9  [V5-BETA] (rgerhards), 2011-03-??
+Version 5.7.9  [V5-BETA] (rgerhards), 2011-03-16
 - improved testbench
   among others, life tests for ommysql (against a test database) have
   been added, valgrind-based testing enhanced, ...
 - enhance: fallback *at runtime* to epoll_create if epoll_create1 is not
   available. Thanks to Michael Biebl for analysis and patch!
+- bugfix: failover did not work correctly if repeated msg reduction was on
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=236
+  affected directive was: $ActionExecOnlyWhenPreviousIsSuspended on
 - bugfix: minor memory leak in omlibdbi (< 1k per instance and run)
 - bugfix: (regression) omhdfs did no longer compile
 - bugfix: omlibdbi did not use password from rsyslog.conf
diff --git a/configure.ac b/configure.ac
index 58b203ff..4f368bcc 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.61)
-AC_INIT([rsyslog],[5.7.8],[rsyslog@lists.adiscon.com])
+AC_INIT([rsyslog],[5.7.9],[rsyslog@lists.adiscon.com])
 AM_INIT_AUTOMAKE
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
diff --git a/doc/manual.html b/doc/manual.html
index 945a5a62..807ac0da 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p>
 <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a>
 to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the
 project goals.</p>
-<p><b>This documentation is for version 5.7.8 (beta branch) of rsyslog.</b>
+<p><b>This documentation is for version 5.7.9 (beta branch) of rsyslog.</b>
 Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b>
 to obtain current version information and project status.
 </p><p><b>If you like rsyslog, you might
-- 
cgit v1.2.3


From 14f3857a0a122aa212b23a59175bd4e504176372 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 17 Mar 2011 12:29:43 +0100
Subject: some minor enhancements to doc set -- point to more examples

---
 doc/rsyslog_conf.html | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html
index 6990c6bd..703e7a6e 100644
--- a/doc/rsyslog_conf.html
+++ b/doc/rsyslog_conf.html
@@ -2,13 +2,16 @@
 <html><head><title>rsyslog.conf file</title></head>
 <body>
 <h1>rsyslog.conf configuration file</h1>
-<p><b>This document is currently being enhanced. Please
-pardon its current appearance.</b></p>
 <p><b>Rsyslogd is configured via the rsyslog.conf file</b>,
 typically found in /etc. By default, rsyslogd reads the file
 /etc/rsyslog.conf. This may be changed by a command line option.</p>
 <p><a href="http://wiki.rsyslog.com/index.php/Configuration_Samples">
-Configuration file examples can be found in the rsyslog wiki</a>.</p>
+Configuration file examples can be found in the rsyslog wiki</a>. Also
+keep the
+<a href="http://www.rsyslog.com/config-snippets/">rsyslog config snippets</a>
+on your mind. These are ready-to-use
+real building blocks for rsyslog configuration.
+</p>
 <p>There is also one sample file provided together with the
 documentation set. If you do not like to read, be sure to have at least
 a quick look at
@@ -74,7 +77,7 @@ such features is available in rsyslogd, only.</p>
 [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p>
 <p><font size="2">This documentation is part of the
 <a href="http://www.rsyslog.com/">rsyslog</a> project.<br>
-Copyright &copy; 2008,2009 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
+Copyright &copy; 2008-2011 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and
 <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL
 version 3 or higher.</font></p>
 </body>
-- 
cgit v1.2.3


From 68bde6252d22cc4e88edc2b51523726bb858c818 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 17 Mar 2011 13:19:38 +0100
Subject: prepared skeleton for new strgen module

---
 ChangeLog                                 |   2 +
 Makefile.am                               |   4 +
 configure.ac                              |  18 ++++
 plugins/sm_cust_bindcdr/Makefile.am       |   6 ++
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 138 ++++++++++++++++++++++++++++++
 5 files changed, 168 insertions(+)
 create mode 100644 plugins/sm_cust_bindcdr/Makefile.am
 create mode 100644 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c

diff --git a/ChangeLog b/ChangeLog
index 13f6608d..3503ed69 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,6 @@
 ---------------------------------------------------------------------------
+Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
+---------------------------------------------------------------------------
 Version 5.7.9  [V5-BETA] (rgerhards), 2011-03-16
 - improved testbench
   among others, life tests for ommysql (against a test database) have
diff --git a/Makefile.am b/Makefile.am
index 53c04922..2d73013b 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -112,6 +112,10 @@ if ENABLE_CUST1
 SUBDIRS += plugins/cust1
 endif
 
+if ENABLE_SMCUSTBINDCDR
+SUBDIRS += plugins/sm_cust_bindcdr
+endif
+
 if ENABLE_IMTEMPLATE
 SUBDIRS += plugins/imtemplate
 endif
diff --git a/configure.ac b/configure.ac
index 4f368bcc..832a5b20 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1081,6 +1081,20 @@ AC_ARG_ENABLE(cust1,
 AM_CONDITIONAL(ENABLE_CUST1, test x$enable_cust1 = xyes)
 
 
+# A custom strgen that also serves as a sample of how to do
+# SQL-generating strgen's
+AC_ARG_ENABLE(smcustbindcdr,
+        [AS_HELP_STRING([--enable-smcustbindcdr],[Compiles smcustbindcdr module @<:@default=no@:>@])],
+        [case "${enableval}" in
+         yes) enable_smcustbindcdr="yes" ;;
+          no) enable_smcustbindcdr="no" ;;
+           *) AC_MSG_ERROR(bad value ${enableval} for --enable-smcustbindcdr) ;;
+         esac],
+        [enable_smcustbindcdr=no]
+)
+AM_CONDITIONAL(ENABLE_SMCUSTBINDCDR, test x$enable_smcustbindcdr = xyes)
+
+
 # settings for the template input module; copy and modify this code
 # if you intend to add your own module. Be sure to replace imtemplate
 # by the actual name of your module.
@@ -1178,6 +1192,7 @@ AC_CONFIG_FILES([Makefile \
 		plugins/omsnmp/Makefile \
 		plugins/omoracle/Makefile \
 		plugins/omudpspoof/Makefile \
+		plugins/sm_cust_bindcdr/Makefile \
 		plugins/cust1/Makefile \
 		java/Makefile \
 		tests/Makefile])
@@ -1223,6 +1238,9 @@ echo "    pmcisconames module will be compiled:     $enable_pmcisconames"
 echo "    pmaixforwardedfrom module w.be compiled:  $enable_pmaixforwardedfrom"
 echo "    pmsnare module will be compiled:          $enable_pmsnare"
 echo
+echo "---{ strgen modules }---"
+echo "    sm_cust_bindcdr module will be compiled:  $enable_sm_cust_bindcdr"
+echo
 echo "---{ database support }---"
 echo "    MySql support enabled:                    $enable_mysql"
 echo "    libdbi support enabled:                   $enable_libdbi"
diff --git a/plugins/sm_cust_bindcdr/Makefile.am b/plugins/sm_cust_bindcdr/Makefile.am
new file mode 100644
index 00000000..1f71d499
--- /dev/null
+++ b/plugins/sm_cust_bindcdr/Makefile.am
@@ -0,0 +1,6 @@
+pkglib_LTLIBRARIES = sm_cust_bindcdr.la
+
+sm_cust_bindcdr_la_SOURCES = sm_cust_bindcdr.c
+sm_cust_bindcdr_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS)
+sm_cust_bindcdr_la_LDFLAGS = -module -avoid-version
+sm_cust_bindcdr_la_LIBADD = 
diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
new file mode 100644
index 00000000..cf3cdeef
--- /dev/null
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -0,0 +1,138 @@
+/* sm_cust_bindcdr.c
+ * This is a custom developed plugin to process bind information into
+ * a specific SQL statement. While the actual processing may be too specific
+ * to be of general use, this module serves as a template on how this type
+ * of processing can be done.
+ *
+ * Format generated:
+ * "%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+ * Note that this is the same as smtradfile.c, except that we do have a RFC3339 timestamp. However,
+ * we have copied over the code from there, it is too simple to go through all the hassle
+ * of having a single code base.
+ *
+ * NOTE: read comments in module-template.h to understand how this file
+ *       works!
+ *
+ * File begun on 2011-03-17 by RGerhards
+ *
+ * Copyright 2011 Rainer Gerhards and Adiscon GmbH.
+ *
+ * This file is part of rsyslog.
+ *
+ * Rsyslog is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Rsyslog is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Rsyslog.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * A copy of the GPL can be found in the file "COPYING" in this distribution.
+ */
+#include "config.h"
+#include "rsyslog.h"
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include "conf.h"
+#include "syslogd-types.h"
+#include "template.h"
+#include "msg.h"
+#include "module-template.h"
+#include "unicode-helper.h"
+
+MODULE_TYPE_STRGEN
+MODULE_TYPE_NOKEEP
+STRGEN_NAME("Custom_BindCDR")
+
+/* internal structures
+ */
+DEF_SMOD_STATIC_DATA
+
+
+/* config data */
+
+
+/* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings
+ * needed (including their length) and then calculating the actual space required. So when we 
+ * finally copy, we know exactly what we need. So we do at most one alloc.
+ */
+BEGINstrgen
+	register int iBuf;
+	uchar *pTimeStamp;
+	size_t lenTimeStamp;
+	uchar *pHOSTNAME;
+	size_t lenHOSTNAME;
+	uchar *pTAG;
+	int lenTAG;
+	uchar *pMSG;
+	size_t lenMSG;
+	size_t lenTotal;
+CODESTARTstrgen
+	/* first obtain all strings and their length (if not fixed) */
+	pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date);
+	lenTimeStamp = ustrlen(pTimeStamp);
+	pHOSTNAME = (uchar*) getHOSTNAME(pMsg);
+	lenHOSTNAME = getHOSTNAMELen(pMsg);
+	getTAG(pMsg, &pTAG, &lenTAG);
+	pMSG = getMSG(pMsg);
+	lenMSG = getMSGLen(pMsg);
+
+	/* calculate len, constants for spaces and similar fixed strings */
+	lenTotal = lenTimeStamp + 1 + lenHOSTNAME + 1 + lenTAG + lenMSG + 2;
+	if(pMSG[0] != ' ')
+		++lenTotal; /* then we need to introduce one additional space */
+
+	/* now make sure buffer is large enough */
+	if(lenTotal  >= *pLenBuf)
+		CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal));
+
+	/* and concatenate the resulting string */
+	memcpy(*ppBuf, pTimeStamp, lenTimeStamp);
+	iBuf = lenTimeStamp;
+	*(*ppBuf + iBuf++) = ' ';
+
+	memcpy(*ppBuf + iBuf, pHOSTNAME, lenHOSTNAME);
+	iBuf += lenHOSTNAME;
+	*(*ppBuf + iBuf++) = ' ';
+
+	memcpy(*ppBuf + iBuf, pTAG, lenTAG);
+	iBuf += lenTAG;
+
+	if(pMSG[0] != ' ')
+		*(*ppBuf + iBuf++) = ' ';
+	memcpy(*ppBuf + iBuf, pMSG, lenMSG);
+	iBuf += lenMSG;
+
+	/* trailer */
+	*(*ppBuf + iBuf++) = '\n';
+	*(*ppBuf + iBuf) = '\0';
+
+finalize_it:
+ENDstrgen
+
+
+BEGINmodExit
+CODESTARTmodExit
+ENDmodExit
+
+
+BEGINqueryEtryPt
+CODESTARTqueryEtryPt
+CODEqueryEtryPt_STD_SMOD_QUERIES
+ENDqueryEtryPt
+
+
+BEGINmodInit(sm_cust_bindcdr)
+CODESTARTmodInit
+	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
+CODEmodInit_QueryRegCFSLineHdlr
+
+	dbgprintf("rsyslog sm_cust_bindcdr called, compiled with version %s\n", VERSION);
+ENDmodInit
-- 
cgit v1.2.3


From 459f0d094fcb48fd58204ad3527f931715839bcb Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 17 Mar 2011 14:59:18 +0100
Subject: setup test environment and test code in order to look at...

bug http://bugzilla.adiscon.com/show_bug.cgi?id=195
That bug currently prevents strgen's to be used with databases
---
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 45 ++++++++++---------------------
 1 file changed, 14 insertions(+), 31 deletions(-)

diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
index cf3cdeef..5ea7e653 100644
--- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -63,56 +63,39 @@ DEF_SMOD_STATIC_DATA
  * needed (including their length) and then calculating the actual space required. So when we 
  * finally copy, we know exactly what we need. So we do at most one alloc.
  */
+//#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('"
+#define SQL_STMT "INSERT INTO bind_test(date,time,client,view,query,ip) VALUES ('"
 BEGINstrgen
 	register int iBuf;
 	uchar *pTimeStamp;
 	size_t lenTimeStamp;
-	uchar *pHOSTNAME;
-	size_t lenHOSTNAME;
-	uchar *pTAG;
-	int lenTAG;
-	uchar *pMSG;
-	size_t lenMSG;
 	size_t lenTotal;
 CODESTARTstrgen
 	/* first obtain all strings and their length (if not fixed) */
 	pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date);
 	lenTimeStamp = ustrlen(pTimeStamp);
-	pHOSTNAME = (uchar*) getHOSTNAME(pMsg);
-	lenHOSTNAME = getHOSTNAMELen(pMsg);
-	getTAG(pMsg, &pTAG, &lenTAG);
-	pMSG = getMSG(pMsg);
-	lenMSG = getMSGLen(pMsg);
 
 	/* calculate len, constants for spaces and similar fixed strings */
-	lenTotal = lenTimeStamp + 1 + lenHOSTNAME + 1 + lenTAG + lenMSG + 2;
-	if(pMSG[0] != ' ')
-		++lenTotal; /* then we need to introduce one additional space */
+	lenTotal = lenTimeStamp + 1 + 200 /* test! */ + 2;
 
 	/* now make sure buffer is large enough */
 	if(lenTotal  >= *pLenBuf)
 		CHKiRet(ExtendBuf(ppBuf, pLenBuf, lenTotal));
 
 	/* and concatenate the resulting string */
-	memcpy(*ppBuf, pTimeStamp, lenTimeStamp);
-	iBuf = lenTimeStamp;
-	*(*ppBuf + iBuf++) = ' ';
+	memcpy(*ppBuf, SQL_STMT, sizeof(SQL_STMT) - 1);
+	iBuf = sizeof(SQL_STMT) - 1;
 
-	memcpy(*ppBuf + iBuf, pHOSTNAME, lenHOSTNAME);
-	iBuf += lenHOSTNAME;
-	*(*ppBuf + iBuf++) = ' ';
+	// SQL content:DATE,TIME,CLIENT,VIEW,QUERY,IP); 
 
-	memcpy(*ppBuf + iBuf, pTAG, lenTAG);
-	iBuf += lenTAG;
+	memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp);
+	iBuf += lenTimeStamp;
+	memcpy(*ppBuf + iBuf, "' , '", sizeof("', '") - 1);
+	iBuf += sizeof("', '") - 1;
 
-	if(pMSG[0] != ' ')
-		*(*ppBuf + iBuf++) = ' ';
-	memcpy(*ppBuf + iBuf, pMSG, lenMSG);
-	iBuf += lenMSG;
-
-	/* trailer */
-	*(*ppBuf + iBuf++) = '\n';
-	*(*ppBuf + iBuf) = '\0';
+	/* end of SQL statement/trailer (NUL is contained in string!) */
+	memcpy(*ppBuf + iBuf, "');", sizeof("');"));
+	iBuf += sizeof("');");
 
 finalize_it:
 ENDstrgen
@@ -129,7 +112,7 @@ CODEqueryEtryPt_STD_SMOD_QUERIES
 ENDqueryEtryPt
 
 
-BEGINmodInit(sm_cust_bindcdr)
+BEGINmodInit()
 CODESTARTmodInit
 	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
 CODEmodInit_QueryRegCFSLineHdlr
-- 
cgit v1.2.3


From e2f13df22c7599fa1fc7d6b4c20081aac78b355a Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Mon, 21 Mar 2011 09:53:55 +0100
Subject: potential fix to issue that strgen's do not support SQL option

needs testig and verification (wrong system for doing that ;))
---
 template.c | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/template.c b/template.c
index 5aaf1bcb..cc339653 100644
--- a/template.c
+++ b/template.c
@@ -856,19 +856,37 @@ tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine)
 {
 	uchar *pSrc, *pDst;
 	uchar szMod[2048];
+	unsigned lenMod;
 	strgen_t *pStrgen;
 	DEFiRet;
 
 	pSrc = *ppRestOfConfLine;
 	pDst = szMod;
-	while(*pSrc && !isspace(*pSrc) && pDst < &(szMod[sizeof(szMod) - 1])) {
-		*pDst++ = *pSrc++;
+	lenMod = 0;
+	while(*pSrc && !isspace(*pSrc) && lenMod < sizeof(szMod) - 1) {
+		szMod[lenMod] = *pSrc++;
+		lenMod++;
+		
 	}
-	*pDst = '\0';
+	szMod[lenMod] = '\0';
 	*ppRestOfConfLine = pSrc;
 	CHKiRet(strgen.FindStrgen(&pStrgen, szMod));
 	pTpl->pStrgen = pStrgen->pModule->mod.sm.strgen;
-	dbgprintf("template bound to strgen '%s'\n", szMod);
+	DBGPRINTF("template bound to strgen '%s'\n", szMod);
+	/* check if the name potentially contains some well-known options
+	 * Note: we have opted to let the name contain all options. This sounds 
+	 * useful, because the strgen MUST actually implement a specific set
+	 * of options. Doing this via the name looks to the enduser as if the
+	 * regular syntax were used, and it make sure the strgen postively
+	 * acknowledged implementing the option. -- rgerhards, 2011-03-21
+	 */
+	if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 4, ",stdsql")) {
+		pTpl->optFormatForSQL = 2;
+		DBGPRINTF("strgen suports the stdsql option\n");
+	} else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 7, ",sql")) {
+		pTpl->optFormatForSQL = 1;
+		DBGPRINTF("strgen suports the sql option\n");
+	}
 
 finalize_it:
 	RETiRet;
@@ -936,8 +954,7 @@ struct template *tplAddLine(char* pName, uchar** ppRestOfConfLine)
 		/* we simply make the template defunct in this case by setting
 		 * its name to a zero-string. We do not free it, as this would
 		 * require additional code and causes only a very small memory
-		 * consumption. Memory is freed, however, in normal operation
-		 * and most importantly by HUPing syslogd.
+		 * consumption.
 		 */
 		*pTpl->pszName = '\0';
 		return NULL;
-- 
cgit v1.2.3


From 41d79b8fead00f85614547606b0c1117f3efa04e Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Mon, 21 Mar 2011 10:34:43 +0100
Subject: bugfix: strgen could not be used together with database outputs

because the sql/stdsql option could not be specified. This has been
solved by permitting the strgen to include the opton inside its name.
closes: http://bugzilla.adiscon.com/show_bug.cgi?id=195
---
 ChangeLog                                 | 6 ++++++
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 2 +-
 template.c                                | 4 ++--
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 3503ed69..70d7d322 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
 ---------------------------------------------------------------------------
 Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
+- new sample plugin for a strgen to generate sql statement consumable
+  by a database plugin
+- bugfix: strgen could not be used together with database outputs
+  because the sql/stdsql option could not be specified. This has been
+  solved by permitting the strgen to include the opton inside its name.
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=195
 ---------------------------------------------------------------------------
 Version 5.7.9  [V5-BETA] (rgerhards), 2011-03-16
 - improved testbench
diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
index 5ea7e653..d1634252 100644
--- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -49,7 +49,7 @@
 
 MODULE_TYPE_STRGEN
 MODULE_TYPE_NOKEEP
-STRGEN_NAME("Custom_BindCDR")
+STRGEN_NAME("Custom_BindCDR,sql")
 
 /* internal structures
  */
diff --git a/template.c b/template.c
index cc339653..3f6942c2 100644
--- a/template.c
+++ b/template.c
@@ -880,10 +880,10 @@ tplAddTplMod(struct template *pTpl, uchar** ppRestOfConfLine)
 	 * regular syntax were used, and it make sure the strgen postively
 	 * acknowledged implementing the option. -- rgerhards, 2011-03-21
 	 */
-	if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 4, ",stdsql")) {
+	if(lenMod > 6 && !strcasecmp((char*) szMod + lenMod - 7, ",stdsql")) {
 		pTpl->optFormatForSQL = 2;
 		DBGPRINTF("strgen suports the stdsql option\n");
-	} else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 7, ",sql")) {
+	} else if(lenMod > 3 && !strcasecmp((char*) szMod+ lenMod - 4, ",sql")) {
 		pTpl->optFormatForSQL = 1;
 		DBGPRINTF("strgen suports the sql option\n");
 	}
-- 
cgit v1.2.3


From 5808af91e7e5e552a5cde4603687685978934626 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 22 Mar 2011 12:17:20 +0100
Subject: added internal processing of BIND records

still some questions outstanding, so actual SQL statement is
not usable at the moment
---
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 121 ++++++++++++++++++++++++++++--
 1 file changed, 115 insertions(+), 6 deletions(-)

diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
index d1634252..be01db77 100644
--- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -37,6 +37,7 @@
 #include "config.h"
 #include "rsyslog.h"
 #include <stdlib.h>
+#include <ctype.h>
 #include <string.h>
 #include <assert.h>
 #include <errno.h>
@@ -64,19 +65,112 @@ DEF_SMOD_STATIC_DATA
  * finally copy, we know exactly what we need. So we do at most one alloc.
  */
 //#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('"
-#define SQL_STMT "INSERT INTO bind_test(date,time,client,view,query,ip) VALUES ('"
+//#define SQL_STMT "INSERT INTO bind_test(`Date`,`time`,client,view,query,ip) VALUES ('"
+#define SQL_STMT "INSERT INTO bind_test(`Date`,ip) VALUES ('"
+#define ADD_SQL_DELIM \
+	memcpy(*ppBuf + iBuf, "', '", sizeof("', '") - 1); \
+	iBuf += sizeof("', '") - 1;
+#define SQL_STMT_END "');\n"
 BEGINstrgen
 	register int iBuf;
+	uchar *psz;
 	uchar *pTimeStamp;
+	uchar szClient[64];
+	unsigned lenClient;
+	uchar szView[64];
+	unsigned lenView;
+	uchar szQuery[64];
+	unsigned lenQuery;
+	uchar szIP[64];
+	unsigned lenIP;
 	size_t lenTimeStamp;
 	size_t lenTotal;
 CODESTARTstrgen
+	/* first create an empty statement. This is to be replaced if
+	 * we have better data to fill in.
+	 */
+	/* now make sure buffer is large enough */
+	if(*pLenBuf < 2)
+		CHKiRet(ExtendBuf(ppBuf, pLenBuf, 2));
+	memcpy(*ppBuf, ";", sizeof(";"));
+
 	/* first obtain all strings and their length (if not fixed) */
 	pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date);
 	lenTimeStamp = ustrlen(pTimeStamp);
+	
+	/* "client" */
+	psz = (uchar*) strstr((char*) getMSG(pMsg), "client ");
+	if(psz == NULL) {
+		dbgprintf("Custom_BindCDR: client part in msg missing\n");
+		FINALIZE;
+	} else {
+		psz += sizeof("client ") - 1; /* skip "label" */
+		for(  lenClient = 0
+		    ; *psz && *psz != '#' && lenClient < sizeof(szClient) - 1
+		    ; ++lenClient) {
+			szClient[lenClient] = *psz++;
+		}
+		szClient[lenClient] = '\0';
+	}
+
+	/* "view" */
+	psz = (uchar*) strstr((char*) getMSG(pMsg), "view ");
+	if(psz == NULL) {
+		dbgprintf("Custom_BindCDR: view part in msg missing\n");
+		FINALIZE;
+	} else {
+		psz += sizeof("view ") - 1; /* skip "label" */
+		for(  lenView = 0
+		    ; *psz && *psz != ':' && lenView < sizeof(szView) - 1
+		    ; ++lenView) {
+			szView[lenView] = *psz++;
+		}
+		szView[lenView] = '\0';
+	}
+
+	/* "query" - we must extract just the number, and in reverse! */
+	psz = (uchar*) strstr((char*) getMSG(pMsg), "query: ");
+	if(psz == NULL) {
+		dbgprintf("Custom_BindCDR: query part in msg missing\n");
+		FINALIZE;
+	} else {
+		psz += sizeof("query: ") - 1; /* skip "label" */
+		/* first find end-of-string to process */
+		while(*psz && (isdigit(*psz) || *psz == '.')) {
+dbgprintf("XXXX: step 1: %c\n", *psz);
+			psz++;
+		}
+		/* now shuffle data */
+		for(  lenQuery = 0
+		    ; *psz && *psz != ' ' && lenQuery < sizeof(szQuery) - 1
+		    ; --psz) {
+			if(isdigit(*psz))
+				szQuery[lenQuery++] = *psz;
+		}
+		szQuery[lenQuery] = '\0';
+	}
+
+	/* "ip" */
+	psz = (uchar*) strstr((char*) getMSG(pMsg), "IN TXT + (");
+	if(psz == NULL) {
+		dbgprintf("Custom_BindCDR: ip part in msg missing\n");
+		FINALIZE;
+	} else {
+		psz += sizeof("IN TXT + (") - 1; /* skip "label" */
+		for(  lenIP = 0
+		    ; *psz && *psz != ')' && lenIP < sizeof(szIP) - 1
+		    ; ++lenIP) {
+			szIP[lenIP] = *psz++;
+		}
+		szIP[lenIP] = '\0';
+	}
+
+
+	/* --- strings extracted ---- */
 
 	/* calculate len, constants for spaces and similar fixed strings */
-	lenTotal = lenTimeStamp + 1 + 200 /* test! */ + 2;
+	lenTotal = lenTimeStamp + lenClient + lenView + lenQuery + lenIP + 5 * 5
+		   + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2;
 
 	/* now make sure buffer is large enough */
 	if(lenTotal  >= *pLenBuf)
@@ -90,12 +184,27 @@ CODESTARTstrgen
 
 	memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp);
 	iBuf += lenTimeStamp;
-	memcpy(*ppBuf + iBuf, "' , '", sizeof("', '") - 1);
-	iBuf += sizeof("', '") - 1;
+	ADD_SQL_DELIM
+
+	memcpy(*ppBuf + iBuf, szClient, lenClient);
+	iBuf += lenClient;
+	ADD_SQL_DELIM
+
+	memcpy(*ppBuf + iBuf, szView, lenView);
+	iBuf += lenView;
+	ADD_SQL_DELIM
+
+	memcpy(*ppBuf + iBuf, szQuery, lenQuery);
+	iBuf += lenQuery;
+	ADD_SQL_DELIM
+
+	memcpy(*ppBuf + iBuf, szIP, lenIP);
+	iBuf += lenIP;
+	ADD_SQL_DELIM
 
 	/* end of SQL statement/trailer (NUL is contained in string!) */
-	memcpy(*ppBuf + iBuf, "');", sizeof("');"));
-	iBuf += sizeof("');");
+	memcpy(*ppBuf + iBuf, SQL_STMT_END, sizeof(SQL_STMT_END));
+	iBuf += sizeof(SQL_STMT_END);
 
 finalize_it:
 ENDstrgen
-- 
cgit v1.2.3


From 0d80f4e4612fb209cc9dda5faf7b71cd261c0c20 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 22 Mar 2011 14:22:38 +0100
Subject: sm_bind_cdr: added capability to configure "allowed IPs"

---
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 83 +++++++++++++++++++++++++++++--
 1 file changed, 80 insertions(+), 3 deletions(-)

diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
index be01db77..005a4ba9 100644
--- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -43,10 +43,12 @@
 #include <errno.h>
 #include "conf.h"
 #include "syslogd-types.h"
+#include "cfsysline.h"
 #include "template.h"
 #include "msg.h"
 #include "module-template.h"
 #include "unicode-helper.h"
+#include "errmsg.h"
 
 MODULE_TYPE_STRGEN
 MODULE_TYPE_NOKEEP
@@ -55,10 +57,65 @@ STRGEN_NAME("Custom_BindCDR,sql")
 /* internal structures
  */
 DEF_SMOD_STATIC_DATA
+DEFobjCurrIf(errmsg)
+
+/* list of "allowed" IPs */
+typedef struct allowedip_s {
+	uchar *pszIP;
+	struct allowedip_s *next;
+} allowedip_t;
+
+static allowedip_t *root;
 
 
 /* config data */
 
+/* check if the provided IP is (already) in the allowed list
+ */
+static int
+isAllowed(uchar *pszIP)
+{
+	allowedip_t *pallow;
+	int ret = 0;
+
+	for(pallow = root ; pallow != NULL ; pallow = pallow->next) {
+		DBGPRINTF("XXXX: checking allowed IP '%s'\n", pallow->pszIP);
+		if(!ustrcmp(pallow->pszIP, pszIP)) {
+			ret = 1;
+			goto finalize_it;
+		}
+	}
+finalize_it: return ret;
+}
+
+/* This function is called to add an additional allowed IP. It adds
+ * the IP to the linked list of them. An error is emitted if the IP
+ * already exists.
+ */
+static rsRetVal addAllowedIP(void __attribute__((unused)) *pVal, uchar *pNewVal)
+{
+	allowedip_t *pNew;
+	DEFiRet;
+
+	if(isAllowed(pNewVal)) {
+		errmsg.LogError(0, NO_ERRCODE, "error: allowed IP '%s' already configured "
+				"duplicate ignored", pNewVal);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+	CHKmalloc(pNew = malloc(sizeof(allowedip_t)));
+	pNew->pszIP = pNewVal;
+	pNew->next = root;
+	root = pNew;
+	DBGPRINTF("sm_cust_bindcdr: allowed IP '%s' added.\n", pNewVal);
+
+finalize_it:
+	if(iRet != RS_RET_OK) {
+		free(pNewVal);
+	}
+
+	RETiRet;
+}
 
 /* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings
  * needed (including their length) and then calculating the actual space required. So when we 
@@ -72,9 +129,10 @@ DEF_SMOD_STATIC_DATA
 	iBuf += sizeof("', '") - 1;
 #define SQL_STMT_END "');\n"
 BEGINstrgen
-	register int iBuf;
+	int iBuf;
 	uchar *psz;
 	uchar *pTimeStamp;
+	size_t lenTimeStamp;
 	uchar szClient[64];
 	unsigned lenClient;
 	uchar szView[64];
@@ -83,7 +141,6 @@ BEGINstrgen
 	unsigned lenQuery;
 	uchar szIP[64];
 	unsigned lenIP;
-	size_t lenTimeStamp;
 	size_t lenTotal;
 CODESTARTstrgen
 	/* first create an empty statement. This is to be replaced if
@@ -137,7 +194,6 @@ CODESTARTstrgen
 		psz += sizeof("query: ") - 1; /* skip "label" */
 		/* first find end-of-string to process */
 		while(*psz && (isdigit(*psz) || *psz == '.')) {
-dbgprintf("XXXX: step 1: %c\n", *psz);
 			psz++;
 		}
 		/* now shuffle data */
@@ -168,6 +224,14 @@ dbgprintf("XXXX: step 1: %c\n", *psz);
 
 	/* --- strings extracted ---- */
 
+	/* now check if the IP is "allowed", in which case we should not
+	 * insert into the database.
+	 */
+	if(isAllowed(szIP)) {
+		DBGPRINTF("sm_cust_bindcdr: message from allowed IP, ignoring\n");
+		FINALIZE;
+	}
+
 	/* calculate len, constants for spaces and similar fixed strings */
 	lenTotal = lenTimeStamp + lenClient + lenView + lenQuery + lenIP + 5 * 5
 		   + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2;
@@ -211,7 +275,16 @@ ENDstrgen
 
 
 BEGINmodExit
+	allowedip_t *pallow, *pdel;
 CODESTARTmodExit
+	for(pallow = root ; pallow != NULL ; ) {
+		pdel = pallow;
+		pallow = pallow->next;
+		free(pdel->pszIP);
+		free(pdel);
+	}
+
+	objRelease(errmsg, CORE_COMPONENT);
 ENDmodExit
 
 
@@ -225,6 +298,10 @@ BEGINmodInit()
 CODESTARTmodInit
 	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
 CODEmodInit_QueryRegCFSLineHdlr
+	CHKiRet(objUse(errmsg, CORE_COMPONENT));
 
+	root = NULL;
+	CHKiRet(omsdRegCFSLineHdlr((uchar *)"sgcustombindcdrallowedip", 0, eCmdHdlrGetWord,
+		addAllowedIP, NULL, STD_LOADABLE_MODULE_ID));
 	dbgprintf("rsyslog sm_cust_bindcdr called, compiled with version %s\n", VERSION);
 ENDmodInit
-- 
cgit v1.2.3


From 27ab3768e2eaefaed7054144939f0730b3fac84f Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 22 Mar 2011 15:02:15 +0100
Subject: bugfix: RFC5424 parser confused by empty structured data

closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237
---
 ChangeLog         | 2 ++
 tools/pmrfc5424.c | 3 +--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 13f6608d..de55c799 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,5 @@
+- bugfix: RFC5424 parser confused by empty structured data
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237
 ---------------------------------------------------------------------------
 Version 5.7.9  [V5-BETA] (rgerhards), 2011-03-16
 - improved testbench
diff --git a/tools/pmrfc5424.c b/tools/pmrfc5424.c
index 2bd18042..b06f1347 100644
--- a/tools/pmrfc5424.c
+++ b/tools/pmrfc5424.c
@@ -142,7 +142,7 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr
 	 * structured data. There may also be \] inside the structured data, which
 	 * do NOT terminate an element.
 	 */
-	if(lenStr == 0 || *p2parse != '[')
+	if(lenStr == 0 || (*p2parse != '[' && *p2parse != '-'))
 		return 1; /* this is NOT structured data! */
 
 	if(*p2parse == '-') { /* empty structured data? */
@@ -211,7 +211,6 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult, int *pLenStr
  *
  * rger, 2005-11-24
  */
-//static int parseRFCSyslogMsg(msg_t *pMsg, int flags)
 BEGINparse
 	uchar *p2parse;
 	uchar *pBuf = NULL;
-- 
cgit v1.2.3


From e69efbb6d7907339ef81a2062f858eb082233f61 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 22 Mar 2011 16:55:11 +0100
Subject: enhance: added $BOM system property to ease writing byte order masks

---
 ChangeLog                  |  1 +
 doc/property_replacer.html |  6 ++++++
 runtime/msg.c              | 10 ++++++++++
 runtime/rsyslog.h          |  1 +
 4 files changed, 18 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index de55c799..846d2850 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,4 @@
+- enhance: added $BOM system property to ease writing byte order masks
 - bugfix: RFC5424 parser confused by empty structured data
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237
 ---------------------------------------------------------------------------
diff --git a/doc/property_replacer.html b/doc/property_replacer.html
index 4d242a34..cd357f67 100644
--- a/doc/property_replacer.html
+++ b/doc/property_replacer.html
@@ -156,6 +156,12 @@ than messages generated somewhere.
 </td>
 </tr>
 <tr>
+<td><b>$bom</b></td>
+<td>The UTF-8 encoded Unicode byte-order mask (BOM). This may be useful
+in templates for RFC5424 support, when the character set is know to be
+Unicode.</td>
+</tr>
+<tr>
 <td><b>$now</b></td>
 <td>The current date stamp in the format YYYY-MM-DD</td>
 </tr>
diff --git a/runtime/msg.c b/runtime/msg.c
index b0261faa..e8e60963 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -444,6 +444,8 @@ rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID)
 		*pPropID = PROP_SYS_MINUTE;
 	} else if(!strcmp((char*) pName, "$myhostname")) {
 		*pPropID = PROP_SYS_MYHOSTNAME;
+	} else if(!strcmp((char*) pName, "$bom")) {
+		*pPropID = PROP_SYS_BOM;
 	} else {
 		*pPropID = PROP_INVALID;
 		iRet = RS_RET_VAR_NOT_FOUND;
@@ -525,6 +527,8 @@ uchar *propIDToName(propid_t propID)
 			return UCHAR_CONSTANT("$MINUTE");
 		case PROP_SYS_MYHOSTNAME:
 			return UCHAR_CONSTANT("$MYHOSTNAME");
+		case PROP_SYS_BOM:
+			return UCHAR_CONSTANT("$BOM");
 		default:
 			return UCHAR_CONSTANT("*invalid property id*");
 	}
@@ -2427,6 +2431,12 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 		case PROP_SYS_MYHOSTNAME:
 			pRes = glbl.GetLocalHostName();
 			break;
+		case PROP_SYS_BOM:
+			if(*pbMustBeFreed == 1)
+				free(pRes);
+			pRes = (uchar*) "\xEF\xBB\xBF";
+			*pbMustBeFreed = 0;
+			break;
 		default:
 			/* there is no point in continuing, we may even otherwise render the
 			 * error message unreadable. rgerhards, 2007-07-10
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index 78e61bcb..d63dbe4f 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -137,6 +137,7 @@ typedef uintTiny	propid_t;
 #define PROP_SYS_QHOUR			156
 #define PROP_SYS_MINUTE			157
 #define PROP_SYS_MYHOSTNAME		158
+#define PROP_SYS_BOM			159
 
 
 /* The error codes below are orginally "borrowed" from
-- 
cgit v1.2.3


From eefc9b6eb69c130015955cfb15dc45e36e431d5a Mon Sep 17 00:00:00 2001
From: Corey Smith <corsmith@gmail.com>
Date: Thu, 24 Mar 2011 17:25:03 +0100
Subject: bugfix: PRI was invalid on Solaris for message from local log socket

Signed-off-by: root <root@localhost.localdomain>
---
 ChangeLog        | 1 +
 runtime/parser.c | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 846d2850..dc6b9872 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,4 @@
+- bugfix: PRI was invalid on Solaris for message from local log socket
 - enhance: added $BOM system property to ease writing byte order masks
 - bugfix: RFC5424 parser confused by empty structured data
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237
diff --git a/runtime/parser.c b/runtime/parser.c
index d3644976..b385c54b 100644
--- a/runtime/parser.c
+++ b/runtime/parser.c
@@ -453,10 +453,10 @@ ParsePRI(msg_t *pMsg)
 			if(pri & ~(LOG_FACMASK|LOG_PRIMASK))
 				pri = DEFUPRI;
 		}
+		pMsg->iFacility = LOG_FAC(pri);
+		pMsg->iSeverity = LOG_PRI(pri);
+		MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg);
 	}
-	pMsg->iFacility = LOG_FAC(pri);
-	pMsg->iSeverity = LOG_PRI(pri);
-	MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg);
 	RETiRet;
 }
 
-- 
cgit v1.2.3


From 0fcbbfbe3b40e2739531cfa3a308c6fc8681ee7f Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Mon, 28 Mar 2011 16:09:07 +0200
Subject: sm_cust_bindcdr: custom date parsing added

---
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 100 +++++++++++++++++++++++++++---
 1 file changed, 90 insertions(+), 10 deletions(-)

diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
index 005a4ba9..fa16acb5 100644
--- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -120,6 +120,8 @@ finalize_it:
 /* This strgen tries to minimize the amount of reallocs be first obtaining pointers to all strings
  * needed (including their length) and then calculating the actual space required. So when we 
  * finally copy, we know exactly what we need. So we do at most one alloc.
+ * An actual message sample for what we intend to parse is (one line):
+  <30>Mar 24 13:01:51 named[6085]: 24-Mar-2011 13:01:51.865 queries: info: client 10.0.0.96#39762: view trusted: query: 8.6.0.9.9.4.1.4.6.1.8.3.mobilecrawler.com IN TXT + (10.0.0.96)
  */
 //#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('"
 //#define SQL_STMT "INSERT INTO bind_test(`Date`,`time`,client,view,query,ip) VALUES ('"
@@ -131,8 +133,12 @@ finalize_it:
 BEGINstrgen
 	int iBuf;
 	uchar *psz;
-	uchar *pTimeStamp;
-	size_t lenTimeStamp;
+	uchar szDate[64];
+	unsigned lenDate;
+	uchar szTime[64];
+	unsigned lenTime;
+	uchar szMSec[64];
+	unsigned lenMSec;
 	uchar szClient[64];
 	unsigned lenClient;
 	uchar szView[64];
@@ -152,9 +158,76 @@ CODESTARTstrgen
 	memcpy(*ppBuf, ";", sizeof(";"));
 
 	/* first obtain all strings and their length (if not fixed) */
-	pTimeStamp = (uchar*) getTimeReported(pMsg, tplFmtRFC3339Date);
-	lenTimeStamp = ustrlen(pTimeStamp);
-	
+	/* Note that there are two date fields present, one in the header
+	 * and one more in the actual message. We use the one from the message
+	 * and parse that our. We check validity based on some fixe fields. In-
+	 * depth verification is probably not worth the effort (CPU time), because
+	 * we do various other checks on the message format below).
+	 */
+	psz = getMSG(pMsg);
+	if(psz[0] == ' ' && psz[3] == '-' && psz[7] == '-') {
+		memcpy(szDate, psz+8,  4);
+		szDate[4] = '-';
+		if(!strncmp((char*)psz+4, "Jan", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '1';
+		} else if(!strncmp((char*)psz+4, "Feb", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '2';
+		} else if(!strncmp((char*)psz+4, "Mar", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '3';
+		} else if(!strncmp((char*)psz+4, "Apr", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '4';
+		} else if(!strncmp((char*)psz+4, "May", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '5';
+		} else if(!strncmp((char*)psz+4, "Jun", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '6';
+		} else if(!strncmp((char*)psz+4, "Jul", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '7';
+		} else if(!strncmp((char*)psz+4, "Aug", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '8';
+		} else if(!strncmp((char*)psz+4, "Sep", 3)) {
+			szDate[5] = '0';
+			szDate[6] = '9';
+		} else if(!strncmp((char*)psz+4, "Oct", 3)) {
+			szDate[5] = '1';
+			szDate[6] = '0';
+		} else if(!strncmp((char*)psz+4, "Nov", 3)) {
+			szDate[5] = '1';
+			szDate[6] = '1';
+		} else if(!strncmp((char*)psz+4, "Dec", 3)) {
+			szDate[5] = '1';
+			szDate[6] = '2';
+		}
+		szDate[7] = '-';
+		szDate[8] = psz[1];
+		szDate[9] = psz[2];
+		szDate[10] = '\0';
+		lenDate = 10;
+	} else {
+		dbgprintf("Custom_BindCDR: date part in msg missing\n");
+		FINALIZE;
+	}
+
+	/* now time (pull both regular time and ms) */
+	if(psz[12] == ' ' && psz[15] == ':' && psz[18] == ':' && psz[21] == '.' && psz[25] == ' ') {
+		memcpy(szTime, (char*)psz+13, 8);
+		szTime[9] = '\0';
+		lenTime = 8;
+		memcpy(szMSec, (char*)psz+22, 3);
+		szMSec[4] = '\0';
+		lenMSec = 3;
+	} else {
+		dbgprintf("Custom_BindCDR: date part in msg missing\n");
+		FINALIZE;
+	}
+
 	/* "client" */
 	psz = (uchar*) strstr((char*) getMSG(pMsg), "client ");
 	if(psz == NULL) {
@@ -233,8 +306,8 @@ CODESTARTstrgen
 	}
 
 	/* calculate len, constants for spaces and similar fixed strings */
-	lenTotal = lenTimeStamp + lenClient + lenView + lenQuery + lenIP + 5 * 5
-		   + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2;
+	lenTotal = lenDate + lenTime + lenMSec + lenClient + lenView + lenQuery
+		   + lenIP + 7 * 5 + sizeof(SQL_STMT) + sizeof(SQL_STMT_END) + 2;
 
 	/* now make sure buffer is large enough */
 	if(lenTotal  >= *pLenBuf)
@@ -246,8 +319,16 @@ CODESTARTstrgen
 
 	// SQL content:DATE,TIME,CLIENT,VIEW,QUERY,IP); 
 
-	memcpy(*ppBuf + iBuf, pTimeStamp, lenTimeStamp);
-	iBuf += lenTimeStamp;
+	memcpy(*ppBuf + iBuf, szDate, lenDate);
+	iBuf += lenDate;
+	ADD_SQL_DELIM
+
+	memcpy(*ppBuf + iBuf, szTime, lenTime);
+	iBuf += lenTime;
+	ADD_SQL_DELIM
+
+	memcpy(*ppBuf + iBuf, szMSec, lenMSec);
+	iBuf += lenMSec;
 	ADD_SQL_DELIM
 
 	memcpy(*ppBuf + iBuf, szClient, lenClient);
@@ -264,7 +345,6 @@ CODESTARTstrgen
 
 	memcpy(*ppBuf + iBuf, szIP, lenIP);
 	iBuf += lenIP;
-	ADD_SQL_DELIM
 
 	/* end of SQL statement/trailer (NUL is contained in string!) */
 	memcpy(*ppBuf + iBuf, SQL_STMT_END, sizeof(SQL_STMT_END));
-- 
cgit v1.2.3


From 6f9773f1faa79c87c4c35c679455151d4e6d9e00 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Mon, 28 Mar 2011 17:40:15 +0200
Subject: bugfix: rsyslog did not build with --disable-regexp configure option

closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243
---
 ChangeLog  |  2 ++
 template.c | 12 +++++++++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index dc6b9872..cbc91b87 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,5 @@
+- bugfix: rsyslog did not build with --disable-regexp configure option
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243
 - bugfix: PRI was invalid on Solaris for message from local log socket
 - enhance: added $BOM system property to ease writing byte order masks
 - bugfix: RFC5424 parser confused by empty structured data
diff --git a/template.c b/template.c
index cc339653..1e1a6c3f 100644
--- a/template.c
+++ b/template.c
@@ -42,10 +42,13 @@
 /* static data */
 DEFobjCurrIf(obj)
 DEFobjCurrIf(errmsg)
-DEFobjCurrIf(regexp)
 DEFobjCurrIf(strgen)
 
+#ifdef FEATURE_REGEXP
+DEFobjCurrIf(regexp)
 static int bFirstRegexpErrmsg = 1; /**< did we already do a "can't load regexp" error message? */
+#endif
+
 static struct template *tplRoot = NULL;	/* the root of the template list */
 static struct template *tplLast = NULL;	/* points to the last element of the template list */
 static struct template *tplLastStatic = NULL; /* last static element of the template list */
@@ -549,10 +552,9 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl)
 	cstr_t *pStrB;
 	struct templateEntry *pTpe;
 	int iNum;	/* to compute numbers */
-	rsRetVal iRetLocal;
-
 #ifdef FEATURE_REGEXP
 	/* APR: variables for regex */
+	rsRetVal iRetLocal;
 	int longitud;
 	unsigned char *regex_char;
 	unsigned char *regex_end;
@@ -1084,11 +1086,13 @@ void tplDeleteAll(void)
 				break;
 			case FIELD:
 				/* check if we have a regexp and, if so, delete it */
+#ifdef FEATURE_REGEXP
 				if(pTpeDel->data.field.has_regex != 0) {
 					if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) {
 						regexp.regfree(&(pTpeDel->data.field.re));
 					}
 				}
+#endif
 				break;
 			}
 			/*dbgprintf("\n");*/
@@ -1137,12 +1141,14 @@ void tplDeleteNew(void)
 				free(pTpeDel->data.constant.pConstant);
 				break;
 			case FIELD:
+#ifdef FEATURE_REGEXP
 				/* check if we have a regexp and, if so, delete it */
 				if(pTpeDel->data.field.has_regex != 0) {
 					if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) {
 						regexp.regfree(&(pTpeDel->data.field.re));
 					}
 				}
+#endif
 				break;
 			}
 			/*dbgprintf("\n");*/
-- 
cgit v1.2.3


From d2cce9852bdd9ba9f93ac94d6c8b0178d5b82bb0 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 29 Mar 2011 11:48:45 +0200
Subject: bugfix: error return from strgen caused abort, now causes action to
 be ignored

...(just like a failed filter)
---
 ChangeLog | 2 ++
 action.c  | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 70d7d322..7d868d64 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,7 @@
 ---------------------------------------------------------------------------
 Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
+- bugfix: error return from strgen caused abort, now causes action to be
+  ignored (just like a failed filter)
 - new sample plugin for a strgen to generate sql statement consumable
   by a database plugin
 - bugfix: strgen could not be used together with database outputs
diff --git a/action.c b/action.c
index ea4358fb..c5bd03cb 100644
--- a/action.c
+++ b/action.c
@@ -1071,7 +1071,8 @@ prepareBatch(action_t *pAction, batch_t *pBatch)
 		pElem = &(pBatch->pElem[i]);
 		if(pElem->bFilterOK && pElem->state != BATCH_STATE_DISC) {
 			pElem->state = BATCH_STATE_RDY;
-			prepareDoActionParams(pAction, pElem);
+			if(prepareDoActionParams(pAction, pElem) != RS_RET_OK)
+				pElem->bFilterOK = FALSE;
 		}
 	}
 	RETiRet;
-- 
cgit v1.2.3


From 030fd145a3d8759367b73cc66e03023ff131c951 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 29 Mar 2011 11:50:32 +0200
Subject: did mapping of data items to database table columns

to facilitate testing
---
 plugins/sm_cust_bindcdr/sm_cust_bindcdr.c | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
index fa16acb5..3fe96ac4 100644
--- a/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
+++ b/plugins/sm_cust_bindcdr/sm_cust_bindcdr.c
@@ -79,7 +79,6 @@ isAllowed(uchar *pszIP)
 	int ret = 0;
 
 	for(pallow = root ; pallow != NULL ; pallow = pallow->next) {
-		DBGPRINTF("XXXX: checking allowed IP '%s'\n", pallow->pszIP);
 		if(!ustrcmp(pallow->pszIP, pszIP)) {
 			ret = 1;
 			goto finalize_it;
@@ -123,9 +122,7 @@ finalize_it:
  * An actual message sample for what we intend to parse is (one line):
   <30>Mar 24 13:01:51 named[6085]: 24-Mar-2011 13:01:51.865 queries: info: client 10.0.0.96#39762: view trusted: query: 8.6.0.9.9.4.1.4.6.1.8.3.mobilecrawler.com IN TXT + (10.0.0.96)
  */
-//#define SQL_STMT "INSERT INTO CDR(date,time,client,view,query,ip) VALUES ('"
-//#define SQL_STMT "INSERT INTO bind_test(`Date`,`time`,client,view,query,ip) VALUES ('"
-#define SQL_STMT "INSERT INTO bind_test(`Date`,ip) VALUES ('"
+#define SQL_STMT "INSERT INTO CDR(`Date`,`Time`, timeMS, client, view, query, ip) VALUES ('"
 #define ADD_SQL_DELIM \
 	memcpy(*ppBuf + iBuf, "', '", sizeof("', '") - 1); \
 	iBuf += sizeof("', '") - 1;
@@ -212,7 +209,7 @@ CODESTARTstrgen
 		lenDate = 10;
 	} else {
 		dbgprintf("Custom_BindCDR: date part in msg missing\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	}
 
 	/* now time (pull both regular time and ms) */
@@ -225,14 +222,14 @@ CODESTARTstrgen
 		lenMSec = 3;
 	} else {
 		dbgprintf("Custom_BindCDR: date part in msg missing\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	}
 
 	/* "client" */
 	psz = (uchar*) strstr((char*) getMSG(pMsg), "client ");
 	if(psz == NULL) {
 		dbgprintf("Custom_BindCDR: client part in msg missing\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	} else {
 		psz += sizeof("client ") - 1; /* skip "label" */
 		for(  lenClient = 0
@@ -247,7 +244,7 @@ CODESTARTstrgen
 	psz = (uchar*) strstr((char*) getMSG(pMsg), "view ");
 	if(psz == NULL) {
 		dbgprintf("Custom_BindCDR: view part in msg missing\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	} else {
 		psz += sizeof("view ") - 1; /* skip "label" */
 		for(  lenView = 0
@@ -262,10 +259,10 @@ CODESTARTstrgen
 	psz = (uchar*) strstr((char*) getMSG(pMsg), "query: ");
 	if(psz == NULL) {
 		dbgprintf("Custom_BindCDR: query part in msg missing\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	} else {
 		psz += sizeof("query: ") - 1; /* skip "label" */
-		/* first find end-of-string to process */
+		/* first find end-of-strihttp://www.rsyslog.com/doc/omruleset.htmlng to process */
 		while(*psz && (isdigit(*psz) || *psz == '.')) {
 			psz++;
 		}
@@ -283,7 +280,7 @@ CODESTARTstrgen
 	psz = (uchar*) strstr((char*) getMSG(pMsg), "IN TXT + (");
 	if(psz == NULL) {
 		dbgprintf("Custom_BindCDR: ip part in msg missing\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	} else {
 		psz += sizeof("IN TXT + (") - 1; /* skip "label" */
 		for(  lenIP = 0
@@ -302,7 +299,7 @@ CODESTARTstrgen
 	 */
 	if(isAllowed(szIP)) {
 		DBGPRINTF("sm_cust_bindcdr: message from allowed IP, ignoring\n");
-		FINALIZE;
+		ABORT_FINALIZE(RS_RET_ERR);
 	}
 
 	/* calculate len, constants for spaces and similar fixed strings */
@@ -317,8 +314,6 @@ CODESTARTstrgen
 	memcpy(*ppBuf, SQL_STMT, sizeof(SQL_STMT) - 1);
 	iBuf = sizeof(SQL_STMT) - 1;
 
-	// SQL content:DATE,TIME,CLIENT,VIEW,QUERY,IP); 
-
 	memcpy(*ppBuf + iBuf, szDate, lenDate);
 	iBuf += lenDate;
 	ADD_SQL_DELIM
-- 
cgit v1.2.3


From 12f126a811dd08553ac9387392c3a506eba3fe55 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 29 Mar 2011 11:59:22 +0200
Subject: manually merging ChangeLog

---
 ChangeLog | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index d59871fe..fdd45aa1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,11 +1,11 @@
+---------------------------------------------------------------------------
+Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
 - bugfix: rsyslog did not build with --disable-regexp configure option
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243
 - bugfix: PRI was invalid on Solaris for message from local log socket
 - enhance: added $BOM system property to ease writing byte order masks
 - bugfix: RFC5424 parser confused by empty structured data
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=237
----------------------------------------------------------------------------
-Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
 - bugfix: error return from strgen caused abort, now causes action to be
   ignored (just like a failed filter)
 - new sample plugin for a strgen to generate sql statement consumable
-- 
cgit v1.2.3


From 9003632c7df5e6020b06b7d19e1c226ea46640de Mon Sep 17 00:00:00 2001
From: Christian Kastner <debian@kvr.at>
Date: Tue, 29 Mar 2011 12:00:56 +0200
Subject: Force ANSI SQL treatment of strings for Postgres

Signed-off-by: Rainer Gerhards <rgerhards@adiscon.com>
---
 plugins/ompgsql/ompgsql.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c
index ab8e4d2c..ea4b4b75 100644
--- a/plugins/ompgsql/ompgsql.c
+++ b/plugins/ompgsql/ompgsql.c
@@ -149,8 +149,13 @@ static rsRetVal initPgSQL(instanceData *pData, int bSilent)
 
 	dbgprintf("host=%s dbname=%s uid=%s\n",pData->f_dbsrv,pData->f_dbname,pData->f_dbuid);
 
+	/* Force PostgreSQL to use ANSI-SQL conforming strings, otherwise we may
+	 * get all sorts of side effects (e.g.: backslash escapes) and warnings
+	 */
+	const char *PgConnectionOptions = "-c standard_conforming_strings=on";
+
 	/* Connect to database */
-	if((pData->f_hpgsql=PQsetdbLogin(pData->f_dbsrv, NULL, NULL, NULL,
+	if((pData->f_hpgsql=PQsetdbLogin(pData->f_dbsrv, NULL, PgConnectionOptions, NULL,
 				pData->f_dbname, pData->f_dbuid, pData->f_dbpwd)) == NULL) {
 		reportDBError(pData, bSilent);
 		closePgSQL(pData); /* ignore any error we may get */
-- 
cgit v1.2.3


From c091e8491c0fa100dce951c09f8da77ee833059a Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 29 Mar 2011 12:02:52 +0200
Subject: doc: adding bugfix to ChangeLog

---
 ChangeLog | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index fdd45aa1..90bc0f2d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,7 @@
 ---------------------------------------------------------------------------
 Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
+- bugfix: ompgsql did not work properly with ANSI SQL strings
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=229
 - bugfix: rsyslog did not build with --disable-regexp configure option
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=243
 - bugfix: PRI was invalid on Solaris for message from local log socket
-- 
cgit v1.2.3


From 0fb11a5a11e3dcb389d6139e24b0d8aa72164dd6 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 29 Mar 2011 14:22:28 +0200
Subject: preparing for 5.7.10

---
 ChangeLog       | 2 +-
 configure.ac    | 2 +-
 doc/manual.html | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 90bc0f2d..e4c25889 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,5 @@
 ---------------------------------------------------------------------------
-Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-??
+Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-29
 - bugfix: ompgsql did not work properly with ANSI SQL strings
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=229
 - bugfix: rsyslog did not build with --disable-regexp configure option
diff --git a/configure.ac b/configure.ac
index 832a5b20..2e563490 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.61)
-AC_INIT([rsyslog],[5.7.9],[rsyslog@lists.adiscon.com])
+AC_INIT([rsyslog],[5.7.10],[rsyslog@lists.adiscon.com])
 AM_INIT_AUTOMAKE
 
 m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
diff --git a/doc/manual.html b/doc/manual.html
index 807ac0da..f28d18d2 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p>
 <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a>
 to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the
 project goals.</p>
-<p><b>This documentation is for version 5.7.9 (beta branch) of rsyslog.</b>
+<p><b>This documentation is for version 5.7.10 (beta branch) of rsyslog.</b>
 Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b>
 to obtain current version information and project status.
 </p><p><b>If you like rsyslog, you might
-- 
cgit v1.2.3


From 5bfa8d2a254e0ead0608b954bdab6dd2a7d9f3e0 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Tue, 29 Mar 2011 14:31:14 +0200
Subject: testbench samples incorrect, corrected

this was due to the old RFC5424 parser being considered correct,
what it was not. As such, the samples generated from it were
incorrect as well. Now with the corrected parsers, the tests failed.
---
 tests/testsuites/rfc5424-1.parse1 | 2 +-
 tests/testsuites/rfc5424-2.parse1 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/testsuites/rfc5424-1.parse1 b/tests/testsuites/rfc5424-1.parse1
index 23836c9f..8a66d7ba 100644
--- a/tests/testsuites/rfc5424-1.parse1
+++ b/tests/testsuites/rfc5424-1.parse1
@@ -1,3 +1,3 @@
 #Example from RFC5424, section 6.5 / sample 1
 <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8
-34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8
+34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,BOM'su root' failed for lonvick on /dev/pts/8
diff --git a/tests/testsuites/rfc5424-2.parse1 b/tests/testsuites/rfc5424-2.parse1
index a86fbc35..f5449e68 100644
--- a/tests/testsuites/rfc5424-2.parse1
+++ b/tests/testsuites/rfc5424-2.parse1
@@ -1,4 +1,4 @@
 <165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts.
-165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],- %% It's time to make the do-nuts.
+165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],%% It's time to make the do-nuts.
 #Example from RFC5424, section 6.5 / sample 2
 #Only the first two lines are important, you may place anything behind them!
-- 
cgit v1.2.3


From b59f8fd65d4f464ce3673439725db1606ec31329 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 30 Mar 2011 11:19:22 +0200
Subject: fixed problem in testbench & added new test (not yet integrated)

The test is a setup scenario for this bug tracker:
http://bugzilla.adiscon.com/show_bug.cgi?id=241
---
 plugins/imdiag/imdiag.c                        |  5 +-
 tests/diag.sh                                  | 14 +++-
 tests/diagtalker.c                             |  6 +-
 tests/rcvr_fail_restore.sh                     | 96 ++++++++++++++++++++++++++
 tests/testsuites/diag-common.conf              |  5 +-
 tests/testsuites/rcvr_fail_restore_rcvr.conf   |  8 +++
 tests/testsuites/rcvr_fail_restore_sender.conf | 15 ++++
 7 files changed, 143 insertions(+), 6 deletions(-)
 create mode 100755 tests/rcvr_fail_restore.sh
 create mode 100644 tests/testsuites/rcvr_fail_restore_rcvr.conf
 create mode 100644 tests/testsuites/rcvr_fail_restore_sender.conf

diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c
index 404cebc7..ed6ef509 100644
--- a/plugins/imdiag/imdiag.c
+++ b/plugins/imdiag/imdiag.c
@@ -205,7 +205,7 @@ doInjectMsg(int iNum)
 	DEFiRet;
 
 	snprintf((char*)szMsg, sizeof(szMsg)/sizeof(uchar),
-		 "<167>Mar  1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", iNum);
+		 "<167>Mar  1 01:00:00 172.20.245.8 tag msgnum:%8.8d:", iNum);
 
 	datetime.getCurrTime(&stTime, &ttGenTime);
 	/* we now create our own message object and submit it to the queue */
@@ -247,6 +247,7 @@ injectMsg(uchar *pszCmd, tcps_sess_t *pSess)
 	}
 
 	CHKiRet(sendResponse(pSess, "%d messages injected\n", nMsgs));
+	DBGPRINTF("imdiag: %d messages injected\n", nMsgs);
 
 finalize_it:
 	RETiRet;
@@ -279,6 +280,7 @@ waitMainQEmpty(tcps_sess_t *pSess)
 	}
 
 	CHKiRet(sendResponse(pSess, "mainqueue empty\n"));
+	DBGPRINTF("imdiag: mainqueue empty\n");
 
 finalize_it:
 	RETiRet;
@@ -314,6 +316,7 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg)
 	if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) {
 		CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize));
 		CHKiRet(sendResponse(pSess, "%d\n", iMsgQueueSize));
+		DBGPRINTF("imdiag: %d messages in main queue\n", iMsgQueueSize);
 	} else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("waitmainqueueempty"))) {
 		CHKiRet(waitMainQEmpty(pSess));
 	} else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("injectmsg"))) {
diff --git a/tests/diag.sh b/tests/diag.sh
index 8fbe42ed..d1242fb1 100755
--- a/tests/diag.sh
+++ b/tests/diag.sh
@@ -84,15 +84,27 @@ case $1 in
 			exit 1
 		fi
 		;;
+   'get-mainqueuesize') # show the current main queue size
+		if [ "$2" == "2" ]
+		then
+			echo getmainmsgqueuesize | ./diagtalker -p13501
+		else
+			echo getmainmsgqueuesize | ./diagtalker
+		fi
+		;;
    'wait-queueempty') # wait for main message queue to be empty. $2 is the instance.
 		if [ "$2" == "2" ]
 		then
-			echo WaitMainQueueEmpty | ./diagtalker
+			echo WaitMainQueueEmpty | ./diagtalker -p13501
 		else
 			echo WaitMainQueueEmpty | ./diagtalker
 		fi
 		;;
    'shutdown-when-empty') # shut rsyslogd down when main queue is empty. $2 is the instance.
+		if [ "$2" == "2" ]
+		then
+		   echo Shutting down instance 2
+		fi
    		$srcdir/diag.sh wait-queueempty $2
 		kill `cat rsyslog$2.pid`
 		# note: we do not wait for the actual termination!
diff --git a/tests/diagtalker.c b/tests/diagtalker.c
index 6a721e47..0d728bda 100644
--- a/tests/diagtalker.c
+++ b/tests/diagtalker.c
@@ -1,7 +1,7 @@
 /* A yet very simple tool to talk to imdiag (this replaces the
  * previous Java implementation in order to get fewer dependencies).
  *
- * Copyright 2010 Rainer Gerhards and Adiscon GmbH.
+ * Copyright 2010,2011 Rainer Gerhards and Adiscon GmbH.
  *
  * This file is part of rsyslog.
  *
@@ -126,7 +126,7 @@ doProcessing()
 		len = strlen(line);
 		sendCmd(fd, line, len);
 		waitRsp(fd, line, sizeof(line));
-		printf("imdiag: %s", line);
+		printf("imdiag[%d]: %s", targetPort, line);
 	}
 }
 
@@ -139,7 +139,7 @@ int main(int argc, char *argv[])
 	int ret = 0;
 	int opt;
 
-	while((opt = getopt(argc, argv, "f:t:p:c:C:m:i:I:P:d:n:M:rB")) != -1) {
+	while((opt = getopt(argc, argv, "t:p:")) != -1) {
 		switch (opt) {
 		case 't':	targetIP = optarg;
 				break;
diff --git a/tests/rcvr_fail_restore.sh b/tests/rcvr_fail_restore.sh
new file mode 100755
index 00000000..e2f0a0d8
--- /dev/null
+++ b/tests/rcvr_fail_restore.sh
@@ -0,0 +1,96 @@
+# Copyright (C) 2011 by Rainer Gerhards
+# This file is part of the rsyslog project, released  under GPLv3
+echo ===============================================================================
+echo \[rcvr_fail_restore.sh\]: test failed receiver restore case
+source $srcdir/diag.sh init
+#
+# STEP1: start both instances and send 1000 messages.
+# Note: receiver is instance 2, sender instance 1.
+#
+# start up the instances. Note that the envrionment settings can be changed to
+# set instance-specific debugging parameters!
+export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUGLOG="log2"
+source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2
+export RSYSLOG_DEBUGLOG="log"
+#valgrind="valgrind"
+source $srcdir/diag.sh startup rcvr_fail_restore_sender.conf
+# re-set params so that new instances do not thrash it...
+unset RSYSLOG_DEBUG
+unset RSYSLOG_DEBUGLOG
+
+# now inject the messages into instance 2. It will connect to instance 1,
+# and that instance will record the data.
+source $srcdir/diag.sh injectmsg  1 1000
+source $srcdir/diag.sh wait-queueempty
+./msleep 3000 # make sure some retries happen (retry interval is set to 1 second)
+#
+# Step 2: shutdown receiver, then send some more data, which then
+# needs to go into the queue.
+#
+source $srcdir/diag.sh shutdown-when-empty 2
+source $srcdir/diag.sh wait-shutdown 2
+
+source $srcdir/diag.sh injectmsg  1001 10000
+sleep 1 # we need to wait, otherwise we may be so fast that the receiver
+# comes up before we have finally suspended the action
+source $srcdir/diag.sh get-mainqueuesize
+ls -l test-spool
+
+#
+# Step 3: restart receiver, wait that the sender drains its queue
+#
+#export RSYSLOG_DEBUGLOG="log2"
+source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2
+echo waiting for sender to drain queue [may need a short while]
+source $srcdir/diag.sh wait-queueempty
+ls -l test-spool
+OLDFILESIZE=$(stat -c%s test-spool/mainq.00000001)
+echo file size to expect is $OLDFILESIZE
+
+
+#
+# Step 4: send new data. Queue files are not permitted to grow now
+# (but one file continous to exist).
+#
+source $srcdir/diag.sh injectmsg  11001 10
+source $srcdir/diag.sh wait-queueempty
+
+# at this point, the queue file shall not have grown. Note
+# that we MUST NOT shut down the instance right now, because it
+# would clean up the queue files! So we need to do our checks
+# first (here!).
+ls -l test-spool
+NEWFILESIZE=$(stat -c%s test-spool/mainq.00000001)
+if [ $NEWFILESIZE != $OLDFILESIZE ]
+then
+   echo file sizes do not match, expected $OLDFILESIZE, actual $NEWFILESIZE
+   echo this means that data has been written to the queue file where it
+   echo no longer should be written.
+   # abort will happen below, because we must ensure proper system shutdown
+   # HOWEVER, during actual testing it may be useful to do an exit here (so
+   # that e.g. the debug log is pointed right at the correct spot).
+   # exit 1
+fi
+
+
+#
+# Queue file size checks done. Now it is time to terminate the system
+# and see if everything could be received (the usual check, done here
+# for completeness, more or less as a bonus).
+#
+source $srcdir/diag.sh shutdown-when-empty
+source $srcdir/diag.sh wait-shutdown
+
+# now it is time to stop the receiver as well
+source $srcdir/diag.sh shutdown-when-empty 2
+source $srcdir/diag.sh wait-shutdown 2
+
+# now abort test if we need to (due to filesize predicate)
+if [ $NEWFILESIZE != $OLDFILESIZE ]
+then
+   exit 1
+fi
+# do the final check
+source $srcdir/diag.sh seq-check 1 11010
+source $srcdir/diag.sh exit
diff --git a/tests/testsuites/diag-common.conf b/tests/testsuites/diag-common.conf
index 9e9e28fe..d76e2d15 100644
--- a/tests/testsuites/diag-common.conf
+++ b/tests/testsuites/diag-common.conf
@@ -13,4 +13,7 @@ $IMDiagServerRun 13500
 $template startupfile,"rsyslogd.started" # trick to use relative path names!
 :syslogtag, contains, "rsyslogd"  ?startupfile
 
-$ErrorMessagesToStderr off
+# I have disabled the directive below, so that we see errors in testcase
+# creation. I am not sure why it was present in the first place, so for
+# now I just leave it commented out -- rgerhards, 2011-03-30
+#$ErrorMessagesToStderr off
diff --git a/tests/testsuites/rcvr_fail_restore_rcvr.conf b/tests/testsuites/rcvr_fail_restore_rcvr.conf
new file mode 100644
index 00000000..38ebad29
--- /dev/null
+++ b/tests/testsuites/rcvr_fail_restore_rcvr.conf
@@ -0,0 +1,8 @@
+$IncludeConfig diag-common2.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+# then SENDER sends to this port (not tcpflood!)
+$InputTCPServerRun 13515
+
+$template outfmt,"%msg:F,58:2%\n"
+:msg, contains, "msgnum:" ./rsyslog.out.log;outfmt
diff --git a/tests/testsuites/rcvr_fail_restore_sender.conf b/tests/testsuites/rcvr_fail_restore_sender.conf
new file mode 100644
index 00000000..6b11aa4a
--- /dev/null
+++ b/tests/testsuites/rcvr_fail_restore_sender.conf
@@ -0,0 +1,15 @@
+$IncludeConfig diag-common.conf
+
+$ModLoad ../plugins/imtcp/.libs/imtcp
+# this listener is for message generation by the test framework!
+$InputTCPServerRun 13514
+
+$WorkDirectory test-spool
+$MainMsgQueueMaxFileSize 1g
+$MainMsgQueueFileName mainq
+
+# we use the shortest resume interval a) to let the test not run too long 
+# and b) make sure some retries happen before the reconnect
+$ActionResumeInterval 1
+$ActionResumeRetryCount -1
+*.*	@@127.0.0.1:13515
-- 
cgit v1.2.3


From 00e1f24187ee814801e6969629b82a7ae030beaf Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Wed, 30 Mar 2011 12:13:14 +0200
Subject: stop adding data to DA queue when low watermark has been reached

potentially closes: http://bugzilla.adiscon.com/show_bug.cgi?id=241
But needs more verification.
---
 runtime/queue.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/runtime/queue.c b/runtime/queue.c
index ef6e843b..50ae307c 100644
--- a/runtime/queue.c
+++ b/runtime/queue.c
@@ -1774,9 +1774,13 @@ qqueueChkStopWrkrDA(qqueue_t *pThis)
 {
 	DEFiRet;
 
+//DBGPRINTF("XXXX: chkStopWrkrDA called, low watermark %d, phys Size %d\n", pThis->iLowWtrMrk, getPhysicalQueueSize(pThis));
 	if(pThis->bEnqOnly) {
 		iRet = RS_RET_TERMINATE_WHEN_IDLE;
 	}
+	if(getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk) {
+		iRet = RS_RET_TERMINATE_NOW;
+	}
 
 	RETiRet;
 }
-- 
cgit v1.2.3


From eccfbf71f9dbc235309c2ad40035d41620f5e5df Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Thu, 31 Mar 2011 15:14:22 +0200
Subject: improved testbench, updated ChangeLog to reflect recent bugfix

After trying out some more things with the new code (after last
bugfix), I now think that the bugfix is OK to use and without
regressions.
---
 ChangeLog                  |  4 ++++
 tests/Makefile.am          |  6 ++++--
 tests/rcvr_fail_restore.sh | 42 ++++++++++++++++++++++++++++++++++--------
 3 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index e4c25889..1eba3db9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,8 @@
 ---------------------------------------------------------------------------
+Version 5.8.0  [V5-stable] (rgerhards), 2011-04-??
+- bugfix: DA queue was never shutdown once it was started
+  closes: http://bugzilla.adiscon.com/show_bug.cgi?id=241
+---------------------------------------------------------------------------
 Version 5.7.10  [V5-BETA] (rgerhards), 2011-03-29
 - bugfix: ompgsql did not work properly with ANSI SQL strings
   closes: http://bugzilla.adiscon.com/show_bug.cgi?id=229
diff --git a/tests/Makefile.am b/tests/Makefile.am
index ebbe0cf6..930aa304 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -56,8 +56,7 @@ TESTS +=  \
 	failover-rptd.sh \
 	failover-no-rptd.sh \
 	failover-no-basic.sh \
-	queue-persist.sh 
-	queue-persist.sh \
+	rcvr_fail_restore.sh \
 	linkedlistqueue.sh
 endif
 
@@ -289,6 +288,9 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \
 	   diag.sh \
 	   testsuites/diag-common.conf \
 	   testsuites/diag-common2.conf \
+           rcvr_fail_restore.sh \
+	   testsuites/rcvr_fail_restore_rcvr.conf \
+	   testsuites/rcvr_fail_restore_sender.conf \
            daqueue-persist.sh \
 	   daqueue-persist-drvr.sh \
 	   queue-persist.sh \
diff --git a/tests/rcvr_fail_restore.sh b/tests/rcvr_fail_restore.sh
index e2f0a0d8..79486f10 100755
--- a/tests/rcvr_fail_restore.sh
+++ b/tests/rcvr_fail_restore.sh
@@ -9,31 +9,32 @@ source $srcdir/diag.sh init
 #
 # start up the instances. Note that the envrionment settings can be changed to
 # set instance-specific debugging parameters!
-export RSYSLOG_DEBUG="debug nostdout"
+#export RSYSLOG_DEBUG="debug nostdout"
 #export RSYSLOG_DEBUGLOG="log2"
 source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2
-export RSYSLOG_DEBUGLOG="log"
+#export RSYSLOG_DEBUGLOG="log"
 #valgrind="valgrind"
 source $srcdir/diag.sh startup rcvr_fail_restore_sender.conf
 # re-set params so that new instances do not thrash it...
-unset RSYSLOG_DEBUG
-unset RSYSLOG_DEBUGLOG
+#unset RSYSLOG_DEBUG
+#unset RSYSLOG_DEBUGLOG
 
 # now inject the messages into instance 2. It will connect to instance 1,
 # and that instance will record the data.
 source $srcdir/diag.sh injectmsg  1 1000
 source $srcdir/diag.sh wait-queueempty
-./msleep 3000 # make sure some retries happen (retry interval is set to 1 second)
+./msleep 1000 # let things settle down a bit
+
 #
 # Step 2: shutdown receiver, then send some more data, which then
 # needs to go into the queue.
 #
+
 source $srcdir/diag.sh shutdown-when-empty 2
 source $srcdir/diag.sh wait-shutdown 2
 
 source $srcdir/diag.sh injectmsg  1001 10000
-sleep 1 # we need to wait, otherwise we may be so fast that the receiver
-# comes up before we have finally suspended the action
+./msleep 3000 # make sure some retries happen (retry interval is set to 3 second)
 source $srcdir/diag.sh get-mainqueuesize
 ls -l test-spool
 
@@ -73,6 +74,31 @@ then
    # exit 1
 fi
 
+#
+# We now do an extra test (so this is two in one ;)) to see if the DA
+# queue can be reactivated after its initial shutdown. In essence, we 
+# redo steps 2 and 3.
+#
+# Step 5: stop receiver again, then send some more data, which then
+# needs to go into the queue.
+#
+echo "*** done primary test *** now checking if DA can be restarted"
+source $srcdir/diag.sh shutdown-when-empty 2
+source $srcdir/diag.sh wait-shutdown 2
+
+source $srcdir/diag.sh injectmsg  11011 10000
+sleep 1 # we need to wait, otherwise we may be so fast that the receiver
+# comes up before we have finally suspended the action
+source $srcdir/diag.sh get-mainqueuesize
+ls -l test-spool
+
+#
+# Step 6: restart receiver, wait that the sender drains its queue
+#
+source $srcdir/diag.sh startup rcvr_fail_restore_rcvr.conf 2
+echo waiting for sender to drain queue [may need a short while]
+source $srcdir/diag.sh wait-queueempty
+ls -l test-spool
 
 #
 # Queue file size checks done. Now it is time to terminate the system
@@ -92,5 +118,5 @@ then
    exit 1
 fi
 # do the final check
-source $srcdir/diag.sh seq-check 1 11010
+source $srcdir/diag.sh seq-check 1 21010
 source $srcdir/diag.sh exit
-- 
cgit v1.2.3