From ab7b1b381a467d6a61760a5cb84ef804a74598f1 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Fri, 24 Aug 2012 15:08:12 +0200
Subject: add base plumbing for template() config object to grammar

---
 grammar/grammar.y      | 14 +++++++++++++-
 grammar/lexer.l        |  8 +++++++-
 grammar/rainerscript.h |  3 +++
 runtime/rsconf.c       | 10 ++++++++++
 4 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/grammar/grammar.y b/grammar/grammar.y
index 402b1a57..35afae30 100644
--- a/grammar/grammar.y
+++ b/grammar/grammar.y
@@ -63,6 +63,9 @@ extern int yyerror(char*);
 %token ENDOBJ
 %token <s> CFSYSLINE
 %token BEGIN_ACTION
+%token BEGIN_PROPERTY
+%token BEGIN_CONSTANT
+%token BEGIN_TPL
 %token STOP
 %token <s> LEGACY_ACTION
 %token <s> PRIFILT
@@ -89,7 +92,7 @@ extern int yyerror(char*);
 %token CMP_STARTSWITHI
 
 %type <nvlst> nv nvlst
-%type <obj> obj
+%type <obj> obj propconst
 %type <actlst> actlst
 %type <actlst> act
 %type <s> cfsysline
@@ -128,6 +131,15 @@ conf:	/* empty (to end recursion) */
 	| conf BSD_HOST_SELECTOR	{ cnfDoBSDHost($2); }
 obj:	  BEGINOBJ nvlst ENDOBJ 	{ $$ = cnfobjNew($1, $2); }
 	| BEGIN_ACTION nvlst ENDOBJ 	{ $$ = cnfobjNew(CNFOBJ_ACTION, $2); }
+        | BEGIN_TPL nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_TPL, $2);  dbgprintf("processing template() without {}\n"); }
+        | BEGIN_TPL nvlst ENDOBJ '{' propconst '}'
+					{ $$ = cnfobjNew(CNFOBJ_TPL, $2); dbgprintf("processing template() WITH {}\n"); }
+/* TODO: NOTE:
+   propconst is the NEXT step. It is just included as an experiment and needs
+   to be replaced.
+*/
+propconst: BEGIN_PROPERTY nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_PROPERTY, $2);
+					  dbgprintf("processed property()\n"); }
 cfsysline: CFSYSLINE	 		{ $$ = $1; }
 nvlst:					{ $$ = NULL; }
 	| nvlst nv 			{ $2->next = $1; $$ = $2; }
diff --git a/grammar/lexer.l b/grammar/lexer.l
index e688ffce..c5e7bf7d 100644
--- a/grammar/lexer.l
+++ b/grammar/lexer.l
@@ -9,7 +9,7 @@
   * cases. So while we hope that cfsysline support can be dropped some time in
   * the future, we will probably keep these useful constructs.
   *
-  * Copyright 2011 Rainer Gerhards and Adiscon GmbH.
+  * Copyright 2011-2012 Rainer Gerhards and Adiscon GmbH.
   *
   * This file is part of the rsyslog runtime library.
   *
@@ -151,6 +151,12 @@ int fileno(FILE *stream);
 				  BEGIN INITIAL; }
 "global"[ \n\t]*"("		{ yylval.objType = CNFOBJ_GLOBAL;
 				  BEGIN INOBJ; return BEGINOBJ; }
+"template"[ \n\t]*"("		{ yylval.objType = CNFOBJ_TPL;
+				  BEGIN INOBJ; return BEGIN_TPL; }
+"property"[ \n\t]*"("		{ yylval.objType = CNFOBJ_PROPERTY;
+				  BEGIN INOBJ; return BEGIN_PROPERTY; }
+"constant"[ \n\t]*"("		{ yylval.objType = CNFOBJ_CONSTANT;
+				  BEGIN INOBJ; return BEGIN_CONSTANT; }
 "input"[ \n\t]*"("		{ yylval.objType = CNFOBJ_INPUT;
 				  BEGIN INOBJ; return BEGINOBJ; }
 "module"[ \n\t]*"("		{ yylval.objType = CNFOBJ_MODULE;
diff --git a/grammar/rainerscript.h b/grammar/rainerscript.h
index 5ff71bee..83a253f7 100644
--- a/grammar/rainerscript.h
+++ b/grammar/rainerscript.h
@@ -16,6 +16,9 @@ enum cnfobjType {
 	CNFOBJ_GLOBAL,
 	CNFOBJ_INPUT,
 	CNFOBJ_MODULE,
+	CNFOBJ_TPL,
+	CNFOBJ_PROPERTY,
+	CNFOBJ_CONSTANT,
 	CNFOBJ_INVALID = 0
 };
 
diff --git a/runtime/rsconf.c b/runtime/rsconf.c
index bd002353..f07ab314 100644
--- a/runtime/rsconf.c
+++ b/runtime/rsconf.c
@@ -386,6 +386,7 @@ yyerror(char *s)
 }
 void cnfDoObj(struct cnfobj *o)
 {
+int bChkUnuse = 1; // TODO: Delete
 	dbgprintf("cnf:global:obj: ");
 	cnfobjPrint(o);
 	switch(o->objType) {
@@ -398,7 +399,16 @@ void cnfDoObj(struct cnfobj *o)
 	case CNFOBJ_ACTION:
 		actionProcessCnf(o);
 		break;
+	case CNFOBJ_TPL:
+		//processTemplate(o);
+bChkUnuse = 0;
+		break;
+	case CNFOBJ_PROPERTY:
+		//processTemplate(o);
+bChkUnuse = 0;
+		break;
 	}
+if(bChkUnuse)
 	nvlstChkUnused(o->nvlst);
 	cnfobjDestruct(o);
 }
-- 
cgit v1.2.3


From 1eac94e11dab1e7caead5e31a57d2cae31b5ad62 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 11:08:38 +0200
Subject: v6 config/templates: legacy types are now supported via template()

---
 runtime/rsconf.c |   4 +-
 template.c       | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 template.h       |   1 +
 3 files changed, 180 insertions(+), 3 deletions(-)

diff --git a/runtime/rsconf.c b/runtime/rsconf.c
index f07ab314..cbc09c11 100644
--- a/runtime/rsconf.c
+++ b/runtime/rsconf.c
@@ -66,6 +66,7 @@
 #include "parserif.h"
 #include "modules.h"
 #include "dirty.h"
+#include "template.h"
 
 /* static data */
 DEFobjStaticHelpers
@@ -400,8 +401,7 @@ int bChkUnuse = 1; // TODO: Delete
 		actionProcessCnf(o);
 		break;
 	case CNFOBJ_TPL:
-		//processTemplate(o);
-bChkUnuse = 0;
+		tplProcessCnf(o);
 		break;
 	case CNFOBJ_PROPERTY:
 		//processTemplate(o);
diff --git a/template.c b/template.c
index 32b8d561..01eab556 100644
--- a/template.c
+++ b/template.c
@@ -2,7 +2,7 @@
  * Please see syslogd.c for license information.
  * begun 2004-11-17 rgerhards
  *
- * Copyright 2004, 2007 Rainer Gerhards and Adiscon
+ * Copyright 2004-2012 Rainer Gerhards and Adiscon
  *
  * This file is part of rsyslog.
  *
@@ -45,6 +45,22 @@ DEFobjCurrIf(obj)
 DEFobjCurrIf(errmsg)
 DEFobjCurrIf(strgen)
 
+/* tables for interfacing with the v6 config system */
+static struct cnfparamdescr cnfparamdescr[] = {
+	{ "name", eCmdHdlrString, 1 },
+	{ "type", eCmdHdlrString, 0 },
+	{ "string", eCmdHdlrString, 0 },
+	{ "plugin", eCmdHdlrString, 0 },
+	{ "option.stdsql", eCmdHdlrBinary, 0 },
+	{ "option.sql", eCmdHdlrBinary, 0 },
+	{ "option.json", eCmdHdlrBinary, 0 }
+};
+static struct cnfparamblk pblk =
+	{ CNFPARAMBLK_VERSION,
+	  sizeof(cnfparamdescr)/sizeof(struct cnfparamdescr),
+	  cnfparamdescr
+	};
+
 #ifdef FEATURE_REGEXP
 DEFobjCurrIf(regexp)
 static int bFirstRegexpErrmsg = 1; /**< did we already do a "can't load regexp" error message? */
@@ -1122,6 +1138,166 @@ struct template *tplAddLine(rsconf_t *conf, char* pName, uchar** ppRestOfConfLin
 }
 
 
+
+// v6 - ASL 2.0!
+/* Add a new template via the v6 config system.
+ */
+rsRetVal
+tplProcessCnf(struct cnfobj *o)
+{
+	struct template *pTpl = NULL;
+	struct cnfparamvals *pvals;
+	int lenName;
+	char *name = NULL;
+	uchar *tplStr = NULL;
+	uchar *p;
+	uchar *plugin;
+	enum { T_STRING, T_PLUGIN, T_LIST } tplType;
+	int i;
+	int o_sql=0, o_stdsql=0, o_json=0; /* options */
+	int numopts;
+	rsRetVal localRet;
+	DEFiRet;
+
+	pvals = nvlstGetParams(o->nvlst, &pblk, NULL);
+	cnfparamsPrint(&pblk, pvals);
+
+	
+	for(i = 0 ; i < pblk.nParams ; ++i) {
+		if(!pvals[i].bUsed)
+			continue;
+		if(!strcmp(pblk.descr[i].name, "name")) {
+			lenName = es_strlen(pvals[i].val.d.estr);
+			name = es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(pblk.descr[i].name, "type")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"string", sizeof("string")-1)) {
+				tplType = T_STRING;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"plugin", sizeof("plugin")-1)) {
+				tplType = T_PLUGIN;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"list", sizeof("list")-1)) {
+				tplType = T_LIST;
+				errmsg.LogError(0, RS_RET_ERR, "template type 'list' is not "
+					"supported in this rsyslog version");
+				ABORT_FINALIZE(RS_RET_ERR);
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid template type '%s'",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblk.descr[i].name, "string")) {
+			tplStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(pblk.descr[i].name, "plugin")) {
+			plugin = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(pblk.descr[i].name, "option.stdsql")) {
+			o_stdsql = pvals[i].val.d.n;
+		} else if(!strcmp(pblk.descr[i].name, "option.sql")) {
+			o_sql = pvals[i].val.d.n;
+		} else if(!strcmp(pblk.descr[i].name, "option.json")) {
+			o_json = pvals[i].val.d.n;
+		} else {
+			dbgprintf("template: program error, non-handled "
+			  "param '%s'\n", pblk.descr[i].name);
+		}
+	}
+
+	/* do config sanity checks */
+	if(tplStr  == NULL) {
+		if(tplType == T_STRING) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' of type string needs "
+				"string parameter", name);
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
+	} else {
+		if(tplType != T_STRING) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' is not a string "
+				"template but has a string specified - ignored", name);
+		}
+	}
+
+	if(plugin  == NULL) {
+		if(tplType == T_PLUGIN) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' of type plugin needs "
+				"plugin parameter - ignored", name);
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
+	} else {
+		if(tplType != T_PLUGIN) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' is not a plugin "
+				"template but has a plugin specified - ignored", name);
+		}
+	}
+
+	numopts = 0;
+	if(o_sql) ++numopts;
+	if(o_stdsql) ++numopts;
+	if(o_json) ++numopts;
+	if(numopts > 1) {
+		errmsg.LogError(0, RS_RET_ERR, "template '%s' has multiple incompatible "
+			"options of sql, stdsql or json specified", name);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+	/* config ok */
+	if((pTpl = tplConstruct(loadConf)) == NULL) {
+		DBGPRINTF("template.c: tplConstruct failed!\n");
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+	pTpl->pszName = name;
+	pTpl->iLenName = lenName;
+	
+	switch(tplType) {
+	case T_STRING:	p = tplStr;
+			while(*p) {
+				switch(*p) {
+					case '%': /* parameter */
+						++p; /* eat '%' */
+						do_Parameter(&p, pTpl);
+						break;
+					default: /* constant */
+						do_Constant(&p, pTpl);
+						break;
+				}
+			}
+			break;
+	case T_PLUGIN:	p = plugin;
+			/* TODO: the use of tplAddTplMod() can be improved! */
+			localRet = tplAddTplMod(pTpl, &p);
+			if(localRet != RS_RET_OK) {
+				errmsg.LogError(0, localRet, "template '%s': error %d "
+						"defining template via plugin (strgen) module",
+						pTpl->pszName, localRet);
+				ABORT_FINALIZE(localRet);
+			}
+	}
+	
+	pTpl->optFormatEscape = NO_ESCAPE;
+	if(o_stdsql)
+		pTpl->optFormatEscape = STDSQL_ESCAPE;
+	else if(o_sql)
+		pTpl->optFormatEscape = SQL_ESCAPE;
+	else if(o_json)
+		pTpl->optFormatEscape = JSON_ESCAPE;
+
+finalize_it:
+	if(iRet != RS_RET_OK) {
+		if(pTpl != NULL) {
+			/* we simply make the template defunct in this case by setting
+			 * its name to a zero-string. We do not free it, as this would
+			 * require additional code and causes only a very small memory
+			 * consumption. TODO: maybe in next iteration...
+			 */
+			*pTpl->pszName = '\0';
+		}
+	}
+
+	RETiRet;
+}
+
+// END v6
+
+
 /* Find a template object based on name. Search
  * currently is case-senstive (should we change?).
  * returns pointer to template object if found and
diff --git a/template.h b/template.h
index e41823e9..9daf2c6b 100644
--- a/template.h
+++ b/template.h
@@ -141,6 +141,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *)
 rsRetVal doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode);
 
 rsRetVal templateInit();
+rsRetVal tplProcessCnf(struct cnfobj *o);
 
 #endif /* #ifndef TEMPLATE_H_INCLUDED */
 /* vim:set ai:
-- 
cgit v1.2.3


From 2f7586bbe415318c35b9ec6d4680c81d443a6660 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 12:35:09 +0200
Subject: cleanup: remove left-over commented-out code

---
 tools/syslogd.c | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/tools/syslogd.c b/tools/syslogd.c
index 90f78d25..81150677 100644
--- a/tools/syslogd.c
+++ b/tools/syslogd.c
@@ -215,16 +215,6 @@ int	repeatinterval[2] = { 30, 60 };	/* # of secs before flush */
 
 static pid_t ppid; /* This is a quick and dirty hack used for spliting main/startup thread */
 
-#if 0
-#warning need this?
-//=======
-typedef struct legacyOptsLL_s {
-	uchar *line;
-	struct legacyOptsLL_s *next;
-} legacyOptsLL_t;
-legacyOptsLL_t *pLegacyOptsLL = NULL;
-#endif
-
 struct queuefilenames_s {
 	struct queuefilenames_s *next;
 	uchar *name;
-- 
cgit v1.2.3


From 6258cb42fd407b9388de63c746634b4df03e78eb Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 13:30:53 +0200
Subject: milestone: base plumbing for LIST-type templates mostly in place

---
 grammar/grammar.y      | 22 +++++++++++++++-------
 grammar/rainerscript.c | 39 +++++++++++++++++++++++++++++++++++++++
 grammar/rainerscript.h | 18 ++++++++++++++++++
 runtime/rsconf.c       |  1 +
 template.c             | 44 +++++++++++++++++++++++++++++++++-----------
 5 files changed, 106 insertions(+), 18 deletions(-)

diff --git a/grammar/grammar.y b/grammar/grammar.y
index 35afae30..d91eb3bb 100644
--- a/grammar/grammar.y
+++ b/grammar/grammar.y
@@ -49,6 +49,7 @@ extern int yyerror(char*);
 	enum cnfobjType objType;
 	struct cnfobj *obj;
 	struct nvlst *nvlst;
+	struct objlst *objlst;
 	struct cnfactlst *actlst;
 	struct cnfexpr *expr;
 	struct cnfrule *rule;
@@ -92,7 +93,8 @@ extern int yyerror(char*);
 %token CMP_STARTSWITHI
 
 %type <nvlst> nv nvlst
-%type <obj> obj propconst
+%type <obj> obj property constant
+%type <objlst> propconst
 %type <actlst> actlst
 %type <actlst> act
 %type <s> cfsysline
@@ -133,13 +135,19 @@ obj:	  BEGINOBJ nvlst ENDOBJ 	{ $$ = cnfobjNew($1, $2); }
 	| BEGIN_ACTION nvlst ENDOBJ 	{ $$ = cnfobjNew(CNFOBJ_ACTION, $2); }
         | BEGIN_TPL nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_TPL, $2);  dbgprintf("processing template() without {}\n"); }
         | BEGIN_TPL nvlst ENDOBJ '{' propconst '}'
-					{ $$ = cnfobjNew(CNFOBJ_TPL, $2); dbgprintf("processing template() WITH {}\n"); }
-/* TODO: NOTE:
-   propconst is the NEXT step. It is just included as an experiment and needs
-   to be replaced.
-*/
-propconst: BEGIN_PROPERTY nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_PROPERTY, $2);
+					{ $$ = cnfobjNew(CNFOBJ_TPL, $2);
+					  $$->subobjs = $5;
+					  dbgprintf("processing template() WITH {}\n"); }
+propconst:				{ $$ = NULL; }
+	| propconst property		{ if($1 == NULL)
+						$$ = objlstNew($2);
+					  else
+						$1->next = objlstNew($2); }
+	| propconst constant		{ /*$2->next = $1; $$ = $2;*/ }
+property: BEGIN_PROPERTY nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_PROPERTY, $2);
 					  dbgprintf("processed property()\n"); }
+constant: BEGIN_CONSTANT nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_CONSTANT, $2);
+					  dbgprintf("processed constant()\n"); }
 cfsysline: CFSYSLINE	 		{ $$ = $1; }
 nvlst:					{ $$ = NULL; }
 	| nvlst nv 			{ $2->next = $1; $$ = $2; }
diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c
index 3bfb2e07..1ccac575 100644
--- a/grammar/rainerscript.c
+++ b/grammar/rainerscript.c
@@ -98,6 +98,44 @@ readConfFile(FILE *fp, es_str_t **str)
 	es_addChar(str, '\0');
 }
 
+struct objlst*
+objlstNew(struct cnfobj *o)
+{
+	struct objlst *lst;
+
+	if((lst = malloc(sizeof(struct objlst))) != NULL) {
+		lst->next = NULL;
+		lst->obj = o;
+	}
+dbgprintf("AAAA: creating new objlst\n");
+cnfobjPrint(o);
+
+	return lst;
+}
+
+void
+objlstDestruct(struct objlst *lst)
+{
+	struct objlst *toDel;
+
+	while(lst != NULL) {
+		toDel = lst;
+		lst = lst->next;
+		// TODO: delete object
+		free(toDel);
+	}
+}
+
+void
+objlstPrint(struct objlst *lst)
+{
+	dbgprintf("objlst %p:\n", lst);
+	while(lst != NULL) {
+		cnfobjPrint(lst->obj);
+		lst = lst->next;
+	}
+}
+
 struct nvlst*
 nvlstNew(es_str_t *name, es_str_t *value)
 {
@@ -581,6 +619,7 @@ cnfobjNew(enum cnfobjType objType, struct nvlst *lst)
 		nvlstChkDupes(lst);
 		o->objType = objType;
 		o->nvlst = lst;
+		o->subobjs = NULL;
 	}
 
 	return o;
diff --git a/grammar/rainerscript.h b/grammar/rainerscript.h
index 83a253f7..4c625cd8 100644
--- a/grammar/rainerscript.h
+++ b/grammar/rainerscript.h
@@ -38,6 +38,15 @@ cnfobjType2str(enum cnfobjType ot)
 	case CNFOBJ_MODULE:
 		return "module";
 		break;
+	case CNFOBJ_TPL:
+		return "template";
+		break;
+	case CNFOBJ_PROPERTY:
+		return "property";
+		break;
+	case CNFOBJ_CONSTANT:
+		return "constant";
+		break;
 	default:return "error: invalid cnfobjType";
 	}
 }
@@ -63,6 +72,12 @@ struct var {
 struct cnfobj {
 	enum cnfobjType objType;
 	struct nvlst *nvlst;
+	struct objlst *subobjs;
+};
+
+struct objlst {
+	struct objlst *next;
+	struct cnfobj *obj;
 };
 
 struct nvlst {
@@ -221,6 +236,9 @@ struct cnfparamvals { /* the values we obtained for param descr. */
 
 int cnfParseBuffer(char *buf, unsigned lenBuf);
 void readConfFile(FILE *fp, es_str_t **str);
+struct objlst* objlstNew(struct cnfobj *obj);
+void objlstDestruct(struct objlst *lst);
+void objlstPrint(struct objlst *lst);
 struct nvlst* nvlstNew(es_str_t *name, es_str_t *value);
 void nvlstDestruct(struct nvlst *lst);
 void nvlstPrint(struct nvlst *lst);
diff --git a/runtime/rsconf.c b/runtime/rsconf.c
index cbc09c11..9c6bf00e 100644
--- a/runtime/rsconf.c
+++ b/runtime/rsconf.c
@@ -404,6 +404,7 @@ int bChkUnuse = 1; // TODO: Delete
 		tplProcessCnf(o);
 		break;
 	case CNFOBJ_PROPERTY:
+	case CNFOBJ_CONSTANT:
 		//processTemplate(o);
 bChkUnuse = 0;
 		break;
diff --git a/template.c b/template.c
index 50db86a3..39f95967 100644
--- a/template.c
+++ b/template.c
@@ -1146,11 +1146,23 @@ struct template *tplAddLine(rsconf_t *conf, char* pName, uchar** ppRestOfConfLin
 	return(pTpl);
 }
 
+/* create a template in list mode, is build from sub-objects */
+static rsRetVal
+createListTpl(struct template *pTpl, struct cnfobj *o)
+{
+	struct objlst *lst;
+	DEFiRet;
 
+	dbgprintf("AAAA: create template from subobjs\n");
+	objlstPrint(o->subobjs);
 
-// v6 - ASL 2.0!
-/* Add a new template via the v6 config system.
- */
+	for(lst = o->subobjs ; lst != NULL ; lst = lst->next) {
+		dbgprintf("AAAA: subjobject entry %p\n", lst);
+	}
+	RETiRet;
+}
+
+/* Add a new template via the v6 config system.  */
 rsRetVal
 tplProcessCnf(struct cnfobj *o)
 {
@@ -1159,8 +1171,8 @@ tplProcessCnf(struct cnfobj *o)
 	int lenName;
 	char *name = NULL;
 	uchar *tplStr = NULL;
+	uchar *plugin = NULL;
 	uchar *p;
-	uchar *plugin;
 	enum { T_STRING, T_PLUGIN, T_LIST } tplType;
 	int i;
 	int o_sql=0, o_stdsql=0, o_json=0; /* options */
@@ -1170,7 +1182,6 @@ tplProcessCnf(struct cnfobj *o)
 
 	pvals = nvlstGetParams(o->nvlst, &pblk, NULL);
 	cnfparamsPrint(&pblk, pvals);
-
 	
 	for(i = 0 ; i < pblk.nParams ; ++i) {
 		if(!pvals[i].bUsed)
@@ -1185,9 +1196,6 @@ tplProcessCnf(struct cnfobj *o)
 				tplType = T_PLUGIN;
 			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"list", sizeof("list")-1)) {
 				tplType = T_LIST;
-				errmsg.LogError(0, RS_RET_ERR, "template type 'list' is not "
-					"supported in this rsyslog version");
-				ABORT_FINALIZE(RS_RET_ERR);
 			} else {
 				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
 				errmsg.LogError(0, RS_RET_ERR, "invalid template type '%s'",
@@ -1228,7 +1236,7 @@ tplProcessCnf(struct cnfobj *o)
 	if(plugin  == NULL) {
 		if(tplType == T_PLUGIN) {
 			errmsg.LogError(0, RS_RET_ERR, "template '%s' of type plugin needs "
-				"plugin parameter - ignored", name);
+				"plugin parameter", name);
 			ABORT_FINALIZE(RS_RET_ERR);
 		}
 	} else {
@@ -1238,6 +1246,19 @@ tplProcessCnf(struct cnfobj *o)
 		}
 	}
 
+	if(o->subobjs  == NULL) {
+		if(tplType == T_LIST) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' of type list has "
+				"has no parameters specified", name);
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
+	} else {
+		if(tplType != T_LIST) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' is not a list "
+				"template but has parameters specified - ignored", name);
+		}
+	}
+
 	numopts = 0;
 	if(o_sql) ++numopts;
 	if(o_stdsql) ++numopts;
@@ -1279,6 +1300,9 @@ tplProcessCnf(struct cnfobj *o)
 						pTpl->pszName, localRet);
 				ABORT_FINALIZE(localRet);
 			}
+			break;
+	case T_LIST:	createListTpl(pTpl, o);
+			break;
 	}
 	
 	pTpl->optFormatEscape = NO_ESCAPE;
@@ -1304,8 +1328,6 @@ finalize_it:
 	RETiRet;
 }
 
-// END v6
-
 
 /* Find a template object based on name. Search
  * currently is case-senstive (should we change?).
-- 
cgit v1.2.3


From 76e9968c0890cb4db068c953db6e6574b6c28da0 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 17:17:27 +0200
Subject: milestone: LIST-type templates work, but no all options yet present

---
 grammar/grammar.y      |   9 +-
 grammar/rainerscript.c |  18 ++++
 runtime/rsconf.c       |  11 +-
 template.c             | 266 ++++++++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 291 insertions(+), 13 deletions(-)

diff --git a/grammar/grammar.y b/grammar/grammar.y
index d91eb3bb..27d0e1d5 100644
--- a/grammar/grammar.y
+++ b/grammar/grammar.y
@@ -137,13 +137,10 @@ obj:	  BEGINOBJ nvlst ENDOBJ 	{ $$ = cnfobjNew($1, $2); }
         | BEGIN_TPL nvlst ENDOBJ '{' propconst '}'
 					{ $$ = cnfobjNew(CNFOBJ_TPL, $2);
 					  $$->subobjs = $5;
-					  dbgprintf("processing template() WITH {}\n"); }
+					  dbgprintf("processing template() WITH {}, subobj=%p\n", $5); }
 propconst:				{ $$ = NULL; }
-	| propconst property		{ if($1 == NULL)
-						$$ = objlstNew($2);
-					  else
-						$1->next = objlstNew($2); }
-	| propconst constant		{ /*$2->next = $1; $$ = $2;*/ }
+	| propconst property		{ $$ = objlstAdd($1, $2); }
+	| propconst constant		{ $$ = objlstAdd($1, $2); }
 property: BEGIN_PROPERTY nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_PROPERTY, $2);
 					  dbgprintf("processed property()\n"); }
 constant: BEGIN_CONSTANT nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_CONSTANT, $2);
diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c
index 1ccac575..33630a76 100644
--- a/grammar/rainerscript.c
+++ b/grammar/rainerscript.c
@@ -113,6 +113,24 @@ cnfobjPrint(o);
 	return lst;
 }
 
+/* add object to end of object list, always returns pointer to root object */
+struct objlst*
+objlstAdd(struct objlst *root, struct cnfobj *o)
+{
+	struct objlst *l;
+	struct objlst *newl;
+	
+	newl = objlstNew(o);
+	if(root == 0) {
+		root = newl;
+	} else { /* find last, linear search ok, as only during config phase */
+		for(l = root ; l->next != NULL ; l = l->next)
+			;
+		l->next = newl;
+	}
+	return root;
+}
+
 void
 objlstDestruct(struct objlst *lst)
 {
diff --git a/runtime/rsconf.c b/runtime/rsconf.c
index 9c6bf00e..5d2407ec 100644
--- a/runtime/rsconf.c
+++ b/runtime/rsconf.c
@@ -387,7 +387,8 @@ yyerror(char *s)
 }
 void cnfDoObj(struct cnfobj *o)
 {
-int bChkUnuse = 1; // TODO: Delete
+	int bChkUnuse = 1;
+
 	dbgprintf("cnf:global:obj: ");
 	cnfobjPrint(o);
 	switch(o->objType) {
@@ -405,12 +406,12 @@ int bChkUnuse = 1; // TODO: Delete
 		break;
 	case CNFOBJ_PROPERTY:
 	case CNFOBJ_CONSTANT:
-		//processTemplate(o);
-bChkUnuse = 0;
+		/* these types are processed at a later stage */
+		bChkUnuse = 0;
 		break;
 	}
-if(bChkUnuse)
-	nvlstChkUnused(o->nvlst);
+	if(bChkUnuse)
+		nvlstChkUnused(o->nvlst);
 	cnfobjDestruct(o);
 }
 
diff --git a/template.c b/template.c
index 39f95967..cdcec305 100644
--- a/template.c
+++ b/template.c
@@ -66,6 +66,35 @@ static struct cnfparamblk pblk =
 	  cnfparamdescr
 	};
 
+static struct cnfparamdescr cnfparamdescrProperty[] = {
+	{ "name", eCmdHdlrString, 1 },
+	{ "outname", eCmdHdlrString, 0 },
+	{ "dateformat", eCmdHdlrString, 0 },
+	{ "caseconversion", eCmdHdlrString, 0 },
+	{ "controlcharacters", eCmdHdlrString, 0 },
+	{ "securepath", eCmdHdlrString, 0 },
+	{ "format", eCmdHdlrString, 0 },
+	{ "droplastlf", eCmdHdlrBinary, 0 },
+	{ "spifno1stsp", eCmdHdlrBinary, 0 }
+};
+static struct cnfparamblk pblkProperty =
+	{ CNFPARAMBLK_VERSION,
+	  sizeof(cnfparamdescrProperty)/sizeof(struct cnfparamdescr),
+	  cnfparamdescrProperty
+	};
+
+static struct cnfparamdescr cnfparamdescrConstant[] = {
+	{ "value", eCmdHdlrString, 1 },
+	{ "outname", eCmdHdlrString, 0 },
+	{ "format", eCmdHdlrString, 0 }
+};
+static struct cnfparamblk pblkConstant =
+	{ CNFPARAMBLK_VERSION,
+	  sizeof(cnfparamdescrConstant)/sizeof(struct cnfparamdescr),
+	  cnfparamdescrConstant
+	};
+
+
 #ifdef FEATURE_REGEXP
 DEFobjCurrIf(regexp)
 static int bFirstRegexpErrmsg = 1; /**< did we already do a "can't load regexp" error message? */
@@ -1146,6 +1175,227 @@ struct template *tplAddLine(rsconf_t *conf, char* pName, uchar** ppRestOfConfLin
 	return(pTpl);
 }
 
+static rsRetVal
+createConstantTpe(struct template *pTpl, struct cnfobj *o)
+{
+	struct templateEntry *pTpe;
+	es_str_t *value;
+	int i;
+	struct cnfparamvals *pvals;
+	DEFiRet;
+
+	/* pull params */
+	pvals = nvlstGetParams(o->nvlst, &pblkConstant, NULL);
+	cnfparamsPrint(&pblkConstant, pvals);
+	
+	for(i = 0 ; i < pblkConstant.nParams ; ++i) {
+		if(!pvals[i].bUsed)
+			continue;
+		if(!strcmp(pblkConstant.descr[i].name, "value")) {
+			value = pvals[i].val.d.estr;
+		} else if(!strcmp(pblkConstant.descr[i].name, "format")) {
+			errmsg.LogError(0, RS_RET_ERR, "paramter 'format' is currently not "
+				"supported for 'constant' template parts - ignored");
+		} else if(!strcmp(pblkConstant.descr[i].name, "outname")) {
+			errmsg.LogError(0, RS_RET_ERR, "paramter 'outname' is currently not "
+				"supported for 'constant' template parts - ignored");
+		} else {
+			dbgprintf("template:constantTpe: program error, non-handled "
+			  "param '%s'\n", pblkConstant.descr[i].name);
+		}
+	}
+
+	/* sanity check */
+
+	/* apply */
+	CHKmalloc(pTpe = tpeConstruct(pTpl));
+	es_unescapeStr(value);
+	pTpe->eEntryType = CONSTANT;
+	pTpe->data.constant.iLenConstant = es_strlen(value);
+	pTpe->data.constant.pConstant = (uchar*)es_str2cstr(value, NULL);
+
+finalize_it:
+	RETiRet;
+}
+
+static rsRetVal
+createPropertyTpe(struct template *pTpl, struct cnfobj *o)
+{
+	struct templateEntry *pTpe;
+	cstr_t *name;
+	es_str_t *estrname;
+	es_str_t *outname = NULL;
+	int i;
+	int droplastlf = 0;
+	int spifno1stsp = 0;
+	struct cnfparamvals *pvals;
+	enum {F_NONE, F_CSV, F_JSON, F_JSONF} formatType = F_NONE;
+	enum {CC_NONE, CC_ESCAPE, CC_SPACE, CC_DROP} controlchr = CC_NONE;
+	enum {SP_NONE, SP_DROP, SP_REPLACE} secpath = SP_NONE;
+	enum tplFormatCaseConvTypes caseconv = tplCaseConvNo;
+	enum tplFormatTypes datefmt = tplFmtDefault;
+	DEFiRet;
+
+	/* pull params */
+	pvals = nvlstGetParams(o->nvlst, &pblkProperty, NULL);
+	cnfparamsPrint(&pblkProperty, pvals);
+	
+	for(i = 0 ; i < pblkProperty.nParams ; ++i) {
+		if(!pvals[i].bUsed)
+			continue;
+		if(!strcmp(pblkProperty.descr[i].name, "name")) {
+			estrname = es_strdup(pvals[i].val.d.estr);
+			/* TODO: unify strings!!! */
+			rsCStrConstructFromszStr(&name,
+				(uchar*)es_str2cstr(pvals[i].val.d.estr, NULL));
+		} else if(!strcmp(pblkProperty.descr[i].name, "droplastlf")) {
+			droplastlf = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "spifno1stsp")) {
+			spifno1stsp = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "outname")) {
+			outname = es_strdup(pvals[i].val.d.estr);
+		} else if(!strcmp(pblkProperty.descr[i].name, "format")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"csv", sizeof("csv")-1)) {
+				formatType = F_CSV;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"json", sizeof("json")-1)) {
+				formatType = F_JSON;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"jsonf", sizeof("jsonf")-1)) {
+				formatType = F_JSONF;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid format type '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblkProperty.descr[i].name, "controlcharacters")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"escape", sizeof("escape")-1)) {
+				controlchr = CC_ESCAPE;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"space", sizeof("space")-1)) {
+				controlchr = CC_SPACE;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"drop", sizeof("drop")-1)) {
+				controlchr = CC_DROP;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid controlcharacter mode '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblkProperty.descr[i].name, "securepath")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"drop", sizeof("drop")-1)) {
+				secpath = SP_DROP;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"replace", sizeof("replace")-1)) {
+				secpath = SP_REPLACE;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid securepath mode '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblkProperty.descr[i].name, "caseconversion")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"lower", sizeof("lower")-1)) {
+				caseconv = tplCaseConvLower;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"upper", sizeof("upper")-1)) {
+				caseconv = tplCaseConvUpper;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid caseconversion type '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblkProperty.descr[i].name, "dateformat")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"mysql", sizeof("mysql")-1)) {
+				datefmt = tplFmtMySQLDate;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"pgsql", sizeof("pgsql")-1)) {
+				datefmt = tplFmtPgSQLDate;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"rfc3164", sizeof("rfc3164")-1)) {
+				datefmt = tplFmtRFC3164Date;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"rfc3164-buggyday", sizeof("rfc3164-buggyday")-1)) {
+				datefmt = tplFmtRFC3164BuggyDate;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"rfc3339", sizeof("rfc3339")-1)) {
+				datefmt = tplFmtRFC3339Date;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"unixtimestamp", sizeof("unixtimestamp")-1)) {
+				datefmt = tplFmtUnixDate;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"subseconds", sizeof("subseconds")-1)) {
+				datefmt = tplFmtSecFrac;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid date format '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else {
+			dbgprintf("template:propertyTpe: program error, non-handled "
+			  "param '%s'\n", pblkProperty.descr[i].name);
+		}
+	}
+	if(outname == NULL)
+		outname = es_strdup(estrname);
+
+	/* sanity check */
+
+	/* apply */
+	CHKmalloc(pTpe = tpeConstruct(pTpl));
+	pTpe->eEntryType = FIELD;
+	CHKiRet(propNameToID(name, &pTpe->data.field.propid));
+	if(pTpe->data.field.propid == PROP_CEE) {
+		/* in CEE case, we need to preserve the actual property name */
+		pTpe->data.field.propName = estrname;
+	} else {
+		es_deleteStr(estrname);
+	}
+	pTpe->data.field.options.bDropLastLF = droplastlf;
+	pTpe->data.field.options.bSPIffNo1stSP = spifno1stsp;
+	pTpe->data.field.eCaseConv = caseconv;
+	switch(formatType) {
+	case F_NONE:
+		/* all set ;) */
+		break;
+	case F_CSV:
+		pTpe->data.field.options.bCSV = 1;
+		break;
+	case F_JSON:
+		pTpe->data.field.options.bJSON = 1;
+		break;
+	case F_JSONF:
+		pTpe->data.field.options.bJSONf = 1;
+		break;
+	}
+	switch(controlchr) {
+	case CC_NONE:
+		/* all set ;) */
+		break;
+	case CC_ESCAPE:
+		pTpe->data.field.options.bEscapeCC = 1;
+		break;
+	case CC_SPACE:
+		pTpe->data.field.options.bSpaceCC = 1;
+		break;
+	case CC_DROP:
+		pTpe->data.field.options.bDropCC = 1;
+		break;
+	}
+	switch(secpath) {
+	case SP_NONE:
+		/* all set ;) */
+		break;
+	case SP_DROP:
+		pTpe->data.field.options.bSecPathDrop = 1;
+		break;
+	case SP_REPLACE:
+		pTpe->data.field.options.bSecPathReplace = 1;
+		break;
+	}
+	pTpe->data.field.fieldName = outname;
+	pTpe->data.field.eDateFormat = datefmt;
+finalize_it:
+	RETiRet;
+}
+
 /* create a template in list mode, is build from sub-objects */
 static rsRetVal
 createListTpl(struct template *pTpl, struct cnfobj *o)
@@ -1153,12 +1403,24 @@ createListTpl(struct template *pTpl, struct cnfobj *o)
 	struct objlst *lst;
 	DEFiRet;
 
-	dbgprintf("AAAA: create template from subobjs\n");
+	dbgprintf("create template from subobjs\n");
 	objlstPrint(o->subobjs);
 
 	for(lst = o->subobjs ; lst != NULL ; lst = lst->next) {
-		dbgprintf("AAAA: subjobject entry %p\n", lst);
+		switch(lst->obj->objType) {
+		case CNFOBJ_PROPERTY:
+			CHKiRet(createPropertyTpe(pTpl, lst->obj));
+			break;
+		case CNFOBJ_CONSTANT:
+			CHKiRet(createConstantTpe(pTpl, lst->obj));
+			break;
+		default:dbgprintf("program error: invalid object type %d "
+				  "in createLstTpl\n", lst->obj->objType);
+			break;
+		}
+		nvlstChkUnused(lst->obj->nvlst);
 	}
+finalize_it:
 	RETiRet;
 }
 
-- 
cgit v1.2.3


From a36b7131ab79204f43c400de7b5663613af62cc6 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 17:48:16 +0200
Subject: milestone: LIST-type template now only missing regex support

---
 template.c | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/template.c b/template.c
index cdcec305..7a4c398f 100644
--- a/template.c
+++ b/template.c
@@ -74,6 +74,17 @@ static struct cnfparamdescr cnfparamdescrProperty[] = {
 	{ "controlcharacters", eCmdHdlrString, 0 },
 	{ "securepath", eCmdHdlrString, 0 },
 	{ "format", eCmdHdlrString, 0 },
+// From here
+	{ "position.from", eCmdHdlrInt, 0 },
+	{ "position.to", eCmdHdlrInt, 0 },
+	{ "field.number", eCmdHdlrInt, 0 },
+	{ "field.delimiter", eCmdHdlrInt, 0 },
+	{ "regex.expression", eCmdHdlrString, 0 },
+	{ "regex.type", eCmdHdlrString, 0 },
+	{ "regex.nomatchmode", eCmdHdlrString, 0 },
+	{ "regex.match", eCmdHdlrInt, 0 },
+	{ "regex.submatch", eCmdHdlrInt, 0 },
+//-- to here
 	{ "droplastlf", eCmdHdlrBinary, 0 },
 	{ "spifno1stsp", eCmdHdlrBinary, 0 }
 };
@@ -1228,6 +1239,10 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 	int i;
 	int droplastlf = 0;
 	int spifno1stsp = 0;
+	int frompos = -1;
+	int topos = -1;
+	int fieldnum = -1;
+	int fielddelim = 9; /* default is HT (USACSII 9) */
 	struct cnfparamvals *pvals;
 	enum {F_NONE, F_CSV, F_JSON, F_JSONF} formatType = F_NONE;
 	enum {CC_NONE, CC_ESCAPE, CC_SPACE, CC_DROP} controlchr = CC_NONE;
@@ -1254,6 +1269,14 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 			spifno1stsp = pvals[i].val.d.n;
 		} else if(!strcmp(pblkProperty.descr[i].name, "outname")) {
 			outname = es_strdup(pvals[i].val.d.estr);
+		} else if(!strcmp(pblkProperty.descr[i].name, "position.from")) {
+			frompos = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "position.to")) {
+			topos = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "field.number")) {
+			fieldnum = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "field.delimiter")) {
+			fielddelim = pvals[i].val.d.n;
 		} else if(!strcmp(pblkProperty.descr[i].name, "format")) {
 			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"csv", sizeof("csv")-1)) {
 				formatType = F_CSV;
@@ -1337,6 +1360,15 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 		outname = es_strdup(estrname);
 
 	/* sanity check */
+	if(topos == -1 && frompos != -1)
+		topos = 2000000000; /* large enough ;) */
+	if(frompos == -1 && topos != -1)
+		frompos = 0;
+	if(topos < frompos) {
+		errmsg.LogError(0, RS_RET_ERR, "position.to=%d is lower than postion.from=%d\n",
+			topos, frompos);
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
 
 	/* apply */
 	CHKmalloc(pTpe = tpeConstruct(pTpl));
@@ -1392,6 +1424,15 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 	}
 	pTpe->data.field.fieldName = outname;
 	pTpe->data.field.eDateFormat = datefmt;
+	if(fieldnum != -1) {
+		pTpe->data.field.has_fields = 1;
+		pTpe->data.field.iFieldNr = fieldnum;
+		pTpe->data.field.field_delim = fielddelim;
+	}
+	if(frompos != -1) {
+		pTpe->data.field.iFromPos = frompos;
+		pTpe->data.field.iToPos = topos;
+	}
 finalize_it:
 	RETiRet;
 }
-- 
cgit v1.2.3


From 00c4f69c559e5ba036e20a095843a1ca6eba57c8 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 19:08:07 +0200
Subject: milestone: LIST-type templates support full option set

---
 template.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 template.h | 10 ++++-----
 2 files changed, 77 insertions(+), 5 deletions(-)

diff --git a/template.c b/template.c
index 7a4c398f..4cad7cb7 100644
--- a/template.c
+++ b/template.c
@@ -1243,12 +1243,17 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 	int topos = -1;
 	int fieldnum = -1;
 	int fielddelim = 9; /* default is HT (USACSII 9) */
+	int re_matchToUse = 0;
+	int re_submatchToUse = 0;
+	char *re_expr = NULL;
 	struct cnfparamvals *pvals;
 	enum {F_NONE, F_CSV, F_JSON, F_JSONF} formatType = F_NONE;
 	enum {CC_NONE, CC_ESCAPE, CC_SPACE, CC_DROP} controlchr = CC_NONE;
 	enum {SP_NONE, SP_DROP, SP_REPLACE} secpath = SP_NONE;
 	enum tplFormatCaseConvTypes caseconv = tplCaseConvNo;
 	enum tplFormatTypes datefmt = tplFmtDefault;
+	enum tplRegexType re_type = TPL_REGEX_BRE;
+	enum tlpRegexNoMatchType re_nomatchType = TPL_REGEX_NOMATCH_USE_DFLTSTR;
 	DEFiRet;
 
 	/* pull params */
@@ -1277,6 +1282,40 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 			fieldnum = pvals[i].val.d.n;
 		} else if(!strcmp(pblkProperty.descr[i].name, "field.delimiter")) {
 			fielddelim = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "regex.expression")) {
+			re_expr = es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(pblkProperty.descr[i].name, "regex.type")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"BRE", sizeof("BRE")-1)) {
+				re_type = TPL_REGEX_BRE;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"ERE", sizeof("ERE")-1)) {
+				re_type = TPL_REGEX_ERE;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid regex.type '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblkProperty.descr[i].name, "regex.nomatchmode")) {
+			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"DFLT", sizeof("DFLT")-1)) {
+				re_nomatchType = TPL_REGEX_NOMATCH_USE_DFLTSTR;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"BLANK", sizeof("BLANK")-1)) {
+				re_nomatchType = TPL_REGEX_NOMATCH_USE_BLANK;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"FIELD", sizeof("FIELD")-1)) {
+				re_nomatchType = TPL_REGEX_NOMATCH_USE_WHOLE_FIELD;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"ZERO", sizeof("ZERO")-1)) {
+				re_nomatchType = TPL_REGEX_NOMATCH_USE_ZERO;
+			} else {
+				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+				errmsg.LogError(0, RS_RET_ERR, "invalid format type '%s' for property",
+					typeStr);
+				free(typeStr);
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else if(!strcmp(pblkProperty.descr[i].name, "regex.match")) {
+			re_matchToUse = pvals[i].val.d.n;
+		} else if(!strcmp(pblkProperty.descr[i].name, "regex.submatch")) {
+			re_submatchToUse = pvals[i].val.d.n;
 		} else if(!strcmp(pblkProperty.descr[i].name, "format")) {
 			if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"csv", sizeof("csv")-1)) {
 				formatType = F_CSV;
@@ -1369,6 +1408,11 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 			topos, frompos);
 		ABORT_FINALIZE(RS_RET_ERR);
 	}
+	if(fieldnum != -1 && re_expr != NULL) {
+		errmsg.LogError(0, RS_RET_ERR, "both field extraction and regex extraction "
+				"specified - this is not possible, remove one");
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
 
 	/* apply */
 	CHKmalloc(pTpe = tpeConstruct(pTpl));
@@ -1433,6 +1477,34 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 		pTpe->data.field.iFromPos = frompos;
 		pTpe->data.field.iToPos = topos;
 	}
+	if(re_expr != NULL) {
+		rsRetVal iRetLocal;
+		pTpe->data.field.typeRegex = re_type;
+		pTpe->data.field.nomatchAction = re_nomatchType;
+		pTpe->data.field.iMatchToUse = re_matchToUse;
+		pTpe->data.field.iSubMatchToUse = re_submatchToUse;
+		pTpe->data.field.has_regex = 1;
+		if((iRetLocal = objUse(regexp, LM_REGEXP_FILENAME)) == RS_RET_OK) {
+			int iOptions;
+			iOptions = (pTpe->data.field.typeRegex == TPL_REGEX_ERE) ? REG_EXTENDED : 0;
+			if(regexp.regcomp(&(pTpe->data.field.re), (char*) re_expr, iOptions) != 0) {
+				dbgprintf("error: can not compile regex: '%s'\n", re_expr);
+				errmsg.LogError(0, NO_ERRCODE, "error compiling regex '%s'", re_expr);
+				pTpe->data.field.has_regex = 2;
+				ABORT_FINALIZE(RS_RET_ERR);
+			}
+		} else {
+			/* regexp object could not be loaded */
+			if(bFirstRegexpErrmsg) { /* prevent flood of messages, maybe even an endless loop! */
+				bFirstRegexpErrmsg = 0;
+				errmsg.LogError(0, NO_ERRCODE, "regexp library could not be loaded (error %d), "
+						"regexp ignored", iRetLocal);
+			}
+			pTpe->data.field.has_regex = 2;
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
+	}
+
 finalize_it:
 	RETiRet;
 }
diff --git a/template.h b/template.h
index c58d452c..65435cd8 100644
--- a/template.h
+++ b/template.h
@@ -58,6 +58,9 @@ enum tplFormatTypes { tplFmtDefault = 0, tplFmtMySQLDate = 1,
                       tplFmtRFC3164Date = 2, tplFmtRFC3339Date = 3, tplFmtPgSQLDate = 4,
 		      tplFmtSecFrac = 5, tplFmtRFC3164BuggyDate = 6, tplFmtUnixDate};
 enum tplFormatCaseConvTypes { tplCaseConvNo = 0, tplCaseConvUpper = 1, tplCaseConvLower = 2 };
+enum tplRegexType { TPL_REGEX_BRE = 0, /* posix BRE */
+		    TPL_REGEX_ERE = 1  /* posix ERE */
+		  };
 
 #include "msg.h"
 
@@ -80,11 +83,8 @@ struct templateEntry {
 			short has_regex;
 			short iMatchToUse;/* which match should be obtained (10 max) */
 			short iSubMatchToUse;/* which submatch should be obtained (10 max) */
-			enum {
-				TPL_REGEX_BRE = 0, /* posix BRE */
-				TPL_REGEX_ERE = 1  /* posix ERE */
-			} typeRegex;
-			enum {
+			enum tplRegexType typeRegex;
+			enum tlpRegexNoMatchType {
 				TPL_REGEX_NOMATCH_USE_DFLTSTR = 0, /* use the (old style) default "**NO MATCH**" string */
 				TPL_REGEX_NOMATCH_USE_BLANK = 1, /* use a blank string */
 				TPL_REGEX_NOMATCH_USE_WHOLE_FIELD = 2, /* use the full field contents that we were searching in*/
-- 
cgit v1.2.3


From 3d56820f130e6c1b674560125e677be3b6a2d8f4 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sat, 25 Aug 2012 19:21:12 +0200
Subject: add capability to configure outname for constant (inside template)

also some cleanup
---
 grammar/grammar.y | 10 ++++------
 runtime/msg.c     |  4 ++--
 template.c        | 21 ++++++++-------------
 template.h        |  2 +-
 4 files changed, 15 insertions(+), 22 deletions(-)

diff --git a/grammar/grammar.y b/grammar/grammar.y
index 27d0e1d5..8371f854 100644
--- a/grammar/grammar.y
+++ b/grammar/grammar.y
@@ -133,18 +133,16 @@ conf:	/* empty (to end recursion) */
 	| conf BSD_HOST_SELECTOR	{ cnfDoBSDHost($2); }
 obj:	  BEGINOBJ nvlst ENDOBJ 	{ $$ = cnfobjNew($1, $2); }
 	| BEGIN_ACTION nvlst ENDOBJ 	{ $$ = cnfobjNew(CNFOBJ_ACTION, $2); }
-        | BEGIN_TPL nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_TPL, $2);  dbgprintf("processing template() without {}\n"); }
+        | BEGIN_TPL nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_TPL, $2); }
         | BEGIN_TPL nvlst ENDOBJ '{' propconst '}'
 					{ $$ = cnfobjNew(CNFOBJ_TPL, $2);
 					  $$->subobjs = $5;
-					  dbgprintf("processing template() WITH {}, subobj=%p\n", $5); }
+					}
 propconst:				{ $$ = NULL; }
 	| propconst property		{ $$ = objlstAdd($1, $2); }
 	| propconst constant		{ $$ = objlstAdd($1, $2); }
-property: BEGIN_PROPERTY nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_PROPERTY, $2);
-					  dbgprintf("processed property()\n"); }
-constant: BEGIN_CONSTANT nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_CONSTANT, $2);
-					  dbgprintf("processed constant()\n"); }
+property: BEGIN_PROPERTY nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_PROPERTY, $2); }
+constant: BEGIN_CONSTANT nvlst ENDOBJ	{ $$ = cnfobjNew(CNFOBJ_CONSTANT, $2); }
 cfsysline: CFSYSLINE	 		{ $$ = $1; }
 nvlst:					{ $$ = NULL; }
 	| nvlst nv 			{ $2->next = $1; $$ = $2; }
diff --git a/runtime/msg.c b/runtime/msg.c
index da751dba..891907ec 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -2515,9 +2515,9 @@ jsonField(struct templateEntry *pTpe, uchar **ppRes, unsigned short *pbMustBeFre
 	pSrc = *ppRes;
 	buflen = (*pBufLen == -1) ? ustrlen(pSrc) : *pBufLen;
 	/* we hope we have only few escapes... */
-	dst = es_newStr(buflen+es_strlen(pTpe->data.field.fieldName)+15);
+	dst = es_newStr(buflen+es_strlen(pTpe->fieldName)+15);
 	es_addChar(&dst, '"');
-	es_addStr(&dst, pTpe->data.field.fieldName);
+	es_addStr(&dst, pTpe->fieldName);
 	es_addBufConstcstr(&dst, "\"=\"");
 	CHKiRet(jsonAddVal(pSrc, buflen, &dst));
 	es_addChar(&dst, '"');
diff --git a/template.c b/template.c
index 4cad7cb7..768608f1 100644
--- a/template.c
+++ b/template.c
@@ -74,7 +74,6 @@ static struct cnfparamdescr cnfparamdescrProperty[] = {
 	{ "controlcharacters", eCmdHdlrString, 0 },
 	{ "securepath", eCmdHdlrString, 0 },
 	{ "format", eCmdHdlrString, 0 },
-// From here
 	{ "position.from", eCmdHdlrInt, 0 },
 	{ "position.to", eCmdHdlrInt, 0 },
 	{ "field.number", eCmdHdlrInt, 0 },
@@ -84,7 +83,6 @@ static struct cnfparamdescr cnfparamdescrProperty[] = {
 	{ "regex.nomatchmode", eCmdHdlrString, 0 },
 	{ "regex.match", eCmdHdlrInt, 0 },
 	{ "regex.submatch", eCmdHdlrInt, 0 },
-//-- to here
 	{ "droplastlf", eCmdHdlrBinary, 0 },
 	{ "spifno1stsp", eCmdHdlrBinary, 0 }
 };
@@ -97,7 +95,6 @@ static struct cnfparamblk pblkProperty =
 static struct cnfparamdescr cnfparamdescrConstant[] = {
 	{ "value", eCmdHdlrString, 1 },
 	{ "outname", eCmdHdlrString, 0 },
-	{ "format", eCmdHdlrString, 0 }
 };
 static struct cnfparamblk pblkConstant =
 	{ CNFPARAMBLK_VERSION,
@@ -980,12 +977,12 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl)
 
 	/* save field name - if none was given, use the property name instead */
 	if(pStrField == NULL) {
-		if((pTpe->data.field.fieldName =
+		if((pTpe->fieldName =
 		      es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrProp), cstrLen(pStrProp))) == NULL) {
 			return 1;
 		}
 	} else {
-		if((pTpe->data.field.fieldName =
+		if((pTpe->fieldName =
 		      es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrField), cstrLen(pStrField))) == NULL) {
 			return 1;
 		}
@@ -1193,6 +1190,7 @@ createConstantTpe(struct template *pTpl, struct cnfobj *o)
 	es_str_t *value;
 	int i;
 	struct cnfparamvals *pvals;
+	es_str_t *outname = NULL;
 	DEFiRet;
 
 	/* pull params */
@@ -1204,12 +1202,8 @@ createConstantTpe(struct template *pTpl, struct cnfobj *o)
 			continue;
 		if(!strcmp(pblkConstant.descr[i].name, "value")) {
 			value = pvals[i].val.d.estr;
-		} else if(!strcmp(pblkConstant.descr[i].name, "format")) {
-			errmsg.LogError(0, RS_RET_ERR, "paramter 'format' is currently not "
-				"supported for 'constant' template parts - ignored");
 		} else if(!strcmp(pblkConstant.descr[i].name, "outname")) {
-			errmsg.LogError(0, RS_RET_ERR, "paramter 'outname' is currently not "
-				"supported for 'constant' template parts - ignored");
+			outname = es_strdup(pvals[i].val.d.estr);
 		} else {
 			dbgprintf("template:constantTpe: program error, non-handled "
 			  "param '%s'\n", pblkConstant.descr[i].name);
@@ -1222,6 +1216,7 @@ createConstantTpe(struct template *pTpl, struct cnfobj *o)
 	CHKmalloc(pTpe = tpeConstruct(pTpl));
 	es_unescapeStr(value);
 	pTpe->eEntryType = CONSTANT;
+	pTpe->fieldName = outname;
 	pTpe->data.constant.iLenConstant = es_strlen(value);
 	pTpe->data.constant.pConstant = (uchar*)es_str2cstr(value, NULL);
 
@@ -1466,7 +1461,7 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o)
 		pTpe->data.field.options.bSecPathReplace = 1;
 		break;
 	}
-	pTpe->data.field.fieldName = outname;
+	pTpe->fieldName = outname;
 	pTpe->data.field.eDateFormat = datefmt;
 	if(fieldnum != -1) {
 		pTpe->data.field.has_fields = 1;
@@ -1767,8 +1762,8 @@ void tplDeleteAll(rsconf_t *conf)
 				}
 				if(pTpeDel->data.field.propName != NULL)
 					es_deleteStr(pTpeDel->data.field.propName);
-				if(pTpeDel->data.field.fieldName != NULL)
-					es_deleteStr(pTpeDel->data.field.fieldName);
+				if(pTpeDel->fieldName != NULL)
+					es_deleteStr(pTpeDel->fieldName);
 #endif
 				break;
 			}
diff --git a/template.h b/template.h
index 65435cd8..9f6a4c33 100644
--- a/template.h
+++ b/template.h
@@ -68,6 +68,7 @@ enum tplRegexType { TPL_REGEX_BRE = 0, /* posix BRE */
 struct templateEntry {
 	struct templateEntry *pNext;
 	enum EntryTypes eEntryType;
+	es_str_t *fieldName;	/**< field name to be used for structured output */
 	union {
 		struct {
 			uchar *pConstant;	/* pointer to constant value */
@@ -99,7 +100,6 @@ struct templateEntry {
 #endif
 
 			es_str_t *propName;	/**< property name (currently being used for CEE only) */
-			es_str_t *fieldName;	/**< field name to be used for structured output */
 
 			enum tplFormatTypes eDateFormat;
 			enum tplFormatCaseConvTypes eCaseConv;
-- 
cgit v1.2.3


From 3fd617f1acb7f1a057edb415cec1b144d210da81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= <mitr@redhat.com>
Date: Fri, 24 Aug 2012 19:28:59 +0200
Subject: INCOMPATIBLE: use :, not =, as separator for jsonf
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

JSON fields are "name":value, not "name"=value.  Therefore change the
jsonf flag to use a colon.

Signed-off-by: Miloslav Trmač <mitr@redhat.com>
---
 doc/property_replacer.html | 2 +-
 runtime/msg.c              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/property_replacer.html b/doc/property_replacer.html
index 4c92bf4c..86a07474 100644
--- a/doc/property_replacer.html
+++ b/doc/property_replacer.html
@@ -368,7 +368,7 @@ The json option cannot be used together with either jsonf or csv options.
 This signifies that the property should be expressed as a json <b>f</b>ield.
 That means not only the property is written, but rather a complete json field in
 the format<br>
-"fieldname"="value"</b>
+"fieldname":"value"</b>
 where "filedname" is the assigend field name (or the property name if none was assigned)
 and value is the end result of property replacer operation. Note that value supports
 all property replacer options, like substrings, case converson and the like.
diff --git a/runtime/msg.c b/runtime/msg.c
index da751dba..61c2f87b 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -2518,7 +2518,7 @@ jsonField(struct templateEntry *pTpe, uchar **ppRes, unsigned short *pbMustBeFre
 	dst = es_newStr(buflen+es_strlen(pTpe->data.field.fieldName)+15);
 	es_addChar(&dst, '"');
 	es_addStr(&dst, pTpe->data.field.fieldName);
-	es_addBufConstcstr(&dst, "\"=\"");
+	es_addBufConstcstr(&dst, "\":\"");
 	CHKiRet(jsonAddVal(pSrc, buflen, &dst));
 	es_addChar(&dst, '"');
 
-- 
cgit v1.2.3


From e1753e3dd1b03f2ca8863bbaee3ab799dca4932e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= <mitr@redhat.com>
Date: Sat, 11 Aug 2012 09:48:10 +0200
Subject: Fix printing of some template options
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Miloslav Trmač <mitr@redhat.com>
---
 template.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/template.c b/template.c
index a80ac6fb..e80015da 100644
--- a/template.c
+++ b/template.c
@@ -1364,7 +1364,10 @@ void tplPrintList(rsconf_t *conf)
 				  	dbgprintf("[format as CSV (RFC4180)]");
 				}
 				if(pTpe->data.field.options.bJSON) {
-				  	dbgprintf("[format as JSON");
+					dbgprintf("[format as JSON] ");
+				}
+				if(pTpe->data.field.options.bJSONf) {
+					dbgprintf("[format as JSON field] ");
 				}
 				if(pTpe->data.field.options.bDropLastLF) {
 				  	dbgprintf("[drop last LF in msg] ");
-- 
cgit v1.2.3


From 7580b8c954ad2291198c52b7bc50ae7a41be65a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= <mitr@redhat.com>
Date: Thu, 9 Aug 2012 17:07:17 +0200
Subject: Remove a no-effect ", NULL";
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

It does nothing, at is just confusing.

Signed-off-by: Miloslav Trmač <mitr@redhat.com>
---
 plugins/ommongodb/ommongodb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/ommongodb/ommongodb.c b/plugins/ommongodb/ommongodb.c
index 39e2e4f9..aa20b33f 100644
--- a/plugins/ommongodb/ommongodb.c
+++ b/plugins/ommongodb/ommongodb.c
@@ -340,7 +340,7 @@ CODESTARTnewActInst
 		if(!strcmp(actpblk.descr[i].name, "server")) {
 			pData->server = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
 		} else if(!strcmp(actpblk.descr[i].name, "serverport")) {
-			pData->port = (int) pvals[i].val.d.n, NULL;
+			pData->port = (int) pvals[i].val.d.n;
 		} else if(!strcmp(actpblk.descr[i].name, "db")) {
 			pData->db = (uchar*)es_str2cstr(pvals[i].val.d.estr, NULL);
 		} else if(!strcmp(actpblk.descr[i].name, "collection")) {
-- 
cgit v1.2.3


From e48728e80e6ce7b94319363581040774067de0fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= <mitr@redhat.com>
Date: Fri, 24 Aug 2012 19:28:59 +0200
Subject: INCOMPATIBLE: use :, not =, as separator for jsonf
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

JSON fields are "name":value, not "name"=value.  Therefore change the
jsonf flag to use a colon.

Signed-off-by: Miloslav Trmač <mitr@redhat.com>
---
 doc/property_replacer.html | 2 +-
 runtime/msg.c              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/property_replacer.html b/doc/property_replacer.html
index 4c92bf4c..86a07474 100644
--- a/doc/property_replacer.html
+++ b/doc/property_replacer.html
@@ -368,7 +368,7 @@ The json option cannot be used together with either jsonf or csv options.
 This signifies that the property should be expressed as a json <b>f</b>ield.
 That means not only the property is written, but rather a complete json field in
 the format<br>
-"fieldname"="value"</b>
+"fieldname":"value"</b>
 where "filedname" is the assigend field name (or the property name if none was assigned)
 and value is the end result of property replacer operation. Note that value supports
 all property replacer options, like substrings, case converson and the like.
diff --git a/runtime/msg.c b/runtime/msg.c
index da751dba..61c2f87b 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -2518,7 +2518,7 @@ jsonField(struct templateEntry *pTpe, uchar **ppRes, unsigned short *pbMustBeFre
 	dst = es_newStr(buflen+es_strlen(pTpe->data.field.fieldName)+15);
 	es_addChar(&dst, '"');
 	es_addStr(&dst, pTpe->data.field.fieldName);
-	es_addBufConstcstr(&dst, "\"=\"");
+	es_addBufConstcstr(&dst, "\":\"");
 	CHKiRet(jsonAddVal(pSrc, buflen, &dst));
 	es_addChar(&dst, '"');
 
-- 
cgit v1.2.3


From 9068d7fa1e1ce57ab55838866ac58b060721193e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miloslav=20Trma=C4=8D?= <mitr@redhat.com>
Date: Fri, 24 Aug 2012 19:33:24 +0200
Subject: INCOMPATIBLE: Return {} for missing $!all-json
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

%$!all-json% will return a {}-wrapped object if there are are any
events, or when there is an attached (empty) pMsg->event, but an empty
string if nothing is attached.  Let it return an empty object "{}" in
that case for consistency.  In particular, this allows

$template MongoTemplate,"{%hostname:::jsonf:sys%, %$!all-json:2:$:%"

to always result in a valid JSON.

Signed-off-by: Miloslav Trmač <mitr@redhat.com>
---
 runtime/msg.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtime/msg.c b/runtime/msg.c
index 61c2f87b..44d36fef 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -2734,7 +2734,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 			if(pMsg->event == NULL) {
 			if(*pbMustBeFreed == 1)
 				free(pRes);
-			pRes = (uchar*) "";
+			pRes = (uchar*) "{}";
 			*pbMustBeFreed = 0;
 			} else {
 				ee_fmtEventToJSON(pMsg->event, &str);
-- 
cgit v1.2.3


From 9a9282b86c785d7c0b67a0b546748e3591a487a2 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sun, 26 Aug 2012 12:27:37 +0200
Subject: doc: add incompatible patch info to ChangeLog

---
 ChangeLog | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index dd165e1d..95cd80e5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -4,6 +4,18 @@ Version 6.4.1  [V6-STABLE] 2012-08-??
   This lead to queue file corruption. While the root cause is a config
   error, it is a bug that this important and hard to find config error
   was not detected by rsyslog.
+- bugfix: "jsonf" property replacer option did generate invalid JSON
+  in JSON, we have "fieldname":"value", but the option emitted 
+  "fieldname"="value". Interestingly, this was accepted by a couple
+  of sinks, most importantly elasticsearch. Now the correct format is
+  emitted, which causes a remote chance that some things that relied on
+  the wrong format will break.
+  Thanks to Miloslav Trmač for the patch
+- change $!all-json did emit an empty (thus non-JSON) string if no libee
+  data was present. It now emits {} and thus valid JSON. There is a
+  small risk that this may break some things that relied on the previous
+  inconsistency.
+  Thanks to Miloslav Trmač for the patch
 ---------------------------------------------------------------------------
 Version 6.4.0  [V6-STABLE] 2012-08-20
 - THIS IS THE FIRST VERSION OF THE 6.4.x STABLE BRANCH
-- 
cgit v1.2.3


From 744d7c426da4aa3229771358a5da27b79e2edf52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Renard?= <jerome.renard@gmail.com>
Date: Sun, 26 Aug 2012 14:13:26 +0200
Subject: add uuid property to message object

---
 configure.ac      | 15 +++++++++++++
 runtime/msg.c     | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++--
 runtime/msg.h     |  2 +-
 runtime/rsyslog.h |  1 +
 tools/Makefile.am |  2 +-
 5 files changed, 79 insertions(+), 4 deletions(-)

diff --git a/configure.ac b/configure.ac
index 80c086f4..78315a80 100644
--- a/configure.ac
+++ b/configure.ac
@@ -674,6 +674,21 @@ AM_CONDITIONAL(ENABLE_OMLIBDBI, test x$enable_libdbi = xyes)
 AC_SUBST(LIBDBI_CFLAGS)
 AC_SUBST(LIBDBI_LIBS)
 
+# libuuid support
+AC_CHECK_HEADERS(
+  [uuid/uuid.h],,
+  [AC_MSG_FAILURE([libuuid is missing])]
+)
+AC_CHECK_LIB(
+  [uuid],
+  [uuid_generate],
+  [LIBUUID_CFLAGS=""
+   LIBUUID_LIBS="-luuid"
+  ],
+  [AC_MSG_FAILURE([libuuid library is missing])]
+)
+AC_SUBST(LIBUUID_CFLAGS)
+AC_SUBST(LIBUUID_LIBS)
 
 # SNMP support
 AC_ARG_ENABLE(snmp,
diff --git a/runtime/msg.c b/runtime/msg.c
index 44d36fef..8d24ea66 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -43,6 +43,7 @@
 #if HAVE_MALLOC_H
 #  include <malloc.h>
 #endif
+#include <uuid/uuid.h>
 #include "rsyslog.h"
 #include "srUtils.h"
 #include "stringbuf.h"
@@ -568,6 +569,8 @@ propNameStrToID(uchar *pName, propid_t *pPropID)
 		*pPropID = PROP_SYS_BOM;
 	} else if(!strcmp((char*) pName, "$uptime")) {
 		*pPropID = PROP_SYS_UPTIME;
+	} else if(!strcmp((char*) pName, "uuid")) {
+		*pPropID = PROP_UUID;
 	} else {
 		*pPropID = PROP_INVALID;
 		iRet = RS_RET_VAR_NOT_FOUND;
@@ -666,6 +669,8 @@ uchar *propIDToName(propid_t propID)
 			return UCHAR_CONSTANT("$!all-json");
 		case PROP_SYS_BOM:
 			return UCHAR_CONSTANT("$BOM");
+		case PROP_UUID:
+			return UCHAR_CONSTANT("uuid");
 		default:
 			return UCHAR_CONSTANT("*invalid property id*");
 	}
@@ -745,6 +750,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis)
 	pM->pszRcvdAt_SecFrac[0] = '\0';
 	pM->pszTIMESTAMP_Unix[0] = '\0';
 	pM->pszRcvdAt_Unix[0] = '\0';
+	pM->pszUUID = NULL;
 
 	/* DEV debugging only! dbgprintf("msgConstruct\t0x%x, ref 1\n", (int)pM);*/
 
@@ -875,6 +881,8 @@ CODESTARTobjDestruct(msg)
 			rsCStrDestruct(&pThis->pCSMSGID);
 		if(pThis->event != NULL)
 			ee_deleteEvent(pThis->event);
+		if(pThis->pszUUID != NULL)
+			free(pThis->pszUUID);
 #	ifndef HAVE_ATOMIC_BUILTINS
 		MsgUnlock(pThis);
 # 	endif
@@ -1080,6 +1088,8 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm)
 	objSerializePTR(pStrm, pCSPROCID, CSTR);
 	objSerializePTR(pStrm, pCSMSGID, CSTR);
 	
+	objSerializePTR(pStrm, pszUUID, PSZ);
+
 	if(pThis->pRuleset != NULL) {
 		rulesetGetName(pThis->pRuleset);
 		CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszRuleset"), PROPTYPE_PSZ,
@@ -1242,6 +1252,54 @@ char *getProtocolVersionString(msg_t *pM)
 	return(pM->iProtocolVersion ? "1" : "0");
 }
 
+void msgSetUUID(msg_t *pM)
+{
+	dbgprintf("[MsgSetUUID] START\n");
+	assert(pM != NULL);
+
+	dbgprintf("[MsgSetUUID] pM NOT null \n");
+
+	size_t lenRes = sizeof(uuid_t) * 2 + 1;
+	char hex_char [] = "0123456789ABCDEF";
+	unsigned int byte_nbr;
+	uuid_t uuid;
+
+	if((pM->pszUUID = (uchar*) MALLOC(lenRes)) == NULL) {
+		pM->pszUUID = (uchar *)"";
+	} else {
+		uuid_generate(uuid);
+		for (byte_nbr = 0; byte_nbr < sizeof (uuid_t); byte_nbr++) {
+			pM->pszUUID[byte_nbr * 2 + 0] = hex_char[uuid [byte_nbr] >> 4];
+			pM->pszUUID[byte_nbr * 2 + 1] = hex_char[uuid [byte_nbr] & 15];
+		}
+
+		dbgprintf("[MsgSetUUID] UUID : %s LEN: %d \n", pM->pszUUID, (int)lenRes);
+		pM->pszUUID[lenRes] = '\0';
+	}
+	dbgprintf("[MsgSetUUID] END\n");
+}
+
+void getUUID(msg_t *pM, uchar **pBuf, int *piLen)
+{
+	dbgprintf("[getUUID] START\n");
+	if(pM == NULL) {
+		dbgprintf("[getUUID] pM is NULL\n");
+		*pBuf=	UCHAR_CONSTANT("");
+		*piLen = 0;
+	} else {
+		if(pM->pszUUID == NULL) {
+			dbgprintf("[getUUID] pM->pszUUID is NULL\n");
+
+			msgSetUUID(pM);
+		} else {
+			/* UUID already there we reuse it */
+			dbgprintf("[getUUID] pM->pszUUID already exists\n");
+		}
+		*pBuf = pM->pszUUID;
+		*piLen = sizeof(uuid_t) * 2;
+	}
+	dbgprintf("[getUUID] END\n");
+}
 
 void
 getRawMsg(msg_t *pM, uchar **pBuf, int *piLen)
@@ -1908,7 +1966,6 @@ static inline char *getStructuredData(msg_t *pM)
 	return (char*) pszRet;
 }
 
-
 /* check if we have a ProgramName, and, if not, try to aquire/emulate it.
  * rgerhards, 2009-06-26
  */
@@ -2232,7 +2289,6 @@ finalize_it:
 	RETiRet;
 }
 
-
 /* set raw message in message object. Size of message is provided.
  * The function makes sure that the stored rawmsg is properly
  * terminated by '\0'.
@@ -2774,6 +2830,9 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 			}
 #			endif
 		break;
+		case PROP_UUID:
+			getUUID(pMsg, &pRes, &bufLen);
+		break;
 		default:
 			/* there is no point in continuing, we may even otherwise render the
 			 * error message unreadable. rgerhards, 2007-07-10
diff --git a/runtime/msg.h b/runtime/msg.h
index ed2e9d04..f6b54a77 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -123,6 +123,7 @@ struct msg {
 	char pszRcvdAt_SecFrac[7];	     /* same as above. Both are fractional seconds for their respective timestamp */
 	char pszTIMESTAMP_Unix[12]; /* almost as small as a pointer! */
 	char pszRcvdAt_Unix[12];
+    uchar *pszUUID; /* The message's UUID */
 };
 
 
@@ -184,7 +185,6 @@ void getRawMsg(msg_t *pM, uchar **pBuf, int *piLen);
 rsRetVal msgGetCEEVar(msg_t *pThis, cstr_t *propName, var_t **ppVar);
 es_str_t* msgGetCEEVarNew(msg_t *pMsg, char *name);
 
-
 /* TODO: remove these five (so far used in action.c) */
 uchar *getMSG(msg_t *pM);
 char *getHOSTNAME(msg_t *pM);
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index d802536a..57e8a05c 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -142,6 +142,7 @@ typedef uintTiny	propid_t;
 #define PROP_CEE_ALL_JSON		201
 #define PROP_SYS_BOM			159
 #define PROP_SYS_UPTIME			160
+#define PROP_UUID               161
 
 
 /* The error codes below are orginally "borrowed" from
diff --git a/tools/Makefile.am b/tools/Makefile.am
index 4e457426..60a2dd61 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -40,7 +40,7 @@ rsyslogd_CPPFLAGS =  $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(CNF_LIBS)
 # note: it looks like librsyslog.la must be explicitely given on LDDADD,
 # otherwise dependencies are not properly calculated (resulting in a 
 # potentially incomplete build, a problem we had several times...)
-rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) $(LIBEE_LIBS) $(LIBLOGNORM_LIBS) $(CNF_LIBS) ../grammar/libgrammar.la ../runtime/librsyslog.la
+rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) $(LIBEE_LIBS) $(LIBLOGNORM_LIBS) $(CNF_LIBS) $(LIBUUID_LIBS) ../grammar/libgrammar.la ../runtime/librsyslog.la
 rsyslogd_LDFLAGS = -export-dynamic
 
 if ENABLE_DIAGTOOLS
-- 
cgit v1.2.3


From ec48c41e27ec15618f57b54209eaafe025e29010 Mon Sep 17 00:00:00 2001
From: Rainer Gerhards <rgerhards@adiscon.com>
Date: Sun, 26 Aug 2012 14:28:13 +0200
Subject: made new uuid property threadsafe

---
 ChangeLog     |  4 ++++
 runtime/msg.c | 36 +++++++++++++++++++++---------------
 2 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 6b96d35b..4ee5beb0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -18,6 +18,10 @@ Version 6.5.0  [devel] 2012-0?-??
   Thanks to Miloslav Trmač for the patch
 - $SystemLogParseTrusted config file option
   Thanks to Milan Bartos for the patch
+- added new uuid message property
+  Thanks to Jérôme Renard for the idea and patches.
+  Note: patches were released under ASL 2.0, see
+  http://bugzilla.adiscon.com/show_bug.cgi?id=353
 ---------------------------------------------------------------------------
 Version 6.4.1  [V6-STABLE] 2012-08-??
 - bugfix: multiple main queues with same queue file name were not detected
diff --git a/runtime/msg.c b/runtime/msg.c
index 8d24ea66..7ba46722 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -542,6 +542,8 @@ propNameStrToID(uchar *pName, propid_t *pPropID)
 		*pPropID = PROP_MSGID;
 	} else if(!strcmp((char*) pName, "parsesuccess")) {
 		*pPropID = PROP_PARSESUCCESS;
+	} else if(!strcmp((char*) pName, "uuid")) {
+		*pPropID = PROP_UUID;
 	/* here start system properties (those, that do not relate to the message itself */
 	} else if(!strcmp((char*) pName, "$now")) {
 		*pPropID = PROP_SYS_NOW;
@@ -569,8 +571,6 @@ propNameStrToID(uchar *pName, propid_t *pPropID)
 		*pPropID = PROP_SYS_BOM;
 	} else if(!strcmp((char*) pName, "$uptime")) {
 		*pPropID = PROP_SYS_UPTIME;
-	} else if(!strcmp((char*) pName, "uuid")) {
-		*pPropID = PROP_UUID;
 	} else {
 		*pPropID = PROP_INVALID;
 		iRet = RS_RET_VAR_NOT_FOUND;
@@ -1252,22 +1252,26 @@ char *getProtocolVersionString(msg_t *pM)
 	return(pM->iProtocolVersion ? "1" : "0");
 }
 
-void msgSetUUID(msg_t *pM)
+/* note: libuuid seems not to be thread-safe, so we need
+ * to get some safeguards in place.
+ */
+static void msgSetUUID(msg_t *pM)
 {
-	dbgprintf("[MsgSetUUID] START\n");
-	assert(pM != NULL);
-
-	dbgprintf("[MsgSetUUID] pM NOT null \n");
-
 	size_t lenRes = sizeof(uuid_t) * 2 + 1;
 	char hex_char [] = "0123456789ABCDEF";
 	unsigned int byte_nbr;
 	uuid_t uuid;
+	static pthread_mutex_t mutUUID = PTHREAD_MUTEX_INITIALIZER;
+
+	dbgprintf("[MsgSetUUID] START\n");
+	assert(pM != NULL);
 
 	if((pM->pszUUID = (uchar*) MALLOC(lenRes)) == NULL) {
 		pM->pszUUID = (uchar *)"";
 	} else {
+		pthread_mutex_lock(&mutUUID);
 		uuid_generate(uuid);
+		pthread_mutex_unlock(&mutUUID);
 		for (byte_nbr = 0; byte_nbr < sizeof (uuid_t); byte_nbr++) {
 			pM->pszUUID[byte_nbr * 2 + 0] = hex_char[uuid [byte_nbr] >> 4];
 			pM->pszUUID[byte_nbr * 2 + 1] = hex_char[uuid [byte_nbr] & 15];
@@ -1289,10 +1293,12 @@ void getUUID(msg_t *pM, uchar **pBuf, int *piLen)
 	} else {
 		if(pM->pszUUID == NULL) {
 			dbgprintf("[getUUID] pM->pszUUID is NULL\n");
-
-			msgSetUUID(pM);
-		} else {
-			/* UUID already there we reuse it */
+			MsgLock(pM);
+			/* re-query, things may have changed in the mean time... */
+			if(pM->pszUUID == NULL)
+				msgSetUUID(pM);
+			MsgUnlock(pM);
+		} else { /* UUID already there we reuse it */
 			dbgprintf("[getUUID] pM->pszUUID already exists\n");
 		}
 		*pBuf = pM->pszUUID;
@@ -2732,6 +2738,9 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 		case PROP_MSGID:
 			pRes = (uchar*)getMSGID(pMsg);
 			break;
+		case PROP_UUID:
+			getUUID(pMsg, &pRes, &bufLen);
+			break;
 		case PROP_PARSESUCCESS:
 			pRes = (uchar*)getParseSuccess(pMsg);
 			break;
@@ -2830,9 +2839,6 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 			}
 #			endif
 		break;
-		case PROP_UUID:
-			getUUID(pMsg, &pRes, &bufLen);
-		break;
 		default:
 			/* there is no point in continuing, we may even otherwise render the
 			 * error message unreadable. rgerhards, 2007-07-10
-- 
cgit v1.2.3