Merge branch 'master' into master-ratelimit

Conflicts:
	ChangeLog
	doc/v7compatibility.html

15 files changed, 117 insertions, 24 deletions

diff --git a/ChangeLog b/ChangeLog
index 6e8dfc97..307abceb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,12 @@
 ---------------------------------------------------------------------------
 Version 7.3.1  [devel] 2012-10-??
 - imudp: support for input batching added (performance improvement)
+- change lumberjack cookie to "@cee:" from "@cee: "
+  CEE originally specified the cookie with SP, whereas other lumberjack
+  tools used it without space. In order to keep interop with lumberjack,
+  we now use the cookie without space as well. I hope this can be changed
+  in CEE as well when it is released at a later time.
+  Thanks to Miloslav Trmač for pointing this out and a similiar v7 patch.
 ---------------------------------------------------------------------------
 Version 7.3.0  [devel] 2012-10-09
 - omlibdbi improvements, added
@@ -13,8 +19,34 @@ Version 7.3.0  [devel] 2012-10-09
   This was achieved by somewhat reducing the robustness of the zip archive.
   This is controlled by the new action parameter "VeryReliableZip".
 ---------------------------------------------------------------------------
-Version 7.1.9  [beta] 2012-10-??
+Version 7.1.11  [beta] 2012-10-??
+- bugfix: do not crash if set statement is used with date field
+  Thanks to Miloslav Trmač for the patch.
+- change lumberjack cookie to "@cee" from "@cee "
+  CEE originally specified the cookie with SP, whereas other lumberjack
+  tools used it without space. In order to keep interop with lumberjack,
+  we now use the cookie without space as well. I hope this can be changed
+  in CEE as well when it is released at a later time.
+  Thanks to Miloslav Trmač for pointing this out and a similiar v7 patch.
+- added deprecated note to omruleset (plus clue to use "call")
+- added deprecated note to discard action (plus clue to use "stop")
+ ---------------------------------------------------------------------------
+Version 7.1.10  [beta] 2012-10-11
+ - bugfix: m4 directory was not present in release tarball
+ - bugfix: small memory leak with string-type templates
+ - bugfix: small memory leak when template was specified in omfile
+ - bugfix: some config processing warning messages were treated as errors
+ - bugfix: small memory leak when processing action() statements
+ - bugfix: unknown action() parameters were not reported
+ ---------------------------------------------------------------------------
+Version 7.1.9  [beta] 2012-10-09
 - bugfix: comments inside objects (e.g. action()) were not properly handled
+- bugfix: in (non)equal comparisons the position of arrays influenced result
+  This behaviour is OK for "contains"-type of comparisons (which have quite
+  different semantics), but not for == and <>, which shall be commutative.
+  This has been fixed now, so there is no difference any longer if the
+  constant string array is the left or right hand operand. We solved this
+  via the optimizer, as it keeps the actual script execution code small.
 ---------------------------------------------------------------------------
 Version 7.1.8  [beta] 2012-10-02
 - bugfix: ruleset(){} directive errornously changed default ruleset
@@ -156,6 +188,12 @@ Version 6.5.0  [devel] 2012-08-28
   http://bugzilla.adiscon.com/show_bug.cgi?id=353
 ---------------------------------------------------------------------------
 Version 6.4.3  [V6-STABLE] 2012-??-??
+- change lumberjack cookie to "@cee" from "@cee "
+  CEE originally specified the cookie with SP, whereas other lumberjack
+  tools used it without space. In order to keep interop with lumberjack,
+  we now use the cookie without space as well. I hope this can be changed
+  in CEE as well when it is released at a later time.
+  Thanks to Miloslav Trmač for pointing this out and a similiar v7 patch.
 - bugfix: comments inside objects (e.g. action()) were not properly handled
 - bugfix: sysklogd-emulating standard template was no longer present in v6
   This was obviously lost during the transition to the new config format.
diff --git a/configure.ac b/configure.ac
index 57ee41ff..f817ca06 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1188,7 +1188,7 @@ AM_CONDITIONAL(ENABLE_PMRFC3164SD, test x$enable_pmrfc3164sd = xyes)
 
 # settings for omruleset
 AC_ARG_ENABLE(omruleset,
-        [AS_HELP_STRING([--enable-omruleset],[Compiles ruleset forwarding module @<:@default=yes@:>@])],
+        [AS_HELP_STRING([--enable-omruleset],[Compiles ruleset forwarding module @<:@default=no@:>@])],
         [case "${enableval}" in
          yes) enable_omruleset="yes" ;;
           no) enable_omruleset="no" ;;
diff --git a/doc/v7compatibility.html b/doc/v7compatibility.html
index e954ca1d..932e2076 100644
--- a/doc/v7compatibility.html
+++ b/doc/v7compatibility.html
@@ -25,6 +25,23 @@ has been implemented. Consequently, situations where the previous behaviour were
 desired need now to be solved differently. We do not think that this will cause any
 problems to anyone, especially as in v6 this was announced as a missing feature.
 
+<h2>omruleset and discard (~) action are deprecated</h2>
+<p>Both continue to work, but have been replaced by better alternatives.
+<p>The discard action (tilde character) has been replaced by the "stop"
+RainerScript directive. It is considered more intuitive and offers slightly
+better performance.
+<p>The omruleset module has been replaced by the "call" RainerScript directive.
+Call permits to execute a ruleset like a subroutine, and does so with much
+higher performance than omruleset did. Note that omruleset could be run off
+an async queue. This was more a side than a desired effect and is not supported
+by the call statement. If that effect was needed, it can simply be simulated by
+running the called rulesets actions asynchronously (what in any case is the right
+way to handle this).
+<p>Note that the deprecated modules emit warning messages when being used.
+They tell that the construct is deprecated and which statement is to be used
+as replacement. This does <b>not</b> affect operations: both modules are still
+fully operational and will not be removed in the v7 timeframe.
+
 <h2>"last message repeated n times" Processing</h2>
 <p>This processing has been optimized and moved to the input side. This results
 in usually far better performance and also de-couples different sources
@@ -57,7 +74,7 @@ parser step cannot be deferred to the main queue processing thread and
 thus must be done during input processing. The changes workload distribution
 and may have (good or bad) effect on the overall performance. If you have
 a very high performance installation, it is suggested to check the performance
-profike before deploying the new version. Note: for high-performance
+profile before deploying the new version. Note: for high-performance
 environments it is highly recommended NOT to use "last message repeated n times"
 processing but rather the other (more efficient) rate-limiting methods. These
 also do NOT require the parsing step to be done during input processing.
diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c
index 7e75326c..9483e116 100644
--- a/grammar/rainerscript.c
+++ b/grammar/rainerscript.c
@@ -1173,6 +1173,8 @@ evalVar(struct cnfvar *var, void *usrptr, struct var *ret)
  * that one one comparison is true, the whole construct is true.
  * TODO: we can obviously optimize this process. One idea is to
  * compile a regex, which should work faster than serial comparison.
+ * Note: compiling a regex does NOT work at all. I experimented with that
+ * and it was generally 5 to 10 times SLOWER than what we do here...
  */
 static int
 evalStrArrayCmp(es_str_t *estr_l, struct cnfarray* ar, int cmpop)
@@ -1756,7 +1758,6 @@ cnfexprPrint(struct cnfexpr *expr, int indent)
 	struct cnffunc *func;
 	int i;
 
-	dbgprintf("expr %p, indent %d, type '%c'\n", expr, indent, expr->nodetype);
 	switch(expr->nodetype) {
 	case CMP_EQ:
 		cnfexprPrint(expr->l, indent+1);
@@ -2185,10 +2186,14 @@ cnfstmtNewAct(struct nvlst *lst)
 {
 	struct cnfstmt* cnfstmt;
 	char namebuf[256];
+	rsRetVal localRet;
 	if((cnfstmt = cnfstmtNew(S_ACT)) == NULL) 
 		goto done;
-	if(actionNewInst(lst, &cnfstmt->d.act) != RS_RET_OK) {
-	// TODO:RS_RET_WARN?
+	localRet = actionNewInst(lst, &cnfstmt->d.act);
+	if(localRet == RS_RET_OK_WARN) {
+		parser_errmsg("warnings occured in file '%s' around line %d",
+			      cnfcurrfn, yylineno);
+	} else if(localRet != RS_RET_OK) {
 		parser_errmsg("errors occured in file '%s' around line %d",
 			      cnfcurrfn, yylineno);
 		cnfstmt->nodetype = S_NOP; /* disable action! */
@@ -2198,6 +2203,8 @@ cnfstmtNewAct(struct nvlst *lst)
 		 modGetName(cnfstmt->d.act->pMod));
 	namebuf[255] = '\0'; /* be on safe side */
 	cnfstmt->printable = (uchar*)strdup(namebuf);
+	nvlstChkUnused(lst);
+	nvlstDestruct(lst);
 done:	return cnfstmt;
 }
 
@@ -2322,6 +2329,7 @@ void
 cnfexprOptimize(struct cnfexpr *expr)
 {
 	long long ln, rn;
+	struct cnfexpr *exprswap;
 
 	dbgprintf("optimize expr %p, type '%c'(%u)\n", expr, expr->nodetype, expr->nodetype);
 	switch(expr->nodetype) {
@@ -2358,6 +2366,19 @@ cnfexprOptimize(struct cnfexpr *expr)
 			((struct cnfnumval*)expr)->val = ln % rn;
 		}
 		break;
+	case CMP_NE:
+	case CMP_EQ:
+		if(expr->l->nodetype == 'A') {
+			if(expr->r->nodetype == 'A') {
+				parser_errmsg("warning: '==' or '<>' "
+				  "comparison of two constant string "
+				  "arrays makes no sense");
+			} else { /* swap for simpler execution step */
+				exprswap = expr->l;
+				expr->l = expr->r;
+				expr->r = exprswap;
+			}
+		}
 	default:/* nodetype we cannot optimize */
 		break;
 	}
diff --git a/grammar/rainerscript.h b/grammar/rainerscript.h
index 8ff0d702..5cfce795 100644
--- a/grammar/rainerscript.h
+++ b/grammar/rainerscript.h
@@ -327,6 +327,7 @@ struct cnfstmt * cnfstmtNewLegaAct(char *actline);
 struct cnfstmt * cnfstmtNewSet(char *var, struct cnfexpr *expr);
 struct cnfstmt * cnfstmtNewUnset(char *var);
 struct cnfstmt * cnfstmtNewCall(es_str_t *name);
+struct cnfstmt * cnfstmtNewContinue(void);
 void cnfstmtDestruct(struct cnfstmt *root);
 void cnfstmtOptimize(struct cnfstmt *root);
 struct cnfarray* cnfarrayNew(es_str_t *val);
diff --git a/plugins/mmjsonparse/mmjsonparse.c b/plugins/mmjsonparse/mmjsonparse.c
index 68f5e74c..da5cfb51 100644
--- a/plugins/mmjsonparse/mmjsonparse.c
+++ b/plugins/mmjsonparse/mmjsonparse.c
@@ -138,7 +138,7 @@ finalize_it:
 	RETiRet;
 }
 
-#define COOKIE "@cee: "
+#define COOKIE "@cee:"
 #define LEN_COOKIE (sizeof(COOKIE)-1)
 BEGINdoAction
 	msg_t *pMsg;
diff --git a/plugins/omruleset/omruleset.c b/plugins/omruleset/omruleset.c
index 67aee97e..6c770c94 100644
--- a/plugins/omruleset/omruleset.c
+++ b/plugins/omruleset/omruleset.c
@@ -165,6 +165,9 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1)
 	p += sizeof(":omruleset:") - 1; /* eat indicator sequence  (-1 because of '\0'!) */
 	CHKiRet(createInstance(&pData));
 
+	errmsg.LogError(0, RS_RET_DEPRECATED, "warning: omruleset is deprecated, consider "
+			"using the 'call' statement instead");
+
 	/* check if a non-standard template is to be applied */
 	if(*(p-1) == ';')
 		--p;
@@ -237,6 +240,9 @@ CODEmodInit_QueryRegCFSLineHdlr
 	CHKiRet(objUse(ruleset, CORE_COMPONENT));
 	CHKiRet(objUse(errmsg, CORE_COMPONENT));
 
+	errmsg.LogError(0, RS_RET_DEPRECATED, "warning: omruleset is deprecated, consider "
+			"using the 'call' statement instead");
+
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomrulesetrulesetname", 0, eCmdHdlrGetWord,
 				    setRuleset, NULL, STD_LOADABLE_MODULE_ID));
 	CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler,
diff --git a/runtime/debug.c b/runtime/debug.c
index edc4a255..307a8bb8 100644
--- a/runtime/debug.c
+++ b/runtime/debug.c
@@ -927,12 +927,12 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg)
 		pszObjName = obj.GetName(pObj);
 	}
 
-//	pthread_mutex_lock(&mutdbgprint);
-//	pthread_cleanup_push(dbgMutexCancelCleanupHdlr, &mutdbgprint);
+	pthread_mutex_lock(&mutdbgprint);
+	pthread_cleanup_push(dbgMutexCancelCleanupHdlr, &mutdbgprint);
 
 	do_dbgprint(pszObjName, pszMsg, lenMsg);
 
-//	pthread_cleanup_pop(1);
+	pthread_cleanup_pop(1);
 }
 #pragma GCC diagnostic warning "-Wempty-body"
 
diff --git a/runtime/msg.c b/runtime/msg.c
index d3c814e2..d874178b 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -2698,6 +2698,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 	uchar *pBuf;
 	int iLen;
 	short iOffs;
+	enum tplFormatTypes datefmt;
 
 	BEGINfunc
 	assert(pMsg != NULL);
@@ -2717,7 +2718,11 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 			bufLen = getMSGLen(pMsg);
 			break;
 		case PROP_TIMESTAMP:
-			pRes = (uchar*)getTimeReported(pMsg, pTpe->data.field.eDateFormat);
+			if (pTpe != NULL)
+				datefmt = pTpe->data.field.eDateFormat;
+			else
+				datefmt = tplFmtDefault;
+			pRes = (uchar*)getTimeReported(pMsg, datefmt);
 			break;
 		case PROP_HOSTNAME:
 			pRes = (uchar*)getHOSTNAME(pMsg);
@@ -2767,7 +2772,11 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
 			pRes = (uchar*)getSeverityStr(pMsg);
 			break;
 		case PROP_TIMEGENERATED:
-			pRes = (uchar*)getTimeGenerated(pMsg, pTpe->data.field.eDateFormat);
+			if (pTpe != NULL)
+				datefmt = pTpe->data.field.eDateFormat;
+			else
+				datefmt = tplFmtDefault;
+			pRes = (uchar*)getTimeGenerated(pMsg, datefmt);
 			break;
 		case PROP_PROGRAMNAME:
 			pRes = getProgramName(pMsg, LOCK_MUTEX);
diff --git a/runtime/objomsr.c b/runtime/objomsr.c
index 9cf3781b..e63eb681 100644
--- a/runtime/objomsr.c
+++ b/runtime/objomsr.c
@@ -42,9 +42,7 @@ rsRetVal OMSRdestruct(omodStringRequest_t *pThis)
 	/* free the strings */
 	if(pThis->ppTplName != NULL) {
 		for(i = 0 ; i < pThis->iNumEntries ; ++i) {
-			if(pThis->ppTplName[i] != NULL) {
-				free(pThis->ppTplName[i]);
-			}
+			free(pThis->ppTplName[i]);
 		}
 		free(pThis->ppTplName);
 	}
diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h
index e6238c23..c02db53c 100644
--- a/runtime/rsyslog.h
+++ b/runtime/rsyslog.h
@@ -393,6 +393,7 @@ enum rsRetVal_				/** return value. All methods return this if not specified oth
 	RS_RET_JNAME_NOTFOUND = -2305, /**< JSON name not found (does not exist) */
 	RS_RET_INVLD_SETOP = -2305, /**< invalid variable set operation, incompatible type */
 	RS_RET_RULESET_EXISTS = -2306,/**< ruleset already exists */
+	RS_RET_DEPRECATED = -2307,/**< deprecated functionality is used */
 
 	/* RainerScript error messages (range 1000.. 1999) */
 	RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */
diff --git a/template.c b/template.c
index 8558dd8a..2fc85e55 100644
--- a/template.c
+++ b/template.c
@@ -320,10 +320,6 @@ tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson)
 	rsRetVal localRet;
 	DEFiRet;
 
-	assert(pTpl != NULL);
-	assert(pMsg != NULL);
-	assert(json != NULL);
-
 	if(pTpl->subtree != NULL){
 		localRet = jsonFind(pMsg, pTpl->subtree, pjson);
 		if(*pjson == NULL) {
@@ -1847,6 +1843,7 @@ tplProcessCnf(struct cnfobj *o)
 		pTpl->optFormatEscape = JSON_ESCAPE;
 
 finalize_it:
+	free(tplStr);
 	if(pvals != NULL)
 		cnfparamvalsDestruct(pvals, &pblk);
 	if(iRet != RS_RET_OK) {
diff --git a/tools/logctl.c b/tools/logctl.c
index df332bc2..1ab8ead0 100644
--- a/tools/logctl.c
+++ b/tools/logctl.c
@@ -143,7 +143,6 @@ struct ofields* get_data(struct results *res)
 	struct ofields *fields;
 	const char *msg;
 	const char *prog;
-	const char *level;
 	const char *syslog_tag;
 	gint64 date_r;
 	bson_cursor *c;
@@ -263,7 +262,7 @@ struct select_doc* create_select()
 struct query_doc* create_query(struct queryopt *opt)
 {
 	struct query_doc *qu_doc;
-	bson  *query_what, *order_what, *order_how, *msg_what, *date_what;
+	bson  *query_what, *order_what, *msg_what, *date_what;
 	struct tm tm;
 	time_t t;
 	gint64  	ts;
@@ -417,7 +416,6 @@ int main (int argc, char *argv[])
 	
 	struct queryopt opt;
 	struct ofields *fields;
-	struct bson_doc *doc;
 	struct select_doc *s_doc;
 	struct query_doc *qu_doc;
 	struct db_connect *db_conn;
diff --git a/tools/omdiscard.c b/tools/omdiscard.c
index 182c4b63..15c6ea82 100644
--- a/tools/omdiscard.c
+++ b/tools/omdiscard.c
@@ -35,6 +35,7 @@
 #include "syslogd-types.h"
 #include "omdiscard.h"
 #include "module-template.h"
+#include "errmsg.h"
 
 MODULE_TYPE_OUTPUT
 MODULE_TYPE_NOKEEP
@@ -42,6 +43,7 @@ MODULE_TYPE_NOKEEP
 /* internal structures
  */
 DEF_OMOD_STATIC_DATA
+DEFobjCurrIf(errmsg);
 
 typedef struct _instanceData {
 	EMPTY_STRUCT
@@ -92,8 +94,10 @@ CODE_STD_STRING_REQUESTparseSelectorAct(0)
 	p = *pp;
 
 	if(*p == '~') {
-		/* TODO: check the rest of the selector line - error reporting */
 		dbgprintf("discard\n");
+		errmsg.LogError(0, RS_RET_DEPRECATED, "warning: ~ action "
+			"is deprecated, consider using the 'stop' "
+			"statement instead");
 	} else {
 		iRet = RS_RET_CONFLINE_UNPROCESSED;
 	}
@@ -103,6 +107,7 @@ ENDparseSelectorAct
 
 BEGINmodExit
 CODESTARTmodExit
+	objRelease(errmsg, CORE_COMPONENT);
 ENDmodExit
 
 
@@ -116,6 +121,7 @@ BEGINmodInit(Discard)
 CODESTARTmodInit
 	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
 CODEmodInit_QueryRegCFSLineHdlr
+	CHKiRet(objUse(errmsg, CORE_COMPONENT));
 ENDmodInit
 /*
  * vi:set ai:
diff --git a/tools/omfile.c b/tools/omfile.c
index 2bb51e99..c7e0dc25 100644
--- a/tools/omfile.c
+++ b/tools/omfile.c
@@ -824,6 +824,7 @@ ENDcreateInstance
 
 BEGINfreeInstance
 CODESTARTfreeInstance
+	free(pData->tplName);
 	free(pData->f_fname);
 	if(pData->bDynamicName) {
 		dynaFileFreeCache(pData);
@@ -969,7 +970,7 @@ CODESTARTnewActInst
 		ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS);
 	}
 
-	tplToUse = (pData->tplName == NULL) ? ustrdup(getDfltTpl()) : pData->tplName;
+	tplToUse = ustrdup((pData->tplName == NULL) ? getDfltTpl() : pData->tplName);
 	CHKiRet(OMSRsetEntry(*ppOMSR, 0, tplToUse, OMSR_NO_RQD_TPL_OPTS));
 
 	if(pData->bDynamicName) {