new ruleengine: implement template type "subtree"

7 files changed, 126 insertions, 30 deletions

diff --git a/plugins/ommongodb/ommongodb.c b/plugins/ommongodb/ommongodb.c
index f2de33df..c1648c50 100644
--- a/plugins/ommongodb/ommongodb.c
+++ b/plugins/ommongodb/ommongodb.c
@@ -224,11 +224,11 @@ static bson *
 getDefaultBSON(msg_t *pMsg)
 {
 	bson *doc = NULL;
-	uchar *procid; short unsigned procid_free; size_t procid_len;
-	uchar *tag; short unsigned tag_free; size_t tag_len;
-	uchar *pid; short unsigned pid_free; size_t pid_len;
-	uchar *sys; short unsigned sys_free; size_t sys_len;
-	uchar *msg; short unsigned msg_free; size_t msg_len;
+	uchar *procid; short unsigned procid_free; rs_size_t procid_len;
+	uchar *tag; short unsigned tag_free; rs_size_t tag_len;
+	uchar *pid; short unsigned pid_free; rs_size_t pid_len;
+	uchar *sys; short unsigned sys_free; rs_size_t sys_len;
+	uchar *msg; short unsigned msg_free; rs_size_t msg_len;
 	int severity, facil;
 	gint64 ts_gen, ts_rcv; /* timestamps: generated, received */
 	int secfrac;
diff --git a/runtime/msg.c b/runtime/msg.c
index 2f6606bb..f4ad7bf4 100644
--- a/runtime/msg.c
+++ b/runtime/msg.c
@@ -2408,8 +2408,8 @@ static uchar *getNOW(eNOWType eNow)
 
 
 /* Get a CEE-Property as string value*/
-static inline rsRetVal
-getCEEPropVal(msg_t *pM, es_str_t *propName, uchar **pRes, int *buflen, unsigned short *pbMustBeFreed)
+rsRetVal
+getCEEPropVal(msg_t *pM, es_str_t *propName, uchar **pRes, rs_size_t *buflen, unsigned short *pbMustBeFreed)
 {
 	uchar *name = NULL;
 	uchar *leaf;
@@ -2431,10 +2431,7 @@ getCEEPropVal(msg_t *pM, es_str_t *propName, uchar **pRes, int *buflen, unsigned
 		CHKiRet(jsonPathFindParent(pM, name, leaf, &parent, 1));
 		field = json_object_object_get(parent, (char*)leaf);
 	}
-	if(field == 0) {
-		*pRes = (uchar*) "";
-		*pbMustBeFreed = 0;
-	} else {
+	if(field != NULL) {
 		*pRes = (uchar*) strdup(json_object_get_string(field));
 		*buflen = (int) ustrlen(*pRes);
 		*pbMustBeFreed = 1;
@@ -2684,11 +2681,11 @@ finalize_it:
 	*pPropLen = sizeof("**OUT OF MEMORY**") - 1; \
 	return(UCHAR_CONSTANT("**OUT OF MEMORY**"));}
 uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
-                 propid_t propid, es_str_t *propName, size_t *pPropLen,
+                 propid_t propid, es_str_t *propName, rs_size_t *pPropLen,
 		 unsigned short *pbMustBeFreed)
 {
 	uchar *pRes; /* result pointer */
-	int bufLen = -1; /* length of string or -1, if not known */
+	rs_size_t bufLen = -1; /* length of string or -1, if not known */
 	uchar *pBufStart;
 	uchar *pBuf;
 	int iLen;
@@ -3528,7 +3525,7 @@ done:
 es_str_t*
 msgGetMsgVarNew(msg_t *pThis, uchar *name)
 {
-	size_t propLen;
+	rs_size_t propLen;
 	uchar *pszProp = NULL;
 	propid_t propid;
 	unsigned short bMustBeFreed = 0;
@@ -3730,6 +3727,36 @@ DBGPRINTF("AAAA jsonMerge adds '%s'\n", it.key);
 	RETiRet;
 }
 
+/* find a JSON structure element (field or container doesn't matter).  */
+rsRetVal
+jsonFind(msg_t *pM, es_str_t *propName, struct json_object **jsonres)
+{
+	uchar *name = NULL;
+	uchar *leaf;
+	struct json_object *parent;
+	struct json_object *field;
+	DEFiRet;
+
+	if(pM->json == NULL) {
+		field = NULL;
+		goto finalize_it;
+	}
+
+	if(!es_strbufcmp(propName, (uchar*)"!", 1)) {
+		field = pM->json;
+	} else {
+		name = (uchar*)es_str2cstr(propName, NULL);
+		leaf = jsonPathGetLeaf(name, ustrlen(name));
+		CHKiRet(jsonPathFindParent(pM, name, leaf, &parent, 0));
+		field = json_object_object_get(parent, (char*)leaf);
+	}
+	*jsonres = field;
+
+finalize_it:
+	free(name);
+	RETiRet;
+}
+
 rsRetVal
 msgAddJSON(msg_t *pM, uchar *name, struct json_object *json)
 {
diff --git a/runtime/msg.h b/runtime/msg.h
index 477d1f1a..396e861f 100644
--- a/runtime/msg.h
+++ b/runtime/msg.h
@@ -173,7 +173,7 @@ void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg);
 rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG);
 uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe,
                   propid_t propid, es_str_t *propName,
-		  size_t *pPropLen, unsigned short *pbMustBeFreed);
+		  rs_size_t *pPropLen, unsigned short *pbMustBeFreed);
 char *textpri(char *pRes, size_t pResLen, int pri);
 rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar);
 es_str_t* msgGetMsgVarNew(msg_t *pThis, uchar *name);
@@ -186,6 +186,7 @@ void getRawMsg(msg_t *pM, uchar **pBuf, int *piLen);
 rsRetVal msgGetCEEVar(msg_t *pThis, cstr_t *propName, var_t **ppVar);
 es_str_t* msgGetCEEVarNew(msg_t *pMsg, char *name);
 rsRetVal msgAddJSON(msg_t *pM, uchar *name, struct json_object *json);
+rsRetVal getCEEPropVal(msg_t *pM, es_str_t *propName, uchar **pRes, rs_size_t *buflen, unsigned short *pbMustBeFreed);
 
 /* TODO: remove these five (so far used in action.c) */
 uchar *getMSG(msg_t *pM);
@@ -204,6 +205,7 @@ uchar *propIDToName(propid_t propID);
 rsRetVal msgGetCEEPropJSON(msg_t *pM, es_str_t *propName, struct json_object **pjson);
 rsRetVal msgSetJSONFromVar(msg_t *pMsg, uchar *varname, struct var *var);
 rsRetVal msgDelJSON(msg_t *pMsg, uchar *varname);
+rsRetVal jsonFind(msg_t *pM, es_str_t *propName, struct json_object **jsonres);
 
 static inline rsRetVal
 msgUnsetJSON(msg_t *pMsg, uchar *varname) {
diff --git a/runtime/ruleset.c b/runtime/ruleset.c
index a40e2ed1..bcccb79d 100644
--- a/runtime/ruleset.c
+++ b/runtime/ruleset.c
@@ -347,7 +347,7 @@ evalPROPFILT(struct cnfstmt *stmt, msg_t *pMsg)
 	unsigned short pbMustBeFreed;
 	uchar *pszPropVal;
 	int bRet = 0;
-	size_t propLen;
+	rs_size_t propLen;
 
 	if(stmt->d.s_propfilt.propID == PROP_INVALID)
 		goto done;
diff --git a/runtime/typedefs.h b/runtime/typedefs.h
index f994cbc4..b53c36f5 100644
--- a/runtime/typedefs.h
+++ b/runtime/typedefs.h
@@ -92,6 +92,8 @@ typedef struct cfgmodules_etry_s cfgmodules_etry_t;
 typedef struct outchannels_s outchannels_t;
 typedef struct modConfData_s modConfData_t;
 typedef struct instanceConf_s instanceConf_t;
+typedef int rs_size_t; /* we do never need more than 2Gig strings, signed permits to
+			* use -1 as a special flag. */
 typedef rsRetVal (*prsf_t)(struct vmstk_s*, int);	/* pointer to a RainerScript function */
 typedef uint64 qDeqID;	/* queue Dequeue order ID. 32 bits is considered dangerously few */
 
diff --git a/template.c b/template.c
index 15d376a3..95c1dc2b 100644
--- a/template.c
+++ b/template.c
@@ -58,6 +58,7 @@ static struct cnfparamdescr cnfparamdescr[] = {
 	{ "type", eCmdHdlrString, 0 },
 	{ "string", eCmdHdlrString, 0 },
 	{ "plugin", eCmdHdlrString, 0 },
+	{ "subtree", eCmdHdlrString, 0 },
 	{ "option.stdsql", eCmdHdlrBinary, 0 },
 	{ "option.sql", eCmdHdlrBinary, 0 },
 	{ "option.json", eCmdHdlrBinary, 0 }
@@ -147,7 +148,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t *
 	size_t iBuf;
 	unsigned short bMustBeFreed = 0;
 	uchar *pVal;
-	size_t iLenVal = 0;
+	rs_size_t iLenVal = 0;
 
 	assert(pTpl != NULL);
 	assert(pMsg != NULL);
@@ -159,6 +160,23 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t *
 		FINALIZE;
 	}
 
+	if(pTpl->subtree != NULL) {
+		/* only a single CEE subtree must be provided */
+		/* note: we could optimize the code below, however, this is
+		 * not worth the effort, as this passing mode is not expected
+		 * in subtree mode and so most probably only used for debug & test.
+		 */
+		getCEEPropVal(pMsg, pTpl->subtree, &pVal, &iLenVal, &bMustBeFreed);
+		if(iLenVal >= (rs_size_t)*pLenBuf) /* we reserve one char for the final \0! */
+			CHKiRet(ExtendBuf(ppBuf, pLenBuf, iLenVal + 1));
+		memcpy(*ppBuf, pVal, iLenVal+1);
+		if(bMustBeFreed)
+			free(pVal);
+		FINALIZE;
+	}
+	
+	/* we have a "regular" template with template entries */
+
 	/* loop through the template. We obtain one value
 	 * and copy it over to our dynamic string buffer. Then, we
 	 * free the obtained value (if requested). We continue this
@@ -233,7 +251,7 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr)
 	struct templateEntry *pTpe;
 	uchar **pArr;
 	int iArr;
-	size_t propLen;
+	rs_size_t propLen;
 	unsigned short bMustBeFreed;
 	uchar *pVal;
 
@@ -241,11 +259,24 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr)
 	assert(pMsg != NULL);
 	assert(ppArr != NULL);
 
+	if(pTpl->subtree) {
+		/* Note: this mode is untested, as there is no official plugin
+		 *       using array passing, so I simply could not test it.
+		 */
+		CHKmalloc(pArr = calloc(2, sizeof(uchar*)));
+		getCEEPropVal(pMsg, pTpl->subtree, &pVal, &propLen, &bMustBeFreed);
+		if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */
+			pArr[0] = pVal; /* ... so we can use it! */
+		} else {
+			CHKmalloc(pArr[0] = (uchar*)strdup((char*) pVal));
+		}
+		FINALIZE;
+	}
+
 	/* loop through the template. We obtain one value, create a
 	 * private copy (if necessary), add it to the string array
 	 * and then on to the next until we have processed everything.
 	 */
-
 	CHKmalloc(pArr = calloc(pTpl->tpenElements + 1, sizeof(uchar*)));
 	iArr = 0;
 
@@ -278,10 +309,11 @@ finalize_it:
  * tpltoString().
  * rgerhards, 2012-08-29
  */
-rsRetVal tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson)
+rsRetVal
+tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson)
 {
 	struct templateEntry *pTpe;
-	size_t propLen;
+	rs_size_t propLen;
 	unsigned short bMustBeFreed;
 	uchar *pVal;
 	struct json_object *json, *jsonf;
@@ -292,6 +324,17 @@ rsRetVal tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjso
 	assert(pMsg != NULL);
 	assert(json != NULL);
 
+	if(pTpl->subtree != NULL){
+		localRet = jsonFind(pMsg, pTpl->subtree, pjson);
+		if(*pjson == NULL) {
+			/* we need to have a root object! */
+			*pjson = json_object_new_object();
+		} else {
+			json_object_get(*pjson); /* inc refcount */
+		}
+		FINALIZE;
+	}
+
 	json = json_object_new_object();
 	for(pTpe = pTpl->pEntryRoot ; pTpe != NULL ; pTpe = pTpe->pNext) {
 		if(pTpe->eEntryType == CONSTANT) {
@@ -325,8 +368,9 @@ rsRetVal tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjso
 			}
 		}
 	}
-
 	*pjson = (iRet == RS_RET_OK) ? json : NULL;
+
+finalize_it:
 	RETiRet;
 }
 
@@ -384,7 +428,7 @@ static void doEmergencyEscape(register uchar *p, int mode)
  * 2005-09-22 rgerhards
  */
 rsRetVal
-doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int mode)
+doEscape(uchar **pp, rs_size_t *pLen, unsigned short *pbMustBeFreed, int mode)
 {
 	DEFiRet;
 	uchar *p = NULL;
@@ -1627,8 +1671,9 @@ tplProcessCnf(struct cnfobj *o)
 	char *name = NULL;
 	uchar *tplStr = NULL;
 	uchar *plugin = NULL;
+	es_str_t *subtree = NULL;
 	uchar *p;
-	enum { T_STRING, T_PLUGIN, T_LIST } tplType;
+	enum { T_STRING, T_PLUGIN, T_LIST, T_SUBTREE } tplType;
 	int i;
 	int o_sql=0, o_stdsql=0, o_json=0; /* options */
 	int numopts;
@@ -1651,6 +1696,8 @@ tplProcessCnf(struct cnfobj *o)
 				tplType = T_PLUGIN;
 			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"list", sizeof("list")-1)) {
 				tplType = T_LIST;
+			} else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"subtree", sizeof("subtree")-1)) {
+				tplType = T_SUBTREE;
 			} else {
 				uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
 				errmsg.LogError(0, RS_RET_ERR, "invalid template type '%s'",
@@ -1660,6 +1707,8 @@ tplProcessCnf(struct cnfobj *o)
 			}
 		} else if(!strcmp(pblk.descr[i].name, "string")) {
 			tplStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
+		} else if(!strcmp(pblk.descr[i].name, "subtree")) {
+			subtree = es_strdup(pvals[i].val.d.estr);
 		} else if(!strcmp(pblk.descr[i].name, "plugin")) {
 			plugin = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL);
 		} else if(!strcmp(pblk.descr[i].name, "option.stdsql")) {
@@ -1701,6 +1750,19 @@ tplProcessCnf(struct cnfobj *o)
 		}
 	}
 
+	if(subtree  == NULL) {
+		if(tplType == T_SUBTREE) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' of type subtree needs "
+				"subtree parameter", name);
+			ABORT_FINALIZE(RS_RET_ERR);
+		}
+	} else {
+		if(tplType != T_SUBTREE) {
+			errmsg.LogError(0, RS_RET_ERR, "template '%s' is not a subtree "
+				"template but has a subtree specified - ignored", name);
+		}
+	}
+
 	if(o->subobjs  == NULL) {
 		if(tplType == T_LIST) {
 			errmsg.LogError(0, RS_RET_ERR, "template '%s' of type list has "
@@ -1758,6 +1820,8 @@ tplProcessCnf(struct cnfobj *o)
 			break;
 	case T_LIST:	createListTpl(pTpl, o);
 			break;
+	case T_SUBTREE:	pTpl->subtree = subtree;
+			break;
 	}
 	
 	pTpl->optFormatEscape = NO_ESCAPE;
@@ -1858,8 +1922,8 @@ void tplDeleteAll(rsconf_t *conf)
 		}
 		pTplDel = pTpl;
 		pTpl = pTpl->pNext;
-		if(pTplDel->pszName != NULL)
-			free(pTplDel->pszName);
+		free(pTplDel->pszName);
+		es_deleteStr(pTplDel->subtree);
 		free(pTplDel);
 	}
 	ENDfunc
@@ -1916,8 +1980,8 @@ void tplDeleteNew(rsconf_t *conf)
 		}
 		pTplDel = pTpl;
 		pTpl = pTpl->pNext;
-		if(pTplDel->pszName != NULL)
-			free(pTplDel->pszName);
+		free(pTplDel->pszName);
+		es_deleteStr(pTplDel->subtree);
 		free(pTplDel);
 	}
 	ENDfunc
diff --git a/template.h b/template.h
index e30f96b4..5a35d274 100644
--- a/template.h
+++ b/template.h
@@ -39,7 +39,8 @@ struct template {
 	struct template *pNext;
 	char *pszName;
 	int iLenName;
-	rsRetVal (*pStrgen)(msg_t*, uchar**, size_t *);	/* name of strgen to use (bound if non-NULL!) */
+	rsRetVal (*pStrgen)(msg_t*, uchar**, size_t *);
+	es_str_t *subtree;	/* subtree name for subtree-type templates */
 	int tpenElements; /* number of elements in templateEntry list */
 	struct templateEntry *pEntryRoot;
 	struct templateEntry *pEntryLast;
@@ -149,7 +150,7 @@ rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize);
 rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr);
 rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *);
 rsRetVal tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **);
-rsRetVal doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode);
+rsRetVal doEscape(uchar **pp, rs_size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode);
 
 rsRetVal templateInit();
 rsRetVal tplProcessCnf(struct cnfobj *o);