diff options
64 files changed, 3736 insertions, 1846 deletions
@@ -1,4 +1,34 @@ --------------------------------------------------------------------------- +<<<<<<< HEAD +Version 7.1.5 [devel] 2012-09-25 +- implemented RainerScript prifield() function +- implemented RainerScript field() function +- added new module imkmsg to process structured kernel log + Thanks to Milan Bartos for contributing this module +- implemented basic RainerScript optimizer, which will speed up script + operations +- bugfix: invalid free if function re_match() was incorrectly used + if the config file parser detected that param 2 was not constant, some + data fields were not initialized. The destructor did not care about that. + This bug happened only if rsyslog startup was unclean. +--------------------------------------------------------------------------- +Version 7.1.4 [devel] 2012-09-19 +- implemented ability for CEE-based properties to be stored in disk queues +- implemented string concatenation in expressions via &-operator +- implemented json subtree copy in variable assignment +- implemented full JSON support for variable manipulation +- introduced "subtree"-type templates +- bugfix: omfile action did not respect "template" parameter + ... and used default template in all cases +- bugfix: MsgDup() did not copy CEE structure + This function was called at various places, most importantly during + "last messages repeated n times" processing and omruleset. If CEE(JSON) + data was present, it was lost as part of the copy process. +- bugfix: debug output indicated improper queue type +--------------------------------------------------------------------------- +Version 7.1.3 [devel] 2012-09-17 +- introduced "set" and "unset" config statements +======= Version 6.5.1 [devel] 2012-08-?? - imfile ported to new v6 config interface - imfile now supports config parameter for maximum number of submits @@ -8,11 +38,50 @@ Version 6.5.1 [devel] 2012-08-?? - bugfix: imtcp could abort on exit due to invalid free() - bugfix: remove invalid socket option call from imuxsock Thanks to Cristian Ionescu-Idbohrn and Jonny Törnbom +>>>>>>> 53df5066688c8472a9f7be5c86bbc253378e6572 - bugfix: missing support for escape sequences in RainerScript only \' was supported. Now the usual set is supported. Note that v5 used \x as escape where x was any character (e.g. "\n" meant "n" and NOT LF). This also means there is some incompatibility to v5 for well-know sequences. Better break it now than later. +- bugfix: invalid property name in property-filter could cause abort + if action chaining (& operator) was used + http://bugzilla.adiscon.com/show_bug.cgi?id=355 + Thanks to pilou@gmx.com for the bug report +--------------------------------------------------------------------------- +Version 7.1.2 [devel] 2012-09-12 +- bugfix: messages were duplicated, sometimes massively + regression from new code in 7.1.1 and reason for early release +- bugfix: remove invalid socket option call from imuxsock + Thanks to Cristian Ionescu-Idbohrn and Jonny Törnbom +- bugfix: abort when invalid property name was configured + in property-based filter +- bugfix: multiple rulesets did no longer work correctly (7.1.1 regression) +--------------------------------------------------------------------------- +Version 7.1.1 [devel] 2012-09-11 +- MAJOR NEW FEATURE: rulengine now fully supports nesting + including if ... then ... else ... constructs. This is a big change + and it obviously has a lot of bug potential. +- BSD-style (filter) blocks are no longer supported + see http://www.rsyslog.com/g/BSD for details and solution +- imuxsock now stores trusted properties by default in the CEE root + This was done in order to keep compatible with other implementations of + the lumberjack schema + Thanks to Miloslav Trmač for pointing to this. +- bugfix: string-generating templates caused abort if CEE field could not + be found +--------------------------------------------------------------------------- +Version 7.1.0 [devel] 2012-09-06 +- added support for hierarchical properties (CEE/lumberjack) +- added pure JSON output plugin parameter passing mode +- ommongodb now supports templates +- bugfix: imtcp could abort on exit due to invalid free() +- imported bugfixes from 6.4.1 +--------------------------------------------------------------------------- +Version 6.5.1 [devel] 2012-08-?? +- added pure JSON output plugin parameter passing mode +- ommongodb now supports templates +- bugfix: imtcp could abort on exit due to invalid free() - bugfix: remove invalid socket option call from imuxsock Thanks to Cristian Ionescu-Idbohrn and Jonny Törnbom - bugfix: missing support for escape sequences in RainerScript diff --git a/Makefile.am b/Makefile.am index 95b432c9..e7aaca55 100644 --- a/Makefile.am +++ b/Makefile.am @@ -80,6 +80,10 @@ if ENABLE_IMKLOG SUBDIRS += plugins/imklog endif +if ENABLE_IMKMSG +SUBDIRS += plugins/imkmsg +endif + if ENABLE_IMPSTATS SUBDIRS += plugins/impstats endif @@ -289,6 +289,11 @@ rsRetVal actionDestruct(action_t *pThis) DEFiRet; ASSERT(pThis != NULL); + if(!strcmp((char*)modGetName(pThis->pMod), "builtin:omdiscard")) { + /* discard actions will be optimized out */ + FINALIZE; + } + if(pThis->pQueue != NULL) { qqueueDestruct(&pThis->pQueue); } @@ -310,8 +315,8 @@ rsRetVal actionDestruct(action_t *pThis) d_free(pThis->pszName); d_free(pThis->ppTpl); +finalize_it: d_free(pThis); - RETiRet; } @@ -362,6 +367,10 @@ actionConstructFinalize(action_t *pThis, struct cnfparamvals *queueParams) ASSERT(pThis != NULL); + if(!strcmp((char*)modGetName(pThis->pMod), "builtin:omdiscard")) { + /* discard actions will be optimized out */ + FINALIZE; + } /* generate a friendly name for us action stats */ if(pThis->pszName == NULL) { snprintf((char*) pszAName, sizeof(pszAName)/sizeof(uchar), "action %d", iActionNbr); @@ -861,7 +870,7 @@ static rsRetVal releaseBatch(action_t *pAction, batch_t *pBatch) for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { pElem = &(pBatch->pElem[i]); - if(pElem->bFilterOK && pElem->state != BATCH_STATE_DISC) { + if(batchIsValidElem(pBatch, i)) { switch(pAction->eParamPassing) { case ACT_ARRAY_PASSING: ppMsgs = (uchar***) pElem->staticActParams; @@ -1059,8 +1068,7 @@ tryDoAction(action_t *pAction, batch_t *pBatch, int *pnElem) /* NOTE: do NOT extend the filter below! Anything else must be done on the * enq side of the queue (see file header comment)! -- rgerhards, 2011-06-15 */ - if( pBatch->pElem[i].bFilterOK - && pBatch->pElem[i].state != BATCH_STATE_DISC) { + if(batchIsValidElem(pBatch, i)) { pMsg = (msg_t*) pBatch->pElem[i].pUsrp; localRet = actionProcessMessage(pAction, pMsg, pBatch->pElem[i].staticActParams, pBatch->pbShutdownImmediate); @@ -1181,6 +1189,29 @@ finalize_it: } +/* copy "active" array of batch, as we need to modify it. The caller + * must make sure the new array is freed and the orginal batch + * pointer is restored (thus the caller must save it). If active + * is currently NULL, this is properly handled. + * Note: the batches active pointer is modified, so it must be + * saved BEFORE calling this function! + * rgerhards, 2012-09-12 + */ +static rsRetVal +copyActive(batch_t *pBatch) +{ + sbool *active; + DEFiRet; + + CHKmalloc(active = malloc(batchNumMsgs(pBatch) * sizeof(sbool))); + if(pBatch->active == NULL) + memset(active, 1, batchNumMsgs(pBatch)); + else + memcpy(active, pBatch->active, batchNumMsgs(pBatch)); + pBatch->active = active; +finalize_it: + RETiRet; +} /* The following function prepares a batch for processing, that it is * reinitializes batch states, generates strings and does everything else @@ -1191,7 +1222,7 @@ finalize_it: * rgerhards, 2010-06-14 */ static inline rsRetVal -prepareBatch(action_t *pAction, batch_t *pBatch) +prepareBatch(action_t *pAction, batch_t *pBatch, sbool **activeSave, int *bMustRestoreActivePtr) { int i; batch_obj_t *pElem; @@ -1200,10 +1231,16 @@ prepareBatch(action_t *pAction, batch_t *pBatch) pBatch->iDoneUpTo = 0; for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { pElem = &(pBatch->pElem[i]); - if(pElem->bFilterOK && pElem->state != BATCH_STATE_DISC) { + if(batchIsValidElem(pBatch, i)) { pElem->state = BATCH_STATE_RDY; - if(prepareDoActionParams(pAction, pElem) != RS_RET_OK) - pElem->bFilterOK = RSFALSE; + if(prepareDoActionParams(pAction, pElem) != RS_RET_OK) { + /* make sure we have our copy of "active" array */ + if(!*bMustRestoreActivePtr) { + *activeSave = pBatch->active; + copyActive(pBatch); + } + pBatch->active[i] = RSFALSE; + } } } RETiRet; @@ -1236,6 +1273,8 @@ static rsRetVal processBatchMain(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate) { int *pbShutdownImmdtSave; + sbool *activeSave; + int bMustRestoreActivePtr = 0; rsRetVal localRet; DEFiRet; @@ -1243,7 +1282,7 @@ processBatchMain(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate) pbShutdownImmdtSave = pBatch->pbShutdownImmediate; pBatch->pbShutdownImmediate = pbShutdownImmediate; - CHKiRet(prepareBatch(pAction, pBatch)); + CHKiRet(prepareBatch(pAction, pBatch, &activeSave, &bMustRestoreActivePtr)); /* We now must guard the output module against execution by multiple threads. The * plugin interface specifies that output modules must not be thread-safe (except @@ -1266,6 +1305,11 @@ processBatchMain(action_t *pAction, batch_t *pBatch, int *pbShutdownImmediate) if(iRet == RS_RET_OK) iRet = localRet; + + if(bMustRestoreActivePtr) { + free(pBatch->active); + pBatch->active = activeSave; + } finalize_it: pBatch->pbShutdownImmediate = pbShutdownImmdtSave; @@ -1572,7 +1616,8 @@ DEFFUNC_llExecFunc(doActivateActions) } actionDisable(pThis); } - DBGPRINTF("Action %p: queue %p started\n", pThis, pThis->pQueue); + DBGPRINTF("Action %s[%p]: queue %p started\n", modGetName(pThis->pMod), + pThis, pThis->pQueue); ENDfunc return RS_RET_OK; /* we ignore errors, we can not do anything either way */ } @@ -1606,22 +1651,15 @@ doSubmitToActionQNotAllMarkBatch(action_t *pAction, batch_t *pBatch) time_t now = 0; time_t lastAct; int i; - int bModifiedFilter; - sbool FilterSave[1024]; - sbool *pFilterSave; + sbool *activeSave; DEFiRet; - if(batchNumMsgs(pBatch) <= (int) (sizeof(FilterSave)/sizeof(sbool))) { - pFilterSave = FilterSave; - } else { - CHKmalloc(pFilterSave = malloc(batchNumMsgs(pBatch) * sizeof(sbool))); - } + activeSave = pBatch->active; + copyActive(pBatch); - bModifiedFilter = 0; for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { - if(!pBatch->pElem[i].bFilterOK) + if((pBatch->pElem[i].state == BATCH_STATE_DISC) || !pBatch->active[i]) continue; - pFilterSave[i] = pBatch->pElem[i].bFilterOK; if(now == 0) { now = datetime.GetTime(NULL); /* good time call - the only one done */ } @@ -1632,15 +1670,15 @@ doSubmitToActionQNotAllMarkBatch(action_t *pAction, batch_t *pBatch) lastAct = pAction->f_time; if(((msg_t*)(pBatch->pElem[i].pUsrp))->msgFlags & MARK) { if((now - lastAct) < MarkInterval / 2) { - pBatch->pElem[i].bFilterOK = 0; - bModifiedFilter = 1; - DBGPRINTF("action was recently called, ignoring mark message\n"); + pBatch->active[i] = 0; + DBGPRINTF("batch item %d: action was recently called, ignoring " + "mark message\n", i); break; /* do not update timestamp for non-written mark messages */ } } } while(ATOMIC_CAS_time_t(&pAction->f_time, lastAct, ((msg_t*)(pBatch->pElem[i].pUsrp))->ttGenTime, &pAction->mutCAS) == 0); - if(pBatch->pElem[i].bFilterOK) { + if(pBatch->active[i]) { DBGPRINTF("Called action(NotAllMark), processing batch[%d] via '%s'\n", i, module.GetStateName(pAction->pMod)); } @@ -1648,17 +1686,8 @@ doSubmitToActionQNotAllMarkBatch(action_t *pAction, batch_t *pBatch) iRet = doSubmitToActionQBatch(pAction, pBatch); - if(bModifiedFilter) { - /* in this case, we need to restore previous state */ - for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { - /* note: clang static code analyzer reports a false positive below */ - pBatch->pElem[i].bFilterOK = pFilterSave[i]; - } - } - -finalize_it: - if(pFilterSave != FilterSave) - free(pFilterSave); + free(pBatch->active); + pBatch->active = activeSave; RETiRet; } @@ -1668,8 +1697,7 @@ countStatsBatchEnq(action_t *pAction, batch_t *pBatch) { int i; for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - if( pBatch->pElem[i].bFilterOK - && pBatch->pElem[i].state != BATCH_STATE_DISC) { + if( batchIsValidElem(pBatch, i)) { STATSCOUNTER_INC(pAction->ctrProcessed, pAction->mutCtrProcessed); } } @@ -1683,18 +1711,13 @@ countStatsBatchEnq(action_t *pAction, batch_t *pBatch) static inline rsRetVal doQueueEnqObjDirectBatch(action_t *pAction, batch_t *pBatch) { - sbool FilterSave[1024]; - sbool *pFilterSave; sbool bNeedSubmit; - sbool bModifiedFilter; + sbool *activeSave; int i; DEFiRet; - if(batchNumMsgs(pBatch) <= (int) (sizeof(FilterSave)/sizeof(sbool))) { - pFilterSave = FilterSave; - } else { - CHKmalloc(pFilterSave = malloc(batchNumMsgs(pBatch) * sizeof(sbool))); - } + activeSave = pBatch->active; + copyActive(pBatch); /* note: for direct mode, we need to adjust the filter property. For non-direct * this is not necessary, because in that case we enqueue only what actually needs @@ -1702,37 +1725,25 @@ doQueueEnqObjDirectBatch(action_t *pAction, batch_t *pBatch) */ if(pAction->bExecWhenPrevSusp) { bNeedSubmit = 0; - bModifiedFilter = 0; for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - pFilterSave[i] = pBatch->pElem[i].bFilterOK; if(!pBatch->pElem[i].bPrevWasSuspended) { - DBGPRINTF("action enq stage: change bFilterOK to 0 due to " + DBGPRINTF("action enq stage: change active to 0 due to " "failover case in elem %d\n", i); - pBatch->pElem[i].bFilterOK = 0; - bModifiedFilter = 1; + pBatch->active[i] = 0; } - if(pBatch->pElem[i].bFilterOK && pBatch->pElem[i].state != BATCH_STATE_DISC) { + if(batchIsValidElem(pBatch, i)) { STATSCOUNTER_INC(pAction->ctrProcessed, pAction->mutCtrProcessed); bNeedSubmit = 1; } - DBGPRINTF("action %p[%d]: filterOK:%d state:%d execWhenPrev:%d prevWasSusp:%d\n", - pAction, i, pBatch->pElem[i].bFilterOK, pBatch->pElem[i].state, + DBGPRINTF("action %p[%d]: valid:%d state:%d execWhenPrev:%d prevWasSusp:%d\n", + pAction, i, batchIsValidElem(pBatch, i), pBatch->pElem[i].state, pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended); } if(bNeedSubmit) { /* note: stats were already computed above */ iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch); } else { - DBGPRINTF("no need to submit batch, all bFilterOK==0 or discarded\n"); - } - if(bModifiedFilter) { - for(i = 0 ; i < batchNumMsgs(pBatch) ; ++i) { - DBGPRINTF("action %p: filterOK:%d state:%d execWhenPrev:%d prevWasSusp:%d\n", - pAction, pBatch->pElem[i].bFilterOK, pBatch->pElem[i].state, - pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended); - /* note: clang static code analyzer reports a false positive below */ - pBatch->pElem[i].bFilterOK = pFilterSave[i]; - } + DBGPRINTF("no need to submit batch, all invalid\n"); } } else { if(GatherStats) @@ -1740,7 +1751,8 @@ doQueueEnqObjDirectBatch(action_t *pAction, batch_t *pBatch) iRet = qqueueEnqObjDirectBatch(pAction->pQueue, pBatch); } -finalize_it: + free(pBatch->active); + pBatch->active = activeSave; RETiRet; } @@ -1763,11 +1775,10 @@ doSubmitToActionQBatch(action_t *pAction, batch_t *pBatch) * TODO: optimize this, we may do at least a multi-submit! */ for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - DBGPRINTF("action %p: filterOK:%d state:%d execWhenPrev:%d prevWasSusp:%d\n", - pAction, pBatch->pElem[i].bFilterOK, pBatch->pElem[i].state, + DBGPRINTF("action %p: valid:%d state:%d execWhenPrev:%d prevWasSusp:%d\n", + pAction, batchIsValidElem(pBatch, i), pBatch->pElem[i].state, pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended); - if( pBatch->pElem[i].bFilterOK - && pBatch->pElem[i].state != BATCH_STATE_DISC + if( batchIsValidElem(pBatch, i) && (pAction->bExecWhenPrevSusp == 0 || pBatch->pElem[i].bPrevWasSuspended == 1)) { doSubmitToActionQ(pAction, (msg_t*)(pBatch->pElem[i].pUsrp)); } @@ -1792,11 +1803,10 @@ helperSubmitToActionQComplexBatch(action_t *pAction, batch_t *pBatch) DBGPRINTF("Called action %p (complex case), logging to %s\n", pAction, module.GetStateName(pAction->pMod)); for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - DBGPRINTF("action %p: filterOK:%d state:%d execWhenPrev:%d prevWasSusp:%d\n", - pAction, pBatch->pElem[i].bFilterOK, pBatch->pElem[i].state, + DBGPRINTF("action %p: valid:%d state:%d execWhenPrev:%d prevWasSusp:%d\n", + pAction, batchIsValidElem(pBatch, i), pBatch->pElem[i].state, pAction->bExecWhenPrevSusp, pBatch->pElem[i].bPrevWasSuspended); - if( pBatch->pElem[i].bFilterOK - && pBatch->pElem[i].state != BATCH_STATE_DISC + if( batchIsValidElem(pBatch, i) && ((pAction->bExecWhenPrevSusp == 0) || pBatch->pElem[i].bPrevWasSuspended) ) { doActionCallAction(pAction, pBatch, i); } @@ -2086,33 +2096,6 @@ finalize_it: RETiRet; } - -/* Process a rsyslog v6 action config object (the now-primary config method). - * rgerhards, 2011-07-19 - */ -rsRetVal -actionProcessCnf(struct cnfobj __attribute__((unused)) *o) -{ - DEFiRet; -#if 0 /* we need to check if we actually need this functionality -- later! */ -// This is for STAND-ALONE actions at the conf file TOP level - struct cnfparamvals *paramvals; - - paramvals = nvlstGetParams(o->nvlst, &pblk, NULL); - if(paramvals == NULL) { - iRet = RS_RET_ERR; - goto finalize_it; - } - DBGPRINTF("action param blk after actionProcessCnf:\n"); - cnfparamsPrint(&pblk, paramvals); - - /* now find module to activate */ -finalize_it: -#endif - RETiRet; -} - - /* TODO: we are not yet a real object, the ClassInit here just looks like it is.. */ rsRetVal actionClassInit(void) diff --git a/configure.ac b/configure.ac index 1d7aec58..b6a44feb 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[6.5.0],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[7.1.5],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) @@ -330,6 +330,18 @@ AM_CONDITIONAL(ENABLE_IMKLOG_BSD, test x$os_type = xbsd) AM_CONDITIONAL(ENABLE_IMKLOG_LINUX, test x$os_type = xlinux) AM_CONDITIONAL(ENABLE_IMKLOG_SOLARIS, test x$os_type = xsolaris) +# kmsg +AC_ARG_ENABLE(kmsg, + [AS_HELP_STRING([--enable-kmsg],[Kmsg structured kernel logs functionality @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_kmsg="yes" ;; + no) enable_kmsg="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-kmsg) ;; + esac], + [enable_kmsg="no"] +) +AM_CONDITIONAL(ENABLE_IMKMSG, test x$enable_kmsg = xyes) + # # SYSLOG_UNIXAF @@ -1358,6 +1370,7 @@ AC_CONFIG_FILES([Makefile \ plugins/imuxsock/Makefile \ plugins/immark/Makefile \ plugins/imklog/Makefile \ + plugins/imkmsg/Makefile \ plugins/omhdfs/Makefile \ plugins/omprog/Makefile \ plugins/omstdout/Makefile \ @@ -1395,9 +1408,9 @@ AC_CONFIG_FILES([Makefile \ plugins/omelasticsearch/Makefile \ plugins/sm_cust_bindcdr/Makefile \ plugins/mmsnmptrapd/Makefile \ - plugins/cust1/Makefile \ java/Makefile \ tests/Makefile]) +#add on demand: plugins/cust1/Makefile AC_OUTPUT echo "****************************************************" @@ -1415,6 +1428,7 @@ echo " Unlimited select() support enabled: $enable_unlimited_select" echo echo "---{ input plugins }---" echo " Klog functionality enabled: $enable_klog ($os_type)" +echo " /dev/kmsg functionality enabled: $enable_kmsg" echo " plain tcp input module enabled: $enable_imptcp" echo " threaded plain tcp input module enabled: $enable_imttcp" echo " imdiag enabled: $enable_imdiag" diff --git a/doc/imkmsg.html b/doc/imkmsg.html new file mode 100644 index 00000000..61068d09 --- /dev/null +++ b/doc/imkmsg.html @@ -0,0 +1,48 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head> +<meta http-equiv="Content-Language" content="en"><title>/dev/kmsg Log Input Module (imkmsg)</title> + +</head> +<body> +<a href="rsyslog_conf_modules.html">back</a> + +<h1>/dev/kmsg Log Input Module</h1> +<p><b>Module Name: imkmsg</b></p> +<p><b>Authors: </b>Rainer Gerhards +<rgerhards@adiscon.com><br /> +Milan Bartos +<mbartos@redhat.com></p> +<p><b>Description</b>:</p> +<p>Reads messages from the /dev/kmsg structured kernel log and submits them to the +syslog engine.</p> +<p> +The printk log buffer constains log records. These records are exported by /dev/kmsg +device as structured data in the following format:<br /> + "level,sequnum,timestamp;<message text>\n"<br /> +There could be continuation lines starting with space that contains key/value pairs.<br /> +<br /> +Log messages are parsed as necessary into rsyslog msg_t structure. Continuation lines are parsed +as json key/value pairs and added into rsyslog's message json representation. +</p> +<p><b>Configuration Directives</b>:</p> +This module has no configuration directives. +<b>Caveats/Known Bugs:</b> +<p>This is Linux specific module and requires /dev/kmsg device with structured kernel logs. +<p><b>Sample:</b></p> +<p>The following sample pulls messages from the /dev/kmsg log device. All +parameters are left by default, which is usually a good idea. Please +note that loading the plugin is sufficient to activate it. No directive +is needed to start pulling messages.<br> +</p> +<textarea rows="15" cols="60">$ModLoad imkmsg +</textarea> +<p>[<a href="rsyslog_conf.html">rsyslog.conf overview</a>] +[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> +project.<br> +Copyright © 2008-2009 by <a href="http://www.gerhards.net/rainer">Rainer +Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. +Released under the GNU GPL version 3 or higher.</font></p> +</body></html> diff --git a/doc/manual.html b/doc/manual.html index 9a8644c9..4e786bd9 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support</a> available directly from the source!</p> <p><b>Please visit the <a href="http://www.rsyslog.com/sponsors">rsyslog sponsor's page</a> to honor the project sponsors or become one yourself!</b> We are very grateful for any help towards the project goals.</p> -<p><b>This documentation is for version 6.5.0 (devel branch) of rsyslog.</b> +<p><b>This documentation is for version 7.1.5 (devel branch) of rsyslog.</b> Visit the <i><a href="http://www.rsyslog.com/status">rsyslog status page</a></i></b> to obtain current version information and project status. </p><p><b>If you like rsyslog, you might @@ -35,6 +35,8 @@ if you upgrade from v4, read the <a href="v5compatibility.html">rsyslog v5 compatibility notes</a>, and if you upgrade from v5, read the <a href="v6compatibility.html">rsyslog v6 compatibility notes</a>. +if you upgrade from v6, read the +<a href="v7compatibility.html">rsyslog v7 compatibility notes</a>. <p>Rsyslog will work even if you do not read the doc, but doing so will definitely improve your experience.</p> <p><b>Follow the links below for the</b></p> diff --git a/doc/rainerscript.html b/doc/rainerscript.html index fcc2674d..84261bdd 100644 --- a/doc/rainerscript.html +++ b/doc/rainerscript.html @@ -61,6 +61,14 @@ variable, if it exists. Returns an empty string if it does not exist. <li>cstr(expr) - converts expr to a string value <li>cnum(expr) - converts expr to a number (integer) <li>re_match(expr, re) - returns 1, if expr matches re, 0 otherwise +<li>field(str, delim, matchnbr) - returns a field-based substring. str is the string +to search, delim is the numerical ascii value of the field delimiter (so that +non-printable characters can by specified) and matchnbr is the match to search +for (the first match starts at 1). This works similar as the field based +property-replacer option. +<li>prifilt(constant) - mimics a traditional PRI-based filter (like "*.*" or +"mail.info"). The traditional filter string must be given as a <b>constant string</b>. +Dynamic string evaluation is not permitted (for performance reasons). </ul> <p>The following example can be used to build a dynamic filter based on some environment variable: diff --git a/doc/rsyslog_conf_filter.html b/doc/rsyslog_conf_filter.html index fbced4a3..3efa3967 100644 --- a/doc/rsyslog_conf_filter.html +++ b/doc/rsyslog_conf_filter.html @@ -4,38 +4,95 @@ <p>This is a part of the rsyslog.conf documentation.</p> <a href="rsyslog_conf.html">back</a> <h2>Filter Conditions</h2> -<p>Rsyslog offers four different types "filter conditions":</p> +<p>Rsyslog offers three different types "filter conditions":</p> <ul> -<li>BSD-style blocks</li> +<li><a href="http://www.rainerscript.com/">RainerScript</a>-based filters</li> <li>"traditional" severity and facility based selectors</li> <li>property-based filters</li> -<li>expression-based filters</li> </ul> -<h3>Blocks</h3> -<p>Rsyslogd supports BSD-style blocks inside rsyslog.conf. Each -block of lines is separated from the previous block by a program or -hostname specification. A block will only log messages corresponding to -the most recent program and hostname specifications given. Thus, a -block which selects ‘ppp’ as the program, directly followed by a block -that selects messages from the hostname ‘dialhost’, then the second -block will only log messages from the ppp program on dialhost. -</p> -<p>A program specification is a line beginning with ‘!prog’ and -the following blocks will be associated with calls to syslog from that -specific program. A program specification for ‘foo’ will also match any -message logged by the kernel with the prefix ‘foo: ’. Alternatively, a -program specification ‘-foo’ causes the following blocks to be applied -to messages from any program but the one specified. A hostname -specification of the form ‘+hostname’ and the following blocks will be -applied to messages received from the specified hostname. -Alternatively, a hostname specification ‘-hostname’ causes the -following blocks to be applied to messages from any host but the one -specified. If the hostname is given as ‘@’, the local hostname will be -used. (NOT YET IMPLEMENTED) A program or hostname specification may be -reset by giving the program or hostname as ‘*’.</p> -<p>Please note that the "#!prog", "#+hostname" and "#-hostname" -syntax available in BSD syslogd is not supported by rsyslogd. By -default, no hostname or program is set.</p> +<h3>RainerScript-Based Filters</h3> +RainerScript based filters are the prime means of creating complex rsyslog configuration. +The permit filtering on arbitrary complex expressions, which can include boolean, +arithmetic and string operations. They also support full nesting of filters, just +as you know from other scripting environments. +<br> +Scripts based filters are indicated by the keyword "if", as usual. +They have this format:<br> +<br> +if expr then block else block +<br> +"If" and "then" are fixed keywords that mus be present. "expr" is a +(potentially quite complex) expression. So the <a href="expression.html">expression documentation</a> for +details. +The keyword "else" and its associated block is optional. Note that a block can contain either +a single action (chain), or an arbitrary complex script enclosed in curly braces, e.g.: +<br> +<pre> +if $programname == 'prog1' then { + action(type="omfile" file="/var/log/prog1.log") + if $msg contains 'test' then + action(type="omfile" file="/var/log/prog1test.log") + else + action(type="omfile" file="/var/log/prog1notest.log") +} +</pre> +<br> +Other types of filtes can also be combined with the pure RainerScript ones. This makes +it particularly easy to migrate from early config files to RainerScript. Also, the traditional +syslog PRI-based filters are a good and easy to use addition. While they are legacy, we still +recommend there use where they are up to the job. We do NOT, however, recommend property-based +filters any longer. As an example, the following is perfectly valid: +<br> +<pre> +if $fromhost == 'host1' then { + mail.* action(type="omfile" file="/var/log/host1/mail.log") + *.err /var/log/host1/errlog # this is also still valid + # + # more "old-style rules" ... + # +} else { + mail.* action(type="omfile" file="/var/log/mail.log") + *.err /var/log/errlog + # + # more "old-style rules" ... + # +} +</pre> +<br> + +Right now, you need to specify numerical values if you would like to +check for facilities and severity. These can be found in <a href="http://www.ietf.org/rfc/rfc3164.txt">RFC 3164</a>. +If you don't like that, you can of course also use the textual property +- just be sure to use the right one. As expression support is enhanced, +this will change. For example, if you would like to filter on message +that have facility local0, start with "DEVNAME" and have either +"error1" or "error0" in their message content, you could use the +following filter:<br> +<br> +<code> +if $syslogfacility-text == 'local0' and $msg +startswith 'DEVNAME' and ($msg contains 'error1' or $msg contains +'error0') then /var/log/somelog<br> +</code> +<br> +Please note that the above <span style="font-weight: bold;">must +all be on one line</span>! And if you would like to store all +messages except those that contain "error1" or "error0", you just need +to add a "not":<br> +<br> +<code> +if $syslogfacility-text == 'local0' and $msg +startswith 'DEVNAME' and <span style="font-weight: bold;">not</span> +($msg contains 'error1' or $msg contains +'error0') then /var/log/somelog<br> +</code> +<br> +If you would like to do case-insensitive comparisons, use +"contains_i" instead of "contains" and "startswith_i" instead of +"startswith".<br> +<br> +Regular expressions are supported via functions (see function list). + <h3>Selectors</h3> <p><b>Selectors are the traditional way of filtering syslog messages.</b> They have been kept in rsyslog with their original @@ -213,71 +270,6 @@ supported (except for "not" as outlined above). Please note that while it is possible to query facility and severity via property-based filters, it is far more advisable to use classic selectors (see above) for those cases.</p> -<h3>Expression-Based Filters</h3> -Expression based filters allow -filtering on arbitrary complex expressions, which can include boolean, -arithmetic and string operations. Expression filters will evolve into a -full configuration scripting language. Unfortunately, their syntax will -slightly change during that process. So if you use them now, you need -to be prepared to change your configuration files some time later. -However, we try to implement the scripting facility as soon as possible -(also in respect to stage work needed). So the window of exposure is -probably not too long.<br> -<br> -Expression based filters are indicated by the keyword "if" in column 1 -of a new line. They have this format:<br> -<br> -if expr then action-part-of-selector-line<br> -<br> -"If" and "then" are fixed keywords that mus be present. "expr" is a -(potentially quite complex) expression. So the <a href="expression.html">expression documentation</a> for -details. "action-part-of-selector-line" is an action, just as you know -it (e.g. "/var/log/logfile" to write to that file).<br> -<br> -A few quick samples:<br> -<br> -<code> -*.* /var/log/file1 # the traditional way<br> -if $msg contains 'error' then /var/log/errlog # the expression-based way<br> -</code> -<br> -Right now, you need to specify numerical values if you would like to -check for facilities and severity. These can be found in <a href="http://www.ietf.org/rfc/rfc3164.txt">RFC 3164</a>. -If you don't like that, you can of course also use the textual property -- just be sure to use the right one. As expression support is enhanced, -this will change. For example, if you would like to filter on message -that have facility local0, start with "DEVNAME" and have either -"error1" or "error0" in their message content, you could use the -following filter:<br> -<br> -<code> -if $syslogfacility-text == 'local0' and $msg -startswith 'DEVNAME' and ($msg contains 'error1' or $msg contains -'error0') then /var/log/somelog<br> -</code> -<br> -Please note that the above <span style="font-weight: bold;">must -all be on one line</span>! And if you would like to store all -messages except those that contain "error1" or "error0", you just need -to add a "not":<br> -<br> -<code> -if $syslogfacility-text == 'local0' and $msg -startswith 'DEVNAME' and <span style="font-weight: bold;">not</span> -($msg contains 'error1' or $msg contains -'error0') then /var/log/somelog<br> -</code> -<br> -If you would like to do case-insensitive comparisons, use -"contains_i" instead of "contains" and "startswith_i" instead of -"startswith".<br> -<br> -Note that regular expressions are currently NOT -supported in expression-based filters. These will be added later when -function support is added to the expression engine (the reason is that -regular expressions will be a separate loadable module, which requires -some more prequisites before it can be implemented).<br> - <p>[<a href="manual.html">manual index</a>] [<a href="rsyslog_conf.html">rsyslog.conf</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html index b97f6609..b5aa687a 100644 --- a/doc/rsyslog_conf_templates.html +++ b/doc/rsyslog_conf_templates.html @@ -3,7 +3,7 @@ <body> <p>This is a part of the rsyslog.conf - documentation.</p> <a href="rsyslog_conf.html">back</a> -<h2>Templates</h2> +<h1>Templates</h1> <p>Templates are a key feature of rsyslog. They allow to specify any format a user might want. They are also used for dynamic file name @@ -16,67 +16,200 @@ compatible with the stock syslogd formats are hardcoded into rsyslogd. So if no template is specified, we use one of these hardcoded templates. Search for "template_" in syslogd.c and you will find the hardcoded ones.</p> -<p>Starting with 5.5.6, there are actually two differnt types of template: +<p>Templates are specified by template() statements. They can also be specified +via $Template legacy statements. Note that these are scheduled for removal in +later versions of rsyslog, so it is probably a good idea to avoid them +for new uses. +<h2>The template() statement</h2> +<p>The template() statement is used to define templates. Note that it is a +<b>static</b> statement, that means all templates are defined when rsyslog +reads the config file. As such, templates are not affected by if-statements +or config nesting. +<p>The basic structure of the template statement is as follows: +<br><br> +<code>template(parameters)</code> +<br><br> +In addition to this simpler syntax, list templates (to be described below) +support an extended syntax: +<br><br> +<code>template(parameters) { list-descriptions }</code> +<p>Each template has a parameter <b>name</b>, which specifies the templates +name, and a parameter <b>type</b>, which specifies the template type. The name +parameter must be unique, and behaviour is unpredictable if it is not. The <b>type</b> +parameter specifies different template types. Different types simply enable +different ways to specify the template content. The template type <b>does not</b> +affect what an (output) plugin can do with it. So use the type that best fits your +needs (from a config writing point of view!). The following types are available: <ul> -<li>string based -<li>string-generator module based +<li>list +<li>subtree +<li>string +<li>plugin </ul> -<p><a href="rsyslog_conf_modules.html#sm">String-generator module</a> based templates -have been introduced in 5.5.6. They permit a string generator, actually a C "program", -the generate a format. Obviously, it is more work required to code such a generator, -but the reward is speed improvement. If you do not need the ultimate throughput, you -can forget about string generators (so most people never need to know what they are). -You may just be interested in learning that for the most important default formats, -rsyslog already contains highly optimized string generators and these are called -without any need to configure anything. But if you have written (or purchased) a -string generator module, you need to know how to call it. Each such module has a name, -which you need to know (look it up in the module doc or ask the developer). Let's assume -that "mystrgen" is the module name. Then you can define a template for that strgen -in the following way: +The various types are described below. -<blockquote><code>template(name="MyTemplateName" type="plugin" string="mystrgen")</code></blockquote> -<p>Legacy example:</p> -<blockquote><code>$template MyTemplateName,=mystrgen</code></blockquote> -(Of course, you must have first loaded the module via $ModLoad). -<p>The important part is the equal sign in the legacy format: it tells the rsyslog config parser that -no string follows but a strgen module name. -<p>There are no additional parameters but the module name supported. This is because -there is no way to customize anything inside such a "template" other than by -modifying the code of the string generator. +<h3>list</h3> +<p>In this case, the template is generated by a list of constant and +variable statements. These follow the template spec in curly braces. This type is +also primarily meant for use with structure-aware outputs, like ommongodb. However, +it also works perfectly with text-based outputs. We recommend to use this mode +if more complex property substitutions needs to be done. In that case, the list-based +template syntax is much clearer than the simple string-based one. +<p>The list template contains the template header (with <b>type="list"</b>) and is followed +by <b>constant</b> and <b>property</b> statements, given in curly braces to signify +the template statement they belong to. As the name says, <b>constant</b> statements +describe constant text and <b>property</b> describes property access. There are many options +to <b>property</b>, described further below. Most of these options are used to extract +only partial property contents or to modify the text obtained (like to change its case +to upper or lower case, only). +<p>To grasp the idea, an actual sample is: +<br><pre><code>template(name="tpl1" type="list") { + constant(value="Syslog MSG is: '") + property(name="msg") + constant(value="', ") + property(name="timereported" dateFormat="rfc3339" caseConversion="lower") + constant(value="\n") + } +</code></pre> +<br>This sample is probably primarily targeted at the usual file-based output.</p> -<p>So for most use cases, string-generator module based templates are <b>not</b> -the route to take. Usually, we use <b>string based templates</b> instead. -This is what the rest of the documentation now talks about. -<p>A template consists of a template directive, a name, the -actual template text and optional options. A sample is:</p> -<blockquote><code>template(name="MyTemplateName" type="string" string="Example: Text %property% some more text\n" options)</code></blockquote> -<p>Legacy example:</p> -<blockquote><code>$template MyTemplateName,"\7Text -%property% some more text\n",<options></code></blockquote> -<p>The "template" (legacy: $template) is the template directive. It tells rsyslog -that this line contains a template. "MyTemplateName" is the template -name. All -other config lines refer to this name. The text within "string" is the -actual template text. The backslash is an escape character, much as it -is in C. It does all these "cool" things. For example, \7 rings the -bell (this is an ASCII value), \n is a new line. C programmers and perl -coders have the advantage of knowing this, but the set in rsyslog is a -bit restricted currently. -</p> -<p>All text in the template is used literally, except for things -within percent signs. These are properties and allow you access to the -contents of the syslog message. Properties are accessed via the -<a href="property_replacer.html">property replacer</a> -(nice name, huh) and it can do cool things, too. For -example, it can pick a substring or do date-specific formatting. More -on this is below, on some lines of the property replacer.<br> -<br> +<h4>constant statement</h4> +<p>This provides a way to specify constant text. The text is used literally. It is +primarily intended for text-based output, so that some constant text can be included. For +example, if a complex template is build for file output, one usually needs to finish it +by a newline, which can be introduced by a constant statement. Here is an actual sample +of that use case from the rsylsog testbench: +<br><pre><code>template(name="outfmt" type="list") { + property(name="$!usr!msgnum") + constant(value="\n") +}</code></pre> +The following escape sequences are recogniced inside the constant text: +<ul> +<li>\\ - single backslash +<li>\n - LF +<li>\ooo - (three octal digits) - represents character with this numerical value (e.g. \101 +equals "A"). Note that three +octal digits must be given (in contrast to some languagues, where between one and three are valid). +While we support octal notation, we recommend to use hex notation as this is better known. +<li>\xhh - (where h is a hex digit) - represents character with this numerical value (e.g. \x41 +equals "A"). Note that two hexadecimal digits must be given (in contrast to some languagues +where one or two are valid). +<li>... some others ... list needs to be extended +</ul> +<p>Note: if an unsupported character follows a backslash, this is treated as an error. Behaviour +is unpredictable in this case. +<p>To aid usage of the same template both for text-based outputs and structured ones, constant +text without an "outname" parameter will be ignored when creating the name/value tree +for structured outputs. So if you want to supply some constant text e.g. to mongodb, you must +include an outname, as can be seen here: +<br><pre><code>template(name="outfmt" type="list") { + property(name="$!usr!msgnum") + constant(value="\n" <b>outname="IWantThisInMyDB"</b>) +}</code></pre> + +The "constant" statement supports the following parameters: +<ul> +<li>value - the constant value to use +<li>outname - output field name (for structured outputs) +</ul> + + +<h4>property statement</h4> +<p>This statement is used to include property text. It can access all properties. Also, +options permit to specify picking only part of a property or modifying it. +It supports the following parameters: +<ul> +<li>name - the name of the property to access +<li>outname - output field name (for structured outputs) +<li>dateformat - date format to use (only for date-related properties) +<li>caseconversion - permits to convert case of the text. supported values are +"lower" and "upper" +<li>controlcharacters - specifies how to handle control characters. Supported values are +"escape", which escapes them, "space", which replaces them by a single space, and +"drop", which simply removes them from the string. +<li>securepath - used for creating pathnames suitable for use in dynafile templates +<li>format - specifiy format on a field basis. Supported values are "csv", for use when +csv-data is generated, "json", which formats proper json content (but without a field +header) and "jsonf", which formats as a complete json field. +<li>position.from - obtain substring starting from this position (1 is the first position) +<li>position.to - obtain substring up to this position +<li>field.number - obtain this field match +<li>field.delimiter - decimal value of delimiter character for field extraction +<li>regex.expression - expression to use +<li>regex.type - either ERE or BRE +<li>regex.nomatchmode - what to do if we have no match +<li>regex.match - match to use +<li>regex.submatch - submatch to use +<li>droplastlf - drop a trailing LF, if it is present +<li>mandatory - signifies a field as mandatory. If set to "on", this field will always +be present in data passed to structured outputs, even if it is empty. If "off" (the default) +empty fields will not be passed to structured outputs. This is especially useful for outputs +that support dynamic schemas (like ommongodb). +<li>spifno1stsp - expert options for RFC3164 template processing +</ul> + + +<h3>subtree</h3> +<p>Available since rsyslog 7.1.4 +<p> +In this case, the template is generated based on a complete +(CEE) subtree. This type of template is most useful for outputs that know how to +process hierarchical structure, like ommongodb. With that type, the parameter +<b>subtree</b> must be specified, which tells which subtree to use. For example +template(name="tpl1" type="subtree" subtree="$!") includes all CEE data, while +template(name="tpl2" type="subtree" subtree="$!usr!tpl2") includes only the +subtree starting at $!usr!tpl2. The core idea when using this type of template +is that the actual data is prefabricated via set and unset script statements, +and the resulting strucuture is then used inside the template. This method MUST +be used if a complete subtree needs to be placed <i>directly</i> into the +object's root. With all other template types, only subcontainers can be generated. +Note that subtree type can also be used with text-based outputs, like omfile. HOWEVER, +you do not have any capability to specify constant text, and as such cannot include +line breaks. As a consequence, using this template type for text outputs is usually +only useful for debugging or very special cases (e.g. where the text is interpreted +by a JSON parser later on). +<h4>Use case</h4> +<p>A typical use case is to first create a custom subtree and then include it into +the template, like in this small example: +<br><blockquote><code>set $!usr!tpl2!msg = $msg; +<br>set $!usr!tpl2!dataflow = field($msg, 58, 2); +<br>template(name="tpl2" type="subtree" subtree="$!usr!tpl2") +</code></blockquote> +<p>Here, we assume that $msg contains various fields, and the data from a field +is to be extracted and stored - together with the message - as field content. +<h3>string</h3> +<p>This closely resembles the legacy template statement. It +has a mandatory parameter <b>string</b>, which holds the template string to be +applied. A template string is a mix of constant text and replacement variables +(see property replacer). These variables are taken from message or other dynamic +content when the final string to be passed to a plugin is generated. String-based +templates are a great way to specify textual content, especially if no complex +manipulation to properties is necessary. Full details on how to specify template +text can be found below. +<br>Config example: +<br><blockquote><code>template(name="tpl3" type="list" string="%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n") +</code></blockquote> +<h3>plugin</h3> +In this case, the template is generated by a plugin (which +is then called +a "strgen" or "string generator"). The format is fix as it is coded. While this +is inflexible, it provides superior performance, and is often used for that +reason (not that "regular" templates are slow - but in very demanding environments +that "last bit" can make a difference). Refer to the plugin's documentation +for further details. For this type, the paramter <b>plugin</b> must be specified and +must contain the name of the plugin as it identifies itself. Note that the +plugin must be loaded prior to being used inside a template. +<br>Config example: +<br><blockquote><code>template(name="tpl4" type="plugin" plugin="mystrgen") +</code></blockquote> + +<h3>options</h3> The <options> part is optional. It carries options -influencing the template as whole. See details below. Be sure NOT to -mistake template options with property options - the latter ones are -processed by the property replacer and apply to a SINGLE property, only -(and not the whole template).<br> +influencing the template as whole and is part of the template parameters. +See details below. Be sure NOT to mistake template options with property +options - the latter ones are processed by the property replacer and +apply to a SINGLE property, only (and not the whole template).<br> <br> Template options are case-insensitive. Currently defined are: </p> <p><b>option.sql</b> - format the string suitable for a SQL @@ -127,50 +260,73 @@ option. Otherwise you will become vulnerable to SQL injection. <br> To escape:<br> % = \%<br> \ = \\ --> '\' is used to escape (as in C)<br> -$template TraditionalFormat,"%timegenerated% %HOSTNAME% %syslogtag%%msg%\n"<br> +template (name="TraditionalFormat" type="string" string="%timegenerated% %HOSTNAME% %syslogtag%%msg%\n"<br> <br> -Properties can be accessed by the <a href="property_replacer.html">property -replacer</a> (see there for details).</p> -<p>Templates can be used in the form of a <b>list</b> as well. This has been -introduced with <b>6.5.0</b> The list consists of two parts which are either -a <b>constant</b> or a <b>property</b>. The constants -are taking the part of "text" that you usually enter in string-based templates. -The properties stay variable, as they are a substitute for different values of a -certain type. This type of template is extremely useful for complicated cases, -as it helps you to easily keep an overview over the template. Though, it has -the disadvantage of needing more effort to create it.</p> -<br>Config example: -<br><blockquote><code>template(name="MyTemplate" type="list" option.json="off") { - <br>constant(value="Test: ") - <br>property(name="msg" outname="mymessage") - <br>constant(value=" --!!!-- ") - <br>property(name="timereported" dateFormat="rfc3339" caseConversion="lower") - <br>constant(value="\n") - <br>} -</code></blockquote> -<p>First, the general template option will be defined. The values of the template -itself get defined in the curly brackets. As it can be seen, we have constants -and properties in exchange. Whereas constants will be filled with a value and probably -some options, properties do direct to a property and the options that could be needed -additional format definitions.</p> -<p>We suggest to use separate lines for all constants and properties. This -helps to keep a good overview over the different parts of the template. -Though, writing it in a single line will work, it is much harder to debug -if anything goes wrong with the template. </p> +<h3>Examples</h3> +<h4>Standard Template for Writing to Files</h4> +<p><pre><code>template(name="FileFormat" type="list") { + property(name="timestamp" dateFormat="rfc3339") + constant(value=" ") + property(name="hostname") + constant(value=" ") + property(name="syslogtag") + constant(value=" ") + property(name="msg" spifno1stsp="on" ) + property(name="msg" droplastlf="on" ) + constant(value="\n") + } +</code></pre> +<p>The equivalent string template looks like this: +<br><pre><code>template(name="FileFormat" type="string" + string= "%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" +)</code></pre> +Note that the template string itself must be on a single line. -<p><b>Please note that templates can also be -used to generate selector lines with dynamic file names.</b> For -example, if you would like to split syslog messages from different -hosts to different files (one per host), you can define the following -template:</p> -<blockquote><code>template (name="DynFile" type="string" string="/var/log/system-%HOSTNAME%.log")</code></blockquote> -<p>Legacy example:</p> -<blockquote><code>$template -DynFile,"/var/log/system-%HOSTNAME%.log"</code></blockquote> -<p>This template can then be used when defining an output -selector line. It will result in something like -"/var/log/system-localhost.log"</p> +<h4>Standard Template for Forwarding to a Remote Host (RFC3164 mode)</h4> +<p><pre><code>template(name="ForwardFormat" type="list") { + constant(value="<") + property(name="PRI") + constant(value="<") + property(name="timestamp" dateFormat="rfc3339") + constant(value=" ") + property(name="hostname") + constant(value=" ") + property(name="syslogtag" position.from="1" position.to="32") + constant(value=" ") + property(name="msg" spifno1stsp="on" ) + } +</code></pre> +<p>The equivalent string template looks like this: +<br><pre><code>template(name="forwardFormat" type="string" + string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%" +)</code></pre> +Note that the template string itself must be on a single line. +<h2>legacy format</h2> +<p>In pre v6-versions of rsyslog, you need to use the <code>$template</code> +statement to configure templates. They provide the equivalent to string- and +plugin-based templates. The legacy syntax continous to work in v7, however +we recommend to avoid legacy format for newly written config files. Legacy and +current config statements can coexist within the same config file. +<p>The general format is +<br><br><code>$template name,param[,options]</code></br></br> +where "name" is the template name and "param" is a single parameter +that specifies template content. The optional "options" part is used to +set template options. +<h3>string</h3> +The parameter is the same string that with the current-style format you +specify in the <b>string</b> parameter, for example: +<br><br><code>$template strtpl,"PRI: %pri%, MSG: %msg%\n"</code> +<p>Note that list templates are not available in legacy format, so you need +to use complex property replacer constructs to do complex things. + +<h3>plugin</h3> +This is equivalent to the "plugin"-type template directive. Here, the +parameter is the plugin name, with an equal sign prepended. An example +is: +<br><br><code>$template plugintpl,=myplugin</code> + +<h2>Reserved Template Names</h2> <p>Template names beginning with "RSYSLOG_" are reserved for rsyslog use. Do NOT use them if, otherwise you may receive a conflict in the future (and @@ -210,12 +366,122 @@ out, but this may happen.</li> is meant to be written to a log file. Do <b>not</b> use for production or remote forwarding.</li> </ul> + +<h2>The following is legacy documentation soon to be integrated.</h2> + +<!--<table> +<tr><td>param name</td><td>meaning</td></tr> +<tr><td>name</td><td>name of the template</td></tr> +</table> +--> + +<p>Starting with 5.5.6, there are actually two different types of template: +<ul> +<li>string based +<li>string-generator module based +</ul> +<p><a href="rsyslog_conf_modules.html#sm">String-generator module</a> based templates +have been introduced in 5.5.6. They permit a string generator, actually a C "program", +the generate a format. Obviously, it is more work required to code such a generator, +but the reward is speed improvement. If you do not need the ultimate throughput, you +can forget about string generators (so most people never need to know what they are). +You may just be interested in learning that for the most important default formats, +rsyslog already contains highly optimized string generators and these are called +without any need to configure anything. But if you have written (or purchased) a +string generator module, you need to know how to call it. Each such module has a name, +which you need to know (look it up in the module doc or ask the developer). Let's assume +that "mystrgen" is the module name. Then you can define a template for that strgen +in the following way: + +<blockquote><code>template(name="MyTemplateName" type="plugin" string="mystrgen")</code></blockquote> +<p>Legacy example:</p> +<blockquote><code>$template MyTemplateName,=mystrgen</code></blockquote> +(Of course, you must have first loaded the module via $ModLoad). +<p>The important part is the equal sign in the legacy format: it tells the rsyslog config parser that +no string follows but a strgen module name. +<p>There are no additional parameters but the module name supported. This is because +there is no way to customize anything inside such a "template" other than by +modifying the code of the string generator. + +<p>So for most use cases, string-generator module based templates are <b>not</b> +the route to take. Usually, we use <b>string based templates</b> instead. +This is what the rest of the documentation now talks about. + +<p>A template consists of a template directive, a name, the +actual template text and optional options. A sample is:</p> +<blockquote><code>template(name="MyTemplateName" type="string" string="Example: Text %property% some more text\n" options)</code></blockquote> +<p>Legacy example:</p> +<blockquote><code>$template MyTemplateName,"\7Text +%property% some more text\n",<options></code></blockquote> +<p>The "template" (legacy: $template) is the template directive. It tells rsyslog +that this line contains a template. "MyTemplateName" is the template +name. All +other config lines refer to this name. The text within "string" is the +actual template text. The backslash is an escape character, much as it +is in C. It does all these "cool" things. For example, \7 rings the +bell (this is an ASCII value), \n is a new line. C programmers and perl +coders have the advantage of knowing this, but the set in rsyslog is a +bit restricted currently. +</p> +<p>All text in the template is used literally, except for things +within percent signs. These are properties and allow you access to the +contents of the syslog message. Properties are accessed via the +<a href="property_replacer.html">property replacer</a> +(nice name, huh) and it can do cool things, too. For +example, it can pick a substring or do date-specific formatting. More +on this is below, on some lines of the property replacer.<br> +<br> + +<br> +Properties can be accessed by the <a href="property_replacer.html">property +replacer</a> (see there for details).</p> +<p>Templates can be used in the form of a <b>list</b> as well. This has been +introduced with <b>6.5.0</b> The list consists of two parts which are either +a <b>constant</b> or a <b>property</b>. The constants +are taking the part of "text" that you usually enter in string-based templates. +The properties stay variable, as they are a substitute for different values of a +certain type. This type of template is extremely useful for complicated cases, +as it helps you to easily keep an overview over the template. Though, it has +the disadvantage of needing more effort to create it.</p> +<br>Config example: +<br><blockquote><code>template(name="MyTemplate" type="list" option.json="off") { + <br>constant(value="Test: ") + <br>property(name="msg" outname="mymessage") + <br>constant(value=" --!!!-- ") + <br>property(name="timereported" dateFormat="rfc3339" caseConversion="lower") + <br>constant(value="\n") + <br>} +</code></blockquote> +<p>First, the general template option will be defined. The values of the template +itself get defined in the curly brackets. As it can be seen, we have constants +and properties in exchange. Whereas constants will be filled with a value and probably +some options, properties do direct to a property and the options that could be needed +additional format definitions.</p> +<p>We suggest to use separate lines for all constants and properties. This +helps to keep a good overview over the different parts of the template. +Though, writing it in a single line will work, it is much harder to debug +if anything goes wrong with the template. </p> + +<p><b>Please note that templates can also be +used to generate selector lines with dynamic file names.</b> For +example, if you would like to split syslog messages from different +hosts to different files (one per host), you can define the following +template:</p> +<blockquote><code>template (name="DynFile" type="string" string="/var/log/system-%HOSTNAME%.log")</code></blockquote> +<p>Legacy example:</p> +<blockquote><code>$template +DynFile,"/var/log/system-%HOSTNAME%.log"</code></blockquote> +<p>This template can then be used when defining an output +selector line. It will result in something like +"/var/log/system-localhost.log"</p> <h3>Legacy String-based Template Samples</h3> -<p>This section provides some sample of what the default formats would -look as a text-based template. Hopefully, their description is self-explanatory. +<p>This section provides some default templates in legacy format, as used in rsyslog +previous to version 6. Note that this format is still supported, so there is no hard need +to upgrade existing configurations. However, it is strongly recommended that the legacy +constructs are not used when crafting new templates. Note that each $Template statement is on a <b>single</b> line, but probably broken accross several lines for display purposes by your browsers. Lines are separated by -empty lines. +empty lines. Keep in mind, that line breaks are important in legacy format. <p><code> $template FileFormat,"%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" <br><br> @@ -233,7 +499,7 @@ $template StdSQLFormat,"insert into SystemEvents (Message, Facility, FromHost, P [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> <p><font size="2">This documentation is part of the <a href="http://www.rsyslog.com/">rsyslog</a> project.<br> -Copyright © 2008 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +Copyright © 2008-2012 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and <a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL version 2 or higher.</font></p> </body> diff --git a/doc/v4compatibility.html b/doc/v4compatibility.html index 72b0f5a9..2a51adea 100644 --- a/doc/v4compatibility.html +++ b/doc/v4compatibility.html @@ -60,7 +60,7 @@ restarting rsyslogd by HUPing it. and most other deamons require that a restart command is typed in if a restart is required. <p>Rsyslog will follow this paradigm in the next versions, resulting in many benefits. In v4, we provide some support for the old-style semantics. We introduced a setting $HUPisRestart -which may be set to "on" (tradional, heavy operationg) +which may be set to "on" (tradional, heavy operation) or "off" (new, lightweight "file close only" operation). The initial versions had the default set to traditional behavior, but starting with 4.5.1 we are now using the new behavior as the default. diff --git a/doc/v7compatibility.html b/doc/v7compatibility.html new file mode 100644 index 00000000..be89f666 --- /dev/null +++ b/doc/v7compatibility.html @@ -0,0 +1,33 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<html><head><title>Compatibility notes for rsyslog v7</title> +</head> +<body> +<h1>Compatibility Notes for rsyslog v7</h1> +This document describes things to keep in mind when moving from v6 to v7. It +does not list enhancements nor does it talk about compatibility concerns introduced +by earlier versions (for this, see their respective compatibility documents). Its focus +is primarily on what you need to know if you used v6 and want to use v7 without hassle. +<p>Version 7 builds on the new config language introduced in v6 and extends it. +Other than v6, it not just only extends the config language, but provides +considerable changes to core elements as well. The result is much more power and +ease of use as well (this time that is not contradictionary). +</p> +<h2>BSD-Style blocks</h2> +BSD style blocks are no longer supported (for good reason). See the +<a href="http://www.rsyslog.com/g/BSD">rsyslog BSD blocks info</a> +page for more information and how to upgrade your config. +<p>[<a href="manual.html">manual index</a>] [<a href="http://www.rsyslog.com/">rsyslog site</a>]</p> + +<h2>CEE-Properties</h2> +In rsyslog v6, CEE properties could not be used across disk-based queues. If this was +done, there content was reset. This was a missing feature in v6. In v7, this feature +has been implemented. Consequently, situations where the previous behaviour were +desired need now to be solved differently. We do not think that this will cause any +problems to anyone, especially as in v6 this was announced as a missing feature. + +<p><font size="2">This documentation is part of the +<a href="http://www.rsyslog.com/">rsyslog</a> project.<br> +Copyright © 2011-2012 by <a href="http://www.gerhards.net/rainer">Rainer Gerhards</a> and +<a href="http://www.adiscon.com/">Adiscon</a>. Released under the GNU GPL +version 2 or higher.</font></p> +</body></html> diff --git a/grammar/grammar.y b/grammar/grammar.y index cdb19c3d..ce749807 100644 --- a/grammar/grammar.y +++ b/grammar/grammar.y @@ -6,10 +6,9 @@ * of course, encouraged to use new constructs only. But it needs to be noted * that some of the legacy constructs (specifically the in-front-of-action * PRI filter) are very hard to beat in ease of use, at least for simpler - * cases. So while we hope that cfsysline support can be dropped some time in - * the future, we will probably keep these useful constructs. + * cases. * - * Copyright 2011 Rainer Gerhards and Adiscon GmbH. + * Copyright 2011-2012 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -37,7 +36,7 @@ #define YYDEBUG 1 extern int yylineno; -/* keep compile rule cleam of errors */ +/* keep compile rule clean of errors */ extern int yylex(void); extern int yyerror(char*); %} @@ -48,11 +47,10 @@ extern int yyerror(char*); es_str_t *estr; enum cnfobjType objType; struct cnfobj *obj; + struct cnfstmt *stmt; struct nvlst *nvlst; struct objlst *objlst; - struct cnfactlst *actlst; struct cnfexpr *expr; - struct cnfrule *rule; struct cnffunc *func; struct cnffparamlst *fparams; } @@ -62,12 +60,13 @@ extern int yyerror(char*); %token <estr> FUNC %token <objType> BEGINOBJ %token ENDOBJ -%token <s> CFSYSLINE %token BEGIN_ACTION %token BEGIN_PROPERTY %token BEGIN_CONSTANT %token BEGIN_TPL %token STOP +%token SET +%token UNSET %token <s> LEGACY_ACTION %token <s> LEGACY_RULESET %token <s> PRIFILT @@ -76,6 +75,7 @@ extern int yyerror(char*); %token <s> BSD_HOST_SELECTOR %token IF %token THEN +%token ELSE %token OR %token AND %token NOT @@ -96,30 +96,27 @@ extern int yyerror(char*); %type <nvlst> nv nvlst %type <obj> obj property constant %type <objlst> propconst -%type <actlst> actlst +/*%type <actlst> actlst %type <actlst> act -%type <s> cfsysline %type <actlst> block +*/ %type <expr> expr +%type <stmt> stmt s_act actlst block script +/* %type <rule> rule %type <rule> scriptfilt +*/ %type <fparams> fparams %left AND OR %left CMP_EQ CMP_NE CMP_LE CMP_GE CMP_LT CMP_GT CMP_CONTAINS CMP_CONTAINSI CMP_STARTSWITH CMP_STARTSWITHI -%left '+' '-' +%left '+' '-' '&' %left '*' '/' '%' %nonassoc UMINUS NOT -%expect 3 -/* these shift/reduce conflicts are created by the CFSYSLINE construct, which we - * unfortunately can not avoid. The problem is that CFSYSLINE can occur both in - * global context as well as within an action. It's not permitted somewhere else, - * but this is suficient for conflicts. The "dangling else" built-in resolution - * works well to solve this issue, so we accept it (it's a wonder that our - * old style grammar doesn't work at all, so we better do not complain...). - * Use "bison -v rscript.y" if more conflicts arise and check rscript.out for - * were exactly these conflicts exits. +%expect 1 /* dangling else */ +/* If more erors show up, Use "bison -v grammar.y" if more conflicts arise and + * check grammar.output for were exactly these conflicts exits. */ %% /* note: we use left recursion below, because that saves stack space AND @@ -128,13 +125,11 @@ extern int yyerror(char*); */ conf: /* empty (to end recursion) */ | conf obj { cnfDoObj($2); } - | conf rule { cnfDoRule($2); } - | conf cfsysline { cnfDoCfsysline($2); } + | conf stmt { cnfDoScript($2); } | conf LEGACY_RULESET { cnfDoCfsysline($2); } | conf BSD_TAG_SELECTOR { cnfDoBSDTag($2); } | conf BSD_HOST_SELECTOR { cnfDoBSDHost($2); } obj: BEGINOBJ nvlst ENDOBJ { $$ = cnfobjNew($1, $2); } - | BEGIN_ACTION nvlst ENDOBJ { $$ = cnfobjNew(CNFOBJ_ACTION, $2); } | BEGIN_TPL nvlst ENDOBJ { $$ = cnfobjNew(CNFOBJ_TPL, $2); } | BEGIN_TPL nvlst ENDOBJ '{' propconst '}' { $$ = cnfobjNew(CNFOBJ_TPL, $2); @@ -145,24 +140,31 @@ propconst: { $$ = NULL; } | propconst constant { $$ = objlstAdd($1, $2); } property: BEGIN_PROPERTY nvlst ENDOBJ { $$ = cnfobjNew(CNFOBJ_PROPERTY, $2); } constant: BEGIN_CONSTANT nvlst ENDOBJ { $$ = cnfobjNew(CNFOBJ_CONSTANT, $2); } -cfsysline: CFSYSLINE { $$ = $1; } nvlst: { $$ = NULL; } | nvlst nv { $2->next = $1; $$ = $2; } nv: NAME '=' VALUE { $$ = nvlstNew($1, $3); } -rule: PRIFILT actlst { $$ = cnfruleNew(CNFFILT_PRI, $2); $$->filt.s = $1; } - | PROPFILT actlst { $$ = cnfruleNew(CNFFILT_PROP, $2); $$->filt.s = $1; } - | scriptfilt { $$ = $1; } - -scriptfilt: IF expr THEN actlst { $$ = cnfruleNew(CNFFILT_SCRIPT, $4); - $$->filt.expr = $2; } -block: actlst { $$ = $1; } - | block actlst { $2->next = $1; $$ = $2; } -actlst: act { $$=$1; } - | actlst '&' act { $3->next = $1; $$ = $3; } - | actlst cfsysline { $$ = cnfactlstAddSysline($1, $2); } - | '{' block '}' { $$ = $2; } -act: BEGIN_ACTION nvlst ENDOBJ { $$ = cnfactlstNew(CNFACT_V2, $2, NULL); } - | LEGACY_ACTION { $$ = cnfactlstNew(CNFACT_LEGACY, NULL, $1); } +script: stmt { $$ = $1; } + | script stmt { $$ = scriptAddStmt($1, $2); } +stmt: actlst { $$ = $1; } + | STOP { $$ = cnfstmtNew(S_STOP); } + | IF expr THEN block { $$ = cnfstmtNew(S_IF); + $$->d.s_if.expr = $2; + $$->d.s_if.t_then = $4; + $$->d.s_if.t_else = NULL; } + | IF expr THEN block ELSE block { $$ = cnfstmtNew(S_IF); + $$->d.s_if.expr = $2; + $$->d.s_if.t_then = $4; + $$->d.s_if.t_else = $6; } + | SET VAR '=' expr ';' { $$ = cnfstmtNewSet($2, $4); } + | UNSET VAR ';' { $$ = cnfstmtNewUnset($2); } + | PRIFILT block { $$ = cnfstmtNewPRIFILT($1, $2); } + | PROPFILT block { $$ = cnfstmtNewPROPFILT($1, $2); } +block: stmt { $$ = $1; } + | '{' script '}' { $$ = $2; } +actlst: s_act { $$ = $1; } + | actlst '&' s_act { $$ = scriptAddStmt($1, $3); } +s_act: BEGIN_ACTION nvlst ENDOBJ { $$ = cnfstmtNewAct($2); } + | LEGACY_ACTION { $$ = cnfstmtNewLegaAct($1); } expr: expr AND expr { $$ = cnfexprNew(AND, $1, $3); } | expr OR expr { $$ = cnfexprNew(OR, $1, $3); } | NOT expr { $$ = cnfexprNew(NOT, NULL, $2); } @@ -176,6 +178,7 @@ expr: expr AND expr { $$ = cnfexprNew(AND, $1, $3); } | expr CMP_CONTAINSI expr { $$ = cnfexprNew(CMP_CONTAINSI, $1, $3); } | expr CMP_STARTSWITH expr { $$ = cnfexprNew(CMP_STARTSWITH, $1, $3); } | expr CMP_STARTSWITHI expr { $$ = cnfexprNew(CMP_STARTSWITHI, $1, $3); } + | expr '&' expr { $$ = cnfexprNew('&', $1, $3); } | expr '+' expr { $$ = cnfexprNew('+', $1, $3); } | expr '-' expr { $$ = cnfexprNew('-', $1, $3); } | expr '*' expr { $$ = cnfexprNew('*', $1, $3); } diff --git a/grammar/lexer.l b/grammar/lexer.l index 4f79c9aa..38703f25 100644 --- a/grammar/lexer.l +++ b/grammar/lexer.l @@ -96,14 +96,17 @@ int fileno(FILE *stream); /* keywords */ "if" { BEGIN EXPR; return IF; } <EXPR>"then" { BEGIN INITIAL; return THEN; } +<EXPR>";" { BEGIN INITIAL; return ';'; } <EXPR>"or" { return OR; } <EXPR>"and" { return AND; } <EXPR>"not" { return NOT; } +<EXPR>"=" | <EXPR>"," | <EXPR>"*" | <EXPR>"/" | <EXPR>"%" | <EXPR>"+" | +<EXPR>"&" | <EXPR>"-" | <EXPR>"(" | <EXPR>")" { return yytext[0]; } @@ -121,7 +124,7 @@ int fileno(FILE *stream); <EXPR>0[0-7]+ | /* octal number */ <EXPR>0x[0-7a-f] | /* hex number, following rule is dec; strtoll handles all! */ <EXPR>([1-9][0-9]*|0) { yylval.n = strtoll(yytext, NULL, 0); return NUMBER; } -<EXPR>\$[$!]{0,1}[a-z][a-z0-9\-_\.]* { yylval.s = strdup(yytext); return VAR; } +<EXPR>\$[$!]{0,1}[a-z][!a-z0-9\-_\.]* { yylval.s = strdup(yytext); return VAR; } <EXPR>\'([^'\\]|\\['"\\$bntr]|\\x[0-9a-f][0-9a-f]|\\[0-7][0-7][0-7])*\' { yytext[yyleng-1] = '\0'; unescapeStr((uchar*)yytext+1, yyleng-2); @@ -143,7 +146,10 @@ int fileno(FILE *stream); /* line number support because the "preprocessor" combines lines and so needs * to tell us the real source line. */ -"stop" { dbgprintf("STOP\n"); return STOP; } +"stop" { return STOP; } +"else" { return ELSE; } +"set" { BEGIN EXPR; return SET; } +"unset" { BEGIN EXPR; return UNSET; } "preprocfilelinenumber(" { BEGIN LINENO; } <LINENO>[0-9]+ { yylineno = atoi(yytext) - 1; } <LINENO>")" { BEGIN INITIAL; } @@ -169,17 +175,15 @@ int fileno(FILE *stream); BEGIN INOBJ; return BEGINOBJ; } "action"[ \n\t]*"(" { BEGIN INOBJ; return BEGIN_ACTION; } ^[ \t]*:\$?[a-z\-]+[ ]*,[ ]*!?[a-z]+[ ]*,[ ]*\".*\" { - yylval.s = strdup(yytext); return PROPFILT; } -^[ \t]*[\*a-z][\*a-z]*[0-7]*[\.,][,!=;\.\*a-z0-7]+ { yylval.s = strdup(yytext); return PRIFILT; } + yylval.s = strdup(rmLeadingSpace(yytext)); return PROPFILT; } +^[ \t]*[\*a-z][\*a-z]*[0-7]*[\.,][,!=;\.\*a-z0-7]+ { yylval.s = strdup(rmLeadingSpace(yytext)); return PRIFILT; } "~" | "*" | \-\/[^*][^\n]* | \/[^*][^\n]* | :[a-z0-9]+:[^\n]* | [\|\.\-\@\^?~>][^\n]+ | -[a-z0-9_][a-z0-9_\-\+,;]* { yylval.s = strdup(yytext); - dbgprintf("lex: LEGA ACT: '%s'\n", yytext); - return LEGACY_ACTION; } +[a-z0-9_][a-z0-9_\-\+,;]* { yylval.s = yytext; return LEGACY_ACTION; } <INOBJ>")" { BEGIN INITIAL; return ENDOBJ; } <INOBJ>[a-z][a-z0-9_\.]* { yylval.estr = es_newStrFromCStr(yytext, yyleng); return NAME; } @@ -202,8 +206,7 @@ int fileno(FILE *stream); yylval.s = strdup(yytext); return LEGACY_RULESET; } else { - yylval.s = strdup(yytext); - return CFSYSLINE; + cnfDoCfsysline(strdup(yytext)); } } ![^ \t\n]+[ \t]*$ { yylval.s = strdup(yytext); return BSD_TAG_SELECTOR; } diff --git a/grammar/parserif.h b/grammar/parserif.h index 597cfe40..aa271ec4 100644 --- a/grammar/parserif.h +++ b/grammar/parserif.h @@ -15,7 +15,7 @@ extern int yylineno; * these functions. */ void cnfDoObj(struct cnfobj *o); -void cnfDoRule(struct cnfrule *rule); +void cnfDoScript(struct cnfstmt *script); void cnfDoCfsysline(char *ln); void cnfDoBSDTag(char *ln); void cnfDoBSDHost(char *ln); diff --git a/grammar/rainerscript.c b/grammar/rainerscript.c index 9e0d04c7..1a14b212 100644 --- a/grammar/rainerscript.c +++ b/grammar/rainerscript.c @@ -2,7 +2,7 @@ * * Module begun 2011-07-01 by Rainer Gerhards * - * Copyright 2011 Rainer Gerhards and Adiscon GmbH. + * Copyright 2011-2012 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -37,16 +37,53 @@ #include <libestr.h> #include "rsyslog.h" #include "rainerscript.h" +#include "conf.h" #include "parserif.h" +#include "rsconf.h" #include "grammar.h" #include "queue.h" #include "srUtils.h" #include "regexp.h" #include "obj.h" +#include "modules.h" DEFobjCurrIf(obj) DEFobjCurrIf(regexp) +void cnfexprOptimize(struct cnfexpr *expr); +static void cnfstmtOptimizePRIFilt(struct cnfstmt *stmt); + +char* +getFIOPName(unsigned iFIOP) +{ + char *pRet; + switch(iFIOP) { + case FIOP_CONTAINS: + pRet = "contains"; + break; + case FIOP_ISEQUAL: + pRet = "isequal"; + break; + case FIOP_STARTSWITH: + pRet = "startswith"; + break; + case FIOP_REGEX: + pRet = "regex"; + break; + case FIOP_EREREGEX: + pRet = "ereregex"; + break; + case FIOP_ISEMPTY: + pRet = "isempty"; + break; + default: + pRet = "NOP"; + break; + } + return pRet; +} + + void readConfFile(FILE *fp, es_str_t **str) { @@ -107,7 +144,6 @@ objlstNew(struct cnfobj *o) lst->next = NULL; lst->obj = o; } -dbgprintf("AAAA: creating new objlst\n"); cnfobjPrint(o); return lst; @@ -131,6 +167,22 @@ objlstAdd(struct objlst *root, struct cnfobj *o) return root; } +/* add stmt to current script, always return root stmt pointer */ +struct cnfstmt* +scriptAddStmt(struct cnfstmt *root, struct cnfstmt *s) +{ + struct cnfstmt *l; + + if(root == NULL) { + root = s; + } else { /* find last, linear search ok, as only during config phase */ + for(l = root ; l->next != NULL ; l = l->next) + ; + l->next = s; + } + return root; +} + void objlstDestruct(struct objlst *lst) { @@ -748,116 +800,6 @@ cnfobjPrint(struct cnfobj *o) } -struct cnfactlst* -cnfactlstNew(enum cnfactType actType, struct nvlst *lst, char *actLine) -{ - struct cnfactlst *actlst; - - if((actlst = malloc(sizeof(struct cnfactlst))) != NULL) { - actlst->next = NULL; - actlst->syslines = NULL; - actlst->actType = actType; - actlst->lineno = yylineno; - actlst->cnfFile = strdup(cnfcurrfn); - if(actType == CNFACT_V2) - actlst->data.lst = lst; - else - actlst->data.legActLine = actLine; - } - return actlst; -} - -struct cnfactlst* -cnfactlstAddSysline(struct cnfactlst* actlst, char *line) -{ - struct cnfcfsyslinelst *cflst; - - if((cflst = malloc(sizeof(struct cnfcfsyslinelst))) != NULL) { - cflst->line = line; - if(actlst->syslines == NULL) { - cflst->next = NULL; - } else { - cflst->next = actlst->syslines; - } - actlst->syslines = cflst; - } - return actlst; -} - - -void -cnfactlstDestruct(struct cnfactlst *actlst) -{ - struct cnfactlst *toDel; - - while(actlst != NULL) { - toDel = actlst; - actlst = actlst->next; - free(toDel->cnfFile); - cnfcfsyslinelstDestruct(toDel->syslines); - if(toDel->actType == CNFACT_V2) - nvlstDestruct(toDel->data.lst); - else - free(toDel->data.legActLine); - free(toDel); - } - -} - -static inline struct cnfcfsyslinelst* -cnfcfsyslinelstReverse(struct cnfcfsyslinelst *lst) -{ - struct cnfcfsyslinelst *curr, *prev; - if(lst == NULL) - return NULL; - prev = NULL; - while(lst != NULL) { - curr = lst; - lst = lst->next; - curr->next = prev; - prev = curr; - } - return prev; -} - -struct cnfactlst* -cnfactlstReverse(struct cnfactlst *actlst) -{ - struct cnfactlst *curr, *prev; - - prev = NULL; - while(actlst != NULL) { - curr = actlst; - actlst = actlst->next; - curr->syslines = cnfcfsyslinelstReverse(curr->syslines); - curr->next = prev; - prev = curr; - } - return prev; -} - -void -cnfactlstPrint(struct cnfactlst *actlst) -{ - struct cnfcfsyslinelst *cflst; - - while(actlst != NULL) { - dbgprintf("aclst %p: ", actlst); - if(actlst->actType == CNFACT_V2) { - dbgprintf("V2 action type: "); - nvlstPrint(actlst->data.lst); - } else { - dbgprintf("legacy action line: '%s'\n", - actlst->data.legActLine); - } - for( cflst = actlst->syslines - ; cflst != NULL ; cflst = cflst->next) { - dbgprintf("action:cfsysline: '%s'\n", cflst->line); - } - actlst = actlst->next; - } -} - struct cnfexpr* cnfexprNew(unsigned nodetype, struct cnfexpr *l, struct cnfexpr *r) { @@ -884,15 +826,19 @@ done: * try to convert it to one. The semantics from es_str2num() * are used (bSuccess tells if the conversion went well or not). */ -static inline long long +static long long var2Number(struct var *r, int *bSuccess) { long long n; if(r->datatype == 'S') { n = es_str2num(r->d.estr, bSuccess); } else { - n = r->d.n; - if(bSuccess) + if(r->datatype == 'J') { + n = (r->d.json == NULL) ? 0 : json_object_get_int(r->d.json); + } else { + n = r->d.n; + } + if(bSuccess != NULL) *bSuccess = 1; } return n; @@ -903,12 +849,88 @@ var2Number(struct var *r, int *bSuccess) static inline es_str_t * var2String(struct var *r, int *bMustFree) { + es_str_t *estr; + char *cstr; + rs_size_t lenstr; if(r->datatype == 'N') { *bMustFree = 1; - return es_newStrFromNumber(r->d.n); + estr = es_newStrFromNumber(r->d.n); + } else if(r->datatype == 'J') { + *bMustFree = 1; + if(r->d.json == NULL) { + cstr = "", + lenstr = 0; + } else { + cstr = (char*)json_object_get_string(r->d.json); + lenstr = strlen(cstr); + } + estr = es_newStrFromCStr(cstr, lenstr); + } else { + *bMustFree = 0; + estr = r->d.estr; } - *bMustFree = 0; - return r->d.estr; + return estr; +} + +static uchar* +var2CString(struct var *r, int *bMustFree) +{ + uchar *cstr; + es_str_t *estr; + estr = var2String(r, bMustFree); + cstr = (uchar*) es_str2cstr(estr, NULL); + if(*bMustFree) + es_deleteStr(estr); + *bMustFree = 1; + return cstr; +} + +rsRetVal +doExtractField(uchar *str, uchar delim, int matchnbr, uchar **resstr) +{ + int iCurrFld; + int iLen; + uchar *pBuf; + uchar *pFld; + uchar *pFldEnd; + DEFiRet; + + /* first, skip to the field in question */ + iCurrFld = 1; + pFld = str; + while(*pFld && iCurrFld < matchnbr) { + /* skip fields until the requested field or end of string is found */ + while(*pFld && (uchar) *pFld != delim) + ++pFld; /* skip to field terminator */ + if(*pFld == delim) { + ++pFld; /* eat it */ + ++iCurrFld; + } + } + dbgprintf("field() field requested %d, field found %d\n", matchnbr, iCurrFld); + + if(iCurrFld == matchnbr) { + /* field found, now extract it */ + /* first of all, we need to find the end */ + pFldEnd = pFld; + while(*pFldEnd && *pFldEnd != delim) + ++pFldEnd; + --pFldEnd; /* we are already at the delimiter - so we need to + * step back a little not to copy it as part of the field. */ + /* we got our end pointer, now do the copy */ + iLen = pFldEnd - pFld + 1; /* the +1 is for an actual char, NOT \0! */ + CHKmalloc(pBuf = MALLOC((iLen + 1) * sizeof(char))); + /* now copy */ + memcpy(pBuf, pFld, iLen); + pBuf[iLen] = '\0'; /* terminate it */ + if(*(pFldEnd+1) != '\0') + ++pFldEnd; /* OK, skip again over delimiter char */ + *resstr = pBuf; + } else { + ABORT_FINALIZE(RS_RET_FIELD_NOT_FOUND); + } +finalize_it: + RETiRet; } /* Perform a function call. This has been moved out of cnfExprEval in order @@ -922,8 +944,13 @@ doFuncCall(struct cnffunc *func, struct var *ret, void* usrptr) int bMustFree; es_str_t *estr; char *str; + uchar *resStr; int retval; struct var r[CNFFUNC_MAX_ARGS]; + int delim; + int matchnbr; + struct funcData_prifilt *pPrifilt; + rsRetVal localRet; dbgprintf("rainerscript: executing function id %d\n", func->fID); switch(func->fID) { @@ -991,8 +1018,7 @@ doFuncCall(struct cnffunc *func, struct var *ret, void* usrptr) break; case CNFFUNC_RE_MATCH: cnfexprEval(func->expr[0], &r[0], usrptr); - estr = var2String(&r[0], &bMustFree); - str = es_str2cstr(estr, NULL); + str = (char*) var2CString(&r[0], &bMustFree); retval = regexp.regexec(func->funcdata, str, 0, NULL, 0); if(retval == 0) ret->d.n = 1; @@ -1003,10 +1029,44 @@ doFuncCall(struct cnffunc *func, struct var *ret, void* usrptr) } } ret->datatype = 'N'; - if(bMustFree) es_deleteStr(estr); + if(bMustFree) free(str); free(str); if(r[0].datatype == 'S') es_deleteStr(r[0].d.estr); break; + case CNFFUNC_FIELD: + cnfexprEval(func->expr[0], &r[0], usrptr); + cnfexprEval(func->expr[1], &r[1], usrptr); + cnfexprEval(func->expr[2], &r[2], usrptr); + str = (char*) var2CString(&r[0], &bMustFree); + delim = var2Number(&r[1], NULL); + matchnbr = var2Number(&r[2], NULL); + localRet = doExtractField((uchar*)str, (char) delim, matchnbr, &resStr); + if(localRet == RS_RET_OK) { + ret->d.estr = es_newStrFromCStr((char*)resStr, strlen((char*)resStr)); + free(resStr); + } else if(localRet == RS_RET_OK) { + ret->d.estr = es_newStrFromCStr("***FIELD NOT FOUND***", + sizeof("***FIELD NOT FOUND***")-1); + } else { + ret->d.estr = es_newStrFromCStr("***ERROR in field() FUNCTION***", + sizeof("***ERROR in field() FUNCTION***")-1); + } + ret->datatype = 'S'; + if(bMustFree) free(str); + if(r[0].datatype == 'S') es_deleteStr(r[0].d.estr); + if(r[1].datatype == 'S') es_deleteStr(r[1].d.estr); + if(r[2].datatype == 'S') es_deleteStr(r[2].d.estr); + break; + case CNFFUNC_PRIFILT: + pPrifilt = (struct funcData_prifilt*) func->funcdata; + if( (pPrifilt->pmask[((msg_t*)usrptr)->iFacility] == TABLE_NOPRI) || + ((pPrifilt->pmask[((msg_t*)usrptr)->iFacility] + & (1<<((msg_t*)usrptr)->iSeverity)) == 0) ) + ret->d.n = 0; + else + ret->d.n = 1; + ret->datatype = 'N'; + break; default: if(Debug) { fname = es_str2cstr(func->fname, NULL); @@ -1019,6 +1079,27 @@ doFuncCall(struct cnffunc *func, struct var *ret, void* usrptr) } } +static inline void +evalVar(struct cnfvar *var, void *usrptr, struct var *ret) +{ + rsRetVal localRet; + es_str_t *estr; + struct json_object *json; + + if(var->name[0] == '$' && var->name[1] == '!') { + /* TODO: unify string libs */ + estr = es_newStrFromBuf(var->name+1, strlen(var->name)-1); + localRet = msgGetCEEPropJSON((msg_t*)usrptr, estr, &json); + es_deleteStr(estr); + ret->datatype = 'J'; + ret->d.json = (localRet == RS_RET_OK) ? json : NULL; + } else { + ret->datatype = 'S'; + ret->d.estr = cnfGetVar(var->name, usrptr); + } + +} + #define FREE_BOTH_RET \ if(r.datatype == 'S') es_deleteStr(r.d.estr); \ if(l.datatype == 'S') es_deleteStr(l.d.estr) @@ -1033,13 +1114,19 @@ doFuncCall(struct cnffunc *func, struct var *ret, void* usrptr) #define PREP_TWO_STRINGS \ cnfexprEval(expr->l, &l, usrptr); \ estr_l = var2String(&l, &bMustFree2); \ - cnfexprEval(expr->r, &r, usrptr); \ - estr_r = var2String(&r, &bMustFree) + if(expr->r->nodetype == 'S') { \ + estr_r = ((struct cnfstringval*)expr->r)->estr;\ + bMustFree = 0; \ + } else { \ + cnfexprEval(expr->r, &r, usrptr); \ + estr_r = var2String(&r, &bMustFree); \ + } #define FREE_TWO_STRINGS \ if(bMustFree) es_deleteStr(estr_r); \ + if(expr->r->nodetype != 'S' && r.datatype == 'S') es_deleteStr(r.d.estr); \ if(bMustFree2) es_deleteStr(estr_l); \ - FREE_BOTH_RET + if(l.datatype == 'S') es_deleteStr(l.d.estr) /* evaluate an expression. * Note that we try to avoid malloc whenever possible (because of @@ -1066,23 +1153,32 @@ cnfexprEval(struct cnfexpr *expr, struct var *ret, void* usrptr) * places flagged with "CMP" need to be changed. */ case CMP_EQ: + /* this is optimized in regard to right param as a PoC for all compOps + * So this is a NOT yet the copy template! + */ cnfexprEval(expr->l, &l, usrptr); - cnfexprEval(expr->r, &r, usrptr); ret->datatype = 'N'; if(l.datatype == 'S') { - if(r.datatype == 'S') { - ret->d.n = !es_strcmp(l.d.estr, r.d.estr); /*CMP*/ + if(expr->r->nodetype == 'S') { + ret->d.n = !es_strcmp(l.d.estr, ((struct cnfstringval*)expr->r)->estr); /*CMP*/ } else { - n_l = var2Number(&l, &convok_l); - if(convok_l) { - ret->d.n = (n_l == r.d.n); /*CMP*/ + cnfexprEval(expr->r, &r, usrptr); + if(r.datatype == 'S') { + ret->d.n = !es_strcmp(l.d.estr, r.d.estr); /*CMP*/ } else { - estr_r = var2String(&r, &bMustFree); - ret->d.n = !es_strcmp(l.d.estr, estr_r); /*CMP*/ - if(bMustFree) es_deleteStr(estr_r); + n_l = var2Number(&l, &convok_l); + if(convok_l) { + ret->d.n = (n_l == r.d.n); /*CMP*/ + } else { + estr_r = var2String(&r, &bMustFree); + ret->d.n = !es_strcmp(l.d.estr, estr_r); /*CMP*/ + if(bMustFree) es_deleteStr(estr_r); + } } + if(r.datatype == 'S') es_deleteStr(r.d.estr); } } else { + cnfexprEval(expr->r, &r, usrptr); if(r.datatype == 'S') { n_r = var2Number(&r, &convok_r); if(convok_r) { @@ -1095,8 +1191,9 @@ cnfexprEval(struct cnfexpr *expr, struct var *ret, void* usrptr) } else { ret->d.n = (l.d.n == r.d.n); /*CMP*/ } + if(r.datatype == 'S') es_deleteStr(r.d.estr); } - FREE_BOTH_RET; + if(l.datatype == 'S') es_deleteStr(l.d.estr); break; case CMP_NE: cnfexprEval(expr->l, &l, usrptr); @@ -1332,8 +1429,15 @@ cnfexprEval(struct cnfexpr *expr, struct var *ret, void* usrptr) ret->d.estr = es_strdup(((struct cnfstringval*)expr)->estr); break; case 'V': + evalVar((struct cnfvar*)expr, usrptr, ret); + break; + case '&': + /* TODO: think about optimization, should be possible ;) */ + PREP_TWO_STRINGS; ret->datatype = 'S'; - ret->d.estr = cnfGetVar(((struct cnfvar*)expr)->name, usrptr); + ret->d.estr = es_strdup(estr_l); + es_addStr(&ret->d.estr, estr_r); + FREE_TWO_STRINGS; break; case '+': COMP_NUM_BINOP(+); @@ -1381,12 +1485,13 @@ cnffuncDestruct(struct cnffunc *func) /* some functions require special destruction */ switch(func->fID) { case CNFFUNC_RE_MATCH: - regexp.regfree(func->funcdata); - free(func->funcdata); - free(func->fname); + if(func->funcdata != NULL) + regexp.regfree(func->funcdata); break; default:break; } + free(func->funcdata); + free(func->fname); } /* Destruct an expression and all sub-expressions contained in it. @@ -1409,6 +1514,7 @@ cnfexprDestruct(struct cnfexpr *expr) case CMP_CONTAINSI: case OR: case AND: + case '&': case '+': case '-': case '*': @@ -1460,6 +1566,22 @@ doIndent(int indent) for(i = 0 ; i < indent ; ++i) dbgprintf(" "); } + +static void +pmaskPrint(uchar *pmask, int indent) +{ + int i; + doIndent(indent); + dbgprintf("pmask: "); + for (i = 0; i <= LOG_NFACILITIES; i++) + if (pmask[i] == TABLE_NOPRI) + dbgprintf(" X "); + else + dbgprintf("%2X ", pmask[i]); + dbgprintf("\n"); +} + + void cnfexprPrint(struct cnfexpr *expr, int indent) { @@ -1563,10 +1685,16 @@ cnfexprPrint(struct cnfexpr *expr, int indent) func = (struct cnffunc*) expr; cstrPrint("function '", func->fname); dbgprintf("' (id:%d, params:%hu)\n", func->fID, func->nParams); + if(func->fID == CNFFUNC_PRIFILT) { + struct funcData_prifilt *pD; + pD = (struct funcData_prifilt*) func->funcdata; + pmaskPrint(pD->pmask, indent+1); + } for(i = 0 ; i < func->nParams ; ++i) { cnfexprPrint(func->expr[i], indent+1); } break; + case '&': case '+': case '-': case '*': @@ -1580,11 +1708,88 @@ cnfexprPrint(struct cnfexpr *expr, int indent) cnfexprPrint(expr->r, indent+1); break; default: - dbgprintf("error: unknown nodetype %u\n", - (unsigned) expr->nodetype); + dbgprintf("error: unknown nodetype %u['%c']\n", + (unsigned) expr->nodetype, (char) expr->nodetype); break; } } +void +cnfstmtPrint(struct cnfstmt *root, int indent) +{ + struct cnfstmt *stmt; + //dbgprintf("stmt %p, indent %d, type '%c'\n", expr, indent, expr->nodetype); + for(stmt = root ; stmt != NULL ; stmt = stmt->next) { + switch(stmt->nodetype) { + case S_NOP: + doIndent(indent); dbgprintf("NOP\n"); + break; + case S_STOP: + doIndent(indent); dbgprintf("STOP\n"); + break; + case S_ACT: + doIndent(indent); dbgprintf("ACTION %p [%s]\n", stmt->d.act, stmt->printable); + break; + case S_IF: + doIndent(indent); dbgprintf("IF\n"); + cnfexprPrint(stmt->d.s_if.expr, indent+1); + doIndent(indent); dbgprintf("THEN\n"); + cnfstmtPrint(stmt->d.s_if.t_then, indent+1); + if(stmt->d.s_if.t_else != NULL) { + doIndent(indent); dbgprintf("ELSE\n"); + cnfstmtPrint(stmt->d.s_if.t_else, indent+1); + } + doIndent(indent); dbgprintf("END IF\n"); + break; + case S_SET: + doIndent(indent); dbgprintf("SET %s =\n", + stmt->d.s_set.varname); + cnfexprPrint(stmt->d.s_set.expr, indent+1); + doIndent(indent); dbgprintf("END SET\n"); + break; + case S_UNSET: + doIndent(indent); dbgprintf("UNSET %s\n", + stmt->d.s_unset.varname); + break; + case S_PRIFILT: + doIndent(indent); dbgprintf("PRIFILT '%s'\n", stmt->printable); + pmaskPrint(stmt->d.s_prifilt.pmask, indent); + cnfstmtPrint(stmt->d.s_prifilt.t_then, indent+1); + if(stmt->d.s_prifilt.t_else != NULL) { + doIndent(indent); dbgprintf("ELSE\n"); + cnfstmtPrint(stmt->d.s_prifilt.t_else, indent+1); + } + doIndent(indent); dbgprintf("END PRIFILT\n"); + break; + case S_PROPFILT: + doIndent(indent); dbgprintf("PROPFILT\n"); + doIndent(indent); dbgprintf("\tProperty.: '%s'\n", + propIDToName(stmt->d.s_propfilt.propID)); + if(stmt->d.s_propfilt.propName != NULL) { + char *cstr; + cstr = es_str2cstr(stmt->d.s_propfilt.propName, NULL); + doIndent(indent); + dbgprintf("\tCEE-Prop.: '%s'\n", cstr); + free(cstr); + } + doIndent(indent); dbgprintf("\tOperation: "); + if(stmt->d.s_propfilt.isNegated) + dbgprintf("NOT "); + dbgprintf("'%s'\n", getFIOPName(stmt->d.s_propfilt.operation)); + if(stmt->d.s_propfilt.pCSCompValue != NULL) { + doIndent(indent); dbgprintf("\tValue....: '%s'\n", + rsCStrGetSzStrNoNULL(stmt->d.s_propfilt.pCSCompValue)); + } + doIndent(indent); dbgprintf("THEN\n"); + cnfstmtPrint(stmt->d.s_propfilt.t_then, indent+1); + doIndent(indent); dbgprintf("END PROPFILT\n"); + break; + default: + dbgprintf("error: unknown stmt type %u\n", + (unsigned) stmt->nodetype); + break; + } + } +} struct cnfnumval* cnfnumvalNew(long long val) @@ -1608,6 +1813,7 @@ cnfstringvalNew(es_str_t *estr) return strval; } + struct cnfvar* cnfvarNew(char *name) { @@ -1619,63 +1825,471 @@ cnfvarNew(char *name) return var; } -struct cnfrule * -cnfruleNew(enum cnfFiltType filttype, struct cnfactlst *actlst) +struct cnfstmt * +cnfstmtNew(unsigned s_type) { - struct cnfrule* cnfrule; - if((cnfrule = malloc(sizeof(struct cnfrule))) != NULL) { - cnfrule->nodetype = 'R'; - cnfrule->filttype = filttype; - cnfrule->actlst = cnfactlstReverse(actlst); + struct cnfstmt* cnfstmt; + if((cnfstmt = malloc(sizeof(struct cnfstmt))) != NULL) { + cnfstmt->nodetype = s_type; + cnfstmt->printable = NULL; + cnfstmt->next = NULL; } - return cnfrule; + return cnfstmt; } void -cnfrulePrint(struct cnfrule *rule) +cnfstmtDestruct(struct cnfstmt *root) +{ + struct cnfstmt *stmt, *todel; + for(stmt = root ; stmt != NULL ; ) { + switch(stmt->nodetype) { + case S_NOP: + case S_STOP: + break; + case S_ACT: + actionDestruct(stmt->d.act); + break; + case S_IF: + cnfexprDestruct(stmt->d.s_if.expr); + if(stmt->d.s_if.t_then != NULL) { + cnfstmtDestruct(stmt->d.s_if.t_then); + } + if(stmt->d.s_if.t_else != NULL) { + cnfstmtDestruct(stmt->d.s_if.t_else); + } + break; + case S_SET: + free(stmt->d.s_set.varname); + cnfexprDestruct(stmt->d.s_set.expr); + break; + case S_UNSET: + free(stmt->d.s_set.varname); + break; + case S_PRIFILT: + cnfstmtDestruct(stmt->d.s_prifilt.t_then); + cnfstmtDestruct(stmt->d.s_prifilt.t_else); + break; + case S_PROPFILT: + if(stmt->d.s_propfilt.propName != NULL) + es_deleteStr(stmt->d.s_propfilt.propName); + if(stmt->d.s_propfilt.regex_cache != NULL) + rsCStrRegexDestruct(&stmt->d.s_propfilt.regex_cache); + if(stmt->d.s_propfilt.pCSCompValue != NULL) + cstrDestruct(&stmt->d.s_propfilt.pCSCompValue); + cnfstmtDestruct(stmt->d.s_propfilt.t_then); + break; + default: + dbgprintf("error: unknown stmt type during destruct %u\n", + (unsigned) stmt->nodetype); + break; + } + free(stmt->printable); + todel = stmt; + stmt = stmt->next; + free(todel); + } +} + +struct cnfstmt * +cnfstmtNewSet(char *var, struct cnfexpr *expr) +{ + struct cnfstmt* cnfstmt; + if((cnfstmt = cnfstmtNew(S_SET)) != NULL) { + cnfstmt->d.s_set.varname = (uchar*) var; + cnfstmt->d.s_set.expr = expr; + } + return cnfstmt; +} + +struct cnfstmt * +cnfstmtNewUnset(char *var) +{ + struct cnfstmt* cnfstmt; + if((cnfstmt = cnfstmtNew(S_UNSET)) != NULL) { + cnfstmt->d.s_unset.varname = (uchar*) var; + } + return cnfstmt; +} + +struct cnfstmt * +cnfstmtNewPRIFILT(char *prifilt, struct cnfstmt *t_then) +{ + struct cnfstmt* cnfstmt; + if((cnfstmt = cnfstmtNew(S_PRIFILT)) != NULL) { + cnfstmt->printable = (uchar*)prifilt; + cnfstmt->d.s_prifilt.t_then = t_then; + cnfstmt->d.s_prifilt.t_else = NULL; + DecodePRIFilter((uchar*)prifilt, cnfstmt->d.s_prifilt.pmask); + } + return cnfstmt; +} + +struct cnfstmt * +cnfstmtNewPROPFILT(char *propfilt, struct cnfstmt *t_then) +{ + struct cnfstmt* cnfstmt; + rsRetVal lRet; + if((cnfstmt = cnfstmtNew(S_PROPFILT)) != NULL) { + cnfstmt->printable = (uchar*)propfilt; + cnfstmt->d.s_propfilt.t_then = t_then; + cnfstmt->d.s_propfilt.propName = NULL; + cnfstmt->d.s_propfilt.regex_cache = NULL; + cnfstmt->d.s_propfilt.pCSCompValue = NULL; + lRet = DecodePropFilter((uchar*)propfilt, cnfstmt); + } + return cnfstmt; +} + +struct cnfstmt * +cnfstmtNewAct(struct nvlst *lst) { - dbgprintf("------ start rule %p:\n", rule); - dbgprintf("%s: ", cnfFiltType2str(rule->filttype)); - switch(rule->filttype) { - case CNFFILT_NONE: + struct cnfstmt* cnfstmt; + char namebuf[256]; + if((cnfstmt = cnfstmtNew(S_ACT)) == NULL) + goto done; + if(actionNewInst(lst, &cnfstmt->d.act) != RS_RET_OK) { + // TODO:RS_RET_WARN? + parser_errmsg("errors occured in file '%s' around line %d", + cnfcurrfn, yylineno); + cnfstmt->nodetype = S_NOP; /* disable action! */ + goto done; + } + snprintf(namebuf, sizeof(namebuf)-1, "action(type=\"%s\" ...)", + modGetName(cnfstmt->d.act->pMod)); + namebuf[255] = '\0'; /* be on safe side */ + cnfstmt->printable = (uchar*)strdup(namebuf); +done: return cnfstmt; +} + +struct cnfstmt * +cnfstmtNewLegaAct(char *actline) +{ + struct cnfstmt* cnfstmt; + rsRetVal localRet; + if((cnfstmt = cnfstmtNew(S_ACT)) == NULL) + goto done; + cnfstmt->printable = (uchar*)strdup((char*)actline); + localRet = cflineDoAction(loadConf, (uchar**)&actline, &cnfstmt->d.act); + if(localRet != RS_RET_OK && localRet != RS_RET_OK_WARN) { + parser_errmsg("%s occured in file '%s' around line %d", + (localRet == RS_RET_OK_WARN) ? "warnings" : "errors", + cnfcurrfn, yylineno); + if(localRet != RS_RET_OK_WARN) { + cnfstmt->nodetype = S_NOP; /* disable action! */ + goto done; + } + } +done: return cnfstmt; +} + + +/* returns 1 if the two expressions are constants, 0 otherwise + * if both are constants, the expression subtrees are destructed + * (this is an aid for constant folding optimizing) + */ +static int +getConstNumber(struct cnfexpr *expr, long long *l, long long *r) +{ + int ret = 0; + cnfexprOptimize(expr->l); + cnfexprOptimize(expr->r); + if(expr->l->nodetype == 'N') { + if(expr->r->nodetype == 'N') { + ret = 1; + *l = ((struct cnfnumval*)expr->l)->val; + *r = ((struct cnfnumval*)expr->r)->val; + cnfexprDestruct(expr->l); + cnfexprDestruct(expr->r); + } else if(expr->r->nodetype == 'S') { + ret = 1; + *l = ((struct cnfnumval*)expr->l)->val; + *r = es_str2num(((struct cnfstringval*)expr->r)->estr, NULL); + cnfexprDestruct(expr->l); + cnfexprDestruct(expr->r); + } + } else if(expr->l->nodetype == 'S') { + if(expr->r->nodetype == 'N') { + ret = 1; + *l = es_str2num(((struct cnfstringval*)expr->l)->estr, NULL); + *r = ((struct cnfnumval*)expr->r)->val; + cnfexprDestruct(expr->l); + cnfexprDestruct(expr->r); + } else if(expr->r->nodetype == 'S') { + ret = 1; + *l = es_str2num(((struct cnfstringval*)expr->l)->estr, NULL); + *r = es_str2num(((struct cnfstringval*)expr->r)->estr, NULL); + cnfexprDestruct(expr->l); + cnfexprDestruct(expr->r); + } + } + return ret; +} + + +/* constant folding for string concatenation */ +static inline void +constFoldConcat(struct cnfexpr *expr) +{ + es_str_t *estr; + cnfexprOptimize(expr->l); + cnfexprOptimize(expr->r); + if(expr->l->nodetype == 'S') { + if(expr->r->nodetype == 'S') { + estr = ((struct cnfstringval*)expr->l)->estr; + ((struct cnfstringval*)expr->l)->estr = NULL; + es_addStr(&estr, ((struct cnfstringval*)expr->r)->estr); + cnfexprDestruct(expr->l); + cnfexprDestruct(expr->r); + expr->nodetype = 'S'; + ((struct cnfstringval*)expr)->estr = estr; + } else if(expr->r->nodetype == 'N') { + es_str_t *numstr; + estr = ((struct cnfstringval*)expr->l)->estr; + ((struct cnfstringval*)expr->l)->estr = NULL; + numstr = es_newStrFromNumber(((struct cnfnumval*)expr->r)->val); + es_addStr(&estr, numstr); + es_deleteStr(numstr); + cnfexprDestruct(expr->l); + cnfexprDestruct(expr->r); + expr->nodetype = 'S'; + ((struct cnfstringval*)expr)->estr = estr; + } + } else if(expr->l->nodetype == 'N') { + if(expr->r->nodetype == 'S') { + estr = es_newStrFromNumber(((struct cnfnumval*)expr->l)->val); + es_addStr(&estr, ((struct cnfstringval*)expr->r)->estr); + cnfexprDestruct(expr->l); + cnfexprDestruct(expr->r); + expr->nodetype = 'S'; + ((struct cnfstringval*)expr)->estr = estr; + } else if(expr->r->nodetype == 'S') { + es_str_t *numstr; + estr = es_newStrFromNumber(((struct cnfnumval*)expr->l)->val); + numstr = es_newStrFromNumber(((struct cnfnumval*)expr->r)->val); + es_addStr(&estr, numstr); + es_deleteStr(numstr); + cnfexprDestruct(expr->l); + cnfexprDestruct(expr->r); + expr->nodetype = 'S'; + ((struct cnfstringval*)expr)->estr = estr; + } + } +} + + +/* (recursively) optimize an expression */ +void +cnfexprOptimize(struct cnfexpr *expr) +{ + long long ln, rn; + + dbgprintf("optimize expr %p, type '%c'(%u)\n", expr, expr->nodetype, expr->nodetype); + switch(expr->nodetype) { + case '&': + constFoldConcat(expr); break; - case CNFFILT_PRI: - case CNFFILT_PROP: - dbgprintf("%s\n", rule->filt.s); + case '+': + if(getConstNumber(expr, &ln, &rn)) { + expr->nodetype = 'N'; + ((struct cnfnumval*)expr)->val = ln + rn; + } break; - case CNFFILT_SCRIPT: - dbgprintf("\n"); - cnfexprPrint(rule->filt.expr, 0); + case '-': + if(getConstNumber(expr, &ln, &rn)) { + expr->nodetype = 'N'; + ((struct cnfnumval*)expr)->val = ln - rn; + } + break; + case '*': + if(getConstNumber(expr, &ln, &rn)) { + expr->nodetype = 'N'; + ((struct cnfnumval*)expr)->val = ln * rn; + } + break; + case '/': + if(getConstNumber(expr, &ln, &rn)) { + expr->nodetype = 'N'; + ((struct cnfnumval*)expr)->val = ln / rn; + } + break; + case '%': + if(getConstNumber(expr, &ln, &rn)) { + expr->nodetype = 'N'; + ((struct cnfnumval*)expr)->val = ln % rn; + } + break; + default:/* nodetype we cannot optimize */ break; } - cnfactlstPrint(rule->actlst); - dbgprintf("------ end rule %p\n", rule); + } -/* note: the sysline itself was already freed during processing - * and as such MUST NOT be freed again! +/* removes NOPs from a statement list and returns the + * first non-NOP entry. */ -void -cnfcfsyslinelstDestruct(struct cnfcfsyslinelst *cfslst) +static inline struct cnfstmt * +removeNOPs(struct cnfstmt *root) { - struct cnfcfsyslinelst *toDel; - while(cfslst != NULL) { - toDel = cfslst; - cfslst = cfslst->next; - free(toDel); + struct cnfstmt *stmt, *toDel, *prevstmt = NULL; + struct cnfstmt *newRoot = NULL; + + if(root == NULL) goto done; + stmt = root; + while(stmt != NULL) { + if(stmt->nodetype == S_NOP) { + if(prevstmt != NULL) + /* end chain, is rebuild if more non-NOPs follow */ + prevstmt->next = NULL; + toDel = stmt; + stmt = stmt->next; + cnfstmtDestruct(toDel); + } else { + if(newRoot == NULL) + newRoot = stmt; + if(prevstmt != NULL) + prevstmt->next = stmt; + prevstmt = stmt; + stmt = stmt->next; + } } +done: return newRoot; } -void -cnfruleDestruct(struct cnfrule *rule) + +static inline void +cnfstmtOptimizeIf(struct cnfstmt *stmt) { - if( rule->filttype == CNFFILT_PRI - || rule->filttype == CNFFILT_PROP) - free(rule->filt.s); - cnfactlstDestruct(rule->actlst); - free(rule); + struct cnfstmt *t_then, *t_else; + struct cnfexpr *expr; + struct cnffunc *func; + struct funcData_prifilt *prifilt; + + expr = stmt->d.s_if.expr; + cnfexprOptimize(expr); + stmt->d.s_if.t_then = removeNOPs(stmt->d.s_if.t_then); + stmt->d.s_if.t_else = removeNOPs(stmt->d.s_if.t_else); + cnfstmtOptimize(stmt->d.s_if.t_then); + cnfstmtOptimize(stmt->d.s_if.t_else); + + if(stmt->d.s_if.expr->nodetype == 'F') { + func = (struct cnffunc*)expr; + if(func->fID == CNFFUNC_PRIFILT) { + DBGPRINTF("optimizer: change IF to PRIFILT\n"); + t_then = stmt->d.s_if.t_then; + t_else = stmt->d.s_if.t_else; + stmt->nodetype = S_PRIFILT; + prifilt = (struct funcData_prifilt*) func->funcdata; + memcpy(stmt->d.s_prifilt.pmask, prifilt->pmask, + sizeof(prifilt->pmask)); + stmt->d.s_prifilt.t_then = t_then; + stmt->d.s_prifilt.t_else = t_else; + stmt->printable = (uchar*) + es_str2cstr(((struct cnfstringval*)func->expr[0])->estr, NULL); + cnfexprDestruct(expr); + cnfstmtOptimizePRIFilt(stmt); + } + } +} + +static inline void +cnfstmtOptimizeAct(struct cnfstmt *stmt) +{ + action_t *pAct; + + pAct = stmt->d.act; + if(!strcmp((char*)modGetName(stmt->d.act->pMod), "builtin:omdiscard")) { + DBGPRINTF("optimizer: replacing omdiscard by STOP\n"); + actionDestruct(stmt->d.act); + stmt->nodetype = S_STOP; + } +} + +static void +cnfstmtOptimizePRIFilt(struct cnfstmt *stmt) +{ + int i; + int isAlways = 1; + struct cnfstmt *subroot, *last; + + stmt->d.s_prifilt.t_then = removeNOPs(stmt->d.s_prifilt.t_then); + cnfstmtOptimize(stmt->d.s_prifilt.t_then); + + for(i = 0; i <= LOG_NFACILITIES; i++) + if(stmt->d.s_prifilt.pmask[i] != 0xff) { + isAlways = 0; + break; + } + if(!isAlways) + goto done; + + DBGPRINTF("optimizer: removing always-true PRIFILT %p\n", stmt); + if(stmt->d.s_prifilt.t_else != NULL) { + parser_errmsg("error: always-true PRI filter has else part!\n"); + cnfstmtDestruct(stmt->d.s_prifilt.t_else); + } + free(stmt->printable); + stmt->printable = NULL; + subroot = stmt->d.s_prifilt.t_then; + if(subroot == NULL) { + /* very strange, we set it to NOP, best we can do + * This case is NOT expected in practice + */ + stmt->nodetype = S_NOP; + goto done; + } + for(last = subroot ; last->next != NULL ; last = last->next) + /* find last node in subtree */; + last->next = stmt->next; + memcpy(stmt, subroot, sizeof(struct cnfstmt)); + free(subroot); + +done: return; +} + +/* (recursively) optimize a statement */ +void +cnfstmtOptimize(struct cnfstmt *root) +{ + struct cnfstmt *stmt; + if(root == NULL) goto done; + for(stmt = root ; stmt != NULL ; stmt = stmt->next) { +dbgprintf("RRRR: stmtOptimize: stmt %p, nodetype %u\n", stmt, stmt->nodetype); + switch(stmt->nodetype) { + case S_IF: + cnfstmtOptimizeIf(stmt); + break; + case S_PRIFILT: + cnfstmtOptimizePRIFilt(stmt); + break; + case S_PROPFILT: + stmt->d.s_propfilt.t_then = removeNOPs(stmt->d.s_propfilt.t_then); + cnfstmtOptimize(stmt->d.s_propfilt.t_then); + break; + case S_SET: + cnfexprOptimize(stmt->d.s_set.expr); + break; + case S_ACT: + cnfstmtOptimizeAct(stmt); + break; + case S_STOP: + if(stmt->next != NULL) + parser_errmsg("STOP is followed by unreachable statements!\n"); + break; + case S_UNSET: /* nothing to do */ + break; + case S_NOP: + DBGPRINTF("optimizer error: we see a NOP, how come?\n"); + break; + default: + dbgprintf("error: unknown stmt type %u during optimizer run\n", + (unsigned) stmt->nodetype); + break; + } + } +done: return; } + struct cnffparamlst * cnffparamlstNew(struct cnfexpr *expr, struct cnffparamlst *next) { @@ -1736,6 +2350,20 @@ funcName2ID(es_str_t *fname, unsigned short nParams) return CNFFUNC_INVALID; } return CNFFUNC_RE_MATCH; + } else if(!es_strbufcmp(fname, (unsigned char*)"field", sizeof("field") - 1)) { + if(nParams != 3) { + parser_errmsg("number of parameters for field() must be three " + "but is %d.", nParams); + return CNFFUNC_INVALID; + } + return CNFFUNC_FIELD; + } else if(!es_strbufcmp(fname, (unsigned char*)"prifilt", sizeof("prifilt") - 1)) { + if(nParams != 1) { + parser_errmsg("number of parameters for prifilt() must be one " + "but is %d.", nParams); + return CNFFUNC_INVALID; + } + return CNFFUNC_PRIFILT; } else { return CNFFUNC_INVALID; } @@ -1776,6 +2404,29 @@ finalize_it: RETiRet; } + +static inline rsRetVal +initFunc_prifilt(struct cnffunc *func) +{ + struct funcData_prifilt *pData; + uchar *cstr; + DEFiRet; + + func->funcdata = NULL; + if(func->expr[0]->nodetype != 'S') { + parser_errmsg("param 1 of prifilt() must be a constant string"); + FINALIZE; + } + + CHKmalloc(pData = calloc(1, sizeof(struct funcData_prifilt))); + func->funcdata = pData; + cstr = (uchar*)es_str2cstr(((struct cnfstringval*) func->expr[0])->estr, NULL); + CHKiRet(DecodePRIFilter(cstr, pData->pmask)); + free(cstr); +finalize_it: + RETiRet; +} + struct cnffunc * cnffuncNew(es_str_t *fname, struct cnffparamlst* paramlst) { @@ -1793,6 +2444,7 @@ cnffuncNew(es_str_t *fname, struct cnffparamlst* paramlst) func->nodetype = 'F'; func->fname = fname; func->nParams = nParams; + func->funcdata = NULL; func->fID = funcName2ID(fname, nParams); /* shuffle params over to array (access speed!) */ param = paramlst; @@ -1808,6 +2460,9 @@ cnffuncNew(es_str_t *fname, struct cnffparamlst* paramlst) /* need to compile the regexp in param 2, so this MUST be a constant */ initFunc_re_match(func); break; + case CNFFUNC_PRIFILT: + initFunc_prifilt(func); + break; default:break; } } @@ -1908,6 +2563,15 @@ cstrPrint(char *text, es_str_t *estr) free(str); } +char * +rmLeadingSpace(char *s) +{ + char *p; + for(p = s ; *p && isspace(*p) ; ++p) + ; + return(p); +} + /* init must be called once before any parsing of the script files start */ rsRetVal initRainerscript(void) diff --git a/grammar/rainerscript.h b/grammar/rainerscript.h index f5627597..902ff4c5 100644 --- a/grammar/rainerscript.h +++ b/grammar/rainerscript.h @@ -3,7 +3,14 @@ #include <stdio.h> #include <libestr.h> #include <typedefs.h> +#include <sys/types.h> +#include <regex.h> +//#include "stringbuf.h" +/* TODO: make this hack cleaner... we have circular definitions, so we need: */ + + +#define LOG_NFACILITIES 24 /* current number of syslog facilities */ #define CNFFUNC_MAX_ARGS 32 /**< maximum number of arguments that any function can have (among * others, this is used to size data structures). @@ -65,8 +72,9 @@ struct var { es_str_t *estr; struct cnfexpr *expr; long long n; + struct json_object *json; } d; - char datatype; /* 'N' number, 'S' string, 'E' expression */ + char datatype; /* 'N' number, 'S' string, 'E' expression, 'J' JSON */ }; struct cnfobj { @@ -91,23 +99,6 @@ struct nvlst { */ }; -struct cnfcfsyslinelst { - struct cnfcfsyslinelst *next; - char *line; -}; - -struct cnfactlst { - struct cnfactlst *next; - struct cnfcfsyslinelst *syslines; - enum cnfactType actType; - union { - struct nvlst *lst; - char *legActLine; - } data; - char *cnfFile; - int lineno; -}; - /* the following structures support expressions, and may (very much later * be the sole foundation for the AST. * @@ -118,7 +109,17 @@ struct cnfactlst { * R - rule * S - string * V - var + * ... plus the S_* #define's below: */ +#define S_STOP 4000 +#define S_PRIFILT 4001 +#define S_PROPFILT 4002 +#define S_IF 4003 +#define S_ACT 4004 +#define S_NOP 4005 /* usually used to disable some statement */ +#define S_SET 4006 +#define S_UNSET 4007 + enum cnfFiltType { CNFFILT_NONE, CNFFILT_PRI, CNFFILT_PROP, CNFFILT_SCRIPT }; static inline char* cnfFiltType2str(enum cnfFiltType filttype) @@ -137,14 +138,40 @@ cnfFiltType2str(enum cnfFiltType filttype) } -struct cnfrule { +struct cnfstmt { unsigned nodetype; - enum cnfFiltType filttype; + struct cnfstmt *next; + uchar *printable; /* printable text for debugging */ union { - char *s; - struct cnfexpr *expr; - } filt; - struct cnfactlst *actlst; + struct { + struct cnfexpr *expr; + struct cnfstmt *t_then; + struct cnfstmt *t_else; + } s_if; + struct { + uchar *varname; + struct cnfexpr *expr; + } s_set; + struct { + uchar *varname; + } s_unset; + struct { + uchar pmask[LOG_NFACILITIES+1]; /* priority mask */ + struct cnfstmt *t_then; + struct cnfstmt *t_else; + } s_prifilt; + struct { + fiop_t operation; + regex_t *regex_cache;/* cache for compiled REs, if used */ + struct cstr_s *pCSCompValue;/* value to "compare" against */ + sbool isNegated; + uintTiny propID;/* ID of the requested property */ + es_str_t *propName;/* name of property for CEE-based filters */ + struct cnfstmt *t_then; + struct cnfstmt *t_else; + } s_propfilt; + struct action_s *act; + } d; }; struct cnfexpr { @@ -182,7 +209,9 @@ enum cnffuncid { CNFFUNC_TOLOWER, CNFFUNC_CSTR, CNFFUNC_CNUM, - CNFFUNC_RE_MATCH + CNFFUNC_RE_MATCH, + CNFFUNC_FIELD, + CNFFUNC_PRIFILT }; struct cnffunc { @@ -233,6 +262,10 @@ struct cnfparamvals { /* the values we obtained for param descr. */ unsigned char bUsed; }; +struct funcData_prifilt { + uchar pmask[LOG_NFACILITIES+1]; /* priority mask */ +}; + int cnfParseBuffer(char *buf, unsigned lenBuf); void readConfFile(FILE *fp, es_str_t **str); @@ -247,11 +280,6 @@ struct nvlst* nvlstFindName(struct nvlst *lst, es_str_t *name); struct cnfobj* cnfobjNew(enum cnfobjType objType, struct nvlst *lst); void cnfobjDestruct(struct cnfobj *o); void cnfobjPrint(struct cnfobj *o); -struct cnfactlst* cnfactlstNew(enum cnfactType actType, struct nvlst *lst, char *actLine); -void cnfactlstDestruct(struct cnfactlst *actlst); -void cnfactlstPrint(struct cnfactlst *actlst); -struct cnfactlst* cnfactlstAddSysline(struct cnfactlst* actlst, char *line); -struct cnfactlst* cnfactlstReverse(struct cnfactlst *actlst); struct cnfexpr* cnfexprNew(unsigned nodetype, struct cnfexpr *l, struct cnfexpr *r); void cnfexprPrint(struct cnfexpr *expr, int indent); void cnfexprEval(struct cnfexpr *expr, struct var *ret, void *pusr); @@ -259,9 +287,6 @@ int cnfexprEvalBool(struct cnfexpr *expr, void *usrptr); void cnfexprDestruct(struct cnfexpr *expr); struct cnfnumval* cnfnumvalNew(long long val); struct cnfstringval* cnfstringvalNew(es_str_t *estr); -struct cnfrule * cnfruleNew(enum cnfFiltType filttype, struct cnfactlst *actlst); -void cnfruleDestruct(struct cnfrule *rule); -void cnfrulePrint(struct cnfrule *rule); struct cnfvar* cnfvarNew(char *name); struct cnffunc * cnffuncNew(es_str_t *fname, struct cnffparamlst* paramlst); struct cnffparamlst * cnffparamlstNew(struct cnfexpr *expr, struct cnffparamlst *next); @@ -272,7 +297,20 @@ struct cnfparamvals* nvlstGetParams(struct nvlst *lst, struct cnfparamblk *param void cnfparamsPrint(struct cnfparamblk *params, struct cnfparamvals *vals); void varDelete(struct var *v); void cnfparamvalsDestruct(struct cnfparamvals *paramvals, struct cnfparamblk *blk); -void cnfcfsyslinelstDestruct(struct cnfcfsyslinelst *cfslst); +struct cnfstmt * cnfstmtNew(unsigned s_type); +void cnfstmtPrint(struct cnfstmt *stmt, int indent); +struct cnfstmt* scriptAddStmt(struct cnfstmt *root, struct cnfstmt *s); +struct objlst* objlstAdd(struct objlst *root, struct cnfobj *o); +char *rmLeadingSpace(char *s); +struct cnfstmt * cnfstmtNewPRIFILT(char *prifilt, struct cnfstmt *t_then); +struct cnfstmt * cnfstmtNewPROPFILT(char *propfilt, struct cnfstmt *t_then); +struct cnfstmt * cnfstmtNewAct(struct nvlst *lst); +struct cnfstmt * cnfstmtNewLegaAct(char *actline); +struct cnfstmt * cnfstmtNewSet(char *var, struct cnfexpr *expr); +struct cnfstmt * cnfstmtNewUnset(char *var); +void cnfstmtDestruct(struct cnfstmt *root); +void cnfstmtOptimize(struct cnfstmt *root); +char* getFIOPName(unsigned iFIOP); rsRetVal initRainerscript(void); void unescapeStr(uchar *s, int len); diff --git a/plugins/cust1/Makefile.am b/plugins/cust1/Makefile.am index d2e075f9..9963d0d6 100644 --- a/plugins/cust1/Makefile.am +++ b/plugins/cust1/Makefile.am @@ -1,6 +1,6 @@ -pkglib_LTLIBRARIES = cust1.la +#pkglib_LTLIBRARIES = cust1.la -cust1_la_SOURCES = cust1.c -cust1_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -cust1_la_LDFLAGS = -module -avoid-version -cust1_la_LIBADD = +#cust1_la_SOURCES = cust1.c +#cust1_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +#cust1_la_LDFLAGS = -module -avoid-version +#cust1_la_LIBADD = diff --git a/plugins/imkmsg/Makefile.am b/plugins/imkmsg/Makefile.am new file mode 100644 index 00000000..87c177d2 --- /dev/null +++ b/plugins/imkmsg/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = imkmsg.la +imkmsg_la_SOURCES = imkmsg.c imkmsg.h + +imkmsg_la_SOURCES += kmsg.c + +imkmsg_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +imkmsg_la_LDFLAGS = -module -avoid-version +imkmsg_la_LIBADD = diff --git a/plugins/imkmsg/imkmsg.c b/plugins/imkmsg/imkmsg.c new file mode 100644 index 00000000..2a97f82d --- /dev/null +++ b/plugins/imkmsg/imkmsg.c @@ -0,0 +1,295 @@ +/* The kernel log module. + * + * This is rsyslog Linux only module for reading structured kernel logs. + * Module is based on imklog module so it retains its structure + * and other part is currently in kmsg.c file instead of this (imkmsg.c) + * For more information see that file. + * + * To test under Linux: + * echo test1 > /dev/kmsg + * + * Copyright (C) 2008-2012 Adiscon GmbH + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include "rsyslog.h" +#include <stdio.h> +#include <assert.h> +#include <string.h> +#include <stdarg.h> +#include <ctype.h> +#include <stdlib.h> +#include <sys/socket.h> + +#include "dirty.h" +#include "cfsysline.h" +#include "obj.h" +#include "msg.h" +#include "module-template.h" +#include "datetime.h" +#include "imkmsg.h" +#include "net.h" +#include "glbl.h" +#include "prop.h" +#include "errmsg.h" +#include "unicode-helper.h" + +MODULE_TYPE_INPUT +MODULE_TYPE_NOKEEP +MODULE_CNFNAME("imkmsg") + +/* Module static data */ +DEF_IMOD_STATIC_DATA +DEFobjCurrIf(datetime) +DEFobjCurrIf(glbl) +DEFobjCurrIf(prop) +DEFobjCurrIf(net) +DEFobjCurrIf(errmsg) + +/* config settings */ +typedef struct configSettings_s { + int iFacilIntMsg; /* the facility to use for internal messages (set by driver) */ +} configSettings_t; +static configSettings_t cs; + +static modConfData_t *loadModConf = NULL;/* modConf ptr to use for the current load process */ +static modConfData_t *runModConf = NULL;/* modConf ptr to use for the current load process */ +static int bLegacyCnfModGlobalsPermitted;/* are legacy module-global config parameters permitted? */ + +static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this module */ +static prop_t *pLocalHostIP = NULL; /* a pseudo-constant propterty for 127.0.0.1 */ + +static inline void +initConfigSettings(void) +{ + cs.iFacilIntMsg = klogFacilIntMsg(); +} + + +/* enqueue the the kernel message into the message queue. + * The provided msg string is not freed - thus must be done + * by the caller. + * rgerhards, 2008-04-12 + */ +static rsRetVal +enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity, struct timeval *tp, struct json_object *json) +{ + struct syslogTime st; + msg_t *pMsg; + DEFiRet; + + assert(msg != NULL); + assert(pszTag != NULL); + + if(tp == NULL) { + CHKiRet(msgConstruct(&pMsg)); + } else { + datetime.timeval2syslogTime(tp, &st); + CHKiRet(msgConstructWithTime(&pMsg, &st, tp->tv_sec)); + } + MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); + MsgSetInputName(pMsg, pInputName); + MsgSetRawMsgWOSize(pMsg, (char*)msg); + MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ + MsgSetRcvFrom(pMsg, glbl.GetLocalHostNameProp()); + MsgSetRcvFromIP(pMsg, pLocalHostIP); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); + MsgSetTAG(pMsg, pszTag, ustrlen(pszTag)); + pMsg->iFacility = iFacility; + pMsg->iSeverity = iSeverity; + pMsg->json = json; + CHKiRet(submitMsg(pMsg)); + +finalize_it: + RETiRet; +} + + +/* log an imkmsg-internal message + * rgerhards, 2008-04-14 + */ +rsRetVal imkmsgLogIntMsg(int priority, char *fmt, ...) +{ + DEFiRet; + va_list ap; + uchar msgBuf[2048]; /* we use the same size as sysklogd to remain compatible */ + + va_start(ap, fmt); + vsnprintf((char*)msgBuf, sizeof(msgBuf) / sizeof(char), fmt, ap); + va_end(ap); + + logmsgInternal(NO_ERRCODE ,priority, msgBuf, 0); + + RETiRet; +} + + +/* log a message from /dev/kmsg + */ +rsRetVal Syslog(int priority, uchar *pMsg, struct timeval *tp, struct json_object *json) +{ + DEFiRet; + iRet = enqMsg((uchar*)pMsg, (uchar*) "kernel:", LOG_FAC(priority), LOG_PRI(priority), tp, json); + RETiRet; +} + + +/* helper for some klog drivers which need to know the MaxLine global setting. They can + * not obtain it themselfs, because they are no modules and can not query the object hander. + * It would probably be a good idea to extend the interface to support it, but so far + * we create a (sufficiently valid) work-around. -- rgerhards, 2008-11-24 + */ +int klog_getMaxLine(void) +{ + return glbl.GetMaxLine(); +} + + +BEGINrunInput +CODESTARTrunInput + /* this is an endless loop - it is terminated when the thread is + * signalled to do so. This, however, is handled by the framework, + * right into the sleep below. + */ + while(!pThrd->bShallStop) { + /* klogLogKMsg() waits for the next kernel message, obtains it + * and then submits it to the rsyslog main queue. + * rgerhards, 2008-04-09 + */ + CHKiRet(klogLogKMsg(runModConf)); + } +finalize_it: +ENDrunInput + + +BEGINbeginCnfLoad +CODESTARTbeginCnfLoad + loadModConf = pModConf; + pModConf->pConf = pConf; + /* init our settings */ + pModConf->iFacilIntMsg = klogFacilIntMsg(); + loadModConf->configSetViaV2Method = 0; + bLegacyCnfModGlobalsPermitted = 1; + /* init legacy config vars */ + initConfigSettings(); +ENDbeginCnfLoad + + +BEGINendCnfLoad +CODESTARTendCnfLoad + if(!loadModConf->configSetViaV2Method) { + /* persist module-specific settings from legacy config system */ + loadModConf->iFacilIntMsg = cs.iFacilIntMsg; + } + + loadModConf = NULL; /* done loading */ +ENDendCnfLoad + + +BEGINcheckCnf +CODESTARTcheckCnf +ENDcheckCnf + + +BEGINactivateCnfPrePrivDrop +CODESTARTactivateCnfPrePrivDrop + runModConf = pModConf; + iRet = klogWillRun(runModConf); +ENDactivateCnfPrePrivDrop + + +BEGINactivateCnf +CODESTARTactivateCnf +ENDactivateCnf + + +BEGINfreeCnf +CODESTARTfreeCnf +ENDfreeCnf + + +BEGINwillRun +CODESTARTwillRun +ENDwillRun + + +BEGINafterRun +CODESTARTafterRun + iRet = klogAfterRun(runModConf); +ENDafterRun + + +BEGINmodExit +CODESTARTmodExit + if(pInputName != NULL) + prop.Destruct(&pInputName); + if(pLocalHostIP != NULL) + prop.Destruct(&pLocalHostIP); + + /* release objects we used */ + objRelease(glbl, CORE_COMPONENT); + objRelease(net, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_IMOD_QUERIES +CODEqueryEtryPt_STD_CONF2_QUERIES +CODEqueryEtryPt_STD_CONF2_PREPRIVDROP_QUERIES +ENDqueryEtryPt + +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + cs.iFacilIntMsg = klogFacilIntMsg(); + return RS_RET_OK; +} + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); + CHKiRet(objUse(net, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.CreateStringProp(&pInputName, UCHAR_CONSTANT("imkmsg"), sizeof("imkmsg") - 1)); + CHKiRet(prop.CreateStringProp(&pLocalHostIP, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + + /* init legacy config settings */ + initConfigSettings(); + + CHKiRet(omsdRegCFSLineHdlr((uchar *)"debugprintkernelsymbols", 0, eCmdHdlrGoneAway, + NULL, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogsymbollookup", 0, eCmdHdlrGoneAway, + NULL, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogsymbolstwice", 0, eCmdHdlrGoneAway, + NULL, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogusesyscallinterface", 0, eCmdHdlrGoneAway, + NULL, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +ENDmodInit +/* vim:set ai: + */ diff --git a/plugins/imkmsg/imkmsg.h b/plugins/imkmsg/imkmsg.h new file mode 100644 index 00000000..220a1634 --- /dev/null +++ b/plugins/imkmsg/imkmsg.h @@ -0,0 +1,64 @@ +/* imkmsg.h + * These are the definitions for the kmsg message generation module. + * + * Copyright 2007-2012 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef IMKLOG_H_INCLUDED +#define IMKLOG_H_INCLUDED 1 + +#include "rsyslog.h" +#include "dirty.h" + +/* we need to have the modConf type present in all submodules */ +struct modConfData_s { + rsconf_t *pConf; + int iFacilIntMsg; + uchar *pszPath; + int console_log_level; + sbool bPermitNonKernel; + sbool configSetViaV2Method; +}; + +/* interface to "drivers" + * the platform specific drivers must implement these entry points. Only one + * driver may be active at any given time, thus we simply rely on the linker + * to resolve the addresses. + * rgerhards, 2008-04-09 + */ +rsRetVal klogLogKMsg(modConfData_t *pModConf); +rsRetVal klogWillRun(modConfData_t *pModConf); +rsRetVal klogAfterRun(modConfData_t *pModConf); +int klogFacilIntMsg(); + +/* the functions below may be called by the drivers */ +rsRetVal imkmsgLogIntMsg(int priority, char *fmt, ...) __attribute__((format(printf,2, 3))); +rsRetVal Syslog(int priority, uchar *msg, struct timeval *tp, struct json_object *json); + +/* prototypes */ +extern int klog_getMaxLine(void); /* work-around for klog drivers to get configured max line size */ +extern int InitKsyms(modConfData_t*); +extern void DeinitKsyms(void); +extern int InitMsyms(void); +extern void DeinitMsyms(void); +extern char * ExpandKadds(char *, char *); +extern void SetParanoiaLevel(int); + +#endif /* #ifndef IMKLOG_H_INCLUDED */ +/* vi:set ai: + */ diff --git a/plugins/imkmsg/kmsg.c b/plugins/imkmsg/kmsg.c new file mode 100644 index 00000000..9ad98da4 --- /dev/null +++ b/plugins/imkmsg/kmsg.c @@ -0,0 +1,239 @@ +/* imkmsg driver for Linux /dev/kmsg structured logging + * + * This contains Linux-specific functionality to read /dev/kmsg + * For a general overview, see head comment in imkmsg.c. + * This is heavily based on imklog bsd.c file. + * + * Copyright 2008-2012 Adiscon GmbH + * + * This file is part of rsyslog. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif +#include <stdlib.h> +#include <time.h> +#include <unistd.h> +#include <fcntl.h> +#include <errno.h> +#include <string.h> +#include <ctype.h> +#include <sys/klog.h> +#include <json/json.h> + +#include "rsyslog.h" +#include "srUtils.h" +#include "debug.h" +#include "imkmsg.h" + +/* globals */ +static int fklog = -1; /* kernel log fd */ + +#ifndef _PATH_KLOG +# define _PATH_KLOG "/dev/kmsg" +#endif + +/* submit a message to imkmsg Syslog() API. In this function, we parse + * necessary information from kernel log line, and make json string + * from the rest. + */ +static void +submitSyslog(uchar *buf) +{ + long offs = 0; + struct timeval tv; + long int timestamp = 0; + struct timespec monotonic; + struct timespec realtime; + char name[1024]; + char value[1024]; + char msg[1024]; + int priority = 0; + long int sequnum = 0; + struct json_object *json = NULL, *jval; + + /* create new json object */ + json = json_object_new_object(); + + /* get priority */ + for (; isdigit(*buf); buf++) { + priority += (priority * 10) + (*buf - '0'); + } + buf++; + + /* get messages sequence number and add it to json */ + for (; isdigit(*buf); buf++) { + sequnum = (sequnum * 10) + (*buf - '0'); + } + buf++; /* skip , */ + jval = json_object_new_int(sequnum); + json_object_object_add(json, "sequnum", jval); + + /* get timestamp */ + for (; isdigit(*buf); buf++) { + timestamp += (timestamp * 10) + (*buf - '0'); + } + buf++; /* skip ; */ + + /* get message */ + offs = 0; + for (; *buf != '\n' && *buf != '\0'; buf++, offs++) { + msg[offs] = *buf; + } + msg[offs] = '\0'; + jval = json_object_new_string((char*)msg); + json_object_object_add(json, "msg", jval); + + if (*buf != '\0') /* message has appended properties, skip \n */ + buf++; + + while (strlen((char *)buf)) { + /* get name of the property */ + buf++; /* skip ' ' */ + offs = 0; + for (; *buf != '=' && *buf != ' '; buf++, offs++) { + name[offs] = *buf; + } + name[offs] = '\0'; + buf++; /* skip = or ' ' */; + + offs = 0; + for (; *buf != '\n' && *buf != '\0'; buf++, offs++) { + value[offs] = *buf; + } + value[offs] = '\0'; + if (*buf != '\0') { + buf++; /* another property, skip \n */ + } + + jval = json_object_new_string((char*)value); + json_object_object_add(json, name, jval); + } + + /* calculate timestamp */ + clock_gettime(CLOCK_MONOTONIC, &monotonic); + clock_gettime(CLOCK_REALTIME, &realtime); + tv.tv_sec = realtime.tv_sec + ((timestamp / 1000000l) - monotonic.tv_sec); + tv.tv_usec = (realtime.tv_nsec + ((timestamp / 1000000000l) - monotonic.tv_nsec)) / 1000; + + Syslog(priority, (uchar *)msg, &tv, json); +} + + +/* open the kernel log - will be called inside the willRun() imkmsg entry point + */ +rsRetVal +klogWillRun(modConfData_t *pModConf) +{ + char errmsg[2048]; + int r; + DEFiRet; + + fklog = open(_PATH_KLOG, O_RDONLY, 0); + if (fklog < 0) { + imkmsgLogIntMsg(RS_RET_ERR_OPEN_KLOG, "imkmsg: cannot open kernel log(%s): %s.", + _PATH_KLOG, rs_strerror_r(errno, errmsg, sizeof(errmsg))); + ABORT_FINALIZE(RS_RET_ERR_OPEN_KLOG); + } + + /* Set level of kernel console messaging.. */ + if(pModConf->console_log_level != -1) { + r = klogctl(8, NULL, pModConf->console_log_level); + if(r != 0) { + imkmsgLogIntMsg(LOG_WARNING, "imkmsg: cannot set console log level: %s", + rs_strerror_r(errno, errmsg, sizeof(errmsg))); + /* make sure we do not try to re-set! */ + pModConf->console_log_level = -1; + } + } + +finalize_it: + RETiRet; +} + +/* Read kernel log while data are available, each read() reads one + * record of printk buffer. + */ +static void +readkmsg(void) +{ + int i; + uchar pRcv[8096+1]; + char errmsg[2048]; + + for (;;) { + dbgprintf("imkmsg waiting for kernel log line\n"); + + /* every read() from the opened device node receives one record of the printk buffer */ + i = read(fklog, pRcv, 8096); + + if (i > 0) { + /* successful read of message of nonzero length */ + pRcv[i] = '\0'; + } else { + /* something went wrong - error or zero length message */ + if (i < 0 && errno != EINTR && errno != EAGAIN) { + /* error occured */ + imkmsgLogIntMsg(LOG_ERR, + "imkmsg: error reading kernel log - shutting down: %s", + rs_strerror_r(errno, errmsg, sizeof(errmsg))); + fklog = -1; + } + break; + } + + submitSyslog(pRcv); + } +} + + +/* to be called in the module's AfterRun entry point + * rgerhards, 2008-04-09 + */ +rsRetVal klogAfterRun(modConfData_t *pModConf) +{ + DEFiRet; + if(fklog != -1) + close(fklog); + /* Turn on logging of messages to console, but only if a log level was speficied */ + if(pModConf->console_log_level != -1) + klogctl(7, NULL, 0); + RETiRet; +} + + +/* to be called in the module's WillRun entry point, this is the main + * "message pull" mechanism. + * rgerhards, 2008-04-09 + */ +rsRetVal klogLogKMsg(modConfData_t __attribute__((unused)) *pModConf) +{ + DEFiRet; + readkmsg(); + RETiRet; +} + + +/* provide the (system-specific) default facility for internal messages + * rgerhards, 2008-04-14 + */ +int +klogFacilIntMsg(void) +{ + return LOG_SYSLOG; +} + diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 29cc19f5..a4933115 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -164,8 +164,6 @@ static int startIndexUxLocalSockets; /* process fd from that index on (used to static int nfd = 1; /* number of Unix sockets open / read-only after startup */ static int sd_fds = 0; /* number of systemd activated sockets */ -static ee_ctx ctxee = NULL; /* library context */ - /* config vars for legacy config system */ #define DFLT_bCreatePath 0 #define DFLT_ratelimitInterval 0 @@ -690,14 +688,12 @@ getTrustedProp(struct ucred *cred, char *propName, uchar *buf, size_t lenBuf, in if((fd = open(namebuf, O_RDONLY)) == -1) { DBGPRINTF("error reading '%s'\n", namebuf); - *lenProp = 0; - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } if((lenRead = read(fd, buf, lenBuf - 1)) == -1) { DBGPRINTF("error reading file data for '%s'\n", namebuf); - *lenProp = 0; close(fd); - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } /* we strip after the first \n */ @@ -733,8 +729,7 @@ getTrustedExe(struct ucred *cred, uchar *buf, size_t lenBuf, int* lenProp) if((lenRead = readlink(namebuf, (char*)buf, lenBuf - 1)) == -1) { DBGPRINTF("error reading link '%s'\n", namebuf); - *lenProp = 0; - FINALIZE; + ABORT_FINALIZE(RS_RET_ERR); } buf[lenRead] = '\0'; @@ -767,6 +762,7 @@ copyescaped(uchar *dstbuf, uchar *inbuf, int inlen) } +#if 0 /* Creates new field to be added to event * used for SystemLogParseTrusted parsing */ @@ -785,6 +781,7 @@ createNewField(char *fieldname, char *value, int lenValue) { return newField; } +#endif /* submit received message to the queue engine @@ -812,7 +809,7 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred, struct tim uchar *pmsgbuf; int toffs; /* offset for trusted properties */ struct syslogTime dummyTS; - struct ee_event *event = NULL; + struct json_object *json = NULL, *jval; DEFiRet; /* TODO: handle format errors?? */ @@ -859,45 +856,27 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred, struct tim } if (pLstn->bParseTrusted) { - struct ee_field *newField; - - if(ctxee == NULL) { - if((ctxee = ee_initCtx()) == NULL) { - errmsg.LogError(0, RS_RET_NO_RULESET, "error: could not initialize libee ctx, cannot " - "activate action"); - ABORT_FINALIZE(RS_RET_ERR_LIBEE_INIT); - } + json = json_object_new_object(); + /* create value string, create field, and add it */ + jval = json_object_new_int(cred->pid); + json_object_object_add(json, "pid", jval); + jval = json_object_new_int(cred->uid); + json_object_object_add(json, "uid", jval); + jval = json_object_new_int(cred->gid); + json_object_object_add(json, "gid", jval); + if(getTrustedProp(cred, "comm", propBuf, sizeof(propBuf), &lenProp) == RS_RET_OK) { + jval = json_object_new_string((char*)propBuf); + json_object_object_add(json, "appname", jval); + } + if(getTrustedExe(cred, propBuf, sizeof(propBuf), &lenProp) == RS_RET_OK) { + jval = json_object_new_string((char*)propBuf); + json_object_object_add(json, "exe", jval); + } + if(getTrustedProp(cred, "cmdline", propBuf, sizeof(propBuf), &lenProp) == RS_RET_OK) { + jval = json_object_new_string((char*)propBuf); + json_object_object_add(json, "cmd", jval); } - - event = ee_newEvent(ctxee); - - /* create value string, create field, and add it to event */ - lenProp = snprintf((char *)propBuf, sizeof(propBuf), "%lu", (long unsigned) cred->pid); - newField = createNewField("pid", (char *)propBuf, lenProp); - ee_addFieldToEvent(event, newField); - - lenProp = snprintf((char *)propBuf, sizeof(propBuf), "%lu", (long unsigned) cred->uid); - newField = createNewField("uid", (char *)propBuf, lenProp); - ee_addFieldToEvent(event, newField); - - lenProp = snprintf((char *)propBuf, sizeof(propBuf), "%lu", (long unsigned) cred->gid); - newField = createNewField("gid", (char *)propBuf, lenProp); - ee_addFieldToEvent(event, newField); - - getTrustedProp(cred, "comm", propBuf, sizeof(propBuf), &lenProp); - newField = createNewField("appname", (char *)propBuf, lenProp); - ee_addFieldToEvent(event, newField); - - getTrustedExe(cred, propBuf, sizeof(propBuf), &lenProp); - newField = createNewField("exe", (char *)propBuf, lenProp); - ee_addFieldToEvent(event, newField); - - getTrustedProp(cred, "cmdline", propBuf, sizeof(propBuf), &lenProp); - newField = createNewField("cmd", (char *)propBuf, lenProp); - ee_addFieldToEvent(event, newField); - } else { - memcpy(pmsgbuf, pRcv, lenRcv); memcpy(pmsgbuf+lenRcv, " @[", 3); toffs = lenRcv + 3; /* next free location */ @@ -907,23 +886,20 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred, struct tim memcpy(pmsgbuf+toffs, propBuf, lenProp); toffs = toffs + lenProp; - getTrustedProp(cred, "comm", propBuf, sizeof(propBuf), &lenProp); - if(lenProp) { + if(getTrustedProp(cred, "comm", propBuf, sizeof(propBuf), &lenProp) == RS_RET_OK) { memcpy(pmsgbuf+toffs, " _COMM=", 7); memcpy(pmsgbuf+toffs+7, propBuf, lenProp); toffs = toffs + 7 + lenProp; } - getTrustedExe(cred, propBuf, sizeof(propBuf), &lenProp); - if(lenProp) { + if(getTrustedExe(cred, propBuf, sizeof(propBuf), &lenProp) == RS_RET_OK) { memcpy(pmsgbuf+toffs, " _EXE=", 6); memcpy(pmsgbuf+toffs+6, propBuf, lenProp); toffs = toffs + 6 + lenProp; } - getTrustedProp(cred, "cmdline", propBuf, sizeof(propBuf), &lenProp); - if(lenProp) { - memcpy(pmsgbuf+toffs, " _CMDLINE=", 9); - toffs = toffs + 9 + - copyescaped(pmsgbuf+toffs+9, propBuf, lenProp); + if(getTrustedProp(cred, "cmdline", propBuf, sizeof(propBuf), &lenProp) == RS_RET_OK) { + memcpy(pmsgbuf+toffs, " _CMDLINE=", 10); + toffs = toffs + 10 + + copyescaped(pmsgbuf+toffs+10, propBuf, lenProp); } /* finalize string */ @@ -949,12 +925,11 @@ SubmitMsg(uchar *pRcv, int lenRcv, lstn_t *pLstn, struct ucred *cred, struct tim parse++; lenMsg--; /* '>' */ - /* event is saved to pMsg */ - if(pMsg->event != NULL) { - ee_deleteEvent(pMsg->event); - } - if (event != NULL) { - pMsg->event = event; + if(json != NULL) { + /* as per lumberjack spec, these properties need to go into + * the CEE root. + */ + msgAddJSON(pMsg, (uchar*)"!", json); } if(ts == NULL) { @@ -1467,10 +1442,6 @@ CODESTARTafterRun discardLogSockets(); nfd = 1; - if(ctxee != NULL) { - ee_exitCtx(ctxee); - ctxee = NULL; - } ENDafterRun diff --git a/plugins/mmaudit/mmaudit.c b/plugins/mmaudit/mmaudit.c index fcefd013..4934312b 100644 --- a/plugins/mmaudit/mmaudit.c +++ b/plugins/mmaudit/mmaudit.c @@ -67,7 +67,6 @@ DEFobjCurrIf(errmsg); DEF_OMOD_STATIC_DATA typedef struct _instanceData { - ee_ctx ctxee; /**< context to be used for libee */ } instanceData; typedef struct configSettings_s { @@ -93,7 +92,6 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance - ee_exitCtx(pData->ctxee); ENDfreeInstance @@ -169,17 +167,20 @@ finalize_it: /* parse the audit record and create libee structure */ static rsRetVal -audit_parse(instanceData *pData, uchar *buf, struct ee_event **event) +audit_parse(uchar *buf, struct json_object **jsonRoot) { - es_str_t *estr; + struct json_object *json; + struct json_object *jval; char name[1024]; char val[1024]; DEFiRet; - *event = ee_newEvent(pData->ctxee); - if(event == NULL) { + *jsonRoot = json_object_new_object(); + if(*jsonRoot == NULL) { ABORT_FINALIZE(RS_RET_ERR); } + json = json_object_new_object(); + json_object_object_add(*jsonRoot, "data", json); while(*buf) { //dbgprintf("audit_parse, buf: '%s'\n", buf); @@ -189,10 +190,8 @@ audit_parse(instanceData *pData, uchar *buf, struct ee_event **event) } ++buf; CHKiRet(parseValue(&buf, val, sizeof(val))); - - estr = es_newStrFromCStr(val, strlen(val)); - ee_addStrFieldToEvent(*event, name, estr); - es_deleteStr(estr); + jval = json_object_new_string(val); + json_object_object_add(json, name, jval); dbgprintf("mmaudit: parsed %s=%s\n", name, val); } @@ -206,9 +205,10 @@ BEGINdoAction msg_t *pMsg; uchar *buf; int typeID; - struct ee_event *event; + struct json_object *jsonRoot; + struct json_object *json; + struct json_object *jval; int i; - es_str_t *estr; char auditID[1024]; int bSuccess = 0; CODESTARTdoAction @@ -252,48 +252,24 @@ dbgprintf("mmaudit: msg is '%s'\n", buf); } buf += 2; -dbgprintf("mmaudit: cookie found, type %d, auditID '%s', rest of message: '%s'\n", typeID, auditID, buf); - audit_parse(pData, buf, &event); - if(event == NULL) { + audit_parse(buf, &jsonRoot); + if(jsonRoot == NULL) { DBGPRINTF("mmaudit: audit parse error, assuming no " "audit message: '%s'\n", buf); FINALIZE; } /* we now need to shuffle the "outer" properties into that stream */ - estr = es_newStrFromCStr(auditID, strlen(auditID)); - ee_addStrFieldToEvent(event, "audithdr.auditid", estr); - es_deleteStr(estr); - - /* we abuse auditID a bit to save space... (TODO: change!) */ - snprintf(auditID, sizeof(auditID), "%d", typeID); - estr = es_newStrFromCStr(auditID, strlen(auditID)); - ee_addStrFieldToEvent(event, "audithdr.type", estr); - es_deleteStr(estr); - - /* TODO: in the long term, we need to think about merging & different - name spaces (probably best to add the newly-obtained event as a child to - the existing event...) - */ - if(pMsg->event != NULL) { - ee_deleteEvent(pMsg->event); - } - pMsg->event = event; + json = json_object_new_object(); + json_object_object_add(jsonRoot, "hdr", json); + jval = json_object_new_string(auditID); + json_object_object_add(json, "auditid", jval); + jval = json_object_new_int(typeID); + json_object_object_add(json, "type", jval); + + msgAddJSON(pMsg, (uchar*)"!audit", jsonRoot); bSuccess = 1; -#if 1 - /***DEBUG***/ // TODO: remove after initial testing - 2010-12-01 - { - char *cstr; - es_str_t *str; - ee_fmtEventToJSON(pMsg->event, &str); - cstr = es_str2cstr(str, NULL); - dbgprintf("mmaudit generated: %s\n", cstr); - free(cstr); - es_deleteStr(str); - } - /***END DEBUG***/ -#endif finalize_it: MsgSetParseSuccess(pMsg, bSuccess); ENDdoAction @@ -318,13 +294,6 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) * the format specified (if any) is always ignored. */ CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_TPL_AS_MSG, (uchar*) "RSYSLOG_FileFormat")); - - /* finally build the instance */ - if((pData->ctxee = ee_initCtx()) == NULL) { - errmsg.LogError(0, RS_RET_NO_RULESET, "error: could not initialize libee ctx, cannot " - "activate action"); - ABORT_FINALIZE(RS_RET_ERR_LIBEE_INIT); - } CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct diff --git a/plugins/mmjsonparse/mmjsonparse.c b/plugins/mmjsonparse/mmjsonparse.c index 03147b59..d419f246 100644 --- a/plugins/mmjsonparse/mmjsonparse.c +++ b/plugins/mmjsonparse/mmjsonparse.c @@ -36,7 +36,7 @@ #include <unistd.h> #include <ctype.h> #include <libestr.h> -#include <libee/libee.h> +#include <json/json.h> #include "conf.h" #include "syslogd-types.h" #include "template.h" @@ -59,7 +59,7 @@ DEFobjCurrIf(errmsg); DEF_OMOD_STATIC_DATA typedef struct _instanceData { - ee_ctx ctxee; /**< context to be used for libee */ + struct json_tokener *tokener; } instanceData; typedef struct configSettings_s { @@ -85,7 +85,8 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance - ee_exitCtx(pData->ctxee); + if(pData->tokener != NULL) + json_tokener_free(pData->tokener); ENDfreeInstance @@ -99,12 +100,54 @@ BEGINtryResume CODESTARTtryResume ENDtryResume + +static rsRetVal +processJSON(instanceData *pData, msg_t *pMsg, char *buf, size_t lenBuf) +{ + struct json_object *json; + const char *errMsg; + DEFiRet; + + dbgprintf("mmjsonparse: toParse: '%s'\n", buf); + json_tokener_reset(pData->tokener); + + json = json_tokener_parse_ex(pData->tokener, buf, lenBuf); + if(Debug) { + errMsg = NULL; + if(json == NULL) { + enum json_tokener_error err; + + err = pData->tokener->err; + if(err != json_tokener_continue) + errMsg = json_tokener_errors[err]; + else + errMsg = "Unterminated input"; + } else if((size_t)pData->tokener->char_offset < lenBuf) + errMsg = "Extra characters after JSON object"; + else if(!json_object_is_type(json, json_type_object)) + errMsg = "JSON value is not an object"; + if(errMsg != NULL) { + dbgprintf("mmjsonparse: Error parsing JSON '%s': %s\n", + buf, errMsg); + } + } + if(json == NULL + || ((size_t)pData->tokener->char_offset < lenBuf) + || (!json_object_is_type(json, json_type_object))) { + FINALIZE; /* just don't set property */ + } + + msgAddJSON(pMsg, (uchar*)"!", json); +dbgprintf("AAAA: The msg json object: %s\n",json_object_to_json_string(pMsg->json)); +finalize_it: + RETiRet; +} + #define COOKIE "@cee: " #define LEN_COOKIE (sizeof(COOKIE)-1) BEGINdoAction msg_t *pMsg; uchar *buf; - struct ee_event *event; int bSuccess = 0; CODESTARTdoAction pMsg = (msg_t*) ppString[0]; @@ -125,35 +168,8 @@ dbgprintf("mmjsonparse: msg is '%s'\n", buf); } buf += LEN_COOKIE; dbgprintf("mmjsonparse: cookie found, rest of message: '%s'\n", buf); - event = ee_newEventFromJSON(pData->ctxee, (char*)buf); - if(event == NULL) { - DBGPRINTF("mmjsonparse: JSON parse error, assuming no " - "JSON-enhanced message: '%s'\n", buf); - FINALIZE; - } - /* TODO: in the long term, we need to think about merging & different - name spaces (probably best to add the newly-obtained event as a child to - the existing event...) - */ - if(pMsg->event != NULL) { - ee_deleteEvent(pMsg->event); - } - pMsg->event = event; + CHKiRet(processJSON(pData, pMsg, (char*) buf, strlen((char*)buf))); bSuccess = 1; - -#if 1 - /***DEBUG***/ // TODO: remove after initial testing - 2010-12-01 - { - char *cstr; - es_str_t *str; - ee_fmtEventToJSON(pMsg->event, &str); - cstr = es_str2cstr(str, NULL); - dbgprintf("mmjsonparse generated: %s\n", cstr); - free(cstr); - es_deleteStr(str); - } - /***END DEBUG***/ -#endif finalize_it: MsgSetParseSuccess(pMsg, bSuccess); ENDdoAction @@ -180,10 +196,11 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_TPL_AS_MSG, (uchar*) "RSYSLOG_FileFormat")); /* finally build the instance */ - if((pData->ctxee = ee_initCtx()) == NULL) { - errmsg.LogError(0, RS_RET_NO_RULESET, "error: could not initialize libee ctx, cannot " - "activate action"); - ABORT_FINALIZE(RS_RET_ERR_LIBEE_INIT); + pData->tokener = json_tokener_new(); + if(pData->tokener == NULL) { + errmsg.LogError(0, RS_RET_ERR, "error: could not create json " + "tokener, cannot activate action"); + ABORT_FINALIZE(RS_RET_ERR); } CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct diff --git a/plugins/mmnormalize/mmnormalize.c b/plugins/mmnormalize/mmnormalize.c index c5b290f4..bf0b9ce6 100644 --- a/plugins/mmnormalize/mmnormalize.c +++ b/plugins/mmnormalize/mmnormalize.c @@ -4,9 +4,12 @@ * * NOTE: read comments in module-template.h for details on the calling interface! * + * TODO: check if we can replace libee via JSON system - currently that part + * is pretty inefficient... rgerhards, 2012-08-27 + * * File begun on 2010-01-01 by RGerhards * - * Copyright 2010 Rainer Gerhards and Adiscon GmbH. + * Copyright 2010-2012 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -37,6 +40,7 @@ #include <unistd.h> #include <libestr.h> #include <libee/libee.h> +#include <json/json.h> #include <liblognorm.h> #include "conf.h" #include "syslogd-types.h" @@ -108,8 +112,12 @@ BEGINdoAction msg_t *pMsg; es_str_t *str; uchar *buf; + char *cstrJSON; int len; int r; + struct ee_event *event = NULL; + struct json_tokener *tokener; + struct json_object *json; CODESTARTdoAction pMsg = (msg_t*) ppString[0]; /* note that we can performance-optimize the interface, but this also @@ -123,7 +131,7 @@ CODESTARTdoAction len = getMSGLen(pMsg); } str = es_newStrFromCStr((char*)buf, len); - r = ln_normalize(pData->ctxln, str, &pMsg->event); + r = ln_normalize(pData->ctxln, str, &event); if(r != 0) { DBGPRINTF("error %d during ln_normalize\n", r); MsgSetParseSuccess(pMsg, 0); @@ -131,16 +139,20 @@ CODESTARTdoAction MsgSetParseSuccess(pMsg, 1); } es_deleteStr(str); - /***DEBUG***/ // TODO: remove after initial testing - 2010-12-01 - { - char *cstr; - ee_fmtEventToJSON(pMsg->event, &str); - cstr = es_str2cstr(str, NULL); - dbgprintf("mmnormalize generated: %s\n", cstr); - free(cstr); - es_deleteStr(str); - } - /***END DEBUG***/ + + /* reformat to our json data struct */ + // TODO: this is all extremly ineffcient! + ee_fmtEventToJSON(event, &str); + cstrJSON = es_str2cstr(str, NULL); + dbgprintf("mmnormalize generated: %s\n", cstrJSON); + + tokener = json_tokener_new(); + json = json_tokener_parse_ex(tokener, cstrJSON, strlen((char*)cstrJSON)); + json_tokener_free(tokener); + msgAddJSON(pMsg, (uchar*)"!", json); + + free(cstrJSON); + es_deleteStr(str); ENDdoAction diff --git a/plugins/ommongodb/ommongodb.c b/plugins/ommongodb/ommongodb.c index f2de33df..2c65f275 100644 --- a/plugins/ommongodb/ommongodb.c +++ b/plugins/ommongodb/ommongodb.c @@ -224,11 +224,11 @@ static bson * getDefaultBSON(msg_t *pMsg) { bson *doc = NULL; - uchar *procid; short unsigned procid_free; size_t procid_len; - uchar *tag; short unsigned tag_free; size_t tag_len; - uchar *pid; short unsigned pid_free; size_t pid_len; - uchar *sys; short unsigned sys_free; size_t sys_len; - uchar *msg; short unsigned msg_free; size_t msg_len; + uchar *procid; short unsigned procid_free; rs_size_t procid_len; + uchar *tag; short unsigned tag_free; rs_size_t tag_len; + uchar *pid; short unsigned pid_free; rs_size_t pid_len; + uchar *sys; short unsigned sys_free; rs_size_t sys_len; + uchar *msg; short unsigned msg_free; rs_size_t msg_len; int severity, facil; gint64 ts_gen, ts_rcv; /* timestamps: generated, received */ int secfrac; @@ -296,7 +296,9 @@ static bson *BSONFromJSONObject(struct json_object *json); static gboolean BSONAppendJSONObject(bson *doc, const gchar *name, struct json_object *json) { - switch(json_object_get_type(json)) { + switch(json != NULL ? json_object_get_type(json) : json_type_null) { + case json_type_null: + return bson_append_null(doc, name); case json_type_boolean: return bson_append_boolean(doc, name, json_object_get_boolean(json)); diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 67e235a0..7af26d2b 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -63,8 +63,6 @@ librsyslog_la_SOURCES = \ queue.h \ ruleset.c \ ruleset.h \ - rule.c \ - rule.h \ prop.c \ prop.h \ cfsysline.c \ diff --git a/runtime/batch.h b/runtime/batch.h index 944889bd..f743c188 100644 --- a/runtime/batch.h +++ b/runtime/batch.h @@ -51,7 +51,6 @@ struct batch_obj_s { /* work variables for action processing; these are reused for each action (or block of * actions) */ - sbool bFilterOK; /* work area for filter processing (per action, reused!) */ sbool bPrevWasSuspended; /* following are caches to save allocs if not absolutely necessary */ uchar *staticActStrings[CONF_OMOD_NUMSTRINGS_MAXSIZE]; /**< for strings */ @@ -83,6 +82,7 @@ struct batch_s { int iDoneUpTo; /* all messages below this index have state other than RDY */ qDeqID deqID; /* ID of dequeue operation that generated this batch */ int *pbShutdownImmediate;/* end processing of this batch immediately if set to 1 */ + sbool *active; /* which messages are active for processing, NULL=all */ sbool bSingleRuleset; /* do all msgs of this batch use a single ruleset? */ batch_obj_t *pElem; /* batch elements */ }; @@ -129,7 +129,8 @@ batchSetElemState(batch_t *pBatch, int i, batch_state_t newState) { */ static inline int batchIsValidElem(batch_t *pBatch, int i) { - return(pBatch->pElem[i].bFilterOK && pBatch->pElem[i].state != BATCH_STATE_DISC); + return( (pBatch->pElem[i].state != BATCH_STATE_DISC) + && (pBatch->active == NULL || pBatch->active[i])); } diff --git a/runtime/conf.c b/runtime/conf.c index 488d1b86..23fb6bbd 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -61,17 +61,16 @@ #include "srUtils.h" #include "errmsg.h" #include "net.h" -#include "rule.h" #include "ruleset.h" #include "rsconf.h" #include "unicode-helper.h" +#include "rainerscript.h" #ifdef OS_SOLARIS # define NAME_MAX MAXNAMELEN #endif /* forward definitions */ -//static rsRetVal cfline(rsconf_t *conf, uchar *line, rule_t **pfCurr); /* static data */ @@ -79,7 +78,6 @@ DEFobjStaticHelpers DEFobjCurrIf(module) DEFobjCurrIf(errmsg) DEFobjCurrIf(net) -DEFobjCurrIf(rule) DEFobjCurrIf(ruleset) int bConfStrictScoping = 0; /* force strict scoping during config processing? */ @@ -326,14 +324,9 @@ cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int } -/* Helper to cfline(). This function takes the filter part of a traditional, PRI - * based line and decodes the PRIs given in the selector line. It processed the - * line up to the beginning of the action part. A pointer to that beginnig is - * passed back to the caller. - * rgerhards 2005-09-15 - */ +/* Decode a traditional PRI filter */ /* GPLv3 - stems back to sysklogd */ -rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) +rsRetVal DecodePRIFilter(uchar *pline, uchar pmask[]) { uchar *p; register uchar *q; @@ -347,22 +340,15 @@ rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) DEFiRet; ASSERT(pline != NULL); - ASSERT(*pline != NULL); - ISOBJ_TYPE_assert(pRule, rule); - dbgprintf(" - traditional PRI filter '%s'\n", *pline); - errno = 0; /* keep strerror_r() stuff out of logerror messages */ + dbgprintf("Decoding traditional PRI filter '%s'\n", pline); - pRule->f_filter_type = FILTER_PRI; - /* Note: file structure is pre-initialized to zero because it was - * created with calloc()! - */ for (i = 0; i <= LOG_NFACILITIES; i++) { - pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; + pmask[i] = TABLE_NOPRI; } /* scan through the list of selectors */ - for (p = *pline; *p && *p != '\t' && *p != ' ';) { + for (p = pline; *p && *p != '\t' && *p != ' ';) { /* find the end of this facility name list */ for (q = p; *q && *q != '\t' && *q++ != '.'; ) continue; @@ -411,28 +397,28 @@ rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) for (i = 0; i <= LOG_NFACILITIES; i++) { if ( pri == INTERNAL_NOPRI ) { if ( ignorepri ) - pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; + pmask[i] = TABLE_ALLPRI; else - pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; + pmask[i] = TABLE_NOPRI; } else if ( singlpri ) { if ( ignorepri ) - pRule->f_filterData.f_pmask[i] &= ~(1<<pri); + pmask[i] &= ~(1<<pri); else - pRule->f_filterData.f_pmask[i] |= (1<<pri); + pmask[i] |= (1<<pri); } else { if ( pri == TABLE_ALLPRI ) { if ( ignorepri ) - pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; + pmask[i] = TABLE_NOPRI; else - pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; + pmask[i] = TABLE_ALLPRI; } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) - pRule->f_filterData.f_pmask[i] &= ~(1<<i2); + pmask[i] &= ~(1<<i2); else for (i2= 0; i2 <= pri; ++i2) - pRule->f_filterData.f_pmask[i] |= (1<<i2); + pmask[i] |= (1<<i2); } } } @@ -447,27 +433,27 @@ rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) if ( pri == INTERNAL_NOPRI ) { if ( ignorepri ) - pRule->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; + pmask[i >> 3] = TABLE_ALLPRI; else - pRule->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; + pmask[i >> 3] = TABLE_NOPRI; } else if ( singlpri ) { if ( ignorepri ) - pRule->f_filterData.f_pmask[i >> 3] &= ~(1<<pri); + pmask[i >> 3] &= ~(1<<pri); else - pRule->f_filterData.f_pmask[i >> 3] |= (1<<pri); + pmask[i >> 3] |= (1<<pri); } else { if ( pri == TABLE_ALLPRI ) { if ( ignorepri ) - pRule->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; + pmask[i >> 3] = TABLE_NOPRI; else - pRule->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; + pmask[i >> 3] = TABLE_ALLPRI; } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) - pRule->f_filterData.f_pmask[i >> 3] &= ~(1<<i2); + pmask[i >> 3] &= ~(1<<i2); else for (i2= 0; i2 <= pri; ++i2) - pRule->f_filterData.f_pmask[i >> 3] |= (1<<i2); + pmask[i >> 3] |= (1<<i2); } } } @@ -478,11 +464,6 @@ rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) p = q; } - /* skip to action part */ - while (*p == '\t' || *p == ' ') - p++; - - *pline = p; RETiRet; } @@ -492,7 +473,7 @@ rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) * of the action part. A pointer to that beginnig is passed back to the caller. * rgerhards 2005-09-15 */ -rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) +rsRetVal DecodePropFilter(uchar *pline, struct cnfstmt *stmt) { rsParsObj *pPars; cstr_t *pCSCompOp; @@ -501,16 +482,11 @@ rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) int iOffset; /* for compare operations */ ASSERT(pline != NULL); - ASSERT(*pline != NULL); - ASSERT(f != NULL); - dbgprintf(" - property-based filter '%s'\n", *pline); - errno = 0; /* keep strerror_r() stuff out of logerror messages */ - - f->f_filter_type = FILTER_PROP; + dbgprintf("Decoding property-based filter '%s'\n", pline); /* create parser object starting with line string without leading colon */ - if((iRet = rsParsConstructFromSz(&pPars, (*pline)+1)) != RS_RET_OK) { + if((iRet = rsParsConstructFromSz(&pPars, pline+1)) != RS_RET_OK) { errmsg.LogError(0, iRet, "Error %d constructing parser object - ignoring selector", iRet); return(iRet); } @@ -522,15 +498,15 @@ rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) rsParsDestruct(pPars); return(iRet); } - iRet = propNameToID(pCSPropName, &f->f_filterData.prop.propID); + iRet = propNameToID(pCSPropName, &stmt->d.s_propfilt.propID); if(iRet != RS_RET_OK) { errmsg.LogError(0, iRet, "error %d parsing filter property - ignoring selector", iRet); rsParsDestruct(pPars); return(iRet); } - if(f->f_filterData.prop.propID == PROP_CEE) { + if(stmt->d.s_propfilt.propID == PROP_CEE) { /* in CEE case, we need to preserve the actual property name */ - if((f->f_filterData.prop.propName = + if((stmt->d.s_propfilt.propName = es_newStrFromBuf((char*)cstrGetSzStrNoNULL(pCSPropName)+2, cstrLen(pCSPropName)-2)) == NULL) { cstrDestruct(&pCSPropName); return(RS_RET_ERR); @@ -553,38 +529,38 @@ rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) */ if(rsCStrLen(pCSCompOp) > 0) { if(*rsCStrGetBufBeg(pCSCompOp) == '!') { - f->f_filterData.prop.isNegated = 1; + stmt->d.s_propfilt.isNegated = 1; iOffset = 1; /* ignore '!' */ } else { - f->f_filterData.prop.isNegated = 0; + stmt->d.s_propfilt.isNegated = 0; iOffset = 0; } } else { - f->f_filterData.prop.isNegated = 0; + stmt->d.s_propfilt.isNegated = 0; iOffset = 0; } if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (uchar*) "contains", 8)) { - f->f_filterData.prop.operation = FIOP_CONTAINS; + stmt->d.s_propfilt.operation = FIOP_CONTAINS; } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (uchar*) "isequal", 7)) { - f->f_filterData.prop.operation = FIOP_ISEQUAL; + stmt->d.s_propfilt.operation = FIOP_ISEQUAL; } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (uchar*) "isempty", 7)) { - f->f_filterData.prop.operation = FIOP_ISEMPTY; + stmt->d.s_propfilt.operation = FIOP_ISEMPTY; } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (uchar*) "startswith", 10)) { - f->f_filterData.prop.operation = FIOP_STARTSWITH; + stmt->d.s_propfilt.operation = FIOP_STARTSWITH; } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (unsigned char*) "regex", 5)) { - f->f_filterData.prop.operation = FIOP_REGEX; + stmt->d.s_propfilt.operation = FIOP_REGEX; } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (unsigned char*) "ereregex", 8)) { - f->f_filterData.prop.operation = FIOP_EREREGEX; + stmt->d.s_propfilt.operation = FIOP_EREREGEX; } else { errmsg.LogError(0, NO_ERRCODE, "error: invalid compare operation '%s' - ignoring selector", (char*) rsCStrGetSzStrNoNULL(pCSCompOp)); } rsCStrDestruct(&pCSCompOp); /* no longer needed */ - if(f->f_filterData.prop.operation != FIOP_ISEMPTY) { + if(stmt->d.s_propfilt.operation != FIOP_ISEMPTY) { /* read compare value */ - iRet = parsQuotedCStr(pPars, &f->f_filterData.prop.pCSCompValue); + iRet = parsQuotedCStr(pPars, &stmt->d.s_propfilt.pCSCompValue); if(iRet != RS_RET_OK) { errmsg.LogError(0, iRet, "error %d compare value property - ignoring selector", iRet); rsParsDestruct(pPars); @@ -592,114 +568,10 @@ rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) } } - /* skip to action part */ - if((iRet = parsSkipWhitespace(pPars)) != RS_RET_OK) { - errmsg.LogError(0, iRet, "error %d skipping to action part - ignoring selector", iRet); - rsParsDestruct(pPars); - return(iRet); - } - - /* cleanup */ - *pline = *pline + rsParsGetParsePointer(pPars) + 1; - /* we are adding one for the skipped initial ":" */ - return rsParsDestruct(pPars); } -/* - * Helper to cfline(). This function interprets a BSD host selector line - * from the config file ("+/-hostname"). It stores it for further reference. - * rgerhards 2005-10-19 - */ -rsRetVal cflineProcessHostSelector(uchar **pline) -{ - DEFiRet; - - ASSERT(pline != NULL); - ASSERT(*pline != NULL); - ASSERT(**pline == '-' || **pline == '+'); - - dbgprintf(" - host selector line\n"); - - /* check include/exclude setting */ - if(**pline == '+') { - eDfltHostnameCmpMode = HN_COMP_MATCH; - } else { /* we do not check for '-', it must be, else we wouldn't be here */ - eDfltHostnameCmpMode = HN_COMP_NOMATCH; - } - (*pline)++; /* eat + or - */ - - /* the below is somewhat of a quick hack, but it is efficient (this is - * why it is in here. "+*" resets the tag selector with BSD syslog. We mimic - * this, too. As it is easy to check that condition, we do not fire up a - * parser process, just make sure we do not address beyond our space. - * Order of conditions in the if-statement is vital! rgerhards 2005-10-18 - */ - if(**pline != '\0' && **pline == '*' && *(*pline+1) == '\0') { - dbgprintf("resetting BSD-like hostname filter\n"); - eDfltHostnameCmpMode = HN_NO_COMP; - if(pDfltHostnameCmp != NULL) { - CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, NULL)); - } - } else { - dbgprintf("setting BSD-like hostname filter to '%s'\n", *pline); - if(pDfltHostnameCmp == NULL) { - /* create string for parser */ - CHKiRet(rsCStrConstructFromszStr(&pDfltHostnameCmp, *pline)); - } else { /* string objects exists, just update... */ - CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, *pline)); - } - } - -finalize_it: - RETiRet; -} - - -/* - * Helper to cfline(). This function interprets a BSD tag selector line - * from the config file ("!tagname"). It stores it for further reference. - * rgerhards 2005-10-18 - */ -rsRetVal cflineProcessTagSelector(uchar **pline) -{ - DEFiRet; - - ASSERT(pline != NULL); - ASSERT(*pline != NULL); - ASSERT(**pline == '!'); - - dbgprintf(" - programname selector line\n"); - - (*pline)++; /* eat '!' */ - - /* the below is somewhat of a quick hack, but it is efficient (this is - * why it is in here. "!*" resets the tag selector with BSD syslog. We mimic - * this, too. As it is easy to check that condition, we do not fire up a - * parser process, just make sure we do not address beyond our space. - * Order of conditions in the if-statement is vital! rgerhards 2005-10-18 - */ - if(**pline != '\0' && **pline == '*' && *(*pline+1) == '\0') { - dbgprintf("resetting programname filter\n"); - if(pDfltProgNameCmp != NULL) { - rsCStrDestruct(&pDfltProgNameCmp); - } - } else { - dbgprintf("setting programname filter to '%s'\n", *pline); - if(pDfltProgNameCmp == NULL) { - /* create string for parser */ - CHKiRet(rsCStrConstructFromszStr(&pDfltProgNameCmp, *pline)); - } else { /* string objects exists, just update... */ - CHKiRet(rsCStrSetSzStr(pDfltProgNameCmp, *pline)); - } - } - -finalize_it: - RETiRet; -} - - /* process the action part of a selector line * rgerhards, 2007-08-01 */ @@ -831,7 +703,6 @@ CODESTARTObjClassExit(conf) objRelease(module, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); - objRelease(rule, CORE_COMPONENT); objRelease(ruleset, CORE_COMPONENT); ENDObjClassExit(conf) @@ -845,7 +716,6 @@ BEGINAbstractObjClassInit(conf, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANG CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* TODO: make this dependcy go away! */ - CHKiRet(objUse(rule, CORE_COMPONENT)); CHKiRet(objUse(ruleset, CORE_COMPONENT)); /* These commands will NOT be supported -- the new v6.3 config system provides diff --git a/runtime/conf.h b/runtime/conf.h index 018d9111..a1bb51ad 100644 --- a/runtime/conf.h +++ b/runtime/conf.h @@ -62,11 +62,8 @@ PROTOTYPEObj(conf); rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl); -/* more dirt to cover the new config interface (will go away...) */ -rsRetVal cflineProcessTagSelector(uchar **pline); -rsRetVal cflineProcessHostSelector(uchar **pline); -rsRetVal cflineProcessTradPRIFilter(uchar **pline, rule_t *pRule); -rsRetVal cflineProcessPropFilter(uchar **pline, rule_t *f); +rsRetVal DecodePRIFilter(uchar *pline, uchar pmask[]); +rsRetVal DecodePropFilter(uchar *pline, struct cnfstmt *stmt); rsRetVal cflineDoAction(rsconf_t *conf, uchar **p, action_t **ppAction); extern EHostnameCmpMode eDfltHostnameCmpMode; extern cstr_t *pDfltHostnameCmp; diff --git a/runtime/modules.c b/runtime/modules.c index 5706685f..9f7ff31c 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -313,7 +313,8 @@ finalize_it: /* get the name of a module */ -static uchar *modGetName(modInfo_t *pThis) +uchar * +modGetName(modInfo_t *pThis) { return((pThis->pszName == NULL) ? (uchar*) "" : pThis->pszName); } diff --git a/runtime/modules.h b/runtime/modules.h index 02e4a699..e42d19e1 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -190,8 +190,11 @@ ENDinterface(module) PROTOTYPEObj(module); /* in v6, we go back to in-core static link for core objects, at least those * that are not called from plugins. + * ... and we need to know that none of the module functions are called from plugins! + * rgerhards, 2012-09-24 */ rsRetVal modulesProcessCnf(struct cnfobj *o); +uchar *modGetName(modInfo_t *pThis); rsRetVal addModToCnfList(cfgmodules_etry_t *pNew, cfgmodules_etry_t *pLast); rsRetVal readyModForCnf(modInfo_t *pThis, cfgmodules_etry_t **ppNew, cfgmodules_etry_t **ppLast); #endif /* #ifndef MODULES_H_INCLUDED */ diff --git a/runtime/msg.c b/runtime/msg.c index 187f0c22..d3c814e2 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -39,7 +39,9 @@ #include <sys/sysinfo.h> #include <netdb.h> #include <libestr.h> -#include <libee/libee.h> +#include <json/json.h> +/* For struct json_object_iter, should not be necessary in future versions */ +#include <json/json_object_private.h> #if HAVE_MALLOC_H # include <malloc.h> #endif @@ -291,6 +293,9 @@ static pthread_mutex_t mutTrimCtr; /* mutex to handle malloc trim */ /* some forward declarations */ static int getAPPNAMELen(msg_t *pM, sbool bLockMutex); +static rsRetVal jsonPathFindParent(msg_t *pM, uchar *name, uchar *leaf, struct json_object **parent, int bCreate); +static uchar * jsonPathGetLeaf(uchar *name, int lenName); +static struct json_object *jsonDeepCopy(struct json_object *src); /* The following functions will support advanced output module @@ -740,7 +745,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->pRcvFromIP = NULL; pM->rcvFrom.pRcvFrom = NULL; pM->pRuleset = NULL; - pM->event = NULL; + pM->json = NULL; memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt)); memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP)); pM->TAG.pszTAG = NULL; @@ -879,8 +884,8 @@ CODESTARTobjDestruct(msg) rsCStrDestruct(&pThis->pCSPROCID); if(pThis->pCSMSGID != NULL) rsCStrDestruct(&pThis->pCSMSGID); - if(pThis->event != NULL) - ee_deleteEvent(pThis->event); + if(pThis->json != NULL) + json_object_put(pThis->json); if(pThis->pszUUID != NULL) free(pThis->pszUUID); # ifndef HAVE_ATOMIC_BUILTINS @@ -1029,6 +1034,9 @@ msg_t* MsgDup(msg_t* pOld) tmpCOPYCSTR(PROCID); tmpCOPYCSTR(MSGID); + if(pOld->json != NULL) + pNew->json = jsonDeepCopy(pOld->json); + /* we do not copy all other cache properties, as we do not even know * if they are needed once again. So we let them re-create if needed. */ @@ -1082,6 +1090,10 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszRcvFrom"), PROPTYPE_PSZ, (void*) psz)); psz = getRcvFromIP(pThis); CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszRcvFromIP"), PROPTYPE_PSZ, (void*) psz)); + if(pThis->json != NULL) { + psz = (uchar*) json_object_get_string(pThis->json); + CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("json"), PROPTYPE_PSZ, (void*) psz)); + } objSerializePTR(pStrm, pCSStrucData, CSTR); objSerializePTR(pStrm, pCSAPPNAME, CSTR); @@ -2403,39 +2415,77 @@ static uchar *getNOW(eNOWType eNow) #undef tmpBUFSIZE /* clean up */ -/* Get a CEE-Property from libee. This function probably should be - * placed somewhere else, but this smells like a big restructuring - * useful in any case. So for the time being, I'll simply leave the - * function here, as the context seems good enough. -- rgerhards, 2010-12-01 - */ -static inline void -getCEEPropVal(msg_t *pMsg, es_str_t *propName, uchar **pRes, int *buflen, unsigned short *pbMustBeFreed) +/* Get a CEE-Property as string value*/ +rsRetVal +getCEEPropVal(msg_t *pM, es_str_t *propName, uchar **pRes, rs_size_t *buflen, unsigned short *pbMustBeFreed) { - es_str_t *str = NULL; - int r; + uchar *name = NULL; + uchar *leaf; + struct json_object *parent; + struct json_object *field; + DEFiRet; if(*pbMustBeFreed) free(*pRes); *pRes = NULL; + // TODO: mutex? + if(pM->json == NULL) goto finalize_it; - if(pMsg->event == NULL) goto finalize_it; - r = ee_getEventFieldAsString(pMsg->event, propName, &str); - - if(r != EE_OK) { - DBGPRINTF("msgGtCEEVar: libee error %d during ee_getEventFieldAsString\n", r); - FINALIZE; + if(!es_strbufcmp(propName, (uchar*)"!", 1)) { + field = pM->json; + } else { + name = (uchar*)es_str2cstr(propName, NULL); + leaf = jsonPathGetLeaf(name, ustrlen(name)); + CHKiRet(jsonPathFindParent(pM, name, leaf, &parent, 1)); + field = json_object_object_get(parent, (char*)leaf); + } + if(field != NULL) { + *pRes = (uchar*) strdup(json_object_get_string(field)); + *buflen = (int) ustrlen(*pRes); + *pbMustBeFreed = 1; } - *pRes = (unsigned char*) es_str2cstr(str, "#000"); - es_deleteStr(str); - *buflen = (int) ustrlen(*pRes); - *pbMustBeFreed = 1; finalize_it: + free(name); if(*pRes == NULL) { /* could not find any value, so set it to empty */ *pRes = (unsigned char*)""; *pbMustBeFreed = 0; } + RETiRet; +} + + +/* Get a CEE-Property as native json object + */ +rsRetVal +msgGetCEEPropJSON(msg_t *pM, es_str_t *propName, struct json_object **pjson) +{ + uchar *name = NULL; + uchar *leaf; + struct json_object *parent; + DEFiRet; + + // TODO: mutex? + if(pM->json == NULL) { + ABORT_FINALIZE(RS_RET_NOT_FOUND); + } + + if(!es_strbufcmp(propName, (uchar*)"!", 1)) { + *pjson = pM->json; + FINALIZE; + } + name = (uchar*)es_str2cstr(propName, NULL); + leaf = jsonPathGetLeaf(name, ustrlen(name)); + CHKiRet(jsonPathFindParent(pM, name, leaf, &parent, 1)); + *pjson = json_object_object_get(parent, (char*)leaf); + if(*pjson == NULL) { + ABORT_FINALIZE(RS_RET_NOT_FOUND); + } + +finalize_it: + free(name); + RETiRet; } @@ -2639,16 +2689,15 @@ finalize_it: *pPropLen = sizeof("**OUT OF MEMORY**") - 1; \ return(UCHAR_CONSTANT("**OUT OF MEMORY**"));} uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - propid_t propid, es_str_t *propName, size_t *pPropLen, + propid_t propid, es_str_t *propName, rs_size_t *pPropLen, unsigned short *pbMustBeFreed) { uchar *pRes; /* result pointer */ - int bufLen = -1; /* length of string or -1, if not known */ + rs_size_t bufLen = -1; /* length of string or -1, if not known */ uchar *pBufStart; uchar *pBuf; int iLen; short iOffs; - es_str_t *str; /* for CEE handling, temp. string */ BEGINfunc assert(pMsg != NULL); @@ -2796,16 +2845,15 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = glbl.GetLocalHostName(); break; case PROP_CEE_ALL_JSON: - if(pMsg->event == NULL) { - if(*pbMustBeFreed == 1) - free(pRes); - pRes = (uchar*) "{}"; - *pbMustBeFreed = 0; + if(pMsg->json == NULL) { + if(*pbMustBeFreed == 1) + free(pRes); + pRes = (uchar*) "{}"; + bufLen = 2; + *pbMustBeFreed = 0; } else { - ee_fmtEventToJSON(pMsg->event, &str); - pRes = (uchar*) es_str2cstr(str, "#000"); - es_deleteStr(str); - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + pRes = (uchar*)strdup(json_object_get_string(pMsg->json)); + *pbMustBeFreed = 1; } break; case PROP_CEE: @@ -3456,29 +3504,25 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, es_str_t* msgGetCEEVarNew(msg_t *pMsg, char *name) { + uchar *leaf; + char *val; es_str_t *estr = NULL; - es_str_t *epropName = NULL; - struct ee_field *field; + struct json_object *json, *parent; ISOBJ_TYPE_assert(pMsg, msg); - if(pMsg->event == NULL) { + if(pMsg->json == NULL) { estr = es_newStr(1); goto done; } - - epropName = es_newStrFromCStr(name, strlen(name)); // TODO: optimize (in grammar!) - field = ee_getEventField(pMsg->event, epropName); - if(field != NULL) { - ee_getFieldAsString(field, &estr); - } - if(estr == NULL) { - DBGPRINTF("msgGetCEEVar: error obtaining var (field=%p, var='%s')\n", - field, name); - estr = es_newStrFromCStr("*ERROR*", sizeof("*ERROR*") - 1); + leaf = jsonPathGetLeaf((uchar*)name, strlen(name)); + if(jsonPathFindParent(pMsg, (uchar*)name, leaf, &parent, 1) != RS_RET_OK) { + estr = es_newStr(1); + goto done; } - es_deleteStr(epropName); - + json = json_object_object_get(parent, (char*)leaf); + val = (char*)json_object_get_string(json); + estr = es_newStrFromCStr(val, strlen(val)); done: return estr; } @@ -3489,7 +3533,7 @@ done: es_str_t* msgGetMsgVarNew(msg_t *pThis, uchar *name) { - size_t propLen; + rs_size_t propLen; uchar *pszProp = NULL; propid_t propid; unsigned short bMustBeFreed = 0; @@ -3522,6 +3566,8 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) prop_t *myProp; prop_t *propRcvFrom = NULL; prop_t *propRcvFromIP = NULL; + struct json_tokener *tokener; + struct json_object *json; DEFiRet; ISOBJ_TYPE_assert(pThis, msg); @@ -3576,6 +3622,12 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) MsgSetRulesetByName(pThis, pProp->val.pStr); } else if(isProp("pszMSG")) { dbgprintf("no longer supported property pszMSG silently ignored\n"); + } else if(isProp("json")) { + tokener = json_tokener_new(); + json = json_tokener_parse_ex(tokener, (char*)rsCStrGetSzStrNoNULL(pProp->val.pStr), + cstrLen(pProp->val.pStr)); + json_tokener_free(tokener); + msgAddJSON(pThis, (uchar*)"!", json); } else { dbgprintf("unknown supported property '%s' silently ignored\n", rsCStrGetSzStrNoNULL(pProp->pcsName)); @@ -3613,6 +3665,285 @@ MsgGetSeverity(obj_t_ptr pThis, int *piSeverity) } +static uchar * +jsonPathGetLeaf(uchar *name, int lenName) +{ + int i; + for(i = lenName ; name[i] != '!' && i >= 0 ; --i) + /* just skip */; + if(name[i] == '!') + ++i; + return name + i; +} + + +static rsRetVal +jsonPathFindNext(struct json_object *root, uchar **name, uchar *leaf, + struct json_object **found, int bCreate) +{ + uchar namebuf[1024]; + struct json_object *json; + size_t i; + uchar *p = *name; + DEFiRet; + + if(*p == '!') + ++p; + for(i = 0 ; *p && *p != '!' && p != leaf && i < sizeof(namebuf)-1 ; ++i, ++p) + namebuf[i] = *p; + if(i > 0) { + namebuf[i] = '\0'; + dbgprintf("AAAA: next JSONPath elt: '%s'\n", namebuf); + json = json_object_object_get(root, (char*)namebuf); + } else + json = root; + if(json == NULL) { + if(!bCreate) { + ABORT_FINALIZE(RS_RET_JNAME_INVALID); + } else { + json = json_object_new_object(); + json_object_object_add(root, (char*)namebuf, json); + } + } + + *name = p; + *found = json; +finalize_it: + RETiRet; +} + +static rsRetVal +jsonPathFindParent(msg_t *pM, uchar *name, uchar *leaf, struct json_object **parent, int bCreate) +{ + DEFiRet; + *parent = pM->json; + while(name < leaf-1) { + jsonPathFindNext(*parent, &name, leaf, parent, bCreate); + } + RETiRet; +} + +static rsRetVal +jsonMerge(struct json_object *existing, struct json_object *json) +{ + /* TODO: check & handle duplicate names */ + DEFiRet; + struct json_object_iter it; + + json_object_object_foreachC(json, it) { +DBGPRINTF("AAAA jsonMerge adds '%s'\n", it.key); + json_object_object_add(existing, it.key, + json_object_get(it.val)); + } + /* note: json-c does ref counting. We added all descandants refcounts + * in the loop above. So when we now free(_put) the root object, only + * root gets freed(). + */ + json_object_put(json); + RETiRet; +} + +/* find a JSON structure element (field or container doesn't matter). */ +rsRetVal +jsonFind(msg_t *pM, es_str_t *propName, struct json_object **jsonres) +{ + uchar *name = NULL; + uchar *leaf; + struct json_object *parent; + struct json_object *field; + DEFiRet; + + if(pM->json == NULL) { + field = NULL; + goto finalize_it; + } + + if(!es_strbufcmp(propName, (uchar*)"!", 1)) { + field = pM->json; + } else { + name = (uchar*)es_str2cstr(propName, NULL); + leaf = jsonPathGetLeaf(name, ustrlen(name)); + CHKiRet(jsonPathFindParent(pM, name, leaf, &parent, 0)); + field = json_object_object_get(parent, (char*)leaf); + } + *jsonres = field; + +finalize_it: + free(name); + RETiRet; +} + +rsRetVal +msgAddJSON(msg_t *pM, uchar *name, struct json_object *json) +{ + /* TODO: error checks! This is a quick&dirty PoC! */ + struct json_object *parent, *leafnode; + uchar *leaf; + DEFiRet; + + MsgLock(pM); + if(name[0] == '!' && name[1] == '\0') { + if(pM->json == NULL) + pM->json = json; + else + CHKiRet(jsonMerge(pM->json, json)); + } else { + if(pM->json == NULL) { + /* now we need a root obj */ + pM->json = json_object_new_object(); + } + leaf = jsonPathGetLeaf(name, ustrlen(name)); + CHKiRet(jsonPathFindParent(pM, name, leaf, &parent, 1)); + leafnode = json_object_object_get(parent, (char*)leaf); + if(leafnode == NULL) { + json_object_object_add(parent, (char*)leaf, json); + } else { + if(json_object_get_type(json) == json_type_object) { + CHKiRet(jsonMerge(pM->json, json)); + } else { +//dbgprintf("AAAA: leafnode already exists, type is %d, update with %d\n", (int)json_object_get_type(leafnode), (int)json_object_get_type(json)); + /* TODO: improve the code below, however, the current + * state is not really bad */ + if(json_object_get_type(leafnode) == json_type_object) { + DBGPRINTF("msgAddJSON: trying to update a container " + "node with a leaf, name is '%s' - " + "forbidden\n", name); + json_object_put(json); + ABORT_FINALIZE(RS_RET_INVLD_SETOP); + } + /* json-c code indicates we can simply replace a + * json type. Unfortunaltely, this is not documented + * as part of the interface spec. We still use it, + * because it speeds up processing. If it does not work + * at some point, use + * json_object_object_del(parent, (char*)leaf); + * before adding. rgerhards, 2012-09-17 + */ + json_object_object_add(parent, (char*)leaf, json); + } + } + } + +finalize_it: + MsgUnlock(pM); + RETiRet; +} + +rsRetVal +msgDelJSON(msg_t *pM, uchar *name) +{ + struct json_object *parent, *leafnode; + uchar *leaf; + DEFiRet; + +dbgprintf("AAAA: unset variable '%s'\n", name); + MsgLock(pM); + if(name[0] == '!' && name[1] == '\0') { + /* strange, but I think we should permit this. After all, + * we trust rsyslog.conf to be written by the admin. + */ + DBGPRINTF("unsetting JSON root object\n"); + json_object_put(pM->json); + pM->json = NULL; + } else { + if(pM->json == NULL) { + /* now we need a root obj */ + pM->json = json_object_new_object(); + } + leaf = jsonPathGetLeaf(name, ustrlen(name)); + CHKiRet(jsonPathFindParent(pM, name, leaf, &parent, 1)); + leafnode = json_object_object_get(parent, (char*)leaf); +DBGPRINTF("AAAA: unset found JSON value path '%s', " "leaf '%s', leafnode %p\n", name, leaf, leafnode); + if(leafnode == NULL) { + DBGPRINTF("unset JSON: could not find '%s'\n", name); + ABORT_FINALIZE(RS_RET_JNAME_NOTFOUND); + } else { + DBGPRINTF("deleting JSON value path '%s', " + "leaf '%s', type %d\n", + name, leaf, json_object_get_type(leafnode)); + json_object_object_del(parent, (char*)leaf); + } + } + +finalize_it: + MsgUnlock(pM); + RETiRet; +} + +static struct json_object * +jsonDeepCopy(struct json_object *src) +{ + struct json_object *dst = NULL, *json; + struct json_object_iter it; + int arrayLen, i; + + if(src == NULL) goto done; + + switch(json_object_get_type(src)) { + case json_type_boolean: + dst = json_object_new_boolean(json_object_get_boolean(src)); + break; + case json_type_double: + dst = json_object_new_double(json_object_get_double(src)); + break; + case json_type_int: + dst = json_object_new_int(json_object_get_int(src)); + break; + case json_type_string: + dst = json_object_new_string(json_object_get_string(src)); + break; + case json_type_object: + dst = json_object_new_object(); + json_object_object_foreachC(src, it) { + json = jsonDeepCopy(it.val); + json_object_object_add(dst, it.key, json); + } + break; + case json_type_array: + arrayLen = json_object_array_length(src); + dst = json_object_new_array(); + for(i = 0 ; i < arrayLen ; ++i) { + json = json_object_array_get_idx(src, i); + json = jsonDeepCopy(json); + json_object_array_add(dst, json); + } + break; + default:DBGPRINTF("jsonDeepCopy(): error unknown type %d\n", + json_object_get_type(src)); + dst = NULL; + break; + } +done: return dst; +} + + +rsRetVal +msgSetJSONFromVar(msg_t *pMsg, uchar *varname, struct var *var) +{ + struct json_object *json = NULL; + char *cstr; + DEFiRet; + switch(var->datatype) { + case 'S':/* string */ + cstr = es_str2cstr(var->d.estr, NULL); + json = json_object_new_string(cstr); + free(cstr); + break; + case 'N':/* number (integer) */ + json = json_object_new_int((int) var->d.n); + break; + case 'J':/* native JSON */ + json = jsonDeepCopy(var->d.json); + break; + default:DBGPRINTF("msgSetJSONFromVar: unsupported datatype %c\n", + var->datatype); + ABORT_FINALIZE(RS_RET_ERR); + } + msgAddJSON(pMsg, varname+1, json); +finalize_it: + RETiRet; +} + /* dummy */ rsRetVal msgQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } diff --git a/runtime/msg.h b/runtime/msg.h index f6b54a77..396e861f 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -3,7 +3,7 @@ * * File begun on 2007-07-13 by RGerhards (extracted from syslogd.c) * - * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2012 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -30,6 +30,7 @@ #include <pthread.h> #include <libestr.h> +#include <json/json.h> #include "obj.h" #include "syslogd-types.h" #include "template.h" @@ -109,7 +110,7 @@ struct msg { it obviously is solved in way or another...). */ struct syslogTime tRcvdAt;/* time the message entered this program */ struct syslogTime tTIMESTAMP;/* (parsed) value of the timestamp */ - struct ee_event *event; /**< libee event */ + struct json_object *json; /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ uchar szHOSTNAME[CONF_HOSTNAME_BUFSIZE]; @@ -172,7 +173,7 @@ void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG); uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, propid_t propid, es_str_t *propName, - size_t *pPropLen, unsigned short *pbMustBeFreed); + rs_size_t *pPropLen, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); es_str_t* msgGetMsgVarNew(msg_t *pThis, uchar *name); @@ -184,6 +185,8 @@ char *getPRI(msg_t *pMsg); void getRawMsg(msg_t *pM, uchar **pBuf, int *piLen); rsRetVal msgGetCEEVar(msg_t *pThis, cstr_t *propName, var_t **ppVar); es_str_t* msgGetCEEVarNew(msg_t *pMsg, char *name); +rsRetVal msgAddJSON(msg_t *pM, uchar *name, struct json_object *json); +rsRetVal getCEEPropVal(msg_t *pM, es_str_t *propName, uchar **pRes, rs_size_t *buflen, unsigned short *pbMustBeFreed); /* TODO: remove these five (so far used in action.c) */ uchar *getMSG(msg_t *pM); @@ -199,6 +202,15 @@ int getProgramNameLen(msg_t *pM, sbool bLockMutex); uchar *getRcvFrom(msg_t *pM); rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID); uchar *propIDToName(propid_t propID); +rsRetVal msgGetCEEPropJSON(msg_t *pM, es_str_t *propName, struct json_object **pjson); +rsRetVal msgSetJSONFromVar(msg_t *pMsg, uchar *varname, struct var *var); +rsRetVal msgDelJSON(msg_t *pMsg, uchar *varname); +rsRetVal jsonFind(msg_t *pM, es_str_t *propName, struct json_object **jsonres); + +static inline rsRetVal +msgUnsetJSON(msg_t *pMsg, uchar *varname) { + return msgDelJSON(pMsg, varname+1); +} /* The MsgPrepareEnqueue() function is a macro for performance reasons. diff --git a/runtime/obj.c b/runtime/obj.c index eb151b67..3ecf9ab2 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -665,7 +665,7 @@ static rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) if(c != '\n') ABORT_FINALIZE(RS_RET_INVALID_PROPFRAME); finalize_it: - if(Debug && iRet != RS_RET_OK) { + if(Debug && iRet != RS_RET_OK && iRet != RS_RET_NO_PROPLINE) { strm.GetCurrOffset(pStrm, &offs); dbgprintf("error %d deserializing property name, offset %lld, step %d\n", iRet, offs, step); diff --git a/runtime/queue.c b/runtime/queue.c index bb9ea060..0cd33701 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -232,12 +232,16 @@ getQueueTypeName(queueType_t t) switch(t) { case QUEUETYPE_FIXED_ARRAY: r = "FixedArray"; + break; case QUEUETYPE_LINKEDLIST: r = "LinkedList"; + break; case QUEUETYPE_DISK: r = "Disk"; + break; case QUEUETYPE_DIRECT: r = "Direct"; + break; } return r; } @@ -976,6 +980,7 @@ static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) { batch_t singleBatch; batch_obj_t batchObj; + sbool active = 1; int i; DEFiRet; @@ -994,9 +999,9 @@ static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) memset(&singleBatch, 0, sizeof(batch_t)); batchObj.state = BATCH_STATE_RDY; batchObj.pUsrp = (obj_t*) pUsr; - batchObj.bFilterOK = 1; singleBatch.nElem = 1; /* there always is only one in direct mode */ singleBatch.pElem = &batchObj; + singleBatch.active = &active; iRet = pThis->pConsumer(pThis->pUsr, &singleBatch, &pThis->bShutdownImmediate); /* delete the batch string params: TODO: create its own "class" for this */ for(i = 0 ; i < CONF_OMOD_NUMSTRINGS_MAXSIZE ; ++i) { @@ -1596,7 +1601,6 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz /* all well, use this element */ pWti->batch.pElem[nDequeued].pUsrp = pUsr; pWti->batch.pElem[nDequeued].state = BATCH_STATE_RDY; - pWti->batch.pElem[nDequeued].bFilterOK = 1; // TODO: think again if we can handle that with more performance ++nDequeued; } @@ -2075,6 +2079,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ pThis->pqParent == NULL ? 0 : 1, pThis->iFullDlyMrk, pThis->iLightDlyMrk, pThis->iDeqBatchSize); + pThis->bQueueStarted = 1; if(pThis->qType == QUEUETYPE_DIRECT) FINALIZE; /* with direct queues, we are already finished... */ @@ -2105,7 +2110,6 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ * the case when a disk queue has been loaded. If we did not start it here, it would never start. */ qqueueAdviseMaxWorkers(pThis); - pThis->bQueueStarted = 1; /* support statistics gathering */ qName = obj.GetName((obj_t*)pThis); @@ -2303,73 +2307,75 @@ DoSaveOnShutdown(qqueue_t *pThis) /* destructor for the queue object */ BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(qqueue) - /* shut down all workers - * We do not need to shutdown workers when we are in enqueue-only mode or we are a - * direct queue - because in both cases we have none... ;) - * with a child! -- rgerhards, 2008-01-28 - */ - if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL - && pThis->pWtpReg != NULL) - ShutdownWorkers(pThis); + if(pThis->bQueueStarted) { + /* shut down all workers + * We do not need to shutdown workers when we are in enqueue-only mode or we are a + * direct queue - because in both cases we have none... ;) + * with a child! -- rgerhards, 2008-01-28 + */ + if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL + && pThis->pWtpReg != NULL) + ShutdownWorkers(pThis); - if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { - CHKiRet(DoSaveOnShutdown(pThis)); - } + if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { + CHKiRet(DoSaveOnShutdown(pThis)); + } - /* finally destruct our (regular) worker thread pool - * Note: currently pWtpReg is never NULL, but if we optimize our logic, this may happen, - * e.g. when they are not created in enqueue-only mode. We already check the condition - * as this may otherwise be very hard to find once we optimize (and have long forgotten - * about this condition here ;) - * rgerhards, 2008-01-25 - */ - if(pThis->qType != QUEUETYPE_DIRECT && pThis->pWtpReg != NULL) { - wtpDestruct(&pThis->pWtpReg); - } + /* finally destruct our (regular) worker thread pool + * Note: currently pWtpReg is never NULL, but if we optimize our logic, this may happen, + * e.g. when they are not created in enqueue-only mode. We already check the condition + * as this may otherwise be very hard to find once we optimize (and have long forgotten + * about this condition here ;) + * rgerhards, 2008-01-25 + */ + if(pThis->qType != QUEUETYPE_DIRECT && pThis->pWtpReg != NULL) { + wtpDestruct(&pThis->pWtpReg); + } - /* Now check if we actually have a DA queue and, if so, destruct it. - * Note that the wtp must be destructed first, it may be in cancel cleanup handler - * *right now* and actually *need* to access the queue object to persist some final - * data (re-queueing case). So we need to destruct the wtp first, which will make - * sure all workers have terminated. Please note that this also generates a situation - * where it is possible that the DA queue has a parent pointer but the parent has - * no WtpDA associated with it - which is perfectly legal thanks to this code here. - */ - if(pThis->pWtpDA != NULL) { - wtpDestruct(&pThis->pWtpDA); - } - if(pThis->pqDA != NULL) { - qqueueDestruct(&pThis->pqDA); - } + /* Now check if we actually have a DA queue and, if so, destruct it. + * Note that the wtp must be destructed first, it may be in cancel cleanup handler + * *right now* and actually *need* to access the queue object to persist some final + * data (re-queueing case). So we need to destruct the wtp first, which will make + * sure all workers have terminated. Please note that this also generates a situation + * where it is possible that the DA queue has a parent pointer but the parent has + * no WtpDA associated with it - which is perfectly legal thanks to this code here. + */ + if(pThis->pWtpDA != NULL) { + wtpDestruct(&pThis->pWtpDA); + } + if(pThis->pqDA != NULL) { + qqueueDestruct(&pThis->pqDA); + } - /* persist the queue (we always do that - queuePersits() does cleanup if the queue is empty) - * This handler is most important for disk queues, it will finally persist the necessary - * on-disk structures. In theory, other queueing modes may implement their other (non-DA) - * methods of persisting a queue between runs, but in practice all of this is done via - * disk queues and DA mode. Anyhow, it doesn't hurt to know that we could extend it here - * if need arises (what I doubt...) -- rgerhards, 2008-01-25 - */ - CHKiRet_Hdlr(qqueuePersist(pThis, QUEUE_NO_CHECKPOINT)) { - DBGOPRINT((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet); - } + /* persist the queue (we always do that - queuePersits() does cleanup if the queue is empty) + * This handler is most important for disk queues, it will finally persist the necessary + * on-disk structures. In theory, other queueing modes may implement their other (non-DA) + * methods of persisting a queue between runs, but in practice all of this is done via + * disk queues and DA mode. Anyhow, it doesn't hurt to know that we could extend it here + * if need arises (what I doubt...) -- rgerhards, 2008-01-25 + */ + CHKiRet_Hdlr(qqueuePersist(pThis, QUEUE_NO_CHECKPOINT)) { + DBGOPRINT((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet); + } - /* finally, clean up some simple things... */ - if(pThis->pqParent == NULL) { - /* if we are not a child, we allocated our own mutex, which we now need to destroy */ - pthread_mutex_destroy(pThis->mut); - free(pThis->mut); - } - pthread_mutex_destroy(&pThis->mutThrdMgmt); - pthread_cond_destroy(&pThis->notFull); - pthread_cond_destroy(&pThis->notEmpty); - pthread_cond_destroy(&pThis->belowFullDlyWtrMrk); - pthread_cond_destroy(&pThis->belowLightDlyWtrMrk); + /* finally, clean up some simple things... */ + if(pThis->pqParent == NULL) { + /* if we are not a child, we allocated our own mutex, which we now need to destroy */ + pthread_mutex_destroy(pThis->mut); + free(pThis->mut); + } + pthread_mutex_destroy(&pThis->mutThrdMgmt); + pthread_cond_destroy(&pThis->notFull); + pthread_cond_destroy(&pThis->notEmpty); + pthread_cond_destroy(&pThis->belowFullDlyWtrMrk); + pthread_cond_destroy(&pThis->belowLightDlyWtrMrk); - DESTROY_ATOMIC_HELPER_MUT(pThis->mutQueueSize); - DESTROY_ATOMIC_HELPER_MUT(pThis->mutLogDeq); + DESTROY_ATOMIC_HELPER_MUT(pThis->mutQueueSize); + DESTROY_ATOMIC_HELPER_MUT(pThis->mutLogDeq); - /* type-specific destructor */ - iRet = pThis->qDestruct(pThis); + /* type-specific destructor */ + iRet = pThis->qDestruct(pThis); + } free(pThis->pszFilePrefix); free(pThis->pszSpoolDir); diff --git a/runtime/rsconf.c b/runtime/rsconf.c index 118e9c11..b45746c4 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -36,7 +36,6 @@ #include "rsyslog.h" #include "obj.h" #include "srUtils.h" -#include "rule.h" #include "ruleset.h" #include "modules.h" #include "conf.h" @@ -70,7 +69,6 @@ /* static data */ DEFobjStaticHelpers -DEFobjCurrIf(rule) DEFobjCurrIf(ruleset) DEFobjCurrIf(module) DEFobjCurrIf(conf) @@ -254,54 +252,6 @@ CODESTARTobjDebugPrint(rsconf) ENDobjDebugPrint(rsconf) -rsRetVal -cnfDoActlst(struct cnfactlst *actlst, rule_t *pRule) -{ - struct cnfcfsyslinelst *cflst; - action_t *pAction; - uchar *str; - rsRetVal localRet; - DEFiRet; - - while(actlst != NULL) { - dbgprintf("aclst %p: ", actlst); - if(actlst->actType == CNFACT_V2) { - dbgprintf("v6+ action object\n"); - if(actionNewInst(actlst->data.lst, &pAction) == RS_RET_OK) { - iRet = llAppend(&(pRule)->llActList, NULL, (void*) pAction); - } else { - errmsg.LogError(0, RS_RET_ERR, "errors occured in file '%s' " - "around line %d", actlst->cnfFile, actlst->lineno); - } - } else { - DBGPRINTF("legacy action line:%s\n", actlst->data.legActLine); - str = (uchar*) actlst->data.legActLine; - if((localRet = cflineDoAction(loadConf, &str, &pAction)) != RS_RET_OK) { - uchar szErrLoc[MAXFNAME + 64]; - if(localRet != RS_RET_OK_WARN) { - DBGPRINTF("legacy action line NOT successfully processed\n"); - } - snprintf((char*)szErrLoc, sizeof(szErrLoc) / sizeof(uchar), - "%s, line %d", actlst->cnfFile, actlst->lineno); - errmsg.LogError(0, NO_ERRCODE, "the last %s occured in %s:\"%s\"", - (localRet == RS_RET_OK_WARN) ? "warning" : "error", - (char*)szErrLoc, (char*)actlst->data.legActLine); - if(localRet != RS_RET_OK_WARN) { - ABORT_FINALIZE(localRet); - } - } - iRet = llAppend(&(pRule)->llActList, NULL, (void*) pAction); - } - for( cflst = actlst->syslines - ; cflst != NULL ; cflst = cflst->next) { - cnfDoCfsysline(cflst->line); - } - actlst = actlst->next; - } -finalize_it: - RETiRet; -} - /* This function returns the current date in different * variants. It is used to construct the $NOW series of * system properties. The returned buffer must be freed @@ -435,9 +385,6 @@ parser_errmsg(char *fmt, ...) va_start(ap, fmt); if(vsnprintf(errBuf, sizeof(errBuf), fmt, ap) == sizeof(errBuf)) errBuf[sizeof(errBuf)-1] = '\0'; -dbgprintf("XXXX: msg: %s\n", errBuf); -dbgprintf("XXXX: cnfcurrfn: %s\n", cnfcurrfn); -dbgprintf("XXXX: yylineno: %d\n", yylineno); errmsg.LogError(0, RS_RET_CONF_PARSE_ERROR, "error during parsing file %s, on or before line %d: %s", cnfcurrfn, yylineno, errBuf); @@ -463,9 +410,6 @@ void cnfDoObj(struct cnfobj *o) case CNFOBJ_MODULE: modulesProcessCnf(o); break; - case CNFOBJ_ACTION: - actionProcessCnf(o); - break; case CNFOBJ_INPUT: inputProcessCnf(o); break; @@ -477,61 +421,20 @@ void cnfDoObj(struct cnfobj *o) /* these types are processed at a later stage */ bChkUnuse = 0; break; + default: + dbgprintf("cnfDoObj program error: unexpected object type %u\n", + o->objType); + break; } if(bChkUnuse) nvlstChkUnused(o->nvlst); cnfobjDestruct(o); } -void cnfDoRule(struct cnfrule *cnfrule) +void cnfDoScript(struct cnfstmt *script) { - rule_t *pRule; - uchar *str; - rsRetVal iRet = RS_RET_OK; //DEFiRet; - - dbgprintf("cnf:global:rule\n"); - cnfrulePrint(cnfrule); - - CHKiRet(rule.Construct(&pRule)); /* create "fresh" selector */ - CHKiRet(rule.SetAssRuleset(pRule, ruleset.GetCurrent(loadConf))); - CHKiRet(rule.ConstructFinalize(pRule)); - - switch(cnfrule->filttype) { - case CNFFILT_NONE: - break; - case CNFFILT_PRI: - str = (uchar*) cnfrule->filt.s; - iRet = cflineProcessTradPRIFilter(&str, pRule); - break; - case CNFFILT_PROP: - dbgprintf("%s\n", cnfrule->filt.s); - str = (uchar*) cnfrule->filt.s; - iRet = cflineProcessPropFilter(&str, pRule); - break; - case CNFFILT_SCRIPT: - pRule->f_filter_type = FILTER_EXPR; - pRule->f_filterData.expr = cnfrule->filt.expr; - break; - } - /* we now check if there are some global (BSD-style) filter conditions - * and, if so, we copy them over. rgerhards, 2005-10-18 - */ - if(pDfltProgNameCmp != NULL) { - CHKiRet(rsCStrConstructFromCStr(&(pRule->pCSProgNameComp), pDfltProgNameCmp)); - } - - if(eDfltHostnameCmpMode != HN_NO_COMP) { - pRule->eHostnameCmpMode = eDfltHostnameCmpMode; - CHKiRet(rsCStrConstructFromCStr(&(pRule->pCSHostnameComp), pDfltHostnameCmp)); - } - - cnfDoActlst(cnfrule->actlst, pRule); - - CHKiRet(ruleset.AddRule(rule.GetAssRuleset(pRule), &pRule)); - -finalize_it: - //TODO: do something with error states - cnfruleDestruct(cnfrule); + dbgprintf("cnf:global:script\n"); + ruleset.AddScript(ruleset.GetCurrent(loadConf), script); } void cnfDoCfsysline(char *ln) @@ -545,13 +448,21 @@ void cnfDoCfsysline(char *ln) void cnfDoBSDTag(char *ln) { DBGPRINTF("cnf:global:BSD tag: %s\n", ln); - cflineProcessTagSelector((uchar**)&ln); + errmsg.LogError(0, RS_RET_BSD_BLOCKS_UNSUPPORTED, + "BSD-style blocks are no longer supported in rsyslog, " + "see http://www.rsyslog.com/g/BSD for details and a " + "solution (Block '%s')", ln); + free(ln); } void cnfDoBSDHost(char *ln) { DBGPRINTF("cnf:global:BSD host: %s\n", ln); - cflineProcessHostSelector((uchar**)&ln); + errmsg.LogError(0, RS_RET_BSD_BLOCKS_UNSUPPORTED, + "BSD-style blocks are no longer supported in rsyslog, " + "see http://www.rsyslog.com/g/BSD for details and a " + "solution (Block '%s')", ln); + free(ln); } es_str_t* @@ -1378,6 +1289,7 @@ ourConf = loadConf; // TODO: remove, once ourConf is gone! ABORT_FINALIZE(RS_RET_NO_ACTIONS); } tellLexEndParsing(); + rulesetOptimizeAll(loadConf); tellCoreConfigLoadDone(); tellModulesConfigLoadDone(); @@ -1436,7 +1348,6 @@ ENDobjQueryInterface(rsconf) BEGINObjClassInit(rsconf, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(ruleset, CORE_COMPONENT)); - CHKiRet(objUse(rule, CORE_COMPONENT)); CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(conf, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); @@ -1453,7 +1364,6 @@ ENDObjClassInit(rsconf) /* De-initialize the rsconf class. */ BEGINObjClassExit(rsconf, OBJ_IS_CORE_MODULE) /* class, version */ - objRelease(rule, CORE_COMPONENT); objRelease(ruleset, CORE_COMPONENT); objRelease(module, CORE_COMPONENT); objRelease(conf, CORE_COMPONENT); diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index cbab06b7..047dfa9b 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -72,7 +72,6 @@ #include "glbl.h" #include "errmsg.h" #include "prop.h" -#include "rule.h" #include "ruleset.h" #include "parser.h" #include "strgen.h" @@ -171,8 +170,6 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(glblClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "rule"; - CHKiRet(ruleClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "ruleset"; CHKiRet(rulesetClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "wti"; @@ -220,7 +217,6 @@ rsrtExit(void) confClassExit(); glblClassExit(); rulesetClassExit(); - ruleClassExit(); objClassExit(); /* *THIS* *MUST/SHOULD?* always be the first class initilizer being called (except debug)! */ } diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index a6e4b100..336ab5bf 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -383,8 +383,17 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_NO_JSON_PASSING = -2223, /**< rsyslog core does not support JSON-passing plugin API */ RS_RET_MOD_NO_INPUT_STMT = -2224, /**< (input) module does not support input() statement */ + /**** up to 2300 is reserved for v6 use ****/ + RS_RET_JNAME_NO_ROOT = -2301, /**< root element is missing in JSON path */ + RS_RET_JNAME_INVALID = -2302, /**< JSON path is invalid */ + RS_RET_JSON_PARSE_ERR = -2303, /**< we had a problem parsing JSON (or extra data) */ + RS_RET_BSD_BLOCKS_UNSUPPORTED = -2304, /**< BSD-style config blocks are no longer supported */ + RS_RET_JNAME_NOTFOUND = -2305, /**< JSON name not found (does not exist) */ + RS_RET_INVLD_SETOP = -2305, /**< invalid variable set operation, incompatible type */ + /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ + RS_RET_FIELD_NOT_FOUND = 1002, /**< field() function did not find requested field */ /* some generic error/status codes */ RS_RET_OK = 0, /**< operation successful */ diff --git a/runtime/rule.c b/runtime/rule.c deleted file mode 100644 index fc1e740f..00000000 --- a/runtime/rule.c +++ /dev/null @@ -1,479 +0,0 @@ -/* rule.c - rsyslog's rule object - * - * See file comment in rule.c for the overall structure of rule processing. - * - * Module begun 2009-06-10 by Rainer Gerhards - * - * Copyright 2009-2012 Adiscon GmbH. - * - * This file is part of the rsyslog runtime library. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * -or- - * see COPYING.ASL20 in the source distribution - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "config.h" -#include <stdlib.h> -#include <string.h> -#include <assert.h> -#include <ctype.h> - -#include "rsyslog.h" -#include "obj.h" -#include "action.h" -#include "rule.h" -#include "errmsg.h" -#include "srUtils.h" -#include "batch.h" -#include "parserif.h" -#include "unicode-helper.h" - -/* static data */ -DEFobjStaticHelpers -DEFobjCurrIf(errmsg) - - -/* support for simple textual representation of FIOP names - * rgerhards, 2005-09-27 - */ -static char* -getFIOPName(unsigned iFIOP) -{ - char *pRet; - switch(iFIOP) { - case FIOP_CONTAINS: - pRet = "contains"; - break; - case FIOP_ISEQUAL: - pRet = "isequal"; - break; - case FIOP_STARTSWITH: - pRet = "startswith"; - break; - case FIOP_REGEX: - pRet = "regex"; - break; - case FIOP_EREREGEX: - pRet = "ereregex"; - break; - case FIOP_ISEMPTY: - pRet = "isempty"; - break; - default: - pRet = "NOP"; - break; - } - return pRet; -} - - -/* iterate over all actions, this is often needed, for example when HUP processing - * must be done or a shutdown is pending. - */ -static rsRetVal -iterateAllActions(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) -{ - return llExecFunc(&pThis->llActList, pFunc, pParam); -} - - -/* helper to processMsg(), used to call the configured actions. It is - * executed from within llExecFunc() of the action list. - * rgerhards, 2007-08-02 - */ -DEFFUNC_llExecFunc(processBatchDoActions) -{ - DEFiRet; - rsRetVal iRetMod; /* return value of module - we do not always pass that back */ - action_t *pAction = (action_t*) pData; - batch_t *pBatch = (batch_t*) pParam; - - DBGPRINTF("Processing next action\n"); - iRetMod = pAction->submitToActQ(pAction, pBatch); - - RETiRet; -} - - -/* This functions looks at the given message and checks if it matches the - * provided filter condition. - */ -static rsRetVal -shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, sbool *bProcessMsg) -{ - DEFiRet; - unsigned short pbMustBeFreed; - uchar *pszPropVal; - int bRet = 0; - size_t propLen; - - ISOBJ_TYPE_assert(pRule, rule); - assert(pMsg != NULL); - - /* we first have a look at the global, BSD-style block filters (for tag - * and host). Only if they match, we evaluate the actual filter. - * rgerhards, 2005-10-18 - */ - if(pRule->eHostnameCmpMode == HN_NO_COMP) { - /* EMPTY BY INTENSION - we check this value first, because - * it is the one most often used, so this saves us time! - */ - } else if(pRule->eHostnameCmpMode == HN_COMP_MATCH) { - if(rsCStrSzStrCmp(pRule->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, so we are already done... */ - DBGPRINTF("hostname filter '+%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(pRule->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } else { /* must be -hostname */ - if(!rsCStrSzStrCmp(pRule->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, SO WE ARe already done... */ - DBGPRINTF("hostname filter '-%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(pRule->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } - - if(pRule->pCSProgNameComp != NULL) { - int bInv = 0, bEqv = 0, offset = 0; - if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp)) == '-') { - if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp) + 1) == '-') - offset = 1; - else { - bInv = 1; - offset = 1; - } - } - if(!rsCStrOffsetSzStrCmp(pRule->pCSProgNameComp, offset, - (uchar*) getProgramName(pMsg, LOCK_MUTEX), getProgramNameLen(pMsg, LOCK_MUTEX))) - bEqv = 1; - - if((!bEqv && !bInv) || (bEqv && bInv)) { - /* not equal or inverted selection, so we are already done... */ - DBGPRINTF("programname filter '%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp), getProgramName(pMsg, LOCK_MUTEX)); - FINALIZE; - } - } - - /* done with the BSD-style block filters */ - - if(pRule->f_filter_type == FILTER_PRI) { - /* skip messages that are incorrect priority */ - if ( (pRule->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ - ((pRule->f_filterData.f_pmask[pMsg->iFacility] & (1<<pMsg->iSeverity)) == 0) ) - bRet = 0; - else - bRet = 1; - DBGPRINTF("testing filter, f_pmask %d, result %d\n", pRule->f_filterData.f_pmask[pMsg->iFacility], bRet); - } else if(pRule->f_filter_type == FILTER_EXPR) { - bRet = cnfexprEvalBool(pRule->f_filterData.expr, pMsg); - DBGPRINTF("result of rainerscript filter evaluation: %d\n", bRet); - } else { - assert(pRule->f_filter_type == FILTER_PROP); /* assert() just in case... */ - if(pRule->f_filterData.prop.propID == PROP_INVALID) { - DBGPRINTF("invalid property ID, filter always returns 0\n"); - bRet = 0; - } else { - pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.propID, - pRule->f_filterData.prop.propName, &propLen, &pbMustBeFreed); - - /* Now do the compares (short list currently ;)) */ - switch(pRule->f_filterData.prop.operation ) { - case FIOP_CONTAINS: - if(rsCStrLocateInSzStr(pRule->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1) - bRet = 1; - break; - case FIOP_ISEMPTY: - if(propLen == 0) - bRet = 1; /* process message! */ - break; - case FIOP_ISEQUAL: - if(rsCStrSzStrCmp(pRule->f_filterData.prop.pCSCompValue, - pszPropVal, ustrlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_STARTSWITH: - if(rsCStrSzStrStartsWithCStr(pRule->f_filterData.prop.pCSCompValue, - pszPropVal, ustrlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_REGEX: - if(rsCStrSzStrMatchRegex(pRule->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 0, &pRule->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - case FIOP_EREREGEX: - if(rsCStrSzStrMatchRegex(pRule->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 1, &pRule->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - default: - /* here, it handles NOP (for performance reasons) */ - assert(pRule->f_filterData.prop.operation == FIOP_NOP); - bRet = 1; /* as good as any other default ;) */ - break; - } - - /* now check if the value must be negated */ - if(pRule->f_filterData.prop.isNegated) - bRet = (bRet == 1) ? 0 : 1; - - if(Debug) { - char *cstr; - if(pRule->f_filterData.prop.propID == PROP_CEE) { - cstr = es_str2cstr(pRule->f_filterData.prop.propName, NULL); - dbgprintf("Filter: check for CEE property '%s' (value '%s') ", - cstr, pszPropVal); - free(cstr); - } else { - dbgprintf("Filter: check for property '%s' (value '%s') ", - propIDToName(pRule->f_filterData.prop.propID), pszPropVal); - } - if(pRule->f_filterData.prop.isNegated) - dbgprintf("NOT "); - if(pRule->f_filterData.prop.operation == FIOP_ISEMPTY) { - dbgprintf("%s : %s\n", - getFIOPName(pRule->f_filterData.prop.operation), - bRet ? "TRUE" : "FALSE"); - } else { - dbgprintf("%s '%s': %s\n", - getFIOPName(pRule->f_filterData.prop.operation), - rsCStrGetSzStrNoNULL(pRule->f_filterData.prop.pCSCompValue), - bRet ? "TRUE" : "FALSE"); - } - } - - /* cleanup */ - if(pbMustBeFreed) - free(pszPropVal); - } - } - -finalize_it: - *bProcessMsg = bRet; - RETiRet; -} - - - -/* Process (consume) a batch of messages. Calls the actions configured. - * rgerhards, 2005-10-13 - */ -static rsRetVal -processBatch(rule_t *pThis, batch_t *pBatch) -{ - int i; - rsRetVal localRet; - DEFiRet; - - ISOBJ_TYPE_assert(pThis, rule); - assert(pBatch != NULL); - - /* first check the filters and reset status variables */ - for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { - localRet = shouldProcessThisMessage(pThis, (msg_t*)(pBatch->pElem[i].pUsrp), - &(pBatch->pElem[i].bFilterOK)); - if(localRet != RS_RET_OK) { - DBGPRINTF("processBatch: iRet %d returned from shouldProcessThisMessage, " - "ignoring message\n", localRet); - pBatch->pElem[i].bFilterOK = 0; - } - if(pBatch->pElem[i].bFilterOK) { - /* re-init only when actually needed (cache write cost!) */ - pBatch->pElem[i].bPrevWasSuspended = 0; - } - } - CHKiRet(llExecFunc(&pThis->llActList, processBatchDoActions, pBatch)); - -finalize_it: - RETiRet; -} - - -/* Standard-Constructor - */ -BEGINobjConstruct(rule) /* be sure to specify the object type also in END macro! */ -ENDobjConstruct(rule) - - -/* ConstructionFinalizer - * rgerhards, 2008-01-09 - */ -static rsRetVal -ruleConstructFinalize(rule_t *pThis) -{ - DEFiRet; - ISOBJ_TYPE_assert(pThis, rule); - - /* note: actionDestruct is from action.c API! */ - CHKiRet(llInit(&pThis->llActList, actionDestruct, NULL, NULL)); - -finalize_it: - RETiRet; -} - - -/* destructor for the rule object */ -BEGINobjDestruct(rule) /* be sure to specify the object type also in END and CODESTART macros! */ -CODESTARTobjDestruct(rule) - if(pThis->pCSHostnameComp != NULL) - rsCStrDestruct(&pThis->pCSHostnameComp); - if(pThis->pCSProgNameComp != NULL) - rsCStrDestruct(&pThis->pCSProgNameComp); - - if(pThis->f_filter_type == FILTER_PROP) { - if(pThis->f_filterData.prop.pCSCompValue != NULL) - rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); - if(pThis->f_filterData.prop.regex_cache != NULL) - rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache); - if(pThis->f_filterData.prop.propName != NULL) - es_deleteStr(pThis->f_filterData.prop.propName); - } else if(pThis->f_filter_type == FILTER_EXPR) { - cnfexprDestruct(pThis->f_filterData.expr); - } - - llDestroy(&pThis->llActList); -ENDobjDestruct(rule) - - -/* set the associated ruleset */ -static rsRetVal -setAssRuleset(rule_t *pThis, ruleset_t *pRuleset) -{ - DEFiRet; - ISOBJ_TYPE_assert(pThis, rule); - ISOBJ_TYPE_assert(pRuleset, ruleset); - pThis->pRuleset = pRuleset; - RETiRet; -} - -/* get the associated ruleset (may be NULL if not set!) */ -static ruleset_t* -getAssRuleset(rule_t *pThis) -{ - ISOBJ_TYPE_assert(pThis, rule); - return pThis->pRuleset; -} - - -/* helper to DebugPrint, to print out all actions via - * the llExecFunc() facility. - */ -DEFFUNC_llExecFunc(dbgPrintInitInfoAction) -{ - DEFiRet; - iRet = actionDbgPrint((action_t*) pData); - dbgprintf("\n"); - RETiRet; -} - - -/* debugprint for the rule object */ -BEGINobjDebugPrint(rule) /* be sure to specify the object type also in END and CODESTART macros! */ - int i; - char *cstr; -CODESTARTobjDebugPrint(rule) - dbgoprint((obj_t*) pThis, "rsyslog rule:\n"); - if(pThis->pCSProgNameComp != NULL) - dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(pThis->pCSProgNameComp)); - if(pThis->eHostnameCmpMode != HN_NO_COMP) - dbgprintf("hostname: %s '%s'\n", - pThis->eHostnameCmpMode == HN_COMP_MATCH ? - "only" : "allbut", - rsCStrGetSzStrNoNULL(pThis->pCSHostnameComp)); - if(pThis->f_filter_type == FILTER_PRI) { - for (i = 0; i <= LOG_NFACILITIES; i++) - if (pThis->f_filterData.f_pmask[i] == TABLE_NOPRI) - dbgprintf(" X "); - else - dbgprintf("%2X ", pThis->f_filterData.f_pmask[i]); - } else if(pThis->f_filter_type == FILTER_EXPR) { - dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); - } else { - dbgprintf("PROPERTY-BASED Filter:\n"); - dbgprintf("\tProperty.: '%s'\n", propIDToName(pThis->f_filterData.prop.propID)); - if(pThis->f_filterData.prop.propID != PROP_INVALID) { - if(pThis->f_filterData.prop.propName != NULL) { - cstr = es_str2cstr(pThis->f_filterData.prop.propName, NULL); - dbgprintf("\tCEE-Prop.: '%s'\n", cstr); - free(cstr); - } - dbgprintf("\tOperation: "); - if(pThis->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("'%s'\n", getFIOPName(pThis->f_filterData.prop.operation)); - dbgprintf("\tValue....: '%s'\n", - rsCStrGetSzStrNoNULL(pThis->f_filterData.prop.pCSCompValue)); - } - dbgprintf("\tAction...: "); - } - - dbgprintf("\nActions:\n"); - llExecFunc(&pThis->llActList, dbgPrintInitInfoAction, NULL); /* actions */ - - dbgprintf("\n"); -ENDobjDebugPrint(rule) - - -/* queryInterface function - * rgerhards, 2008-02-21 - */ -BEGINobjQueryInterface(rule) -CODESTARTobjQueryInterface(rule) - if(pIf->ifVersion != ruleCURR_IF_VERSION) { /* check for current version, increment on each change */ - ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); - } - - /* ok, we have the right interface, so let's fill it - * Please note that we may also do some backwards-compatibility - * work here (if we can support an older interface version - that, - * of course, also affects the "if" above). - */ - pIf->Construct = ruleConstruct; - pIf->ConstructFinalize = ruleConstructFinalize; - pIf->Destruct = ruleDestruct; - pIf->DebugPrint = ruleDebugPrint; - - pIf->IterateAllActions = iterateAllActions; - pIf->ProcessBatch = processBatch; - pIf->SetAssRuleset = setAssRuleset; - pIf->GetAssRuleset = getAssRuleset; -finalize_it: -ENDobjQueryInterface(rule) - - -/* Exit the rule class. - * rgerhards, 2009-04-06 - */ -BEGINObjClassExit(rule, OBJ_IS_CORE_MODULE) /* class, version */ - objRelease(errmsg, CORE_COMPONENT); -ENDObjClassExit(rule) - - -/* Initialize the rule class. Must be called as the very first method - * before anything else is called inside this class. - * rgerhards, 2008-02-19 - */ -BEGINObjClassInit(rule, 1, OBJ_IS_CORE_MODULE) /* class, version */ - /* request objects we use */ - CHKiRet(objUse(errmsg, CORE_COMPONENT)); - - /* set our own handlers */ - OBJSetMethodHandler(objMethod_DEBUGPRINT, ruleDebugPrint); - OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, ruleConstructFinalize); -ENDObjClassInit(rule) - -/* vi:set ai: - */ diff --git a/runtime/rule.h b/runtime/rule.h deleted file mode 100644 index 1b07279b..00000000 --- a/runtime/rule.h +++ /dev/null @@ -1,78 +0,0 @@ -/* The rule object. - * - * This implements rules within rsyslog. - * - * Copyright 2009-2012 Adiscon GmbH. - * - * This file is part of the rsyslog runtime library. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * -or- - * see COPYING.ASL20 in the source distribution - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef INCLUDED_RULE_H -#define INCLUDED_RULE_H - -#include "libestr.h" -#include "linkedlist.h" -#include "regexp.h" -#include "rainerscript.h" - -/* the rule object */ -struct rule_s { - BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ - /* filter properties */ - enum { - FILTER_PRI = 0, /* traditional PRI based filer */ - FILTER_PROP = 1, /* extended filter, property based */ - FILTER_EXPR = 2 /* extended filter, expression based */ - } f_filter_type; - EHostnameCmpMode eHostnameCmpMode; - cstr_t *pCSHostnameComp; /* hostname to check */ - cstr_t *pCSProgNameComp; /* tag to check or NULL, if not to be checked */ - union { - u_char f_pmask[LOG_NFACILITIES+1]; /* priority mask */ - struct { - fiop_t operation; - regex_t *regex_cache; /* cache for compiled REs, if such are used */ - cstr_t *pCSCompValue; /* value to "compare" against */ - sbool isNegated; - propid_t propID; /* ID of the requested property */ - es_str_t *propName; /* name of property for CEE-based filters */ - } prop; - struct cnfexpr *expr; /* expression object */ - } f_filterData; - - ruleset_t *pRuleset; /* associated ruleset */ - linkedList_t llActList; /* list of configured actions */ -}; - -/* interfaces */ -BEGINinterface(rule) /* name must also be changed in ENDinterface macro! */ - INTERFACEObjDebugPrint(rule); - rsRetVal (*Construct)(rule_t **ppThis); - rsRetVal (*ConstructFinalize)(rule_t __attribute__((unused)) *pThis); - rsRetVal (*Destruct)(rule_t **ppThis); - rsRetVal (*IterateAllActions)(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void *pParam); - rsRetVal (*ProcessBatch)(rule_t *pThis, batch_t *pBatch); - rsRetVal (*SetAssRuleset)(rule_t *pThis, ruleset_t*); - ruleset_t* (*GetAssRuleset)(rule_t *pThis); -ENDinterface(rule) -#define ruleCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ -/* change for v2: ProcessMsg replaced by ProcessBatch - 2010-06-10 */ - - -/* prototypes */ -PROTOTYPEObj(rule); - -#endif /* #ifndef INCLUDED_RULE_H */ diff --git a/runtime/ruleset.c b/runtime/ruleset.c index 5cb34148..2af44bc4 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -11,25 +11,23 @@ * * Module begun 2009-06-10 by Rainer Gerhards * - * Copyright 2009-2011 Rainer Gerhards and Adiscon GmbH. + * Copyright 2009-2012 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * - * The rsyslog runtime library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The rsyslog runtime library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the rsyslog runtime library. If not, see <http://www.gnu.org/licenses/>. - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #include "config.h" @@ -42,22 +40,25 @@ #include "cfsysline.h" #include "msg.h" #include "ruleset.h" -#include "rule.h" #include "errmsg.h" #include "parser.h" #include "batch.h" #include "unicode-helper.h" #include "rsconf.h" +#include "action.h" +#include "rainerscript.h" +#include "srUtils.h" +#include "modules.h" #include "dirty.h" /* for main ruleset queue creation */ /* static data */ DEFobjStaticHelpers DEFobjCurrIf(errmsg) -DEFobjCurrIf(rule) DEFobjCurrIf(parser) /* forward definitions */ static rsRetVal processBatch(batch_t *pBatch); +static rsRetVal scriptExec(struct cnfstmt *root, batch_t *pBatch, sbool *active); /* ---------- linked-list key handling functions (ruleset) ---------- */ @@ -73,45 +74,60 @@ rulesetKeyDestruct(void __attribute__((unused)) *pData) /* ---------- END linked-list key handling functions (ruleset) ---------- */ +/* iterate over all actions in a script (stmt subtree) */ +static void +scriptIterateAllActions(struct cnfstmt *root, rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + struct cnfstmt *stmt; + for(stmt = root ; stmt != NULL ; stmt = stmt->next) { + switch(stmt->nodetype) { + case S_NOP: + case S_STOP: + break; + case S_ACT: + DBGPRINTF("iterateAllActions calling into action %p\n", stmt->d.act); + pFunc(stmt->d.act, pParam); + break; + case S_IF: + if(stmt->d.s_if.t_then != NULL) + scriptIterateAllActions(stmt->d.s_if.t_then, + pFunc, pParam); + if(stmt->d.s_if.t_else != NULL) + scriptIterateAllActions(stmt->d.s_if.t_else, + pFunc, pParam); + break; + case S_PRIFILT: + if(stmt->d.s_prifilt.t_then != NULL) + scriptIterateAllActions(stmt->d.s_prifilt.t_then, + pFunc, pParam); + if(stmt->d.s_prifilt.t_else != NULL) + scriptIterateAllActions(stmt->d.s_prifilt.t_else, + pFunc, pParam); + break; + case S_PROPFILT: + scriptIterateAllActions(stmt->d.s_propfilt.t_then, + pFunc, pParam); + break; + default: + dbgprintf("error: unknown stmt type %u during iterateAll\n", + (unsigned) stmt->nodetype); + break; + } + } +} /* driver to iterate over all of this ruleset actions */ typedef struct iterateAllActions_s { rsRetVal (*pFunc)(void*, void*); void *pParam; } iterateAllActions_t; -DEFFUNC_llExecFunc(doIterateRulesetActions) -{ - DEFiRet; - rule_t* pRule = (rule_t*) pData; - iterateAllActions_t *pMyParam = (iterateAllActions_t*) pParam; - iRet = rule.IterateAllActions(pRule, pMyParam->pFunc, pMyParam->pParam); - RETiRet; -} -/* iterate over all actions of THIS rule set. - */ -static rsRetVal -iterateRulesetAllActions(ruleset_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) -{ - iterateAllActions_t params; - DEFiRet; - assert(pFunc != NULL); - - params.pFunc = pFunc; - params.pParam = pParam; - CHKiRet(llExecFunc(&(pThis->llRules), doIterateRulesetActions, ¶ms)); - -finalize_it: - RETiRet; -} - - /* driver to iterate over all actions */ DEFFUNC_llExecFunc(doIterateAllActions) { DEFiRet; ruleset_t* pThis = (ruleset_t*) pData; iterateAllActions_t *pMyParam = (iterateAllActions_t*) pParam; - iRet = iterateRulesetAllActions(pThis, pMyParam->pFunc, pMyParam->pParam); + scriptIterateAllActions(pThis->root, pMyParam->pFunc, pMyParam->pParam); RETiRet; } /* iterate over ALL actions present in the WHOLE system. @@ -134,30 +150,10 @@ finalize_it: } - -/* helper to processBatch(), used to call the configured actions. It is - * executed from within llExecFunc() of the action list. - * rgerhards, 2007-08-02 - */ -DEFFUNC_llExecFunc(processBatchDoRules) -{ - rsRetVal iRet; - ISOBJ_TYPE_assert(pData, rule); - DBGPRINTF("Processing next rule\n"); - iRet = rule.ProcessBatch((rule_t*) pData, (batch_t*) pParam); - DBGPRINTF("ruleset: get iRet %d from rule.ProcessMsg()\n", iRet); - return iRet; -} - - - /* This function is similar to processBatch(), but works on a batch that * contains rules from multiple rulesets. In this case, we can not push * the whole batch through the ruleset. Instead, we examine it and * partition it into sub-rulesets which we then push through the system. - * Note that when we evaluate which message must be processed, we do NOT need - * to look at bFilterOK, because this value is only set in a later processing - * stage. Doing so caused a bug during development ;) * rgerhards, 2010-06-15 */ static inline rsRetVal @@ -207,6 +203,322 @@ finalize_it: RETiRet; } +/* return a new "active" structure for the batch. Free with freeActive(). */ +static inline sbool *newActive(batch_t *pBatch) +{ + return malloc(sizeof(sbool) * batchNumMsgs(pBatch)); + +} +static inline void freeActive(sbool *active) { free(active); } + + +/* for details, see scriptExec() header comment! */ +/* call action for all messages with filter on */ +static rsRetVal +execAct(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +{ + DEFiRet; +dbgprintf("RRRR: execAct [%s]: batch of %d elements, active %p\n", modGetName(stmt->d.act->pMod), batchNumMsgs(pBatch), active); + pBatch->active = active; + stmt->d.act->submitToActQ(stmt->d.act, pBatch); + RETiRet; +} + +static rsRetVal +execSet(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +{ + int i; + struct var result; + DEFiRet; + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + if( pBatch->pElem[i].state != BATCH_STATE_DISC + && (active == NULL || active[i])) { + cnfexprEval(stmt->d.s_set.expr, &result, pBatch->pElem[i].pUsrp); + msgSetJSONFromVar((msg_t*)pBatch->pElem[i].pUsrp, stmt->d.s_set.varname, + &result); + varDelete(&result); + } + } + RETiRet; +} + +static rsRetVal +execUnset(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +{ + int i; + DEFiRet; + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + if( pBatch->pElem[i].state != BATCH_STATE_DISC + && (active == NULL || active[i])) { + msgUnsetJSON((msg_t*)pBatch->pElem[i].pUsrp, stmt->d.s_unset.varname); + } + } + RETiRet; +} + +/* for details, see scriptExec() header comment! */ +/* "stop" simply discards the filtered items - it's just a (hopefully more intuitive + * shortcut for users. + */ +static rsRetVal +execStop(batch_t *pBatch, sbool *active) +{ + int i; + DEFiRet; + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + if( pBatch->pElem[i].state != BATCH_STATE_DISC + && (active == NULL || active[i])) { + pBatch->pElem[i].state = BATCH_STATE_DISC; + } + } + RETiRet; +} + +/* for details, see scriptExec() header comment! */ +// save current filter, evaluate new one +// perform then (if any message) +// if ELSE given: +// set new filter, inverted +// perform else (if any messages) +static rsRetVal +execIf(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +{ + sbool *newAct; + int i; + sbool bRet; + DEFiRet; + newAct = newActive(pBatch); + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + if(pBatch->pElem[i].state == BATCH_STATE_DISC) + continue; /* will be ignored in any case */ + if(active == NULL || active[i]) { + bRet = cnfexprEvalBool(stmt->d.s_if.expr, + (msg_t*)(pBatch->pElem[i].pUsrp)); + } else + bRet = 0; + newAct[i] = bRet; + DBGPRINTF("batch: item %d: expr eval: %d\n", i, bRet); + } + + if(stmt->d.s_if.t_then != NULL) { + scriptExec(stmt->d.s_if.t_then, pBatch, newAct); + } + if(stmt->d.s_if.t_else != NULL) { + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) + ; ++i) + if(pBatch->pElem[i].state != BATCH_STATE_DISC) + newAct[i] = !newAct[i]; + scriptExec(stmt->d.s_if.t_else, pBatch, newAct); + } + freeActive(newAct); + RETiRet; +} + +/* for details, see scriptExec() header comment! */ +static void +execPRIFILT(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +{ + sbool *newAct; + msg_t *pMsg; + int bRet; + int i; + newAct = newActive(pBatch); + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + if(pBatch->pElem[i].state == BATCH_STATE_DISC) + continue; /* will be ignored in any case */ + pMsg = (msg_t*)(pBatch->pElem[i].pUsrp); + if(active == NULL || active[i]) { + if( (stmt->d.s_prifilt.pmask[pMsg->iFacility] == TABLE_NOPRI) || + ((stmt->d.s_prifilt.pmask[pMsg->iFacility] + & (1<<pMsg->iSeverity)) == 0) ) + bRet = 0; + else + bRet = 1; + } else + bRet = 0; + newAct[i] = bRet; + DBGPRINTF("batch: item %d PRIFILT %d\n", i, newAct[i]); + } + + if(stmt->d.s_prifilt.t_then != NULL) { + scriptExec(stmt->d.s_prifilt.t_then, pBatch, newAct); + } + if(stmt->d.s_prifilt.t_else != NULL) { + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) + ; ++i) + if(pBatch->pElem[i].state != BATCH_STATE_DISC) + newAct[i] = !newAct[i]; + scriptExec(stmt->d.s_prifilt.t_else, pBatch, newAct); + } + freeActive(newAct); +} + + +/* helper to execPROPFILT(), as the evaluation itself is quite lengthy */ +static int +evalPROPFILT(struct cnfstmt *stmt, msg_t *pMsg) +{ + unsigned short pbMustBeFreed; + uchar *pszPropVal; + int bRet = 0; + rs_size_t propLen; + + if(stmt->d.s_propfilt.propID == PROP_INVALID) + goto done; + + pszPropVal = MsgGetProp(pMsg, NULL, stmt->d.s_propfilt.propID, + stmt->d.s_propfilt.propName, &propLen, &pbMustBeFreed); + + /* Now do the compares (short list currently ;)) */ + switch(stmt->d.s_propfilt.operation ) { + case FIOP_CONTAINS: + if(rsCStrLocateInSzStr(stmt->d.s_propfilt.pCSCompValue, (uchar*) pszPropVal) != -1) + bRet = 1; + break; + case FIOP_ISEMPTY: + if(propLen == 0) + bRet = 1; /* process message! */ + break; + case FIOP_ISEQUAL: + if(rsCStrSzStrCmp(stmt->d.s_propfilt.pCSCompValue, + pszPropVal, ustrlen(pszPropVal)) == 0) + bRet = 1; /* process message! */ + break; + case FIOP_STARTSWITH: + if(rsCStrSzStrStartsWithCStr(stmt->d.s_propfilt.pCSCompValue, + pszPropVal, ustrlen(pszPropVal)) == 0) + bRet = 1; /* process message! */ + break; + case FIOP_REGEX: + if(rsCStrSzStrMatchRegex(stmt->d.s_propfilt.pCSCompValue, + (unsigned char*) pszPropVal, 0, &stmt->d.s_propfilt.regex_cache) == RS_RET_OK) + bRet = 1; + break; + case FIOP_EREREGEX: + if(rsCStrSzStrMatchRegex(stmt->d.s_propfilt.pCSCompValue, + (unsigned char*) pszPropVal, 1, &stmt->d.s_propfilt.regex_cache) == RS_RET_OK) + bRet = 1; + break; + default: + /* here, it handles NOP (for performance reasons) */ + assert(stmt->d.s_propfilt.operation == FIOP_NOP); + bRet = 1; /* as good as any other default ;) */ + break; + } + + /* now check if the value must be negated */ + if(stmt->d.s_propfilt.isNegated) + bRet = (bRet == 1) ? 0 : 1; + + if(Debug) { + char *cstr; + if(stmt->d.s_propfilt.propID == PROP_CEE) { + cstr = es_str2cstr(stmt->d.s_propfilt.propName, NULL); + DBGPRINTF("Filter: check for CEE property '%s' (value '%s') ", + cstr, pszPropVal); + free(cstr); + } else { + DBGPRINTF("Filter: check for property '%s' (value '%s') ", + propIDToName(stmt->d.s_propfilt.propID), pszPropVal); + } + if(stmt->d.s_propfilt.isNegated) + DBGPRINTF("NOT "); + if(stmt->d.s_propfilt.operation == FIOP_ISEMPTY) { + DBGPRINTF("%s : %s\n", + getFIOPName(stmt->d.s_propfilt.operation), + bRet ? "TRUE" : "FALSE"); + } else { + DBGPRINTF("%s '%s': %s\n", + getFIOPName(stmt->d.s_propfilt.operation), + rsCStrGetSzStrNoNULL(stmt->d.s_propfilt.pCSCompValue), + bRet ? "TRUE" : "FALSE"); + } + } + + /* cleanup */ + if(pbMustBeFreed) + free(pszPropVal); +done: + return bRet; +} + +/* for details, see scriptExec() header comment! */ +static void +execPROPFILT(struct cnfstmt *stmt, batch_t *pBatch, sbool *active) +{ + sbool *thenAct; + msg_t *pMsg; + sbool bRet; + int i; + thenAct = newActive(pBatch); + for(i = 0 ; i < batchNumMsgs(pBatch) && !*(pBatch->pbShutdownImmediate) ; ++i) { + if(pBatch->pElem[i].state == BATCH_STATE_DISC) + continue; /* will be ignored in any case */ + pMsg = (msg_t*)(pBatch->pElem[i].pUsrp); + if(active == NULL || active[i]) { + bRet = evalPROPFILT(stmt, (msg_t*)(pBatch->pElem[i].pUsrp)); + } else + bRet = 0; + thenAct[i] = bRet; + DBGPRINTF("batch: item %d PROPFILT %d\n", i, thenAct[i]); + } + + scriptExec(stmt->d.s_propfilt.t_then, pBatch, thenAct); + freeActive(thenAct); +} + +/* The rainerscript execution engine. It is debatable if that would be better + * contained in grammer/rainerscript.c, HOWEVER, that file focusses primarily + * on the parsing and object creation part. So as an actual executor, it is + * better suited here. + * param active: if NULL, all messages are active (to be processed), if non-null + * this is an array of the same size as the batch. If 1, the message + * is to be processed, otherwise not. + * NOTE: this function must receive batches which contain a single ruleset ONLY! + * rgerhards, 2012-09-04 + */ +static rsRetVal +scriptExec(struct cnfstmt *root, batch_t *pBatch, sbool *active) +{ + DEFiRet; + struct cnfstmt *stmt; + + for(stmt = root ; stmt != NULL ; stmt = stmt->next) { +dbgprintf("RRRR: scriptExec: batch of %d elements, active %p, stmt %p, nodetype %u\n", batchNumMsgs(pBatch), active, stmt, stmt->nodetype); + switch(stmt->nodetype) { + case S_NOP: + break; + case S_STOP: + execStop(pBatch, active); + break; + case S_ACT: + execAct(stmt, pBatch, active); + break; + case S_SET: + execSet(stmt, pBatch, active); + break; + case S_UNSET: + execUnset(stmt, pBatch, active); + break; + case S_IF: + execIf(stmt, pBatch, active); + break; + case S_PRIFILT: + execPRIFILT(stmt, pBatch, active); + break; + case S_PROPFILT: + execPROPFILT(stmt, pBatch, active); + break; + default: + dbgprintf("error: unknown stmt type %u during exec\n", + (unsigned) stmt->nodetype); + break; + } + } + RETiRet; +} + + /* Process (consume) a batch of messages. Calls the actions configured. * If the whole batch uses a singel ruleset, we can process the batch as * a whole. Otherwise, we need to process it slower, on a message-by-message @@ -226,7 +538,7 @@ processBatch(batch_t *pBatch) if(pThis == NULL) pThis = ourConf->rulesets.pDflt; ISOBJ_TYPE_assert(pThis, ruleset); - CHKiRet(llExecFunc(&pThis->llRules, processBatchDoRules, pBatch)); + CHKiRet(scriptExec(pThis->root, pBatch, NULL)); } else { CHKiRet(processBatchMultiRuleset(pBatch)); } @@ -248,29 +560,16 @@ GetParserList(rsconf_t *conf, msg_t *pMsg) } -/* Add a new rule to the end of the current rule set. We do a number - * of checks and ignore the rule if it does not pass them. - */ -static rsRetVal -addRule(ruleset_t *pThis, rule_t **ppRule) +/* Add a script block to the current ruleset */ +static void +addScript(ruleset_t *pThis, struct cnfstmt *script) { - int iActionCnt; - DEFiRet; - - ISOBJ_TYPE_assert(pThis, ruleset); - ISOBJ_TYPE_assert(*ppRule, rule); - - CHKiRet(llGetNumElts(&(*ppRule)->llActList, &iActionCnt)); - if(iActionCnt == 0) { - errmsg.LogError(0, NO_ERRCODE, "warning: selector line without actions will be discarded"); - rule.Destruct(ppRule); - } else { - CHKiRet(llAppend(&pThis->llRules, NULL, *ppRule)); - DBGPRINTF("selector line successfully processed, %d actions\n", iActionCnt); + if(pThis->last == NULL) + pThis->root = pThis->last = script; + else { + pThis->last->next = script; + pThis->last = script; } - -finalize_it: - RETiRet; } @@ -362,23 +661,11 @@ finalize_it: } -/* destructor we need to destruct rules inside our linked list contents. - */ -static rsRetVal -doRuleDestruct(void *pData) -{ - rule_t *pRule = (rule_t *) pData; - DEFiRet; - rule.Destruct(&pRule); - RETiRet; -} - - /* Standard-Constructor */ BEGINobjConstruct(ruleset) /* be sure to specify the object type also in END macro! */ - CHKiRet(llInit(&pThis->llRules, doRuleDestruct, NULL, NULL)); -finalize_it: + pThis->root = NULL; + pThis->last = NULL; ENDobjConstruct(ruleset) @@ -421,8 +708,8 @@ CODESTARTobjDestruct(ruleset) if(pThis->pParserLst != NULL) { parser.DestructParserList(&pThis->pParserLst); } - llDestroy(&pThis->llRules); free(pThis->pszName); + cnfstmtDestruct(pThis->root); ENDobjDestruct(ruleset) @@ -456,16 +743,11 @@ rulesetDestructForLinkedList(void *pData) return rulesetDestruct(&pThis); } -/* helper for debugPrint(), initiates rule printing */ -DEFFUNC_llExecFunc(doDebugPrintRule) -{ - return rule.DebugPrint((rule_t*) pData); -} /* debugprint for the ruleset object */ BEGINobjDebugPrint(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDebugPrint(ruleset) dbgoprint((obj_t*) pThis, "rsyslog ruleset %s:\n", pThis->pszName); - llExecFunc(&pThis->llRules, doDebugPrintRule, NULL); + cnfstmtPrint(pThis->root, 0); ENDobjDebugPrint(ruleset) @@ -486,6 +768,40 @@ debugPrintAll(rsconf_t *conf) RETiRet; } +static inline void +rulesetOptimize(ruleset_t *pRuleset) +{ + if(Debug) { + dbgprintf("ruleset '%s' before optimization:\n", + pRuleset->pszName); + rulesetDebugPrint((ruleset_t*) pRuleset); + } + cnfstmtOptimize(pRuleset->root); + if(Debug) { + dbgprintf("ruleset '%s' after optimization:\n", + pRuleset->pszName); + rulesetDebugPrint((ruleset_t*) pRuleset); + } +} + +/* helper for rulsetOptimizeAll(), optimizes a single ruleset */ +DEFFUNC_llExecFunc(doRulesetOptimizeAll) +{ + rulesetOptimize((ruleset_t*) pData); + return RS_RET_OK; +} +/* optimize all rulesets + */ +rsRetVal +rulesetOptimizeAll(rsconf_t *conf) +{ + DEFiRet; + dbgprintf("begin ruleset optimization phase\n"); + llExecFunc(&(conf->rulesets.llRulesets), doRulesetOptimizeAll, NULL); + dbgprintf("ruleset optimization phase finished.\n"); + RETiRet; +} + /* Create a ruleset-specific "main" queue for this ruleset. If one is already * defined, an error message is emitted but nothing else is done. @@ -595,7 +911,7 @@ CODESTARTobjQueryInterface(ruleset) pIf->IterateAllActions = iterateAllActions; pIf->DestructAllActions = destructAllActions; - pIf->AddRule = addRule; + pIf->AddScript = addScript; pIf->ProcessBatch = processBatch; pIf->SetName = setName; pIf->DebugPrintAll = debugPrintAll; @@ -614,7 +930,6 @@ ENDobjQueryInterface(ruleset) */ BEGINObjClassExit(ruleset, OBJ_IS_CORE_MODULE) /* class, version */ objRelease(errmsg, CORE_COMPONENT); - objRelease(rule, CORE_COMPONENT); objRelease(parser, CORE_COMPONENT); ENDObjClassExit(ruleset) @@ -626,7 +941,6 @@ ENDObjClassExit(ruleset) BEGINObjClassInit(ruleset, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); - CHKiRet(objUse(rule, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, rulesetDebugPrint); diff --git a/runtime/ruleset.h b/runtime/ruleset.h index f4443e18..1f8f948f 100644 --- a/runtime/ruleset.h +++ b/runtime/ruleset.h @@ -29,9 +29,10 @@ /* the ruleset object */ struct ruleset_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ - linkedList_t llRules; /* this is NOT a pointer - no typo here ;) */ uchar *pszName; /* name of our ruleset */ qqueue_t *pQueue; /* "main" message queue, if the ruleset has its own (else NULL) */ + struct cnfstmt *root; + struct cnfstmt *last; parserList_t *pParserLst;/* list of parsers to use for this ruleset */ }; @@ -42,9 +43,7 @@ BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Construct)(ruleset_t **ppThis); rsRetVal (*ConstructFinalize)(rsconf_t *conf, ruleset_t __attribute__((unused)) *pThis); rsRetVal (*Destruct)(ruleset_t **ppThis); - rsRetVal (*IterateAllActions)(rsconf_t *conf, rsRetVal (*pFunc)(void*, void*), void* pParam); rsRetVal (*DestructAllActions)(rsconf_t *conf); - rsRetVal (*AddRule)(ruleset_t *pThis, rule_t **ppRule); rsRetVal (*SetName)(ruleset_t *pThis, uchar *pszName); rsRetVal (*ProcessBatch)(batch_t*); rsRetVal (*GetRuleset)(rsconf_t *conf, ruleset_t **ppThis, uchar*); @@ -60,8 +59,12 @@ BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ * removed conf ptr from SetName, AddRule as the flex/bison based * system uses globals in any case. */ + /* v7, 2012-09-04 */ + /* AddRule() removed */ + /*TODO:REMOVE*/rsRetVal (*IterateAllActions)(rsconf_t *conf, rsRetVal (*pFunc)(void*, void*), void* pParam); + void (*AddScript)(ruleset_t *pThis, struct cnfstmt *script); ENDinterface(ruleset) -#define rulesetCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ +#define rulesetCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */ /* prototypes */ @@ -87,5 +90,9 @@ rulesetGetName(ruleset_t *pRuleset) } +/* we will most probably convert this module back to traditional C + * calling sequence, so here we go... + */ rsRetVal rulesetGetRuleset(rsconf_t *conf, ruleset_t **ppRuleset, uchar *pszName); +rsRetVal rulesetOptimizeAll(rsconf_t *conf); #endif /* #ifndef INCLUDED_RULESET_H */ diff --git a/runtime/typedefs.h b/runtime/typedefs.h index 4e7f1622..b99daed7 100644 --- a/runtime/typedefs.h +++ b/runtime/typedefs.h @@ -92,6 +92,8 @@ typedef struct cfgmodules_etry_s cfgmodules_etry_t; typedef struct outchannels_s outchannels_t; typedef struct modConfData_s modConfData_t; typedef struct instanceConf_s instanceConf_t; +typedef int rs_size_t; /* we do never need more than 2Gig strings, signed permits to + * use -1 as a special flag. */ typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ @@ -44,6 +44,7 @@ #include "errmsg.h" #include "strgen.h" #include "rsconf.h" +#include "msg.h" #include "unicode-helper.h" /* static data */ @@ -57,6 +58,7 @@ static struct cnfparamdescr cnfparamdescr[] = { { "type", eCmdHdlrString, 0 }, { "string", eCmdHdlrString, 0 }, { "plugin", eCmdHdlrString, 0 }, + { "subtree", eCmdHdlrString, 0 }, { "option.stdsql", eCmdHdlrBinary, 0 }, { "option.sql", eCmdHdlrBinary, 0 }, { "option.json", eCmdHdlrBinary, 0 } @@ -146,7 +148,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * size_t iBuf; unsigned short bMustBeFreed = 0; uchar *pVal; - size_t iLenVal = 0; + rs_size_t iLenVal = 0; assert(pTpl != NULL); assert(pMsg != NULL); @@ -158,6 +160,23 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * FINALIZE; } + if(pTpl->subtree != NULL) { + /* only a single CEE subtree must be provided */ + /* note: we could optimize the code below, however, this is + * not worth the effort, as this passing mode is not expected + * in subtree mode and so most probably only used for debug & test. + */ + getCEEPropVal(pMsg, pTpl->subtree, &pVal, &iLenVal, &bMustBeFreed); + if(iLenVal >= (rs_size_t)*pLenBuf) /* we reserve one char for the final \0! */ + CHKiRet(ExtendBuf(ppBuf, pLenBuf, iLenVal + 1)); + memcpy(*ppBuf, pVal, iLenVal+1); + if(bMustBeFreed) + free(pVal); + FINALIZE; + } + + /* we have a "regular" template with template entries */ + /* loop through the template. We obtain one value * and copy it over to our dynamic string buffer. Then, we * free the obtained value (if requested). We continue this @@ -232,7 +251,7 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) struct templateEntry *pTpe; uchar **pArr; int iArr; - size_t propLen; + rs_size_t propLen; unsigned short bMustBeFreed; uchar *pVal; @@ -240,11 +259,24 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) assert(pMsg != NULL); assert(ppArr != NULL); + if(pTpl->subtree) { + /* Note: this mode is untested, as there is no official plugin + * using array passing, so I simply could not test it. + */ + CHKmalloc(pArr = calloc(2, sizeof(uchar*))); + getCEEPropVal(pMsg, pTpl->subtree, &pVal, &propLen, &bMustBeFreed); + if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */ + pArr[0] = pVal; /* ... so we can use it! */ + } else { + CHKmalloc(pArr[0] = (uchar*)strdup((char*) pVal)); + } + FINALIZE; + } + /* loop through the template. We obtain one value, create a * private copy (if necessary), add it to the string array * and then on to the next until we have processed everything. */ - CHKmalloc(pArr = calloc(pTpl->tpenElements + 1, sizeof(uchar*))); iArr = 0; @@ -277,19 +309,32 @@ finalize_it: * tpltoString(). * rgerhards, 2012-08-29 */ -rsRetVal tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson) +rsRetVal +tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjson) { struct templateEntry *pTpe; - size_t propLen; + rs_size_t propLen; unsigned short bMustBeFreed; uchar *pVal; struct json_object *json, *jsonf; + rsRetVal localRet; DEFiRet; assert(pTpl != NULL); assert(pMsg != NULL); assert(json != NULL); + if(pTpl->subtree != NULL){ + localRet = jsonFind(pMsg, pTpl->subtree, pjson); + if(*pjson == NULL) { + /* we need to have a root object! */ + *pjson = json_object_new_object(); + } else { + json_object_get(*pjson); /* inc refcount */ + } + FINALIZE; + } + json = json_object_new_object(); for(pTpe = pTpl->pEntryRoot ; pTpe != NULL ; pTpe = pTpe->pNext) { if(pTpe->eEntryType == CONSTANT) { @@ -298,19 +343,34 @@ rsRetVal tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **pjso jsonf = json_object_new_string((char*) pTpe->data.constant.pConstant); json_object_object_add(json, (char*)pTpe->fieldName, jsonf); } else if(pTpe->eEntryType == FIELD) { - pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, - pTpe->data.field.propName, &propLen, &bMustBeFreed); - if(pTpe->data.field.options.bMandatory || propLen > 0) { - jsonf = json_object_new_string_len((char*)pVal, propLen); - json_object_object_add(json, (char*)pTpe->fieldName, jsonf); - } - if(bMustBeFreed) { /* json-c makes its own private copy! */ - free(pVal); + if(pTpe->data.field.propid == PROP_CEE) { + localRet = msgGetCEEPropJSON(pMsg, pTpe->data.field.propName, &jsonf); + if(localRet == RS_RET_OK) { + json_object_object_add(json, (char*)pTpe->fieldName, json_object_get(jsonf)); + } else { + DBGPRINTF("tplToJSON: error %d looking up property\n", + localRet); + if(pTpe->data.field.options.bMandatory) { + json_object_object_add(json, (char*)pTpe->fieldName, NULL); + } + } + } else { + pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, + pTpe->data.field.propName, &propLen, + &bMustBeFreed); + if(pTpe->data.field.options.bMandatory || propLen > 0) { + jsonf = json_object_new_string_len((char*)pVal, propLen); + json_object_object_add(json, (char*)pTpe->fieldName, jsonf); + } + if(bMustBeFreed) { /* json-c makes its own private copy! */ + free(pVal); + } } } } - *pjson = (iRet == RS_RET_OK) ? json : NULL; + +finalize_it: RETiRet; } @@ -368,7 +428,7 @@ static void doEmergencyEscape(register uchar *p, int mode) * 2005-09-22 rgerhards */ rsRetVal -doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int mode) +doEscape(uchar **pp, rs_size_t *pLen, unsigned short *pbMustBeFreed, int mode) { DEFiRet; uchar *p = NULL; @@ -727,7 +787,7 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) } if(pTpe->data.field.propid == PROP_CEE) { /* in CEE case, we need to preserve the actual property name */ - if((pTpe->data.field.propName = es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrProp)+2, cstrLen(pStrProp)-2)) == NULL) { + if((pTpe->data.field.propName = es_newStrFromCStr((char*)cstrGetSzStrNoNULL(pStrProp)+1, cstrLen(pStrProp)-1)) == NULL) { cstrDestruct(&pStrProp); return 1; } @@ -1478,8 +1538,8 @@ createPropertyTpe(struct template *pTpl, struct cnfobj *o) CHKiRet(propNameToID(name, &pTpe->data.field.propid)); if(pTpe->data.field.propid == PROP_CEE) { /* in CEE case, we need to preserve the actual property name */ - pTpe->data.field.propName = es_newStrFromCStr((char*)cstrGetSzStrNoNULL(name)+2, - cstrLen(name)-2); + pTpe->data.field.propName = es_newStrFromCStr((char*)cstrGetSzStrNoNULL(name)+1, + cstrLen(name)-1); } pTpe->data.field.options.bDropLastLF = droplastlf; pTpe->data.field.options.bSPIffNo1stSP = spifno1stsp; @@ -1611,8 +1671,9 @@ tplProcessCnf(struct cnfobj *o) char *name = NULL; uchar *tplStr = NULL; uchar *plugin = NULL; + es_str_t *subtree = NULL; uchar *p; - enum { T_STRING, T_PLUGIN, T_LIST } tplType; + enum { T_STRING, T_PLUGIN, T_LIST, T_SUBTREE } tplType; int i; int o_sql=0, o_stdsql=0, o_json=0; /* options */ int numopts; @@ -1635,6 +1696,8 @@ tplProcessCnf(struct cnfobj *o) tplType = T_PLUGIN; } else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"list", sizeof("list")-1)) { tplType = T_LIST; + } else if(!es_strbufcmp(pvals[i].val.d.estr, (uchar*)"subtree", sizeof("subtree")-1)) { + tplType = T_SUBTREE; } else { uchar *typeStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); errmsg.LogError(0, RS_RET_ERR, "invalid template type '%s'", @@ -1644,6 +1707,22 @@ tplProcessCnf(struct cnfobj *o) } } else if(!strcmp(pblk.descr[i].name, "string")) { tplStr = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(pblk.descr[i].name, "subtree")) { + uchar *st_str = es_getBufAddr(pvals[i].val.d.estr); + if(st_str[0] != '$' || st_str[1] != '!') { + char *cstr = es_str2cstr(pvals[i].val.d.estr, NULL); + errmsg.LogError(0, RS_RET_ERR, "invalid subtree " + "parameter, variable must start with '$!' but " + "var name is '%s'", cstr); + free(cstr); + free(name); /* overall assigned */ + ABORT_FINALIZE(RS_RET_ERR); + } else { + /* TODO: unify strings! */ + char *cstr = es_str2cstr(pvals[i].val.d.estr, NULL); + subtree = es_newStrFromBuf(cstr+1, es_strlen(pvals[i].val.d.estr)-1); + free(cstr); + } } else if(!strcmp(pblk.descr[i].name, "plugin")) { plugin = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); } else if(!strcmp(pblk.descr[i].name, "option.stdsql")) { @@ -1685,6 +1764,19 @@ tplProcessCnf(struct cnfobj *o) } } + if(subtree == NULL) { + if(tplType == T_SUBTREE) { + errmsg.LogError(0, RS_RET_ERR, "template '%s' of type subtree needs " + "subtree parameter", name); + ABORT_FINALIZE(RS_RET_ERR); + } + } else { + if(tplType != T_SUBTREE) { + errmsg.LogError(0, RS_RET_ERR, "template '%s' is not a subtree " + "template but has a subtree specified - ignored", name); + } + } + if(o->subobjs == NULL) { if(tplType == T_LIST) { errmsg.LogError(0, RS_RET_ERR, "template '%s' of type list has " @@ -1742,6 +1834,8 @@ tplProcessCnf(struct cnfobj *o) break; case T_LIST: createListTpl(pTpl, o); break; + case T_SUBTREE: pTpl->subtree = subtree; + break; } pTpl->optFormatEscape = NO_ESCAPE; @@ -1842,8 +1936,9 @@ void tplDeleteAll(rsconf_t *conf) } pTplDel = pTpl; pTpl = pTpl->pNext; - if(pTplDel->pszName != NULL) - free(pTplDel->pszName); + free(pTplDel->pszName); + if(pTplDel->subtree != NULL) + es_deleteStr(pTplDel->subtree); free(pTplDel); } ENDfunc @@ -1900,8 +1995,9 @@ void tplDeleteNew(rsconf_t *conf) } pTplDel = pTpl; pTpl = pTpl->pNext; - if(pTplDel->pszName != NULL) - free(pTplDel->pszName); + free(pTplDel->pszName); + if(pTplDel->subtree != NULL) + es_deleteStr(pTplDel->subtree); free(pTplDel); } ENDfunc @@ -39,7 +39,8 @@ struct template { struct template *pNext; char *pszName; int iLenName; - rsRetVal (*pStrgen)(msg_t*, uchar**, size_t *); /* name of strgen to use (bound if non-NULL!) */ + rsRetVal (*pStrgen)(msg_t*, uchar**, size_t *); + es_str_t *subtree; /* subtree name for subtree-type templates */ int tpenElements; /* number of elements in templateEntry list */ struct templateEntry *pEntryRoot; struct templateEntry *pEntryLast; @@ -149,7 +150,7 @@ rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize); rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr); rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *); rsRetVal tplToJSON(struct template *pTpl, msg_t *pMsg, struct json_object **); -rsRetVal doEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); +rsRetVal doEscape(uchar **pp, rs_size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); rsRetVal templateInit(); rsRetVal tplProcessCnf(struct cnfobj *o); diff --git a/tests/Makefile.am b/tests/Makefile.am index a9369929..715c024d 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -65,6 +65,13 @@ TESTS += \ failover-no-basic.sh \ rcvr_fail_restore.sh \ rscript_contains.sh \ + rscript_field.sh \ + rscript_stop.sh \ + rscript_stop2.sh \ + rscript_prifilt.sh \ + rscript_optimizer1.sh \ + cee_simple.sh \ + cee_diskqueue.sh \ linkedlistqueue.sh if HAVE_VALGRIND @@ -267,6 +274,20 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/arrayqueue.conf \ rscript_contains.sh \ testsuites/rscript_contains.conf \ + rscript_field.sh \ + testsuites/rscript_field.conf \ + rscript_stop.sh \ + testsuites/rscript_stop.conf \ + rscript_stop2.sh \ + testsuites/rscript_stop2.conf \ + rscript_prifilt.sh \ + testsuites/rscript_prifilt.conf \ + rscript_optimizer1.sh \ + testsuites/rscript_optimizer1.conf \ + cee_simple.sh \ + testsuites/cee_simple.conf \ + cee_diskqueue.sh \ + testsuites/cee_diskqueue.conf \ linkedlistqueue.sh \ testsuites/linkedlistqueue.conf \ da-mainmsg-q.sh \ diff --git a/tests/cee_diskqueue.sh b/tests/cee_diskqueue.sh new file mode 100755 index 00000000..4e19855b --- /dev/null +++ b/tests/cee_diskqueue.sh @@ -0,0 +1,14 @@ +# check if CEE properties are properly saved & restored to/from disk queue +# added 2012-09-19 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[cee_diskqueue.sh\]: CEE and diskqueue test +source $srcdir/diag.sh init +source $srcdir/diag.sh startup cee_diskqueue.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/cee_simple.sh b/tests/cee_simple.sh new file mode 100755 index 00000000..32f56393 --- /dev/null +++ b/tests/cee_simple.sh @@ -0,0 +1,13 @@ +# added 2012-09-19 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[cee_simple.sh\]: basic CEE property test +source $srcdir/diag.sh init +source $srcdir/diag.sh startup cee_simple.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/rscript_field.sh b/tests/rscript_field.sh new file mode 100755 index 00000000..e989e666 --- /dev/null +++ b/tests/rscript_field.sh @@ -0,0 +1,13 @@ +# added 2012-09-20 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[rscript_field.sh\]: testing rainerscript field\(\) function +source $srcdir/diag.sh init +source $srcdir/diag.sh startup rscript_field.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/rscript_optimizer1.sh b/tests/rscript_optimizer1.sh new file mode 100755 index 00000000..1d2dcf87 --- /dev/null +++ b/tests/rscript_optimizer1.sh @@ -0,0 +1,13 @@ +# added 2012-09-20 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[rscript_optimizer1.sh\]: testing rainerscript optimizer +source $srcdir/diag.sh init +source $srcdir/diag.sh startup rscript_optimizer1.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/rscript_prifilt.sh b/tests/rscript_prifilt.sh new file mode 100755 index 00000000..815492ab --- /dev/null +++ b/tests/rscript_prifilt.sh @@ -0,0 +1,13 @@ +# added 2012-09-20 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[rscript_prifilt.sh\]: testing rainerscript prifield\(\) function +source $srcdir/diag.sh init +source $srcdir/diag.sh startup rscript_prifilt.conf +source $srcdir/diag.sh injectmsg 0 5000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/rscript_stop.sh b/tests/rscript_stop.sh new file mode 100755 index 00000000..e532a522 --- /dev/null +++ b/tests/rscript_stop.sh @@ -0,0 +1,13 @@ +# added 2012-09-20 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[rscript_stop.sh\]: testing rainerscript STOP statement +source $srcdir/diag.sh init +source $srcdir/diag.sh startup rscript_stop.conf +source $srcdir/diag.sh injectmsg 0 8000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/rscript_stop2.sh b/tests/rscript_stop2.sh new file mode 100755 index 00000000..eae36cce --- /dev/null +++ b/tests/rscript_stop2.sh @@ -0,0 +1,13 @@ +# added 2012-09-20 by rgerhards +# This file is part of the rsyslog project, released under ASL 2.0 +echo =============================================================================== +echo \[rscript_stop2.sh\]: testing rainerscript STOP statement, alternate method +source $srcdir/diag.sh init +source $srcdir/diag.sh startup rscript_stop2.conf +source $srcdir/diag.sh injectmsg 0 8000 +echo doing shutdown +source $srcdir/diag.sh shutdown-when-empty +echo wait on shutdown +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/cee_diskqueue.conf b/tests/testsuites/cee_diskqueue.conf new file mode 100644 index 00000000..a9b98e80 --- /dev/null +++ b/tests/testsuites/cee_diskqueue.conf @@ -0,0 +1,9 @@ +$IncludeConfig diag-common.conf + +global(workDirectory="/tmp") +template(name="outfmt" type="string" string="%$!usr!msg:F,58:2%\n") + +set $!usr!msg = $msg; +if $msg contains 'msgnum' then + action(type="omfile" file="./rsyslog.out.log" template="outfmt" + queue.type="disk" queue.filename="rsyslog-act1") diff --git a/tests/testsuites/cee_simple.conf b/tests/testsuites/cee_simple.conf new file mode 100644 index 00000000..1bcf83c1 --- /dev/null +++ b/tests/testsuites/cee_simple.conf @@ -0,0 +1,6 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="string" string="%$!usr!msg:F,58:2%\n") +set $!usr!msg = $msg; +if $msg contains 'msgnum' then + action(type="omfile" file="./rsyslog.out.log" template="outfmt") diff --git a/tests/testsuites/rscript_field.conf b/tests/testsuites/rscript_field.conf new file mode 100644 index 00000000..d7eb9066 --- /dev/null +++ b/tests/testsuites/rscript_field.conf @@ -0,0 +1,11 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="list") { + property(name="$!usr!msgnum") + constant(value="\n") +} + +if $msg contains 'msgnum' then { + set $!usr!msgnum = field($msg, 58, 2); + action(type="omfile" file="./rsyslog.out.log" template="outfmt") +} diff --git a/tests/testsuites/rscript_optimizer1.conf b/tests/testsuites/rscript_optimizer1.conf new file mode 100644 index 00000000..7720af7a --- /dev/null +++ b/tests/testsuites/rscript_optimizer1.conf @@ -0,0 +1,12 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="list") { + property(name="msg" field.delimiter="58" field.number="2") + constant(value="\n") +} + +/* tcpflood uses local4.=debug */ +if prifilt("syslog.*") then + stop # it actually doesn't matter what we do here +else + action(type="omfile" file="./rsyslog.out.log" template="outfmt") diff --git a/tests/testsuites/rscript_prifilt.conf b/tests/testsuites/rscript_prifilt.conf new file mode 100644 index 00000000..8cb13a0f --- /dev/null +++ b/tests/testsuites/rscript_prifilt.conf @@ -0,0 +1,10 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="list") { + property(name="msg" field.delimiter="58" field.number="2") + constant(value="\n") +} + +/* tcpflood uses local4.=debug, we use a bit more generic filter */ +if prifilt("local4.*") then + action(type="omfile" file="./rsyslog.out.log" template="outfmt") diff --git a/tests/testsuites/rscript_stop.conf b/tests/testsuites/rscript_stop.conf new file mode 100644 index 00000000..ab9569e5 --- /dev/null +++ b/tests/testsuites/rscript_stop.conf @@ -0,0 +1,13 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="list") { + property(name="$!usr!msgnum") + constant(value="\n") +} + +if $msg contains 'msgnum' then { + set $!usr!msgnum = field($msg, 58, 2); + if cnum($!usr!msgnum) >= 5000 then + stop + action(type="omfile" file="./rsyslog.out.log" template="outfmt") +} diff --git a/tests/testsuites/rscript_stop2.conf b/tests/testsuites/rscript_stop2.conf new file mode 100644 index 00000000..9ac9143e --- /dev/null +++ b/tests/testsuites/rscript_stop2.conf @@ -0,0 +1,18 @@ +$IncludeConfig diag-common.conf + +template(name="outfmt" type="list") { + property(name="$!usr!msgnum") + constant(value="\n") +} + +if not ($msg contains 'msgnum') then + stop + +set $!usr!msgnum = field($msg, 58, 2); +if cnum($!usr!msgnum) >= 5000 then + stop +/* We could use yet another method, but we like to have the action statement + * without a filter in rsyslog.conf top level hierarchy - so this test, as + * a side-effect, also tests this ability. + */ +action(type="omfile" file="./rsyslog.out.log" template="outfmt") diff --git a/tools/Makefile.am b/tools/Makefile.am index 60a2dd61..c37162ba 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -40,7 +40,7 @@ rsyslogd_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) $(CNF_LIBS) # note: it looks like librsyslog.la must be explicitely given on LDDADD, # otherwise dependencies are not properly calculated (resulting in a # potentially incomplete build, a problem we had several times...) -rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) $(LIBEE_LIBS) $(LIBLOGNORM_LIBS) $(CNF_LIBS) $(LIBUUID_LIBS) ../grammar/libgrammar.la ../runtime/librsyslog.la +rsyslogd_LDADD = ../grammar/libgrammar.la ../runtime/librsyslog.la $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) $(LIBEE_LIBS) $(LIBLOGNORM_LIBS) $(CNF_LIBS) $(LIBUUID_LIBS) rsyslogd_LDFLAGS = -export-dynamic if ENABLE_DIAGTOOLS diff --git a/tools/omfile.c b/tools/omfile.c index 1a36343f..1ccd4a56 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -885,6 +885,7 @@ setInstParamDefaults(instanceData *pData) BEGINnewActInst struct cnfparamvals *pvals; + uchar *tplToUse; int i; CODESTARTnewActInst DBGPRINTF("newActInst (omfile)\n"); @@ -959,7 +960,8 @@ CODESTARTnewActInst ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); } - CHKiRet(OMSRsetEntry(*ppOMSR, 0, ustrdup(getDfltTpl()), OMSR_NO_RQD_TPL_OPTS)); + tplToUse = (pData->tplName == NULL) ? ustrdup(getDfltTpl()) : pData->tplName; + CHKiRet(OMSRsetEntry(*ppOMSR, 0, tplToUse, OMSR_NO_RQD_TPL_OPTS)); if(pData->bDynamicName) { /* "filename" is actually a template name, we need this as string 1. So let's add it diff --git a/tools/syslogd.c b/tools/syslogd.c index 152c27ae..8ba8edd3 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -119,7 +119,6 @@ #include "batch.h" #include "unicode-helper.h" #include "ruleset.h" -#include "rule.h" #include "net.h" #include "prop.h" #include "rsconf.h" @@ -134,7 +133,6 @@ DEFobjCurrIf(datetime) /* TODO: make go away! */ DEFobjCurrIf(conf) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) -DEFobjCurrIf(rule) DEFobjCurrIf(ruleset) DEFobjCurrIf(prop) DEFobjCurrIf(parser) @@ -1453,8 +1451,6 @@ InitGlobalClasses(void) CHKiRet(objUse(module, CORE_COMPONENT)); pErrObj = "datetime"; CHKiRet(objUse(datetime, CORE_COMPONENT)); - pErrObj = "rule"; - CHKiRet(objUse(rule, CORE_COMPONENT)); pErrObj = "ruleset"; CHKiRet(objUse(ruleset, CORE_COMPONENT)); pErrObj = "conf"; @@ -1508,7 +1504,6 @@ GlobalClassExit(void) objRelease(prop, CORE_COMPONENT); objRelease(conf, CORE_COMPONENT); objRelease(ruleset, CORE_COMPONENT); - objRelease(rule, CORE_COMPONENT); parserClassExit(); /* this is hack, currently core_modules do not get this automatically called */ rsconfClassExit(); /* this is hack, currently core_modules do not get this automatically called */ objRelease(datetime, CORE_COMPONENT); |