From 060187d21c0009f2fd7181980de3b38df9064024 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 Nov 2013 15:03:50 +0100 Subject: queue workers: eliminate unnecessary queue mutex operations --- runtime/wti.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/runtime/wti.c b/runtime/wti.c index bbefc537..2c6c5123 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -303,15 +303,21 @@ wtiWorker(wti_t *pThis) pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); dbgprintf("DDDD: wti %p: worker starting\n", pThis); - /* now we have our identity, on to real processing */ - while(1) { /* loop will be broken below - need to do mutex locks */ + + /* note: in this loop, the mutex is "never" unlocked. Of course, + * this is not true: it actually is unlocked when the actual processing + * is done, as part of pWtp->pfDoWork() processing. Note that this + * function is required to re-lock it when done. We cannot do the + * lock/unlock here ourselfs, as pfDoWork() needs to access queue + * structures itself. -- rgerhards, 2013-11-20 + */ + d_pthread_mutex_lock(pWtp->pmutUsr); + while(1) { /* loop will be broken below */ if(pWtp->pfRateLimiter != NULL) { /* call rate-limiter, if defined */ pWtp->pfRateLimiter(pWtp->pUsr); } - d_pthread_mutex_lock(pWtp->pmutUsr); - /* first check if we are in shutdown process (but evaluate a bit later) */ terminateRet = wtpChkStopWrkr(pWtp, MUTEX_ALREADY_LOCKED); if(terminateRet == RS_RET_TERMINATE_NOW) { @@ -319,36 +325,31 @@ dbgprintf("DDDD: wti %p: worker starting\n", pThis); localRet = pWtp->pfObjProcessed(pWtp->pUsr, pThis); DBGOPRINT((obj_t*) pThis, "terminating worker because of TERMINATE_NOW mode, del iRet %d\n", localRet); - d_pthread_mutex_unlock(pWtp->pmutUsr); break; } /* try to execute and process whatever we have */ - /* Note that this function releases and re-aquires the mutex. The returned - * information on idle state must be processed before releasing the mutex again. + /* Note that this function releases and re-aquires the mutex. */ localRet = pWtp->pfDoWork(pWtp->pUsr, pThis); if(localRet == RS_RET_ERR_QUEUE_EMERGENCY) { - d_pthread_mutex_unlock(pWtp->pmutUsr); break; /* end of loop */ } else if(localRet == RS_RET_IDLE) { if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE || bInactivityTOOccured) { - d_pthread_mutex_unlock(pWtp->pmutUsr); DBGOPRINT((obj_t*) pThis, "terminating worker terminateRet=%d, bInactivityTOOccured=%d\n", terminateRet, bInactivityTOOccured); break; /* end of loop */ } doIdleProcessing(pThis, pWtp, &bInactivityTOOccured); - d_pthread_mutex_unlock(pWtp->pmutUsr); continue; /* request next iteration */ } - d_pthread_mutex_unlock(pWtp->pmutUsr); - bInactivityTOOccured = 0; /* reset for next run */ } + d_pthread_mutex_unlock(pWtp->pmutUsr); + DBGPRINTF("DDDD: wti %p: worker cleanup up action instances\n", pThis); for(i = 0 ; i < iActionNbr ; ++i) { dbgprintf("wti %p, action %d, ptr %p\n", pThis, i, pThis->actWrkrInfo[i].actWrkrData); -- cgit v1.2.3 From ede2e4ccf225b6f02312ba7d5523dca8268acd67 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 Nov 2013 17:01:39 +0100 Subject: regression fix: prevent queue stall if ratelimiter is used Thanks to Pavel Levshin for alerting us. This regression was introduced roughly 2 hours ago and was never released. --- runtime/queue.c | 5 +++++ runtime/wti.c | 9 ++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index 5770eae9..0fe95876 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1679,6 +1679,9 @@ DequeueConsumable(qqueue_t *pThis, wti_t *pWti) /* The rate limiter + * + * IMPORTANT: the rate-limiter MUST unlock and re-lock the queue when + * it actually delays processing. Otherwise inputs are stalled. * * Here we may wait if a dequeue time window is defined or if we are * rate-limited. TODO: If we do so, we should also look into the @@ -1765,8 +1768,10 @@ RateLimiter(qqueue_t *pThis) } if(iDelay > 0) { + pthread_mutex_unlock(pThis->mut); DBGOPRINT((obj_t*) pThis, "outside dequeue time window, delaying %d seconds\n", iDelay); srSleep(iDelay, 0); + pthread_mutex_lock(pThis->mut); } RETiRet; diff --git a/runtime/wti.c b/runtime/wti.c index 2c6c5123..c02d0573 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -310,7 +310,12 @@ dbgprintf("DDDD: wti %p: worker starting\n", pThis); * is done, as part of pWtp->pfDoWork() processing. Note that this * function is required to re-lock it when done. We cannot do the * lock/unlock here ourselfs, as pfDoWork() needs to access queue - * structures itself. -- rgerhards, 2013-11-20 + * structures itself. + * The same goes for pfRateLimiter(). While we could unlock/lock when + * we call it, in practice the function is often called without any + * ratelimiting actually done. Only the rate limiter itself knows + * that. As such, it needs to bear the burden of doing the locking + * when required. -- rgerhards, 2013-11-20 */ d_pthread_mutex_lock(pWtp->pmutUsr); while(1) { /* loop will be broken below */ @@ -329,8 +334,6 @@ dbgprintf("DDDD: wti %p: worker starting\n", pThis); } /* try to execute and process whatever we have */ - /* Note that this function releases and re-aquires the mutex. - */ localRet = pWtp->pfDoWork(pWtp->pUsr, pThis); if(localRet == RS_RET_ERR_QUEUE_EMERGENCY) { -- cgit v1.2.3 From c6828f7a5d15c26ef87e175a7228836132e64ded Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 Nov 2013 18:09:06 +0100 Subject: refactor queue spool directory config handling --- runtime/queue.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index 6098eeee..74efab20 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -731,7 +731,7 @@ qqueueLoadPersStrmInfoFixup(strm_t *pStrm, qqueue_t __attribute__((unused)) *pTh DEFiRet; ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_TYPE_assert(pThis, qqueue); - CHKiRet(strm.SetDir(pStrm, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetDir(pStrm, pThis->pszSpoolDir, pThis->lenSpoolDir)); finalize_it: RETiRet; } @@ -830,7 +830,7 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) } else { CHKiRet(strm.Construct(&pThis->tVars.disk.pWrite)); CHKiRet(strm.SetbSync(pThis->tVars.disk.pWrite, pThis->bSyncQueueFiles)); - CHKiRet(strm.SetDir(pThis->tVars.disk.pWrite, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetDir(pThis->tVars.disk.pWrite, pThis->pszSpoolDir, pThis->lenSpoolDir)); CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pWrite, 10000000)); CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE)); CHKiRet(strm.SetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR)); @@ -838,7 +838,7 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDeq)); CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); - CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDeq, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDeq, pThis->pszSpoolDir, pThis->lenSpoolDir)); CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDeq, 10000000)); CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDeq, STREAMMODE_READ)); CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDeq, STREAMTYPE_FILE_CIRCULAR)); @@ -847,7 +847,7 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) CHKiRet(strm.Construct(&pThis->tVars.disk.pReadDel)); CHKiRet(strm.SetbSync(pThis->tVars.disk.pReadDel, pThis->bSyncQueueFiles)); CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1)); - CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDel, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetDir(pThis->tVars.disk.pReadDel, pThis->pszSpoolDir, pThis->lenSpoolDir)); CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pReadDel, 10000000)); CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pReadDel, STREAMMODE_READ)); CHKiRet(strm.SetsType(pThis->tVars.disk.pReadDel, STREAMTYPE_FILE_CIRCULAR)); @@ -2070,7 +2070,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ pThis->iNumWorkerThreads = 1; /* we need exactly one worker */ /* pre-construct file name for .qi file */ pThis->lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), - "%s/%s.qi", (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); + "%s/%s.qi", (char*) pThis->pszSpoolDir, (char*)pThis->pszFilePrefix); pThis->pszQIFNam = ustrdup(pszQIFNam); DBGOPRINT((obj_t*) pThis, ".qi file name is '%s', len %d\n", pThis->pszQIFNam, (int) pThis->lenQIFNam); @@ -2135,9 +2135,9 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ pThis->iFullDlyMrk = wrk; } - DBGOPRINT((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, maxQSize %d, lqsize %d, pqsize %d, child %d, " + DBGOPRINT((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, spoolDir '%s', maxFileSz %lld, maxQSize %d, lqsize %d, pqsize %d, child %d, " "full delay %d, light delay %d, deq batch size %d starting, high wtrrmrk %d, low wtrmrk %d\n", - pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize, pThis->iMaxQueueSize, + pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->pszSpoolDir, pThis->iMaxFileSize, pThis->iMaxQueueSize, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1, pThis->iFullDlyMrk, pThis->iLightDlyMrk, pThis->iDeqBatchSize, pThis->iHighWtrMrk, pThis->iLowWtrMrk); -- cgit v1.2.3 From 778568eb79642bed5064b6223594982d81813895 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 21 Nov 2013 09:56:17 +0100 Subject: queue: spool directory setting was not properly propagated to DA queue --- runtime/queue.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/runtime/queue.c b/runtime/queue.c index 74efab20..a152af1e 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -12,7 +12,7 @@ * function names - this makes it really hard to read and does not provide much * benefit, at least I (now) think so... * - * Copyright 2008-2011 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008-2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -87,6 +87,7 @@ static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis); static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis); static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis); static rsRetVal qDestructDisk(qqueue_t *pThis); +rsRetVal qqueueSetSpoolDir(qqueue_t *pThis, uchar *pszSpoolDir, int lenSpoolDir); /* some constants for queuePersist () */ #define QUEUE_CHECKPOINT 1 @@ -418,6 +419,7 @@ StartDA(qqueue_t *pThis) CHKiRet(qqueueSetiDeqSlowdown(pThis->pqDA, pThis->iDeqSlowdown)); CHKiRet(qqueueSetMaxFileSize(pThis->pqDA, pThis->iMaxFileSize)); CHKiRet(qqueueSetFilePrefix(pThis->pqDA, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(qqueueSetSpoolDir(pThis->pqDA, pThis->pszSpoolDir, pThis->lenSpoolDir)); CHKiRet(qqueueSetiPersistUpdCnt(pThis->pqDA, pThis->iPersistUpdCnt)); CHKiRet(qqueueSetbSyncQueueFiles(pThis->pqDA, pThis->bSyncQueueFiles)); CHKiRet(qqueueSettoActShutdown(pThis->pqDA, pThis->toActShutdown)); @@ -2442,6 +2444,24 @@ CODESTARTobjDestruct(qqueue) ENDobjDestruct(qqueue) +/* set the queue's spool directory. The directory MUST NOT be NULL. + * The passed-in string is duplicated. So if the caller does not need + * it any longer, it must free it. + */ +rsRetVal +qqueueSetSpoolDir(qqueue_t *pThis, uchar *pszSpoolDir, int lenSpoolDir) +{ + DEFiRet; + + free(pThis->pszSpoolDir); + CHKmalloc(pThis->pszSpoolDir = ustrdup(pszSpoolDir)); + pThis->lenSpoolDir = lenSpoolDir; + +finalize_it: + RETiRet; +} + + /* set the queue's file prefix * The passed-in string is duplicated. So if the caller does not need * it any longer, it must free it. -- cgit v1.2.3 From 54add6e24b5d10283e08a32c22565e6249204567 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 21 Nov 2013 11:09:25 +0100 Subject: doc: maintain ChangeLog --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index bd4a0b39..df14178d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,8 @@ --------------------------------------------------------------------------- Version 7.4.7 [v7.4-stable] 2013-11-?? +- bugfix: disk queues created files in wrong working directory + if the $WorkDirectory was changed multiple times, all queues only + used the last value set. - bugfix: legacy directive $ActionQueueWorkerThreads was not honored - bugfix: segfault on startup when certain script constructs are used e.g. "if not $msg ..." -- cgit v1.2.3 From 5529113265633700f2220277943ad1b332c954f2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 21 Nov 2013 09:17:19 +0100 Subject: queue: add config parameter queue.spooldirectory Conflicts: runtime/queue.c --- runtime/queue.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/runtime/queue.c b/runtime/queue.c index a6928910..b1938ac1 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -59,6 +59,7 @@ #include "datetime.h" #include "unicode-helper.h" #include "statsobj.h" +#include "parserif.h" #ifdef OS_SOLARIS # include @@ -96,6 +97,7 @@ rsRetVal qqueueSetSpoolDir(qqueue_t *pThis, uchar *pszSpoolDir, int lenSpoolDir) /* tables for interfacing with the v6 config system */ static struct cnfparamdescr cnfpdescr[] = { { "queue.filename", eCmdHdlrGetWord, 0 }, + { "queue.spooldirectory", eCmdHdlrGetWord, 0 }, { "queue.size", eCmdHdlrSize, 0 }, { "queue.dequeuebatchsize", eCmdHdlrInt, 0 }, { "queue.maxdiskspace", eCmdHdlrSize, 0 }, @@ -2883,6 +2885,16 @@ qqueueApplyCnfParam(qqueue_t *pThis, struct nvlst *lst) pThis->lenFilePrefix = es_strlen(pvals[i].val.d.estr); } else if(!strcmp(pblk.descr[i].name, "queue.cry.provider")) { pThis->cryprovName = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); + } else if(!strcmp(pblk.descr[i].name, "queue.spooldirectory")) { + free(pThis->pszSpoolDir); + pThis->pszSpoolDir = (uchar*) es_str2cstr(pvals[i].val.d.estr, NULL); + pThis->lenSpoolDir = es_strlen(pvals[i].val.d.estr); + if(pThis->pszSpoolDir[pThis->lenSpoolDir-1] == '/') { + pThis->pszSpoolDir[pThis->lenSpoolDir-1] = '\0'; + --pThis->lenSpoolDir; + parser_errmsg("queue.spooldirectory must not end with '/', " + "corrected to '%s'", pThis->pszSpoolDir); + } } else if(!strcmp(pblk.descr[i].name, "queue.size")) { pThis->iMaxQueueSize = pvals[i].val.d.n; } else if(!strcmp(pblk.descr[i].name, "queue.dequeuebatchsize")) { -- cgit v1.2.3 From df31131f966b62962eebc42a711c5cf7ed0861bf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 21 Nov 2013 11:15:35 +0100 Subject: doc: add new queue parameter --- doc/queue_parameters.html | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/doc/queue_parameters.html b/doc/queue_parameters.html index 0237a475..049c467d 100644 --- a/doc/queue_parameters.html +++ b/doc/queue_parameters.html @@ -15,7 +15,19 @@ Queues need to be configured in the action or ruleset it should affect. If nothi default values will be used. Thus, the default ruleset has only the default main queue. Specific Action queues are not set up by default.

    -
  • queue.filename name
    • +
  • queue.filename name
    + File name to be used for the queue files. Please note that + this is actually just the file name. A directory can NOT be + specified in this paramter. If the files shall be + created in a specific directory, specify + queue.spoolDirectory for this. The filename is used to build + to complete path for queue files.
    • +
  • queue.spoolDirectory name
    + This is the directory into which queue files will be stored. + Note that the directory must exist, it is NOT automatically + created by rsyslog. If no spoolDirectory is specified, the + work directory is used. +
  • queue.size number
    This is the maximum size of the queue in number of messages. Note that setting the queue size to very small values (roughly -- cgit v1.2.3 From b72a26b304a83b1f1ab69784c364ba975851eed5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 21 Nov 2013 11:34:33 +0100 Subject: queue: guard various config param defaults against very low queue sizes --- runtime/queue.c | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index b1938ac1..336e6ba5 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2139,23 +2139,42 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ } } - if(pThis->iHighWtrMrk < 2 || pThis->iHighWtrMrk > pThis->iMaxQueueSize) + if(pThis->iHighWtrMrk < 2 || pThis->iHighWtrMrk > pThis->iMaxQueueSize) { pThis->iHighWtrMrk = (pThis->iMaxQueueSize / 100) * 90; + if(pThis->iHighWtrMrk == 0) { /* guard against very low max queue sizes! */ + pThis->iHighWtrMrk = pThis->iMaxQueueSize; + } + } if( pThis->iLowWtrMrk < 2 || pThis->iLowWtrMrk > pThis->iMaxQueueSize - || pThis->iLowWtrMrk > pThis->iHighWtrMrk ) + || pThis->iLowWtrMrk > pThis->iHighWtrMrk ) { pThis->iLowWtrMrk = (pThis->iMaxQueueSize / 100) * 70; + if(pThis->iLowWtrMrk == 0) { + pThis->iLowWtrMrk = 1; + } + } if(pThis->iNumWorkerThreads > 1) { if( pThis->iMinMsgsPerWrkr < 1 || pThis->iMinMsgsPerWrkr > pThis->iMaxQueueSize ) pThis->iMinMsgsPerWrkr = pThis->iMaxQueueSize / pThis->iNumWorkerThreads; } - if(pThis->iFullDlyMrk == -1 || pThis->iFullDlyMrk > pThis->iMaxQueueSize) + if(pThis->iFullDlyMrk == -1 || pThis->iFullDlyMrk > pThis->iMaxQueueSize) { pThis->iFullDlyMrk = (pThis->iMaxQueueSize / 100) * 97; - if(pThis->iLightDlyMrk == -1 || pThis->iLightDlyMrk > pThis->iMaxQueueSize) + if(pThis->iFullDlyMrk == 0) { + pThis->iFullDlyMrk = + (pThis->iMaxQueueSize == 1) ? 1 : pThis->iMaxQueueSize - 1; + } + } + if(pThis->iLightDlyMrk == -1 || pThis->iLightDlyMrk > pThis->iMaxQueueSize) { pThis->iLightDlyMrk = (pThis->iMaxQueueSize / 100) * 70; - if(pThis->iMaxQueueSize > 0 && pThis->iDeqBatchSize > pThis->iMaxQueueSize) + if(pThis->iLightDlyMrk == 0) { + pThis->iLightDlyMrk = + (pThis->iMaxQueueSize == 1) ? 1 : pThis->iMaxQueueSize - 1; + } + } + if(pThis->iMaxQueueSize > 0 && pThis->iDeqBatchSize > pThis->iMaxQueueSize) { pThis->iDeqBatchSize = pThis->iMaxQueueSize; + } /* finalize some initializations that could not yet be done because it is * influenced by properties which might have been set after queueConstruct () @@ -2189,14 +2208,14 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ DBGOPRINT((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, spoolDir '%s', maxFileSz %lld, " "maxQSize %d, lqsize %d, pqsize %d, child %d, full delay %d, " - "light delay %d, deq batch size %d starting, high wtrmrk %d, low wtrmrk %d, " - "max wrkr %d, min msgs f. wrkr %d\n", + "light delay %d, deq batch size %d, high wtrmrk %d, low wtrmrk %d, " + "discardmrk %d, max wrkr %d, min msgs f. wrkr %d starting\n", pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->pszSpoolDir, pThis->iMaxFileSize, pThis->iMaxQueueSize, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1, pThis->iFullDlyMrk, pThis->iLightDlyMrk, pThis->iDeqBatchSize, pThis->iHighWtrMrk, pThis->iLowWtrMrk, - pThis->iNumWorkerThreads, pThis->iMinMsgsPerWrkr); + pThis->iDiscardMrk, pThis->iNumWorkerThreads, pThis->iMinMsgsPerWrkr); pThis->bQueueStarted = 1; if(pThis->qType == QUEUETYPE_DIRECT) -- cgit v1.2.3 From 639afa3d0aa9df1b12c2ab143be29c8281145c19 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 21 Nov 2013 12:00:41 +0100 Subject: queue: dynamic default for discardMark, emit warning if set very low --- ChangeLog | 3 +++ runtime/queue.c | 33 +++++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 84efed67..0ed1622a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,7 +3,10 @@ Version 7.5.7 [v7-devel] 2013-11-?? - queue defaults have changed * high water mark is now dynamically 90% of queue size * low water makr is now dynamically 70% of queue size + * queue.discardMark is now dynamically 98% of queue size * queue.workerThreadMinimumMessage set to queue.size / num workers + For queues with very low queue.maxSize (< 100), "emergency" defaults + will be used. - bugfix: RainerScript optimizer did not optimize PRI filters things like "if $syslogfacility-text == "local3"" were not converted to PRIFILT. This was a regression introduced in 7.5.6. diff --git a/runtime/queue.c b/runtime/queue.c index 336e6ba5..e6dd6d69 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1379,7 +1379,7 @@ qqueueSetDefaultsActionQueue(qqueue_t *pThis) pThis->iDeqBatchSize = 128; /* default batch size */ pThis->iHighWtrMrk = -1; /* high water mark for disk-assisted queues */ pThis->iLowWtrMrk = -1; /* low water mark for disk-assisted queues */ - pThis->iDiscardMrk = 980; /* begin to discard messages */ + pThis->iDiscardMrk = -1; /* begin to discard messages */ pThis->iDiscardSeverity = 8; /* turn off */ pThis->iNumWorkerThreads = 1; /* number of worker threads for the mm queue above */ pThis->iMaxFileSize = 1024*1024; @@ -1409,7 +1409,7 @@ qqueueSetDefaultsRulesetQueue(qqueue_t *pThis) pThis->iDeqBatchSize = 1024; /* default batch size */ pThis->iHighWtrMrk = -1; /* high water mark for disk-assisted queues */ pThis->iLowWtrMrk = -1; /* low water mark for disk-assisted queues */ - pThis->iDiscardMrk = 49500; /* begin to discard messages */ + pThis->iDiscardMrk = -1; /* begin to discard messages */ pThis->iDiscardSeverity = 8; /* turn off */ pThis->iNumWorkerThreads = 1; /* number of worker threads for the mm queue above */ pThis->iMaxFileSize = 16*1024*1024; @@ -2139,6 +2139,22 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ } } + if(pThis->iDiscardMrk > pThis->iMaxQueueSize) { + errmsg.LogError(0, RS_RET_CONF_PARSE_WARNING, "queue \"%s\": " + "queue.discardMark %d is set larger than queue.size", + obj.GetName((obj_t*) pThis), pThis->iDiscardMrk); + } + + goodval = (pThis->iMaxQueueSize / 100) * 80; + if(pThis->iDiscardMrk != -1 && pThis->iDiscardMrk < goodval) { + errmsg.LogError(0, RS_RET_CONF_PARSE_WARNING, "queue \"%s\": queue.discardMark " + "is set quite low at %d. You should only set it below " + "80%% (%d) if you have a good reason for this.", + obj.GetName((obj_t*) pThis), pThis->iDiscardMrk, goodval); + } + + + /* now come parameter corrections and defaults */ if(pThis->iHighWtrMrk < 2 || pThis->iHighWtrMrk > pThis->iMaxQueueSize) { pThis->iHighWtrMrk = (pThis->iMaxQueueSize / 100) * 90; if(pThis->iHighWtrMrk == 0) { /* guard against very low max queue sizes! */ @@ -2172,6 +2188,15 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ (pThis->iMaxQueueSize == 1) ? 1 : pThis->iMaxQueueSize - 1; } } + + if(pThis->iDiscardMrk < 1 || pThis->iDiscardMrk > pThis->iMaxQueueSize) { + pThis->iDiscardMrk = (pThis->iMaxQueueSize / 100) * 98; + if(pThis->iDiscardMrk == 0) { + /* for very small queues, we disable this by default */ + pThis->iDiscardMrk = pThis->iMaxQueueSize; + } + } + if(pThis->iMaxQueueSize > 0 && pThis->iDeqBatchSize > pThis->iMaxQueueSize) { pThis->iDeqBatchSize = pThis->iMaxQueueSize; } @@ -2206,10 +2231,10 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ pThis->iFullDlyMrk = wrk; } - DBGOPRINT((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, spoolDir '%s', maxFileSz %lld, " + DBGOPRINT((obj_t*) pThis, "params: type %d, enq-only %d, disk assisted %d, spoolDir '%s', maxFileSz %lld, " "maxQSize %d, lqsize %d, pqsize %d, child %d, full delay %d, " "light delay %d, deq batch size %d, high wtrmrk %d, low wtrmrk %d, " - "discardmrk %d, max wrkr %d, min msgs f. wrkr %d starting\n", + "discardmrk %d, max wrkr %d, min msgs f. wrkr %d\n", pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->pszSpoolDir, pThis->iMaxFileSize, pThis->iMaxQueueSize, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis), -- cgit v1.2.3 From f09010e15e4d824ebeef1e092340c283826a3887 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 21 Nov 2013 14:27:16 +0100 Subject: bugfix: mmpstrucdata generated inaccessible properties --- ChangeLog | 1 + doc/mmpstrucdata.html | 23 +++++++++++++++++++++++ plugins/mmpstrucdata/mmpstrucdata.c | 6 ++++-- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0ed1622a..7e0d0052 100644 --- a/ChangeLog +++ b/ChangeLog @@ -7,6 +7,7 @@ Version 7.5.7 [v7-devel] 2013-11-?? * queue.workerThreadMinimumMessage set to queue.size / num workers For queues with very low queue.maxSize (< 100), "emergency" defaults will be used. +- bugfix: mmpstrucdata generated inaccessible properties - bugfix: RainerScript optimizer did not optimize PRI filters things like "if $syslogfacility-text == "local3"" were not converted to PRIFILT. This was a regression introduced in 7.5.6. diff --git a/doc/mmpstrucdata.html b/doc/mmpstrucdata.html index b4003062..8197d94a 100644 --- a/doc/mmpstrucdata.html +++ b/doc/mmpstrucdata.html @@ -13,6 +13,7 @@

    Description:

    The mmpstrucdata parses RFC5424 structured data into the message json variable tree. +The data parsed, if available, is stored under "jsonRoot!rfc5424-sd!...".

     

    Module Configuration Parameters:

    @@ -33,6 +34,10 @@ Specifies into which json container the data shall be parsed to.

    Caveats/Known Bugs:

    • this module is currently experimental; feedback is appreciated +
    • property names are treated case-insensitive in rsyslog. As such, +RFC5424 names are treated case-insensitive as well. If such names +only differ in case (what is not recommended anyways), problems will +occur.
    • structured data with duplicate SD-IDs and SD-PARAMS is not properly processed
    @@ -48,6 +53,24 @@ template(name="jsondump" type="string" string="%msg%: %$!%\n") action(type="omfile" file="/path/to/log" template="jsondump") +

    A more practical one: +

    Take this example message (inspired by RFC5424 sample;)): +

    <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"][id@2 test="tast"] BOM'su root' failed for lonvick on /dev/pts/8 +

    We apply this configuration: +

    +

    This will output: +

    ALL: { "rfc5424-sd": { "examplesdid@32473": { "iut": "3", "eventsource": "Application", "eventid": "1011" }, "id@2": { "test": "tast" } } }
    +SD: { "examplesdid@32473": { "iut": "3", "eventsource": "Application", "eventid": "1011" }, "id@2": { "test": "tast" } }
    +IUT:3
    +RAWMSG: <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"][id@2 test="tast"] BOM'su root' failed for lonvick on /dev/pts/8     +

    As you can seem, you can address each of the individual items. Note that the +case of the RFC5424 parameter names has been converted to lower case. +

    [rsyslog.conf overview] [manual index] [rsyslog site]

    This documentation is part of the diff --git a/plugins/mmpstrucdata/mmpstrucdata.c b/plugins/mmpstrucdata/mmpstrucdata.c index 4b2a985b..123363bc 100644 --- a/plugins/mmpstrucdata/mmpstrucdata.c +++ b/plugins/mmpstrucdata/mmpstrucdata.c @@ -31,6 +31,7 @@ #include #include #include +#include #include "conf.h" #include "syslogd-types.h" #include "srUtils.h" @@ -206,7 +207,8 @@ dbgprintf("DDDD: parseSD_NAME %s\n", sdbuf+*curridx); if( sdbuf[i] == '=' || sdbuf[i] == '"' || sdbuf[i] == ']' || sdbuf[i] == ' ') break; - namebuf[j] = sdbuf[i++]; + namebuf[j] = tolower(sdbuf[i]); + ++i; } namebuf[j] = '\0'; dbgprintf("DDDD: parseSD_NAME, NAME: '%s'\n", namebuf); @@ -337,7 +339,7 @@ dbgprintf("DDDD: json: '%s'\n", json_object_get_string(json)); if(jroot == NULL) { ABORT_FINALIZE(RS_RET_ERR); } - json_object_object_add(jroot, "RFC5424-SD", json); + json_object_object_add(jroot, "rfc5424-sd", json); msgAddJSON(pMsg, pData->jsonRoot, jroot); finalize_it: RETiRet; -- cgit v1.2.3 From 3ee66d9c28a9510a06572f958f2c680729a4cc6c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 Nov 2013 10:51:54 +0100 Subject: testbench: add test for mmpstrucdata --- tests/Makefile.am | 7 +++++++ tests/mmpstrucdata.sh | 12 ++++++++++++ tests/testsuites/mmpstrucdata.conf | 12 ++++++++++++ 3 files changed, 31 insertions(+) create mode 100755 tests/mmpstrucdata.sh create mode 100644 tests/testsuites/mmpstrucdata.conf diff --git a/tests/Makefile.am b/tests/Makefile.am index 13d87dec..23da9696 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -119,6 +119,11 @@ TESTS += \ imptcp_conndrop.sh endif +if ENABLE_MMPSTRUCDATA +TESTS += \ + mmpstrucdata.sh +endif + if ENABLE_GNUTLS # TODO: re-enable in newer version #TESTS += \ @@ -517,6 +522,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ mysql-asyn.sh \ mysql-asyn-vg.sh \ testsuites/mysql-asyn.conf \ + mmpstrucdata.sh \ + testsuites/mmpstrucdata.conf \ cfg.sh # TODO: re-enable diff --git a/tests/mmpstrucdata.sh b/tests/mmpstrucdata.sh new file mode 100755 index 00000000..62b6ba96 --- /dev/null +++ b/tests/mmpstrucdata.sh @@ -0,0 +1,12 @@ +# This file is part of the rsyslog project, released under ASL 2.0 +# rgerhards, 2013-11-22 +echo =============================================================================== +echo \[mmpstrucdata.sh\]: testing mmpstrucdata +source $srcdir/diag.sh init +source $srcdir/diag.sh startup mmpstrucdata.conf +sleep 1 +source $srcdir/diag.sh tcpflood -m100 -y +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 99 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/mmpstrucdata.conf b/tests/testsuites/mmpstrucdata.conf new file mode 100644 index 00000000..fd18fd99 --- /dev/null +++ b/tests/testsuites/mmpstrucdata.conf @@ -0,0 +1,12 @@ +$IncludeConfig diag-common.conf + +module(load="../plugins/mmpstrucdata/.libs/mmpstrucdata") +module(load="../plugins/imtcp/.libs/imtcp") + +template(name="outfmt" type="string" string="%$!rfc5424-sd!tcpflood@32473!msgnum%\n") + +input(type="imtcp" port="13514") + +action(type="mmpstrucdata") +if $msg contains "msgnum" then + action(type="omfile" template="outfmt" file="rsyslog.out.log") -- cgit v1.2.3 From 3cdb796bad3c0480dc390d44966a7dd4a9c1eeb6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 Nov 2013 10:52:05 +0100 Subject: testbench: enable test tools to support RFC5424 natively --- tests/chkseq.c | 29 ++++++++++++++++++++++++----- tests/tcpflood.c | 28 ++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/tests/chkseq.c b/tests/chkseq.c index bea9f83a..bd8597e8 100644 --- a/tests/chkseq.c +++ b/tests/chkseq.c @@ -51,6 +51,7 @@ int main(int argc, char *argv[]) int reachedEOF; int edLen; /* length of extra data */ static char edBuf[500*1024]; /* buffer for extra data (pretty large to be on the save side...) */ + static char ioBuf[sizeof(edBuf)+1024]; char *file = NULL; while((opt = getopt(argc, argv, "e:f:ds:vE")) != EOF) { @@ -103,14 +104,22 @@ int main(int argc, char *argv[]) for(i = start ; i < end+1 ; ++i) { if(bHaveExtraData) { - scanfOK = fscanf(fp, "%d,%d,%s\n", &val, &edLen, edBuf) == 3 ? 1 : 0; + if(fgets(ioBuf, sizeof(ioBuf), fp) == NULL) { + scanfOK = 0; + } else { + scanfOK = sscanf(ioBuf, "%d,%d,%s\n", &val, &edLen, edBuf) == 3 ? 1 : 0; + } if(edLen != (int) strlen(edBuf)) { printf("extra data length specified %d, but actually is %ld in record %d\n", edLen, (long) strlen(edBuf), i); exit(1); } } else { - scanfOK = fscanf(fp, "%d\n", &val) == 1 ? 1 : 0; + if(fgets(ioBuf, sizeof(ioBuf), fp) == NULL) { + scanfOK = 0; + } else { + scanfOK = sscanf(ioBuf, "%d\n", &val) == 1 ? 1 : 0; + } } if(!scanfOK) { printf("scanf error in index i=%d\n", i); @@ -132,9 +141,11 @@ int main(int argc, char *argv[]) exit(1); } - if(feof(fp)) { + int c = getc(fp); + if(c == EOF) { reachedEOF = 1; } else { + ungetc(c, fp); /* if duplicates are permitted, we need to do a final check if we have duplicates at the * end of file. */ @@ -142,14 +153,22 @@ int main(int argc, char *argv[]) i = end; while(!feof(fp)) { if(bHaveExtraData) { - scanfOK = fscanf(fp, "%d,%d,%s\n", &val, &edLen, edBuf) == 3 ? 1 : 0; + if(fgets(ioBuf, sizeof(ioBuf), fp) == NULL) { + scanfOK = 0; + } else { + scanfOK = sscanf(ioBuf, "%d,%d,%s\n", &val, &edLen, edBuf) == 3 ? 1 : 0; + } if(edLen != (int) strlen(edBuf)) { printf("extra data length specified %d, but actually is %ld in record %d\n", edLen, (long) strlen(edBuf), i); exit(1); } } else { - scanfOK = fscanf(fp, "%d\n", &val) == 1 ? 1 : 0; + if(fgets(ioBuf, sizeof(ioBuf), fp) == NULL) { + scanfOK = 0; + } else { + scanfOK = sscanf(ioBuf, "%d\n", &val) == 1 ? 1 : 0; + } } if(val != i) { diff --git a/tests/tcpflood.c b/tests/tcpflood.c index b3cef2e0..f17363f2 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -48,13 +48,14 @@ * -b number of messages within a batch (default: 100,000,000 millions) * -Y use multiple threads, one per connection (which means 1 if one only connection * is configured!) + * -y use RFC5424 style test message * -z private key file for TLS mode * -Z cert (public key) file for TLS mode * -L loglevel to use for GnuTLS troubleshooting (0-off to 10-all, 0 default) * * Part of the testbench for rsyslog. * - * Copyright 2009, 2010 Rainer Gerhards and Adiscon GmbH. + * Copyright 2009, 2013 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -111,6 +112,7 @@ static int targetPort = 13514; static int numTargetPorts = 1; static int dynFileIDs = 0; static int extraDataLen = 0; /* amount of extra data to add to message */ +static int useRFC5424Format = 0; /* should the test message be in RFC5424 format? */ static int bRandomizeExtraData = 0; /* randomize amount of extra data added */ static int numMsgsToSend; /* number of messages to send */ static unsigned numConnections = 1; /* number of connections to create */ @@ -363,8 +365,14 @@ genMsg(char *buf, size_t maxBuf, int *pLenBuf, struct instdata *inst) snprintf(dynFileIDBuf, sizeof(dynFileIDBuf), "%d:", rand() % dynFileIDs); } if(extraDataLen == 0) { - *pLenBuf = snprintf(buf, maxBuf, "<%s>Mar 1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%c", - msgPRI, dynFileIDBuf, msgNum, frameDelim); + if(useRFC5424Format) { + *pLenBuf = snprintf(buf, maxBuf, "<%s>1 2003-03-01T01:00:00.000Z mymachine.example.com tcpflood " + "- tag [tcpflood@32473 MSGNUM=\"%8.8d\"] msgnum:%s%8.8d:%c", + msgPRI, msgNum, dynFileIDBuf, msgNum, frameDelim); + } else { + *pLenBuf = snprintf(buf, maxBuf, "<%s>Mar 1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%c", + msgPRI, dynFileIDBuf, msgNum, frameDelim); + } } else { if(bRandomizeExtraData) edLen = ((long) rand() + extraDataLen) % extraDataLen + 1; @@ -372,8 +380,14 @@ genMsg(char *buf, size_t maxBuf, int *pLenBuf, struct instdata *inst) edLen = extraDataLen; memset(extraData, 'X', edLen); extraData[edLen] = '\0'; - *pLenBuf = snprintf(buf, maxBuf, "<%s>Mar 1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%d:%s%c", - msgPRI, dynFileIDBuf, msgNum, edLen, extraData, frameDelim); + if(useRFC5424Format) { + *pLenBuf = snprintf(buf, maxBuf, "<%s>1 2003-03-01T01:00:00.000Z mymachine.example.com tcpflood " + "- tag [tcpflood@32473 MSGNUM=\"%8.8d\"] msgnum:%s%8.8d:%c", + msgPRI, msgNum, dynFileIDBuf, msgNum, frameDelim); + } else { + *pLenBuf = snprintf(buf, maxBuf, "<%s>Mar 1 01:00:00 172.20.245.8 tag msgnum:%s%8.8d:%d:%s%c", + msgPRI, dynFileIDBuf, msgNum, edLen, extraData, frameDelim); + } } } else { /* use fixed message format from command line */ @@ -830,7 +844,7 @@ int main(int argc, char *argv[]) setvbuf(stdout, buf, _IONBF, 48); - while((opt = getopt(argc, argv, "b:ef:F:t:p:c:C:m:i:I:P:d:Dn:L:M:rsBR:S:T:XW:Yz:Z:")) != -1) { + while((opt = getopt(argc, argv, "b:ef:F:t:p:c:C:m:i:I:P:d:Dn:L:M:rsBR:S:T:XW:yYz:Z:")) != -1) { switch (opt) { case 'b': batchsize = atoll(optarg); break; @@ -908,6 +922,8 @@ int main(int argc, char *argv[]) break; case 'Y': runMultithreaded = 1; break; + case 'y': useRFC5424Format = 1; + break; case 'z': tlsKeyFile = optarg; break; case 'Z': tlsCertFile = optarg; -- cgit v1.2.3 From 13a18895b657bbbefaa16d89b25192facbf3b7f3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 Nov 2013 11:10:08 +0100 Subject: testbench: add test for rfc5424 parser --- tests/Makefile.am | 3 +++ tests/rfc5424parser.sh | 12 ++++++++++++ tests/testsuites/rfc5424parser.conf | 10 ++++++++++ 3 files changed, 25 insertions(+) create mode 100755 tests/rfc5424parser.sh create mode 100644 tests/testsuites/rfc5424parser.conf diff --git a/tests/Makefile.am b/tests/Makefile.am index 23da9696..a0decacc 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -7,6 +7,7 @@ TESTS = $(TESTRUNS) if ENABLE_IMDIAG TESTS += \ stop-localvar.sh \ + rfc5424parser.sh \ arrayqueue.sh \ da-mainmsg-q.sh \ validation-run.sh \ @@ -298,6 +299,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rscript_stop2.conf \ stop-localvar.sh \ testsuites/stop-localvar.conf \ + rfc5424parser.sh \ + testsuites/rfc5424parser.conf \ rs_optimizer_pri.sh \ testsuites/rs_optimizer_pr.conf \ rscript_prifilt.sh \ diff --git a/tests/rfc5424parser.sh b/tests/rfc5424parser.sh new file mode 100755 index 00000000..3f5be497 --- /dev/null +++ b/tests/rfc5424parser.sh @@ -0,0 +1,12 @@ +# This file is part of the rsyslog project, released under ASL 2.0 +# rgerhards, 2013-11-22 +echo =============================================================================== +echo \[rfc5424parser.sh\]: testing mmpstrucdata +source $srcdir/diag.sh init +source $srcdir/diag.sh startup rfc5424parser.conf +sleep 1 +source $srcdir/diag.sh tcpflood -m100 -y +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh seq-check 0 99 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/rfc5424parser.conf b/tests/testsuites/rfc5424parser.conf new file mode 100644 index 00000000..cd90d120 --- /dev/null +++ b/tests/testsuites/rfc5424parser.conf @@ -0,0 +1,10 @@ +$IncludeConfig diag-common.conf + +module(load="../plugins/imtcp/.libs/imtcp") + +template(name="outfmt" type="string" string="%msg:F,58:2%\n") + +input(type="imtcp" port="13514") + +if $msg contains "msgnum" then + action(type="omfile" template="outfmt" file="rsyslog.out.log") -- cgit v1.2.3 From c76ec71d6f15fb6a94613aad1d0b8ecfa75a37ca Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 Nov 2013 11:23:45 +0100 Subject: typo fix --- tests/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 5465774d..ac964640 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -309,7 +309,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ rfc5424parser.sh \ testsuites/rfc5424parser.conf \ rs_optimizer_pri.sh \ - testsuites/rs_optimizer_pr.conf \ + testsuites/rs_optimizer_pri.conf \ rscript_prifilt.sh \ testsuites/rscript_prifilt.conf \ rscript_optimizer1.sh \ -- cgit v1.2.3 From dbcdca7e4742d33220eeac535501ddd6e185a7e3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 Nov 2013 17:33:43 +0100 Subject: doc: add link to new online impstats analyzer tool --- doc/impstats.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/impstats.html b/doc/impstats.html index 392fc431..770f67a3 100644 --- a/doc/impstats.html +++ b/doc/impstats.html @@ -24,6 +24,13 @@ settings, this impact may be noticable (for high-load environments).

    The rsyslog website has an updated overview of available rsyslog statistic counters.

    +

    Note that there is a +rsyslog statistics +online analyzer available. It can be given a impstats-generated file and +will return problems it detects. Note that the analyzer cannot replace a +human in getting things right, but it is expected to be a good aid in starting +to understand and gain information from the pstats logs. +<7p>

    Module Confguration Parameters:

    This module supports module parameters, only.

      -- cgit v1.2.3 From a2b4a8ac920434afc11d55a447a121a1236409e1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 24 Nov 2013 10:14:16 +0100 Subject: prepare for upcoming liblognorm 1.0.0 release The new liblognorm relase has an incompatible API, so we cannot use it to compile this version of rsyslog. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index ee3394b5..2fc51a67 100644 --- a/configure.ac +++ b/configure.ac @@ -920,7 +920,7 @@ AC_ARG_ENABLE(mmnormalize, ) if test "x$enable_mmnormalize" = "xyes"; then PKG_CHECK_MODULES(LIBEE, libee >= 0.4.0) - PKG_CHECK_MODULES(LIBLOGNORM, lognorm >= 0.3.1) + PKG_CHECK_MODULES(LIBLOGNORM, lognorm >= 0.3.1 lognorm < 1.0.0) fi AM_CONDITIONAL(ENABLE_MMNORMALIZE, test x$enable_mmnormalize = xyes) -- cgit v1.2.3 From 68db6cc0140d0a746c0ea6c8176ce415c9bdd997 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 24 Nov 2013 18:13:13 +0100 Subject: testbench: stop-localvar did actually test msg vars - now fixed --- tests/testsuites/stop-localvar.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/testsuites/stop-localvar.conf b/tests/testsuites/stop-localvar.conf index 020ebd87..63df6509 100644 --- a/tests/testsuites/stop-localvar.conf +++ b/tests/testsuites/stop-localvar.conf @@ -5,17 +5,17 @@ * rgerhards, 2013-11-19 */ $IncludeConfig diag-common.conf -template(name="outfmt" type="string" string="%$!nbr%\n") +template(name="outfmt" type="string" string="%$.nbr%\n") module(load="../plugins/imtcp/.libs/imtcp") input(type="imtcp" port="13514") if $msg contains "msgnum:" then { - set $!nbr = field($msg, 58, 2); - if cnum($!nbr) < 100 then + set $.nbr = field($msg, 58, 2); + if cnum($.nbr) < 100 then stop /* check is intentionally more complex than needed! */ - else if not (cnum($!nbr) > 999) then { + else if not (cnum($.nbr) > 999) then { action(type="omfile" file="rsyslog.out.log" template="outfmt") } } -- cgit v1.2.3 From 8592955c3ff97ece4e21c96454ee681743557a76 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 24 Nov 2013 18:14:23 +0100 Subject: testbench: add (corrected) test for msg variables --- tests/Makefile.am | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/Makefile.am b/tests/Makefile.am index ac964640..5232e3ef 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -7,6 +7,7 @@ TESTS = $(TESTRUNS) if ENABLE_IMDIAG TESTS += \ stop-localvar.sh \ + stop-msgvar.sh \ rfc5424parser.sh \ arrayqueue.sh \ global_vars.sh \ @@ -304,6 +305,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/stop.conf \ stop-localvar.sh \ testsuites/stop-localvar.conf \ + stop-msgvar.sh \ + testsuites/stop-msgvar.conf \ global_vars.sh \ testsuites/global_vars.conf \ rfc5424parser.sh \ -- cgit v1.2.3 From 0248e97a5ec34b5a73cb4af73d97cf7baa29ad64 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 24 Nov 2013 18:13:13 +0100 Subject: testbench: stop-localvar did actually test msg vars - now fixed --- tests/testsuites/stop-localvar.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/testsuites/stop-localvar.conf b/tests/testsuites/stop-localvar.conf index 020ebd87..63df6509 100644 --- a/tests/testsuites/stop-localvar.conf +++ b/tests/testsuites/stop-localvar.conf @@ -5,17 +5,17 @@ * rgerhards, 2013-11-19 */ $IncludeConfig diag-common.conf -template(name="outfmt" type="string" string="%$!nbr%\n") +template(name="outfmt" type="string" string="%$.nbr%\n") module(load="../plugins/imtcp/.libs/imtcp") input(type="imtcp" port="13514") if $msg contains "msgnum:" then { - set $!nbr = field($msg, 58, 2); - if cnum($!nbr) < 100 then + set $.nbr = field($msg, 58, 2); + if cnum($.nbr) < 100 then stop /* check is intentionally more complex than needed! */ - else if not (cnum($!nbr) > 999) then { + else if not (cnum($.nbr) > 999) then { action(type="omfile" file="rsyslog.out.log" template="outfmt") } } -- cgit v1.2.3 From 78a919444daec19b4f2713a78453f308b94adb15 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 24 Nov 2013 18:14:23 +0100 Subject: testbench: add (corrected) test for msg variables Conflicts: tests/Makefile.am --- tests/Makefile.am | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/Makefile.am b/tests/Makefile.am index a0decacc..fe1c8204 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -7,6 +7,7 @@ TESTS = $(TESTRUNS) if ENABLE_IMDIAG TESTS += \ stop-localvar.sh \ + stop-msgvar.sh \ rfc5424parser.sh \ arrayqueue.sh \ da-mainmsg-q.sh \ @@ -299,6 +300,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rscript_stop2.conf \ stop-localvar.sh \ testsuites/stop-localvar.conf \ + stop-msgvar.sh \ + testsuites/stop-msgvar.conf \ rfc5424parser.sh \ testsuites/rfc5424parser.conf \ rs_optimizer_pri.sh \ -- cgit v1.2.3