From 4269e4578118f089021bc15cdd82bb0182a97aaf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Oct 2012 18:54:25 +0200 Subject: new ratelimit: interface plumbing added no actual implementation yet done --- plugins/imfile/imfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'plugins/imfile/imfile.c') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 453b6b05..64a7e032 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -191,7 +191,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) MsgSetRuleset(pMsg, pInfo->pRuleset); pInfo->multiSub.ppMsgs[pInfo->multiSub.nElem++] = pMsg; if(pInfo->multiSub.nElem == pInfo->multiSub.maxElem) - CHKiRet(multiSubmitMsg(&pInfo->multiSub)); + CHKiRet(multiSubmitMsg2(&pInfo->multiSub, NULL)); finalize_it: RETiRet; } @@ -306,7 +306,7 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) finalize_it: if(pThis->multiSub.nElem > 0) { /* submit everything that was not yet submitted */ - CHKiRet(multiSubmitMsg(&pThis->multiSub)); + CHKiRet(multiSubmitMsg2(&pThis->multiSub, NULL)); } ; /*EMPTY STATEMENT - needed to keep compiler happy - see below! */ /* Note: the problem above is that pthread:cleanup_pop() is a macro which -- cgit v1.2.3 From 2a6ff7d53d1c30f0d7efd022cf321736870c5629 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 15 Oct 2012 08:55:12 +0200 Subject: interface cleanup (probably not final) --- plugins/imfile/imfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins/imfile/imfile.c') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 64a7e032..d6e6ad0e 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -191,7 +191,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) MsgSetRuleset(pMsg, pInfo->pRuleset); pInfo->multiSub.ppMsgs[pInfo->multiSub.nElem++] = pMsg; if(pInfo->multiSub.nElem == pInfo->multiSub.maxElem) - CHKiRet(multiSubmitMsg2(&pInfo->multiSub, NULL)); + CHKiRet(multiSubmitMsg(&pInfo->multiSub)); finalize_it: RETiRet; } -- cgit v1.2.3 From 3806643baa9fe0d50fb36080e4ab3a078b8a5952 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 15 Oct 2012 11:19:22 +0200 Subject: ratelimiter: enable thread-safe mode & (related) API changes --- plugins/imfile/imfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins/imfile/imfile.c') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index d6e6ad0e..453b6b05 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -306,7 +306,7 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) finalize_it: if(pThis->multiSub.nElem > 0) { /* submit everything that was not yet submitted */ - CHKiRet(multiSubmitMsg2(&pThis->multiSub, NULL)); + CHKiRet(multiSubmitMsg(&pThis->multiSub)); } ; /*EMPTY STATEMENT - needed to keep compiler happy - see below! */ /* Note: the problem above is that pthread:cleanup_pop() is a macro which -- cgit v1.2.3 From a3a3a322d8c7d40dfed0765457d26a484a624bce Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 15 Oct 2012 12:22:16 +0200 Subject: imfile: converted to new ratelimit interface --- plugins/imfile/imfile.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) (limited to 'plugins/imfile/imfile.c') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 453b6b05..99210b76 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -48,6 +48,7 @@ #include "prop.h" #include "stringbuf.h" #include "ruleset.h" +#include "ratelimit.h" MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ MODULE_TYPE_NOKEEP @@ -82,6 +83,7 @@ typedef struct fileInfo_s { strm_t *pStrm; /* its stream (NULL if not assigned) */ int readMode; /* which mode to use in ReadMulteLine call? */ ruleset_t *pRuleset; /* ruleset to bind listener to (use system default if unspecified) */ + ratelimit_t *ratelimiter; multi_submit_t multiSub; } fileInfo_t; @@ -189,9 +191,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) pMsg->iFacility = LOG_FAC(pInfo->iFacility); pMsg->iSeverity = LOG_PRI(pInfo->iSeverity); MsgSetRuleset(pMsg, pInfo->pRuleset); - pInfo->multiSub.ppMsgs[pInfo->multiSub.nElem++] = pMsg; - if(pInfo->multiSub.nElem == pInfo->multiSub.maxElem) - CHKiRet(multiSubmitMsg(&pInfo->multiSub)); + ratelimitAddMsg(pInfo->ratelimiter, &pInfo->multiSub, pMsg); finalize_it: RETiRet; } @@ -304,18 +304,7 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) } finalize_it: - if(pThis->multiSub.nElem > 0) { - /* submit everything that was not yet submitted */ - CHKiRet(multiSubmitMsg(&pThis->multiSub)); - } - ; /*EMPTY STATEMENT - needed to keep compiler happy - see below! */ - /* Note: the problem above is that pthread:cleanup_pop() is a macro which - * evaluates to something like "} while(0);". So the code would become - * "finalize_it: }", that is a label without a statement. The C standard does - * not permit this. So we add an empty statement "finalize_it: ; }" and - * everybody is happy. Note that without the ;, an error is reported only - * on some platforms/compiler versions. -- rgerhards, 2008-08-15 - */ + multiSubmitFlush(&pThis->multiSub); pthread_cleanup_pop(0); if(pCStr != NULL) { @@ -423,6 +412,7 @@ addListner(instanceConf_t *inst) pThis->lenTag = ustrlen(pThis->pszTag); pThis->pszStateFile = (uchar*) strdup((char*) inst->pszStateFile); + CHKiRet(ratelimitNew(&pThis->ratelimiter)); CHKmalloc(pThis->multiSub.ppMsgs = MALLOC(inst->nMultiSub * sizeof(msg_t*))); pThis->multiSub.maxElem = inst->nMultiSub; pThis->multiSub.nElem = 0; @@ -773,6 +763,8 @@ CODESTARTafterRun persistStrmState(&files[i]); strm.Destruct(&(files[i].pStrm)); } + ratelimitDestruct(files[i].ratelimiter); + free(files[i].multiSub.ppMsgs); free(files[i].pszFileName); free(files[i].pszTag); free(files[i].pszStateFile); -- cgit v1.2.3 From 34a88a7e9a9501593b6fe9f79cc96963c4da7cde Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 15 Oct 2012 16:39:36 +0200 Subject: ratelimit: added linux-like ratelimiter type --- plugins/imfile/imfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins/imfile/imfile.c') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 99210b76..9a8a0373 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -412,7 +412,7 @@ addListner(instanceConf_t *inst) pThis->lenTag = ustrlen(pThis->pszTag); pThis->pszStateFile = (uchar*) strdup((char*) inst->pszStateFile); - CHKiRet(ratelimitNew(&pThis->ratelimiter)); + CHKiRet(ratelimitNew(&pThis->ratelimiter, "imfile", inst->pszFileName)); CHKmalloc(pThis->multiSub.ppMsgs = MALLOC(inst->nMultiSub * sizeof(msg_t*))); pThis->multiSub.maxElem = inst->nMultiSub; pThis->multiSub.nElem = 0; -- cgit v1.2.3 From 68414e7815813cf7a433b9b8a8cfbb28ec27c637 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 15 Oct 2012 17:19:12 +0200 Subject: cosmetic: remove compiler warning --- plugins/imfile/imfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins/imfile/imfile.c') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 9a8a0373..d50f917e 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -412,7 +412,7 @@ addListner(instanceConf_t *inst) pThis->lenTag = ustrlen(pThis->pszTag); pThis->pszStateFile = (uchar*) strdup((char*) inst->pszStateFile); - CHKiRet(ratelimitNew(&pThis->ratelimiter, "imfile", inst->pszFileName)); + CHKiRet(ratelimitNew(&pThis->ratelimiter, "imfile", (char*)inst->pszFileName)); CHKmalloc(pThis->multiSub.ppMsgs = MALLOC(inst->nMultiSub * sizeof(msg_t*))); pThis->multiSub.maxElem = inst->nMultiSub; pThis->multiSub.nElem = 0; -- cgit v1.2.3 From f522a3fc64cf95d36609ad4f0f9b119c3fba06a2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 14 Dec 2012 15:08:30 +0100 Subject: bugfix: very minimal bugfix in imfile not really relevant for production, a few KB were lost during initialization phase if no state file was present --- plugins/imfile/imfile.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'plugins/imfile/imfile.c') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index d50f917e..0f155c10 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -246,6 +246,8 @@ finalize_it: strm.Destruct(&psSF); if(iRet != RS_RET_OK) { + if(pThis->pStrm != NULL) + strm.Destruct(&pThis->pStrm); CHKiRet(strm.Construct(&pThis->pStrm)); CHKiRet(strm.SettOperationsMode(pThis->pStrm, STREAMMODE_READ)); CHKiRet(strm.SetsType(pThis->pStrm, STREAMTYPE_FILE_MONITOR)); -- cgit v1.2.3 From 35bafbef04a8f7ba56abebbed337f79b8d6dd67f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 Mar 2013 17:05:18 +0100 Subject: imfile: now detects file change when rsyslog was inactive Previosly, this case could not be detected, so if a file was overwritten or rotated away while rsyslog was stopped, some data was missing. This is now detected and the new file being forwarded right from the beginning. closes: http://bugzilla.adiscon.com/show_bug.cgi?id=228 --- plugins/imfile/imfile.c | 1 + 1 file changed, 1 insertion(+) (limited to 'plugins/imfile/imfile.c') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 0f155c10..6ce08fd4 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -235,6 +235,7 @@ openFile(fileInfo_t *pThis) /* read back in the object */ CHKiRet(obj.Deserialize(&pThis->pStrm, (uchar*) "strm", psSF, NULL, pThis)); + strm.CheckFileChange(pThis->pStrm); CHKiRet(strm.SeekCurrOffs(pThis->pStrm)); /* note: we do not delete the state file, so that the last position remains -- cgit v1.2.3 From b96674bbc9439f107c700d75e69268ee75e0387e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 Mar 2013 18:10:17 +0100 Subject: imfile: errors persisting state file are now reported closes: http://bugzilla.adiscon.com/show_bug.cgi?id=292 --- plugins/imfile/imfile.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'plugins/imfile/imfile.c') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 6ce08fd4..349acead 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -739,12 +739,20 @@ persistStrmState(fileInfo_t *pInfo) CHKiRet(strm.ConstructFinalize(psSF)); CHKiRet(strm.Serialize(pInfo->pStrm, psSF)); + CHKiRet(strm.Flush(psSF)); CHKiRet(strm.Destruct(&psSF)); finalize_it: if(psSF != NULL) strm.Destruct(&psSF); + + if(iRet != RS_RET_OK) { + errmsg.LogError(0, iRet, "imfile: could not persist state " + "file %s - data may be repeated on next " + "startup. Is WorkDirectory set?", + pInfo->pszStateFile); + } RETiRet; } -- cgit v1.2.3 From 0f4ec4ad04011c9eb63754d10bbd0d912a227671 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Tue, 25 Jun 2013 09:52:56 +0200 Subject: typo fix ("listner" vs. "listener") --- plugins/imfile/imfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins/imfile/imfile.c') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 349acead..89846dad 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -596,7 +596,7 @@ CODESTARTactivateCnf for(inst = runModConf->root ; inst != NULL ; inst = inst->next) { addListner(inst); } - /* if we could not set up any listners, there is no point in running... */ + /* if we could not set up any listeners, there is no point in running... */ if(iFilPtr == 0) { errmsg.LogError(0, NO_ERRCODE, "imfile: no file monitors could be started, " "input not activated.\n"); -- cgit v1.2.3 From bb22ba16d69eada5910a6ce604387ad35f0d9d4a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 25 Jun 2013 12:54:09 +0200 Subject: imfile: remove redundant error mesage --- plugins/imfile/imfile.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'plugins/imfile/imfile.c') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 89846dad..2e80ffc8 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -449,8 +449,6 @@ CODESTARTnewInpInst pvals = nvlstGetParams(lst, &inppblk, NULL); if(pvals == NULL) { - errmsg.LogError(0, RS_RET_MISSING_CNFPARAMS, - "imfile: required parameter are missing\n"); ABORT_FINALIZE(RS_RET_MISSING_CNFPARAMS); } -- cgit v1.2.3 From 9d945773a757ce87b587e5c6642f63e5626ca44c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 2 Sep 2013 15:11:58 +0200 Subject: bugfix: imfile parameter "persistStateInterval" was unusable due to a case typo in imfile; work-around was to use legacy config Thanks to Brandon Murphy for reporting this bug. --- plugins/imfile/imfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'plugins/imfile/imfile.c') diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 2e80ffc8..45882fb2 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -478,7 +478,7 @@ CODESTARTnewInpInst inst->readMode = pvals[i].val.d.n; } else if(!strcmp(inppblk.descr[i].name, "maxlinesatonce")) { inst->maxLinesAtOnce = pvals[i].val.d.n; - } else if(!strcmp(inppblk.descr[i].name, "persistStateInterval")) { + } else if(!strcmp(inppblk.descr[i].name, "persiststateinterval")) { inst->iPersistStateInterval = pvals[i].val.d.n; } else if(!strcmp(inppblk.descr[i].name, "maxsubmitatonce")) { inst->nMultiSub = pvals[i].val.d.n; -- cgit v1.2.3