diff options
Diffstat (limited to 'runtime')
-rw-r--r-- | runtime/Makefile.am | 3 | ||||
-rw-r--r-- | runtime/conf.c | 9 | ||||
-rw-r--r-- | runtime/glbl.c | 2 | ||||
-rw-r--r-- | runtime/msg.c | 96 | ||||
-rw-r--r-- | runtime/msg.h | 2 | ||||
-rw-r--r-- | runtime/queue.c | 2 | ||||
-rw-r--r-- | runtime/ratelimit.c | 359 | ||||
-rw-r--r-- | runtime/ratelimit.h | 51 | ||||
-rw-r--r-- | runtime/rsconf.c | 4 | ||||
-rw-r--r-- | runtime/rsyslog.h | 2 | ||||
-rw-r--r-- | runtime/ruleset.c | 3 | ||||
-rw-r--r-- | runtime/stream.c | 142 | ||||
-rw-r--r-- | runtime/stream.h | 7 | ||||
-rw-r--r-- | runtime/typedefs.h | 1 |
14 files changed, 577 insertions, 106 deletions
diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 7af26d2b..fbc92d9c 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -65,12 +65,13 @@ librsyslog_la_SOURCES = \ ruleset.h \ prop.c \ prop.h \ + ratelimit.c \ + ratelimit.h \ cfsysline.c \ cfsysline.h \ sd-daemon.c \ sd-daemon.h \ \ - \ ../action.h \ ../action.c \ ../threads.c \ diff --git a/runtime/conf.c b/runtime/conf.c index 23fb6bbd..c97391c6 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -607,13 +607,8 @@ rsRetVal cflineDoAction(rsconf_t *conf, uchar **p, action_t **ppAction) if(iRet == RS_RET_OK || iRet == RS_RET_SUSPENDED) { if((iRet = addAction(&pAction, pMod, pModData, pOMSR, NULL, NULL, (iRet == RS_RET_SUSPENDED)? 1 : 0)) == RS_RET_OK) { - /* now check if the module is compatible with select features */ - if(pMod->isCompatibleWithFeature(sFEATURERepeatedMsgReduction) == RS_RET_OK) - pAction->f_ReduceRepeated = loadConf->globals.bReduceRepeatMsgs; - else { - dbgprintf("module is incompatible with RepeatedMsgReduction - turned off\n"); - pAction->f_ReduceRepeated = 0; - } + /* here check if the module is compatible with select features + * (currently, we have no such features!) */ pAction->eState = ACT_STATE_RDY; /* action is enabled */ conf->actions.nbrActions++; /* one more active action! */ } diff --git a/runtime/glbl.c b/runtime/glbl.c index a0997829..0e5cac20 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -210,7 +210,7 @@ setLocalHostIPIF(void __attribute__((unused)) *pVal, uchar *pNewVal) if(propLocalIPIF != NULL) { errmsg.LogError(0, RS_RET_ERR, "$LocalHostIPIF is already set " "and cannot be reset; place it at TOP OF rsyslog.conf!"); - ABORT_FINALIZE(RS_RET_ERR_WRKDIR); + ABORT_FINALIZE(RS_RET_ERR); } localRet = net.GetIFIPAddr(pNewVal, AF_UNSPEC, myIP, (int) sizeof(myIP)); diff --git a/runtime/msg.c b/runtime/msg.c index d874178b..b0b93f98 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -2373,40 +2373,38 @@ char *textpri(char *pRes, size_t pResLen, int pri) */ typedef enum ENOWType { NOW_NOW, NOW_YEAR, NOW_MONTH, NOW_DAY, NOW_HOUR, NOW_HHOUR, NOW_QHOUR, NOW_MINUTE } eNOWType; #define tmpBUFSIZE 16 /* size of formatting buffer */ -static uchar *getNOW(eNOWType eNow) +static uchar *getNOW(eNOWType eNow, struct syslogTime *t) { uchar *pBuf; - struct syslogTime t; if((pBuf = (uchar*) MALLOC(sizeof(uchar) * tmpBUFSIZE)) == NULL) { return NULL; } - datetime.getCurrTime(&t, NULL); switch(eNow) { case NOW_NOW: - snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d-%2.2d-%2.2d", t.year, t.month, t.day); + snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d-%2.2d-%2.2d", t->year, t->month, t->day); break; case NOW_YEAR: - snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d", t.year); + snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d", t->year); break; case NOW_MONTH: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.month); + snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t->month); break; case NOW_DAY: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.day); + snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t->day); break; case NOW_HOUR: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.hour); + snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t->hour); break; case NOW_HHOUR: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.minute / 30); + snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t->minute / 30); break; case NOW_QHOUR: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.minute / 15); + snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t->minute / 15); break; case NOW_MINUTE: - snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t.minute); + snprintf((char*) pBuf, tmpBUFSIZE, "%2.2d", t->minute); break; } @@ -2673,7 +2671,7 @@ finalize_it: * Parameter "bMustBeFreed" is set by this function. It tells the * caller whether or not the string returned must be freed by the * caller itself. It is is 0, the caller MUST NOT free it. If it is - * 1, the caller MUST free 1. Handling this wrongly leads to either + * 1, the caller MUST free it. Handling this wrongly leads to either * a memory leak of a program abort (do to double-frees or frees on * the constant memory pool). So be careful to do it right. * rgerhards 2004-11-23 @@ -2690,7 +2688,7 @@ finalize_it: return(UCHAR_CONSTANT("**OUT OF MEMORY**"));} uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, propid_t propid, es_str_t *propName, rs_size_t *pPropLen, - unsigned short *pbMustBeFreed) + unsigned short *pbMustBeFreed, struct syslogTime *ttNow) { uchar *pRes; /* result pointer */ rs_size_t bufLen = -1; /* length of string or -1, if not known */ @@ -2803,52 +2801,68 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = (uchar*)getParseSuccess(pMsg); break; case PROP_SYS_NOW: - if((pRes = getNOW(NOW_NOW)) == NULL) { + if((pRes = getNOW(NOW_NOW, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 10; + } break; case PROP_SYS_YEAR: - if((pRes = getNOW(NOW_YEAR)) == NULL) { + if((pRes = getNOW(NOW_YEAR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 4; + } break; case PROP_SYS_MONTH: - if((pRes = getNOW(NOW_MONTH)) == NULL) { + if((pRes = getNOW(NOW_MONTH, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } break; case PROP_SYS_DAY: - if((pRes = getNOW(NOW_DAY)) == NULL) { + if((pRes = getNOW(NOW_DAY, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } break; case PROP_SYS_HOUR: - if((pRes = getNOW(NOW_HOUR)) == NULL) { + if((pRes = getNOW(NOW_HOUR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } break; case PROP_SYS_HHOUR: - if((pRes = getNOW(NOW_HHOUR)) == NULL) { + if((pRes = getNOW(NOW_HHOUR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } break; case PROP_SYS_QHOUR: - if((pRes = getNOW(NOW_QHOUR)) == NULL) { + if((pRes = getNOW(NOW_QHOUR, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } break; case PROP_SYS_MINUTE: - if((pRes = getNOW(NOW_MINUTE)) == NULL) { + if((pRes = getNOW(NOW_MINUTE, ttNow)) == NULL) { RET_OUT_OF_MEMORY; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + } else { + *pbMustBeFreed = 1; + bufLen = 2; + } break; case PROP_SYS_MYHOSTNAME: pRes = glbl.GetLocalHostName(); @@ -2907,7 +2921,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } /* If we did not receive a template pointer, we are already done... */ - if(pTpe == NULL) { + if(pTpe == NULL || !pTpe->bComplexProcessing) { *pPropLen = (bufLen == -1) ? ustrlen(pRes) : bufLen; return pRes; } @@ -3494,9 +3508,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, jsonField(pTpe, &pRes, pbMustBeFreed, &bufLen); } - if(bufLen == -1) - bufLen = ustrlen(pRes); - *pPropLen = bufLen; + *pPropLen = (bufLen == -1) ? ustrlen(pRes) : bufLen; ENDfunc return(pRes); @@ -3553,7 +3565,7 @@ msgGetMsgVarNew(msg_t *pThis, uchar *name) /* always call MsgGetProp() without a template specifier */ /* TODO: optimize propNameToID() call -- rgerhards, 2009-06-26 */ propNameStrToID(name, &propid); - pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, NULL, &propLen, &bMustBeFreed); + pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, NULL, &propLen, &bMustBeFreed, NULL); estr = es_newStrFromCStr((char*)pszProp, propLen); if(bMustBeFreed) diff --git a/runtime/msg.h b/runtime/msg.h index 396e861f..172ae0da 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -173,7 +173,7 @@ void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG); uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, propid_t propid, es_str_t *propName, - rs_size_t *pPropLen, unsigned short *pbMustBeFreed); + rs_size_t *pPropLen, unsigned short *pbMustBeFreed, struct syslogTime *ttNow); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); es_str_t* msgGetMsgVarNew(msg_t *pThis, uchar *name); diff --git a/runtime/queue.c b/runtime/queue.c index 0cd33701..fbf77108 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -127,7 +127,7 @@ static struct cnfparamblk pblk = }; /* debug aid */ -static void displayBatchState(batch_t *pBatch) +static inline void displayBatchState(batch_t *pBatch) { int i; for(i = 0 ; i < pBatch->nElem ; ++i) { diff --git a/runtime/ratelimit.c b/runtime/ratelimit.c new file mode 100644 index 00000000..4b618fb5 --- /dev/null +++ b/runtime/ratelimit.c @@ -0,0 +1,359 @@ +/* ratelimit.c + * support for rate-limiting sources, including "last message + * repeated n times" processing. + * + * Copyright 2012 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "config.h" +#include <stdlib.h> +#include <string.h> +#include <assert.h> + +#include "rsyslog.h" +#include "errmsg.h" +#include "ratelimit.h" +#include "datetime.h" +#include "parser.h" +#include "unicode-helper.h" +#include "msg.h" +#include "rsconf.h" +#include "dirty.h" + +/* definitions for objects we access */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(glbl) +DEFobjCurrIf(datetime) +DEFobjCurrIf(parser) + +/* static data */ + +/* generate a "repeated n times" message */ +static inline msg_t * +ratelimitGenRepMsg(ratelimit_t *ratelimit) +{ + msg_t *repMsg; + size_t lenRepMsg; + uchar szRepMsg[1024]; + + if(ratelimit->nsupp == 1) { /* we simply use the original message! */ + repMsg = MsgAddRef(ratelimit->pMsg); + } else {/* we need to duplicate, original message may still be in use in other + * parts of the system! */ + if((repMsg = MsgDup(ratelimit->pMsg)) == NULL) { + DBGPRINTF("Message duplication failed, dropping repeat message.\n"); + goto done; + } + lenRepMsg = snprintf((char*)szRepMsg, sizeof(szRepMsg), + " message repeated %d times: [%.800s]", + ratelimit->nsupp, getMSG(ratelimit->pMsg)); + MsgReplaceMSG(repMsg, szRepMsg, lenRepMsg); + } + +done: return repMsg; +} + +static inline rsRetVal +doLastMessageRepeatedNTimes(ratelimit_t *ratelimit, msg_t *pMsg, msg_t **ppRepMsg) +{ + int bNeedUnlockMutex = 0; + rsRetVal localRet; + DEFiRet; + + if((pMsg->msgFlags & NEEDS_PARSING) != 0) { + if((localRet = parser.ParseMsg(pMsg)) != RS_RET_OK) { + DBGPRINTF("Message discarded, parsing error %d\n", localRet); + ABORT_FINALIZE(RS_RET_DISCARDMSG); + } + } + + if(ratelimit->bThreadSafe) { + pthread_mutex_lock(&ratelimit->mut); + bNeedUnlockMutex = 1; + } + + if( ratelimit->pMsg != NULL && + getMSGLen(pMsg) == getMSGLen(ratelimit->pMsg) && + !ustrcmp(getMSG(pMsg), getMSG(ratelimit->pMsg)) && + !strcmp(getHOSTNAME(pMsg), getHOSTNAME(ratelimit->pMsg)) && + !strcmp(getPROCID(pMsg, LOCK_MUTEX), getPROCID(ratelimit->pMsg, LOCK_MUTEX)) && + !strcmp(getAPPNAME(pMsg, LOCK_MUTEX), getAPPNAME(ratelimit->pMsg, LOCK_MUTEX))) { + ratelimit->nsupp++; + DBGPRINTF("msg repeated %d times\n", ratelimit->nsupp); + /* use current message, so we have the new timestamp + * (means we need to discard previous one) */ + msgDestruct(&ratelimit->pMsg); + ratelimit->pMsg = pMsg; + ABORT_FINALIZE(RS_RET_DISCARDMSG); + } else {/* new message, do "repeat processing" & save it */ + if(ratelimit->pMsg != NULL) { + if(ratelimit->nsupp > 0) { + *ppRepMsg = ratelimitGenRepMsg(ratelimit); + ratelimit->nsupp = 0; + } + msgDestruct(&ratelimit->pMsg); + } + ratelimit->pMsg = MsgAddRef(pMsg); + } + +finalize_it: + if(bNeedUnlockMutex) + pthread_mutex_unlock(&ratelimit->mut); + RETiRet; +} + + +/* helper: tell how many messages we lost due to linux-like ratelimiting */ +static inline void +tellLostCnt(ratelimit_t *ratelimit) +{ + uchar msgbuf[1024]; + if(ratelimit->missed) { + snprintf((char*)msgbuf, sizeof(msgbuf), + "%s: %u messages lost due to rate-limiting", + ratelimit->name, ratelimit->missed); + logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + ratelimit->missed = 0; + } +} + +/* Linux-like ratelimiting, modelled after the linux kernel + * returns 1 if message is within rate limit and shall be + * processed, 0 otherwise. + * This implementation is NOT THREAD-SAFE and must not + * be called concurrently. + */ +static inline int +withinRatelimit(ratelimit_t *ratelimit, time_t tt) +{ + int ret; + uchar msgbuf[1024]; + + if(ratelimit->interval == 0) { + ret = 1; + goto finalize_it; + } + + assert(ratelimit->burst != 0); + + if(ratelimit->begin == 0) + ratelimit->begin = tt; + + /* resume if we go out of out time window */ + if(tt > ratelimit->begin + ratelimit->interval) { + tellLostCnt(ratelimit); + ratelimit->begin = 0; + ratelimit->done = 0; + } + + /* do actual limit check */ + if(ratelimit->burst > ratelimit->done) { + ratelimit->done++; + ret = 1; + } else { + if(ratelimit->missed == 0) { + snprintf((char*)msgbuf, sizeof(msgbuf), + "%s: begin to drop messages due to rate-limiting", + ratelimit->name); + logmsgInternal(RS_RET_RATE_LIMITED, LOG_SYSLOG|LOG_INFO, msgbuf, 0); + } + ratelimit->missed++; + ret = 0; + } + +finalize_it: + return ret; +} + + +/* ratelimit a message, that means: + * - handle "last message repeated n times" logic + * - handle actual (discarding) rate-limiting + * This function returns RS_RET_OK, if the caller shall process + * the message regularly and RS_RET_DISCARD if the caller must + * discard the message. The caller should also discard the message + * if another return status occurs. This places some burden on the + * caller logic, but provides best performance. Demanding this + * cooperative mode can enable a faulty caller to thrash up part + * of the system, but we accept that risk (a faulty caller can + * always do all sorts of evil, so...) + * If *ppRepMsg != NULL on return, the caller must enqueue that + * message before the original message. + */ +rsRetVal +ratelimitMsg(ratelimit_t *ratelimit, msg_t *pMsg, msg_t **ppRepMsg) +{ + DEFiRet; + + *ppRepMsg = NULL; + if(ratelimit->interval) { + if(withinRatelimit(ratelimit, pMsg->ttGenTime) == 0) { + msgDestruct(&pMsg); + ABORT_FINALIZE(RS_RET_DISCARDMSG); + } + } + if(ratelimit->bReduceRepeatMsgs) { + CHKiRet(doLastMessageRepeatedNTimes(ratelimit, pMsg, ppRepMsg)); + } +finalize_it: + RETiRet; +} + +/* returns 1, if the ratelimiter performs any checks and 0 otherwise */ +int +ratelimitChecked(ratelimit_t *ratelimit) +{ + return ratelimit->interval || ratelimit->bReduceRepeatMsgs; +} + + +/* add a message to a ratelimiter/multisubmit structure. + * ratelimiting is automatically handled according to the ratelimit + * settings. + * if pMultiSub == NULL, a single-message enqueue happens (under reconsideration) + */ +rsRetVal +ratelimitAddMsg(ratelimit_t *ratelimit, multi_submit_t *pMultiSub, msg_t *pMsg) +{ + rsRetVal localRet; + msg_t *repMsg; + DEFiRet; + + if(pMultiSub == NULL) { + localRet = ratelimitMsg(ratelimit, pMsg, &repMsg); + if(repMsg != NULL) + CHKiRet(submitMsg2(repMsg)); + if(localRet == RS_RET_OK) + CHKiRet(submitMsg2(pMsg)); + } else { + localRet = ratelimitMsg(ratelimit, pMsg, &repMsg); + if(repMsg != NULL) { + pMultiSub->ppMsgs[pMultiSub->nElem++] = repMsg; + if(pMultiSub->nElem == pMultiSub->maxElem) + CHKiRet(multiSubmitMsg2(pMultiSub)); + } + if(localRet == RS_RET_OK) { + pMultiSub->ppMsgs[pMultiSub->nElem++] = pMsg; + if(pMultiSub->nElem == pMultiSub->maxElem) + CHKiRet(multiSubmitMsg2(pMultiSub)); + } + } + +finalize_it: + RETiRet; +} + + +/* modname must be a static name (usually expected to be the module + * name and MUST be present. dynname may be NULL and can be used for + * dynamic information, e.g. PID or listener IP, ... + * Both values should be kept brief. + */ +rsRetVal +ratelimitNew(ratelimit_t **ppThis, char *modname, char *dynname) +{ + ratelimit_t *pThis; + char namebuf[256]; + DEFiRet; + + CHKmalloc(pThis = calloc(1, sizeof(ratelimit_t))); + if(modname == NULL) + modname ="*ERROR:MODULE NAME MISSING*"; + + if(dynname == NULL) { + pThis->name = strdup(modname); + } else { + snprintf(namebuf, sizeof(namebuf), "%s[%s]", + modname, dynname); + namebuf[sizeof(namebuf)-1] = '\0'; /* to be on safe side */ + pThis->name = strdup(namebuf); + } + pThis->bReduceRepeatMsgs = loadConf->globals.bReduceRepeatMsgs; + *ppThis = pThis; +finalize_it: + RETiRet; +} + + +/* enable linux-like ratelimiting */ +void +ratelimitSetLinuxLike(ratelimit_t *ratelimit, unsigned short interval, unsigned short burst) +{ + ratelimit->interval = interval; + ratelimit->burst = burst; + ratelimit->done = 0; + ratelimit->missed = 0; + ratelimit->begin = 0; +} + + +/* enable thread-safe operations mode. This make sure that + * a single ratelimiter can be called from multiple threads. As + * this causes some overhead and is not always required, it needs + * to be explicitely enabled. This operation cannot be undone + * (think: why should one do that???) + */ +void +ratelimitSetThreadSafe(ratelimit_t *ratelimit) +{ + ratelimit->bThreadSafe = 1; + pthread_mutex_init(&ratelimit->mut, NULL); +} + +void +ratelimitDestruct(ratelimit_t *ratelimit) +{ + msg_t *pMsg; + if(ratelimit->pMsg != NULL) { + if(ratelimit->nsupp > 0) { + pMsg = ratelimitGenRepMsg(ratelimit); + if(pMsg != NULL) + submitMsg2(pMsg); + } + msgDestruct(&ratelimit->pMsg); + } + tellLostCnt(ratelimit); + if(ratelimit->bThreadSafe) + pthread_mutex_destroy(&ratelimit->mut); + free(ratelimit->name); + free(ratelimit); +} + +void +ratelimitModExit(void) +{ + objRelease(datetime, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); +} + +rsRetVal +ratelimitModInit(void) +{ + DEFiRet; + CHKiRet(objGetObjInterface(&obj)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(parser, CORE_COMPONENT)); +finalize_it: + RETiRet; +} + diff --git a/runtime/ratelimit.h b/runtime/ratelimit.h new file mode 100644 index 00000000..820817bc --- /dev/null +++ b/runtime/ratelimit.h @@ -0,0 +1,51 @@ +/* header for ratelimit.c + * + * Copyright 2012 Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * -or- + * see COPYING.ASL20 in the source distribution + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef INCLUDED_RATELIMIT_H +#define INCLUDED_RATELIMIT_H + +struct ratelimit_s { + char *name; /**< rate limiter name, e.g. for user messages */ + /* support for Linux kernel-type ratelimiting */ + unsigned short interval; + unsigned short burst; + unsigned done; + unsigned missed; + time_t begin; + /* support for "last message repeated n times */ + int bReduceRepeatMsgs; /**< shall we do "last message repeated n times" processing? */ + unsigned nsupp; /**< nbr of msgs suppressed */ + msg_t *pMsg; + sbool bThreadSafe; /**< do we need to operate in Thread-Safe mode? */ + pthread_mutex_t mut; /**< mutex if thread-safe operation desired */ +}; + +/* prototypes */ +rsRetVal ratelimitNew(ratelimit_t **ppThis, char *modname, char *dynname); +void ratelimitSetThreadSafe(ratelimit_t *ratelimit); +void ratelimitSetLinuxLike(ratelimit_t *ratelimit, unsigned short interval, unsigned short burst); +rsRetVal ratelimitMsg(ratelimit_t *ratelimit, msg_t *pMsg, msg_t **ppRep); +rsRetVal ratelimitAddMsg(ratelimit_t *ratelimit, multi_submit_t *pMultiSub, msg_t *pMsg); +void ratelimitDestruct(ratelimit_t *pThis); +int ratelimitChecked(ratelimit_t *ratelimit); +rsRetVal ratelimitModInit(void); +void ratelimitModExit(void); + +#endif /* #ifndef INCLUDED_RATELIMIT_H */ diff --git a/runtime/rsconf.c b/runtime/rsconf.c index dcaa1ad9..0fc67499 100644 --- a/runtime/rsconf.c +++ b/runtime/rsconf.c @@ -115,8 +115,8 @@ void cnfDoCfsysline(char *ln); */ BEGINobjConstruct(rsconf) /* be sure to specify the object type also in END macro! */ pThis->globals.bDebugPrintTemplateList = 1; - pThis->globals.bDebugPrintModuleList = 1; - pThis->globals.bDebugPrintCfSysLineHandlerList = 1; + pThis->globals.bDebugPrintModuleList = 0; + pThis->globals.bDebugPrintCfSysLineHandlerList = 0; pThis->globals.bLogStatusMsgs = DFLT_bLogStatusMsgs; pThis->globals.bErrMsgToStderr = 1; pThis->globals.umask = -1; diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 07d58d68..0bf0642a 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -69,6 +69,7 @@ * approach taken here is considered appropriate. * rgerhards, 2010-06-24 */ +#define CONF_NUM_MULTISUB 1024 /* default number of messages per multisub structure */ /* ############################################################# * * # End Config Settings # * @@ -376,6 +377,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_LEGA_ACT_NOT_SUPPORTED = -2215, /**< the module (no longer) supports legacy action syntax */ RS_RET_MAX_OMSR_REACHED = -2216, /**< max nbr of string requests reached, not supported by core */ RS_RET_UID_MISSING = -2217, /**< a user id is missing (but e.g. a password provided) */ + RS_RET_DATAFAIL = -2218, /**< data passed to action caused failure */ /* reserved for pre-v6.5 */ RS_RET_DUP_PARAM = -2220, /**< config parameter is given more than once */ RS_RET_MODULE_ALREADY_IN_CONF = -2221, /**< module already in current configuration */ diff --git a/runtime/ruleset.c b/runtime/ruleset.c index 8d2bb924..24d8279c 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -378,7 +378,8 @@ evalPROPFILT(struct cnfstmt *stmt, msg_t *pMsg) goto done; pszPropVal = MsgGetProp(pMsg, NULL, stmt->d.s_propfilt.propID, - stmt->d.s_propfilt.propName, &propLen, &pbMustBeFreed); + stmt->d.s_propfilt.propName, &propLen, + &pbMustBeFreed, NULL); /* Now do the compares (short list currently ;)) */ switch(stmt->d.s_propfilt.operation ) { diff --git a/runtime/stream.c b/runtime/stream.c index 906a45fc..b8dd5150 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -74,11 +74,12 @@ DEFobjStaticHelpers DEFobjCurrIf(zlibw) /* forward definitions */ -static rsRetVal strmFlushInternal(strm_t *pThis); +static rsRetVal strmFlushInternal(strm_t *pThis, int bFlushZip); static rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); static rsRetVal strmCloseFile(strm_t *pThis); static void *asyncWriterThread(void *pPtr); -static rsRetVal doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); +static rsRetVal doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf, int bFlush); +static rsRetVal doZipFinish(strm_t *pThis); static rsRetVal strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); @@ -341,7 +342,7 @@ static rsRetVal strmCloseFile(strm_t *pThis) (pThis->pszFName == NULL) ? "N/A" : (char*)pThis->pszFName); if(pThis->tOperationsMode != STREAMMODE_READ) { - strmFlushInternal(pThis); + strmFlushInternal(pThis, 0); if(pThis->bAsyncWrite) { strmWaitAsyncWriterDone(pThis); } @@ -675,6 +676,7 @@ BEGINobjConstruct(strm) /* be sure to specify the object type also in END macro! pThis->fd = -1; pThis->fdDir = -1; pThis->iUngetC = -1; + pThis->bVeryReliableZip = 0; pThis->sType = STREAMTYPE_FILE_SINGLE; pThis->sIOBufSize = glblGetIOBufSize(); pThis->tOpenMode = 0600; @@ -777,6 +779,10 @@ stopWriter(strm_t *pThis) BEGINobjDestruct(strm) /* be sure to specify the object type also in END and CODESTART macros! */ int i; CODESTARTobjDestruct(strm) + /* we need to stop the ZIP writer */ + if(pThis->iZipLevel) { + doZipFinish(pThis); + } if(pThis->bAsyncWrite) /* Note: mutex will be unlocked in stopWriter! */ d_pthread_mutex_lock(&pThis->mut); @@ -919,14 +925,14 @@ finalize_it: /* write memory buffer to a stream object. */ static inline rsRetVal -doWriteInternal(strm_t *pThis, uchar *pBuf, size_t lenBuf) +doWriteInternal(strm_t *pThis, uchar *pBuf, size_t lenBuf, int bFlush) { DEFiRet; ASSERT(pThis != NULL); if(pThis->iZipLevel) { - CHKiRet(doZipWrite(pThis, pBuf, lenBuf)); + CHKiRet(doZipWrite(pThis, pBuf, lenBuf, bFlush)); } else { /* write without zipping */ CHKiRet(strmPhysWrite(pThis, pBuf, lenBuf)); @@ -971,7 +977,7 @@ doAsyncWriteInternal(strm_t *pThis, size_t lenBuf) * the background thread. -- rgerhards, 2009-07-07 */ static rsRetVal -strmSchedWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +strmSchedWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf, int bFlushZip) { DEFiRet; @@ -990,7 +996,7 @@ strmSchedWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) if(pThis->bAsyncWrite) { CHKiRet(doAsyncWriteInternal(pThis, lenBuf)); } else { - CHKiRet(doWriteInternal(pThis, pBuf, lenBuf)); + CHKiRet(doWriteInternal(pThis, pBuf, lenBuf, bFlushZip)); } @@ -1030,7 +1036,7 @@ asyncWriterThread(void *pPtr) } if(bTimedOut && pThis->iBufPtr > 0) { /* if we timed out, we need to flush pending data */ - strmFlushInternal(pThis); + strmFlushInternal(pThis, 0); bTimedOut = 0; continue; /* now we should have data */ } @@ -1056,7 +1062,7 @@ asyncWriterThread(void *pPtr) bTimedOut = 0; /* we may have timed out, but there *is* work to do... */ iDeq = pThis->iDeq++ % STREAM_ASYNC_NUMBUFS; - doWriteInternal(pThis, pThis->asyncBuf[iDeq].pBuf, pThis->asyncBuf[iDeq].lenBuf); + doWriteInternal(pThis, pThis->asyncBuf[iDeq].pBuf, pThis->asyncBuf[iDeq].lenBuf, 0); // TODO: flush state // TODO: error check????? 2009-07-06 --pThis->iCnt; @@ -1167,63 +1173,99 @@ finalize_it: * rgerhards, 2009-06-04 */ static rsRetVal -doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf, int bFlush) { - z_stream zstrm; int zRet; /* zlib return state */ - sbool bzInitDone = RSFALSE; DEFiRet; + unsigned outavail; assert(pThis != NULL); assert(pBuf != NULL); - /* allocate deflate state */ - zstrm.zalloc = Z_NULL; - zstrm.zfree = Z_NULL; - zstrm.opaque = Z_NULL; - zstrm.next_in = (Bytef*) pBuf; /* as of zlib doc, this must be set BEFORE DeflateInit2 */ - /* see note in file header for the params we use with deflateInit2() */ - zRet = zlibw.DeflateInit2(&zstrm, pThis->iZipLevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); - if(zRet != Z_OK) { - DBGPRINTF("error %d returned from zlib/deflateInit2()\n", zRet); - ABORT_FINALIZE(RS_RET_ZLIB_ERR); + if(!pThis->bzInitDone) { + /* allocate deflate state */ + pThis->zstrm.zalloc = Z_NULL; + pThis->zstrm.zfree = Z_NULL; + pThis->zstrm.opaque = Z_NULL; + /* see note in file header for the params we use with deflateInit2() */ + zRet = zlibw.DeflateInit2(&pThis->zstrm, pThis->iZipLevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + if(zRet != Z_OK) { + DBGPRINTF("error %d returned from zlib/deflateInit2()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } + pThis->bzInitDone = RSTRUE; } - bzInitDone = RSTRUE; /* now doing the compression */ - zstrm.next_in = (Bytef*) pBuf; /* as of zlib doc, this must be set BEFORE DeflateInit2 */ - zstrm.avail_in = lenBuf; + pThis->zstrm.next_in = (Bytef*) pBuf; + pThis->zstrm.avail_in = lenBuf; /* run deflate() on buffer until everything has been compressed */ do { - DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", zstrm.avail_in, zstrm.total_in); - zstrm.avail_out = pThis->sIOBufSize; - zstrm.next_out = pThis->pZipBuf; - zRet = zlibw.Deflate(&zstrm, Z_FINISH); /* no bad return value */ - DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, zstrm.avail_out); - assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ - if(zstrm.avail_out == pThis->sIOBufSize) - break; /* this is valid, indicates end of compression --> see zlib howto */ - CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, pThis->sIOBufSize - zstrm.avail_out)); - } while (zstrm.avail_out == 0); - assert(zstrm.avail_in == 0); /* all input will be used */ + DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", pThis->zstrm.avail_in, pThis->zstrm.total_in); + pThis->zstrm.avail_out = pThis->sIOBufSize; + pThis->zstrm.next_out = pThis->pZipBuf; + zRet = zlibw.Deflate(&pThis->zstrm, bFlush ? Z_SYNC_FLUSH : Z_NO_FLUSH); /* no bad return value */ + DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, pThis->zstrm.avail_out); + outavail =pThis->sIOBufSize - pThis->zstrm.avail_out; + if(outavail != 0) { + CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, outavail)); + } + } while (pThis->zstrm.avail_out == 0); finalize_it: - if(bzInitDone) { - zRet = zlibw.DeflateEnd(&zstrm); - if(zRet != Z_OK) { - DBGPRINTF("error %d returned from zlib/deflateEnd()\n", zRet); + if(pThis->bzInitDone && pThis->bVeryReliableZip) { + doZipFinish(pThis); + } + RETiRet; +} + + + +/* finish zlib buffer, to be called before closing the ZIP file (if + * running in stream mode). + */ +static rsRetVal +doZipFinish(strm_t *pThis) +{ + int zRet; /* zlib return state */ + DEFiRet; + unsigned outavail; + assert(pThis != NULL); + + if(!pThis->bzInitDone) { + FINALIZE; + } + +dbgprintf("AAAA: doZipFinish() called\n"); + pThis->zstrm.avail_in = 0; + /* run deflate() on buffer until everything has been compressed */ + do { + DBGPRINTF("in deflate() loop, avail_in %d, total_in %ld\n", pThis->zstrm.avail_in, pThis->zstrm.total_in); + pThis->zstrm.avail_out = pThis->sIOBufSize; + pThis->zstrm.next_out = pThis->pZipBuf; + zRet = zlibw.Deflate(&pThis->zstrm, Z_FINISH); /* no bad return value */ + DBGPRINTF("after deflate, ret %d, avail_out %d\n", zRet, pThis->zstrm.avail_out); + outavail = pThis->sIOBufSize - pThis->zstrm.avail_out; + if(outavail != 0) { + CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, outavail)); } + } while (pThis->zstrm.avail_out == 0); + +finalize_it: + zRet = zlibw.DeflateEnd(&pThis->zstrm); + if(zRet != Z_OK) { + DBGPRINTF("error %d returned from zlib/deflateEnd()\n", zRet); } + pThis->bzInitDone = 0; RETiRet; } - /* flush stream output buffer to persistent storage. This can be called at any time * and is automatically called when the output buffer is full. * rgerhards, 2008-01-10 */ static rsRetVal -strmFlushInternal(strm_t *pThis) +strmFlushInternal(strm_t *pThis, int bFlushZip) { DEFiRet; @@ -1233,7 +1275,7 @@ strmFlushInternal(strm_t *pThis) (long) pThis->iBufPtr, (pThis->iBufPtr == 0) ? " (no need to flush)" : ""); if(pThis->tOperationsMode != STREAMMODE_READ && pThis->iBufPtr > 0) { - iRet = strmSchedWrite(pThis, pThis->pIOBuf, pThis->iBufPtr); + iRet = strmSchedWrite(pThis, pThis->pIOBuf, pThis->iBufPtr, bFlushZip); } RETiRet; @@ -1255,7 +1297,7 @@ strmFlush(strm_t *pThis) if(pThis->bAsyncWrite) d_pthread_mutex_lock(&pThis->mut); - CHKiRet(strmFlushInternal(pThis)); + CHKiRet(strmFlushInternal(pThis, 1)); finalize_it: if(pThis->bAsyncWrite) @@ -1278,7 +1320,7 @@ static rsRetVal strmSeek(strm_t *pThis, off64_t offs) if(pThis->fd == -1) { CHKiRet(strmOpenFile(pThis)); } else { - CHKiRet(strmFlushInternal(pThis)); + CHKiRet(strmFlushInternal(pThis, 0)); } long long i; DBGOPRINT((obj_t*) pThis, "file %d seek, pos %llu\n", pThis->fd, (long long unsigned) offs); @@ -1321,7 +1363,7 @@ static rsRetVal strmWriteChar(strm_t *pThis, uchar c) /* if the buffer is full, we need to flush before we can write */ if(pThis->iBufPtr == pThis->sIOBufSize) { - CHKiRet(strmFlushInternal(pThis)); + CHKiRet(strmFlushInternal(pThis, 0)); } /* we now always have space for one character, so we simply copy it */ *(pThis->pIOBuf + pThis->iBufPtr) = c; @@ -1391,7 +1433,7 @@ strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) iOffset = 0; do { if(pThis->iBufPtr == pThis->sIOBufSize) { - CHKiRet(strmFlushInternal(pThis)); /* get a new buffer for rest of data */ + CHKiRet(strmFlushInternal(pThis, 0)); /* get a new buffer for rest of data */ } iWrite = pThis->sIOBufSize - pThis->iBufPtr; /* this fits in current buf */ if(iWrite > lenBuf) @@ -1406,7 +1448,7 @@ strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) * write it. This seems more natural than waiting (hours?) for the next message... */ if(pThis->iBufPtr == pThis->sIOBufSize) { - CHKiRet(strmFlushInternal(pThis)); /* get a new buffer for rest of data */ + CHKiRet(strmFlushInternal(pThis, 0)); /* get a new buffer for rest of data */ } finalize_it: @@ -1434,6 +1476,7 @@ DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) DEFpropSetMeth(strm, sType, strmType_t) DEFpropSetMeth(strm, iZipLevel, int) +DEFpropSetMeth(strm, bVeryReliableZip, int) DEFpropSetMeth(strm, bSync, int) DEFpropSetMeth(strm, sIOBufSize, size_t) DEFpropSetMeth(strm, iSizeLimit, off_t) @@ -1565,7 +1608,7 @@ static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) ISOBJ_TYPE_assert(pThis, strm); ISOBJ_TYPE_assert(pStrm, strm); - strmFlushInternal(pThis); + strmFlushInternal(pThis, 0); CHKiRet(obj.BeginSerialize(pStrm, (obj_t*) pThis)); objSerializeSCALAR(pStrm, iCurrFNum, INT); @@ -1758,6 +1801,7 @@ CODESTARTobjQueryInterface(strm) pIf->SettOpenMode = strmSettOpenMode; pIf->SetsType = strmSetsType; pIf->SetiZipLevel = strmSetiZipLevel; + pIf->SetbVeryReliableZip = strmSetbVeryReliableZip; pIf->SetbSync = strmSetbSync; pIf->SetsIOBufSize = strmSetsIOBufSize; pIf->SetiSizeLimit = strmSetiSizeLimit; diff --git a/runtime/stream.h b/runtime/stream.h index a01929f2..fdfefaa3 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -124,6 +124,8 @@ typedef struct strm_s { sbool bAsyncWrite; /* do asynchronous writes (always if a flush interval is given) */ sbool bStopWriter; /* shall writer thread terminate? */ sbool bDoTimedWait; /* instruct writer thread to do a times wait to support flush timeouts */ + sbool bzInitDone; /* did we do an init of zstrm already? */ + sbool bVeryReliableZip; /* shall we write interim headers to create a very reliable ZIP file? */ int iFlushInterval; /* flush in which interval - 0, no flushing */ pthread_mutex_t mut;/* mutex for flush in async mode */ pthread_cond_t notFull; @@ -132,6 +134,7 @@ typedef struct strm_s { unsigned short iEnq; /* this MUST be unsigned as we use module arithmetic (else invalid indexing happens!) */ unsigned short iDeq; /* this MUST be unsigned as we use module arithmetic (else invalid indexing happens!) */ short iCnt; /* current nbr of elements in buffer */ + z_stream zstrm; /* zip stream to use */ struct { uchar *pBuf; size_t lenBuf; @@ -181,8 +184,10 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, pszSizeLimitCmd, uchar*); /* v6 added */ rsRetVal (*ReadLine)(strm_t *pThis, cstr_t **ppCStr, int mode); + /* v7 added 2012-09-14 */ + INTERFACEpropSetMeth(strm, bVeryReliableZip, int); ENDinterface(strm) -#define strmCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ +#define strmCURR_IF_VERSION 7 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/typedefs.h b/runtime/typedefs.h index ccae08b2..1e0cb466 100644 --- a/runtime/typedefs.h +++ b/runtime/typedefs.h @@ -92,6 +92,7 @@ typedef struct cfgmodules_etry_s cfgmodules_etry_t; typedef struct outchannels_s outchannels_t; typedef struct modConfData_s modConfData_t; typedef struct instanceConf_s instanceConf_t; +typedef struct ratelimit_s ratelimit_t; typedef int rs_size_t; /* we do never need more than 2Gig strings, signed permits to * use -1 as a special flag. */ typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ |