From 31344728fe6f83d4f02ce0e5868c331b4e25d659 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 20 Dec 2008 12:56:41 +0100 Subject: bufgix: $PreserveFQDN was not properly handled for locally emitted messages --- ChangeLog | 2 ++ runtime/glbl.c | 27 ++++++++++++++++++++++++++- runtime/glbl.h | 1 + tools/syslogd.c | 5 ++++- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5056f68d..120ef935 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +- bufgix: $PreserveFQDN was not properly handled for locally emitted + messages --------------------------------------------------------------------------- Version 4.1.3 [DEVEL] (rgerhards), 2008-12-17 - added $InputTCPServerAddtlFrameDelimiter config directive, which diff --git a/runtime/glbl.c b/runtime/glbl.c index d06c88ff..28f14320 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -60,6 +60,7 @@ static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records static int option_DisallowWarning = 1; /* complain if message from disallowed sender is received */ static int bDisableDNS = 0; /* don't look up IP addresses of remote messages */ static uchar *LocalHostName = NULL;/* our hostname - read-only after startup */ +static uchar *LocalFQDNName = NULL;/* our hostname as FQDN - read-only after startup */ static uchar *LocalDomain; /* our local domain name - read-only after startup */ static char **StripDomains = NULL;/* these domains may be stripped before writing logs - r/o after s.u., never touched by init */ static char **LocalHosts = NULL;/* these hosts are logged with their hostname - read-only after startup, never touched by init */ @@ -100,6 +101,7 @@ SIMP_PROP(LocalDomain, LocalDomain, uchar*) SIMP_PROP(StripDomains, StripDomains, char**) SIMP_PROP(LocalHosts, LocalHosts, char**) +SIMP_PROP_SET(LocalFQDNName, LocalFQDNName, uchar*) SIMP_PROP_SET(LocalHostName, LocalHostName, uchar*) SIMP_PROP_SET(DfltNetstrmDrvr, pszDfltNetstrmDrvr, uchar*) /* TODO: use custom function which frees existing value */ SIMP_PROP_SET(DfltNetstrmDrvrCAF, pszDfltNetstrmDrvrCAF, uchar*) /* TODO: use custom function which frees existing value */ @@ -116,7 +118,27 @@ SIMP_PROP_SET(DfltNetstrmDrvrCertFile, pszDfltNetstrmDrvrCertFile, uchar*) /* TO static uchar* GetLocalHostName(void) { - return(LocalHostName == NULL ? (uchar*) "[localhost]" : LocalHostName); + uchar *pszRet; + + if(LocalHostName == NULL) + pszRet = (uchar*) "[localhost]"; + else { + if(GetPreserveFQDN() == 1) + pszRet = LocalFQDNName; + else + pszRet = LocalHostName; + } + return(pszRet); +} + + +/* return the current localhost name as FQDN (requires FQDN to be set) + * TODO: we should set the FQDN ourselfs in here! + */ +static uchar* +GetLocalFQDNName(void) +{ + return(LocalFQDNName == NULL ? (uchar*) "[localhost]" : LocalFQDNName); } @@ -186,6 +208,7 @@ CODESTARTobjQueryInterface(glbl) SIMP_PROP(DropMalPTRMsgs); SIMP_PROP(Option_DisallowWarning); SIMP_PROP(DisableDNS); + SIMP_PROP(LocalFQDNName) SIMP_PROP(LocalHostName) SIMP_PROP(LocalDomain) SIMP_PROP(StripDomains) @@ -270,6 +293,8 @@ BEGINObjClassExit(glbl, OBJ_IS_CORE_MODULE) /* class, version */ free(pszWorkDir); if(LocalHostName != NULL) free(LocalHostName); + if(LocalFQDNName != NULL) + free(LocalFQDNName); ENDObjClassExit(glbl) /* vi:set ai: diff --git a/runtime/glbl.h b/runtime/glbl.h index 205a5212..5bdf4f57 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -48,6 +48,7 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ SIMP_PROP(DropMalPTRMsgs, int) SIMP_PROP(Option_DisallowWarning, int) SIMP_PROP(DisableDNS, int) + SIMP_PROP(LocalFQDNName, uchar*) SIMP_PROP(LocalHostName, uchar*) SIMP_PROP(LocalDomain, uchar*) SIMP_PROP(StripDomains, char**) diff --git a/tools/syslogd.c b/tools/syslogd.c index 138bdfd8..2cac8fe4 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3288,6 +3288,7 @@ int realMain(int argc, char **argv) uchar legacyConfLine[80]; uchar *LocalHostName; uchar *LocalDomain; + uchar *LocalFQDNName; /* first, parse the command line options. We do not carry out any actual work, just * see what we should do. This relieves us from certain anomalies and we can process @@ -3392,7 +3393,9 @@ int realMain(int argc, char **argv) /* get our host and domain names - we need to do this early as we may emit * error log messages, which need the correct hostname. -- rgerhards, 2008-04-04 */ - net.getLocalHostname(&LocalHostName); + net.getLocalHostname(&LocalFQDNName); + CHKmalloc(LocalHostName = (uchar*) strdup((char*)LocalFQDNName)); + glbl.SetLocalFQDNName(LocalFQDNName); /* set the FQDN before we modify it */ if((p = (uchar*)strchr((char*)LocalHostName, '.'))) { *p++ = '\0'; LocalDomain = p; -- cgit v1.2.3 From de1da2c06aaae81fe44cb7d8df38931bc210d05a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sun, 23 Dec 2007 11:43:55 +0100 Subject: doc bugfix: duplicate and invalid link to regex check tool --- doc/manual.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual.html b/doc/manual.html index e8842de6..bc4c0bc5 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -36,9 +36,8 @@ the links below for the

Where <size_nbr> is specified above, diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c index b01dd98b..f3771237 100644 --- a/plugins/imrelp/imrelp.c +++ b/plugins/imrelp/imrelp.c @@ -84,7 +84,7 @@ onSyslogRcv(uchar *pHostname, uchar __attribute__((unused)) *pIP, uchar *pMsg, s { DEFiRet; parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, MSG_PARSE_HOSTNAME, - NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp"); + NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL); RETiRet; } diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index f4830a47..1865d777 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -40,6 +40,7 @@ #include "srUtils.h" #include "errmsg.h" #include "glbl.h" +#include "datetime.h" MODULE_TYPE_INPUT @@ -50,6 +51,7 @@ DEF_IMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(net) +DEFobjCurrIf(datetime) static int iMaxLine; /* maximum UDP message size supported */ static int *udpLstnSocks = NULL; /* Internet datagram sockets, first element is nbr of elements @@ -59,6 +61,8 @@ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a * it so that we can check available memory in willRun() and request * termination if we can not get it. -- rgerhards, 2007-12-27 */ +#define TIME_REQUERY_DFLT 2 +static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ /* config settings */ @@ -141,6 +145,8 @@ BEGINrunInput uchar fromHostIP[NI_MAXHOST]; uchar fromHostFQDN[NI_MAXHOST]; ssize_t l; + struct syslogTime stTime; + int iNbrTimeUsed; CODESTARTrunInput /* this is an endless loop - it is terminated when the thread is * signalled to do so. This, however, is handled by the framework, @@ -179,6 +185,7 @@ CODESTARTrunInput for (i = 0; nfds && i < *udpLstnSocks; i++) { if (FD_ISSET(udpLstnSocks[i+1], &readfds)) { socklen = sizeof(frominet); + iNbrTimeUsed = 0; do { /* we now try to read from the file descriptor until there * is no more data. This is done in the hope to get better performance @@ -203,8 +210,11 @@ CODESTARTrunInput */ if(net.isAllowedSender(net.pAllowedSenders_UDP, (struct sockaddr *)&frominet, (char*)fromHostFQDN)) { + if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { + datetime.getCurrTime(&stTime); + } parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp"); + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", &stTime); } else { dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN); if(glbl.GetOption_DisallowWarning) { @@ -274,6 +284,7 @@ CODESTARTmodExit /* release what we no longer need */ objRelease(errmsg, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); ENDmodExit @@ -293,6 +304,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a net.closeUDPListenSockets(udpLstnSocks); udpLstnSocks = NULL; } + iTimeRequery = TIME_REQUERY_DFLT;/* the default is to query only every second time */ return RS_RET_OK; } @@ -303,6 +315,7 @@ CODESTARTmodInit CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* register config file handlers */ @@ -310,9 +323,10 @@ CODEmodInit_QueryRegCFSLineHdlr addListner, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord, NULL, &pszBindAddr, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpservertimerequery", 0, eCmdHdlrInt, + NULL, &iTimeRequery, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit -/* - * vi:set ai: +/* vim:set ai: */ diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 55b8b2df..77d347d9 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -221,7 +221,7 @@ static rsRetVal readSocket(int fd, int iSock) if (iRcvd > 0) { parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock], (uchar*)"127.0.0.1", pRcv, - iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock"); + iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock", NULL); } else if (iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); diff --git a/runtime/msg.c b/runtime/msg.c index 69296710..df8c1572 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -239,12 +239,21 @@ rsRetVal MsgEnableThreadSafety(void) /* end locking functions */ -/* "Constructor" for a msg "object". Returns a pointer to +/* This is common code for all Constructors. It is defined in an + * inline'able function so that we can save a function call in the + * actual constructors (otherwise, the msgConstruct would need + * to call msgConstructWithTime(), which would require a + * function call). Now, both can use this inline function. This + * enables us to be optimal, but still have the code just once. * the new object or NULL if no such object could be allocated. * An object constructed via this function should only be destroyed - * via "msgDestruct()". + * via "msgDestruct()". This constructor does not query system time + * itself but rather uses a user-supplied value. This enables the caller + * to do some tricks to save processing time (done, for example, in the + * udp input). + * rgerhards, 2008-10-06 */ -rsRetVal msgConstruct(msg_t **ppThis) +static inline rsRetVal msgBaseConstruct(msg_t **ppThis) { DEFiRet; msg_t *pM; @@ -257,21 +266,58 @@ rsRetVal msgConstruct(msg_t **ppThis) pM->iRefCount = 1; pM->iSeverity = -1; pM->iFacility = -1; + objConstructSetObjInfo(pM); + + /* DEV debugging only! dbgprintf("msgConstruct\t0x%x, ref 1\n", (int)pM);*/ + + *ppThis = pM; + +finalize_it: + RETiRet; +} + + +/* "Constructor" for a msg "object". Returns a pointer to + * the new object or NULL if no such object could be allocated. + * An object constructed via this function should only be destroyed + * via "msgDestruct()". This constructor does not query system time + * itself but rather uses a user-supplied value. This enables the caller + * to do some tricks to save processing time (done, for example, in the + * udp input). + * rgerhards, 2008-10-06 + */ +rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime) +{ + DEFiRet; + + CHKiRet(msgBaseConstruct(ppThis)); + memcpy(&(*ppThis)->tRcvdAt, stTime, sizeof(struct syslogTime)); + memcpy(&(*ppThis)->tTIMESTAMP, stTime, sizeof(struct syslogTime)); +finalize_it: + RETiRet; +} + + +/* "Constructor" for a msg "object". Returns a pointer to + * the new object or NULL if no such object could be allocated. + * An object constructed via this function should only be destroyed + * via "msgDestruct()". This constructor, for historical reasons, + * also sets the two timestamps to the current time. + */ +rsRetVal msgConstruct(msg_t **ppThis) +{ + DEFiRet; + + CHKiRet(msgBaseConstruct(ppThis)); /* we initialize both timestamps to contain the current time, so that they * are consistent. Also, this saves us from doing any further time calls just * to obtain a timestamp. The memcpy() should not really make a difference, * especially as I think there is no codepath currently where it would not be * required (after I have cleaned up the pathes ;)). -- rgerhards, 2008-10-02 */ - datetime.getCurrTime(&(pM->tRcvdAt)); - memcpy(&pM->tTIMESTAMP, &pM->tRcvdAt, sizeof(struct syslogTime)); - - objConstructSetObjInfo(pM); - - /* DEV debugging only! dbgprintf("msgConstruct\t0x%x, ref 1\n", (int)pM);*/ - - *ppThis = pM; + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + memcpy(&(*ppThis)->tTIMESTAMP, &(*ppThis)->tRcvdAt, sizeof(struct syslogTime)); finalize_it: RETiRet; @@ -2492,7 +2538,5 @@ BEGINObjClassInit(msg, 1, OBJ_IS_CORE_MODULE) funcDeleteMutex = MsgLockingDummy; funcMsgPrepareEnqueue = MsgLockingDummy; ENDObjClassInit(msg) - -/* - * vi:set ai: +/* vim:set ai: */ diff --git a/runtime/msg.h b/runtime/msg.h index 21cb2c64..d3c0718b 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -119,6 +119,7 @@ struct msg { PROTOTYPEObjClassInit(msg); char* getProgramName(msg_t*); rsRetVal msgConstruct(msg_t **ppThis); +rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); @@ -180,6 +181,5 @@ extern void (*funcMsgPrepareEnqueue)(msg_t *pMsg); #define MsgPrepareEnqueue(pMsg) funcMsgPrepareEnqueue(pMsg) #endif /* #ifndef MSG_H_INCLUDED */ -/* - * vi:set ai: +/* vim:set ai: */ diff --git a/tcps_sess.c b/tcps_sess.c index f5420fc0..7ce9f2f8 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -230,7 +230,7 @@ PrepareClose(tcps_sess_t *pThis) */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME, - NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL); /* TODO: add real InputName */ + NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ pThis->bAtStrtOfFram = 1; } @@ -313,7 +313,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL); /* TODO: add real InputName */ + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ pThis->iMsg = 0; /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good @@ -324,7 +324,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(c == '\n' && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delemiter? */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL); /* TODO: add real InputName */ + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } else { @@ -343,7 +343,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL); /* TODO: add real InputName */ + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } diff --git a/tools/syslogd.c b/tools/syslogd.c index b576bb6d..a6e17d8f 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -581,9 +581,15 @@ void untty(void) * Interface change: added new parameter "InputName", permits the input to provide * a string that identifies it. May be NULL, but must be a valid char* pointer if * non-NULL. + * + * rgerhards, 2008-10-06: + * Interface change: added new parameter "stTime", which enables the caller to provide + * a timestamp that is to be used as timegenerated instead of the current system time. + * This is meant to facilitate performance optimization. Some inputs support such modes. + * If stTime is NULL, the current system time is used. */ -rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int flags, flowControl_t flowCtlType, - uchar *pszInputName) +static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int flags, flowControl_t flowCtlType, + uchar *pszInputName, struct syslogTime *stTime) { DEFiRet; register uchar *p; @@ -591,7 +597,11 @@ rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int msg_t *pMsg; /* Now it is time to create the message object (rgerhards) */ - CHKiRet(msgConstruct(&pMsg)); + if(stTime == NULL) { + CHKiRet(msgConstruct(&pMsg)); + } else { + CHKiRet(msgConstructWithTime(&pMsg, stTime)); + } if(pszInputName != NULL) MsgSetInputName(pMsg, (char*) pszInputName); MsgSetFlowControlType(pMsg, flowCtlType); @@ -684,10 +694,16 @@ finalize_it: * Interface change: added new parameter "InputName", permits the input to provide * a string that identifies it. May be NULL, but must be a valid char* pointer if * non-NULL. + * + * rgerhards, 2008-10-06: + * Interface change: added new parameter "stTime", which enables the caller to provide + * a timestamp that is to be used as timegenerated instead of the current system time. + * This is meant to facilitate performance optimization. Some inputs support such modes. + * If stTime is NULL, the current system time is used. */ rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlType, - uchar *pszInputName) + uchar *pszInputName, struct syslogTime *stTime) { DEFiRet; register int iMsg; @@ -714,9 +730,6 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa * TODO: optimize buffer handling */ iMaxLine = glbl.GetMaxLine(); CHKmalloc(tmpline = malloc(sizeof(uchar) * (iMaxLine + 1))); -# ifdef USE_NETZIP - CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1))); -# endif /* we first check if we have a NUL character at the very end of the * message. This seems to be a frequent problem with a number of senders. @@ -762,6 +775,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa */ int ret; iLenDefBuf = iMaxLine; + CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1))); ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) msg+1, len-1); dbgprintf("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", ret, (long) iLenDefBuf, len-1); @@ -800,7 +814,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa */ if(iMsg == iMaxLine) { *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ - printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName); + printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime); } else { /* This case in theory never can happen. If it happens, we have * a logic error. I am checking for it, because if I would not, @@ -852,7 +866,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ /* typically, we should end up here! */ - printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName); + printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime); finalize_it: if(tmpline != NULL) @@ -1344,6 +1358,13 @@ static int parseRFCSyslogMsg(msg_t *pMsg, int flags) } else { /* we can not parse, so we get the system we * received the data from. + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt)); */ MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } -- cgit v1.2.3 From 7b3c05da9f126063384c80e9dd6fd5b2ae610074 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Oct 2008 10:28:44 +0200 Subject: simple (yet efficient) name caching added to imudp --- plugins/imudp/imudp.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 1865d777..d13314e8 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -133,6 +133,15 @@ finalize_it: /* This function is called to gather input. * Note that udpLstnSocks must be non-NULL because otherwise we would not have * indicated that we want to run (or we have a programming error ;)). -- rgerhards, 2008-10-02 + * rgerhards, 2008-10-07: I have implemented a very simple, yet in most cases probably + * highly efficient "name caching". Before querying a name, I now check if the name to be + * queried is the same as the one queried in the last message processed. If that is the + * case, we can simple re-use the previous value. This algorithm works quite well with + * few sender, especially if they emit messages in bursts. The more sender and the + * more intermixed messages arrive, the less this algorithm works, but the overhead + * is so minimal (a simple memory compare and move) that this does not hurt. Even + * with a real name lookup cache, this optimization here is useful as it is quicker + * than even a cache lookup). */ BEGINrunInput int maxfds; @@ -140,6 +149,7 @@ BEGINrunInput int i; fd_set readfds; struct sockaddr_storage frominet; + struct sockaddr_storage frominetPrev; socklen_t socklen; uchar fromHost[NI_MAXHOST]; uchar fromHostIP[NI_MAXHOST]; @@ -148,6 +158,10 @@ BEGINrunInput struct syslogTime stTime; int iNbrTimeUsed; CODESTARTrunInput + /* start "name caching" algo by making sure the previous system indicator + * is invalidated. + */ + memset(&frominetPrev, 0, sizeof(frominetPrev)); /* this is an endless loop - it is terminated when the thread is * signalled to do so. This, however, is handled by the framework, * right into the sleep below. @@ -184,7 +198,7 @@ CODESTARTrunInput for (i = 0; nfds && i < *udpLstnSocks; i++) { if (FD_ISSET(udpLstnSocks[i+1], &readfds)) { - socklen = sizeof(frominet); + socklen = sizeof(frominet); iNbrTimeUsed = 0; do { /* we now try to read from the file descriptor until there @@ -199,7 +213,9 @@ CODESTARTrunInput l = recvfrom(udpLstnSocks[i+1], (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); if(l > 0) { - if(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) { + if(memcmp(&frominet, &frominetPrev, socklen) == 0 || + net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) { + memcpy(&frominetPrev, &frominet, socklen); dbgprintf("Message from inetd socket: #%d, host: %s\n", udpLstnSocks[i+1], fromHost); /* Here we check if a host is permitted to send us -- cgit v1.2.3 From cdecd7e524a5114ccff4f2909b32738bdb33c6ea Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Oct 2008 11:46:46 +0200 Subject: slightly improved lock contention situation by moving out of the critical section what could so with acceptable consequences --- runtime/queue.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index 76c2f10f..f5f770b3 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2101,6 +2101,15 @@ queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr) ISOBJ_TYPE_assert(pThis, queue); + /* first check if we need to discard this message (which will cause CHKiRet() to exit) + * rgerhards, 2008-10-07: It is OK to do this outside of mutex protection. The iQueueSize + * and bRunsDA parameters may not reflect the correct settings here, but they are + * "good enough" in the sense that they can be used to drive the decision. Valgrind's + * threading tools may point this access to be an error, but this is done + * intentional. I do not see this causes problems to us. + */ + CHKiRet(queueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); + /* Please note that this function is not cancel-safe and consequently * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE * during its execution. If that is not done, race conditions occur if the @@ -2112,9 +2121,6 @@ queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr) d_pthread_mutex_lock(pThis->mut); } - /* first check if we need to discard this message (which will cause CHKiRet() to exit) */ - CHKiRet(queueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); - /* then check if we need to add an assistance disk queue */ if(pThis->bIsDA) CHKiRet(queueChkStrtDA(pThis)); -- cgit v1.2.3 From 8528344ef58b5d2907bba8809f63d0bca2ce8d38 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Oct 2008 14:26:41 +0200 Subject: "output" timestamp now taken from mesg's time generated This enhances performance and, as some have pointed out, is probably also more consistent with what users expect how the various output-timestamp related function should work. This commit needs some more testing. --- ChangeLog | 9 +++++++++ action.c | 6 +++++- dirty.h | 2 +- plugins/imrelp/imrelp.c | 2 +- plugins/imudp/imudp.c | 6 ++++-- plugins/imuxsock/imuxsock.c | 2 +- runtime/datetime.c | 12 ++++++++++-- runtime/datetime.h | 2 +- runtime/msg.c | 15 +++++++++------ runtime/msg.h | 9 ++++++++- runtime/sysvar.c | 2 +- tcps_sess.c | 8 ++++---- tools/syslogd.c | 17 +++++------------ 13 files changed, 59 insertions(+), 33 deletions(-) diff --git a/ChangeLog b/ChangeLog index 346ff39c..0f3d4591 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,14 @@ --------------------------------------------------------------------------- Version 3.21.6 [DEVEL] (rgerhards), 2008-10-?? + +********************************* WARNING ********************************* +This version has a slightly different on-disk format for message entries. +As a consequence, old queue files being read by this version may have +an invalid output timestamp, which could result to some malfunction inside +the output driver. It is recommended to drain queues with the previous +version before switching to this one. +********************************* WARNING ********************************* + - added $UDPServerTimeRequery option which enables to work with less acurate timestamps in favor of performance. This enables querying of the time only every n-th time if imudp is running in the tight diff --git a/action.c b/action.c index a25eca23..cc62f028 100644 --- a/action.c +++ b/action.c @@ -602,7 +602,7 @@ actionWriteToAction(action_t *pAction) * ... RAWMSG is a problem ... Please note that digital * signatures inside the message are also invalidated. */ - datetime.getCurrTime(&(pMsg->tRcvdAt)); + datetime.getCurrTime(&(pMsg->tRcvdAt), &(pMsg->ttGenTime)); memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); MsgSetMSG(pMsg, (char*)szRepMsg); MsgSetRawMsg(pMsg, (char*)szRepMsg); @@ -640,6 +640,7 @@ actionWriteToAction(action_t *pAction) * while in the on-disk queue. Also need to think about a few other implications. * rgerhards, 2008-09-17 */ +#if 0 { struct tm tTm; tTm.tm_sec = pAction->f_pMsg->tRcvdAt.second; @@ -655,7 +656,10 @@ actionWriteToAction(action_t *pAction) /* note: mktime seems to do a stat(/etc/localtime), so this is also sub-optimal! */ dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL)); } +#endif + pAction->f_time = pAction->f_pMsg->ttGenTime; +dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL)); //pAction->f_time = getActNow(pAction); /* re-init time flags */ /* Note: tLastExec could be set in the if block above, but f_time causes us a hard time * so far, I do not see a solution to getting rid of it. -- rgerhards, 2008-09-16 diff --git a/dirty.h b/dirty.h index 58c4ea39..8e7ffd84 100644 --- a/dirty.h +++ b/dirty.h @@ -40,7 +40,7 @@ rsRetVal submitMsg(msg_t *pMsg); rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); -rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime); +rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); /* TODO: the following 2 need to go in conf obj interface... */ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c index f3771237..4515acd7 100644 --- a/plugins/imrelp/imrelp.c +++ b/plugins/imrelp/imrelp.c @@ -84,7 +84,7 @@ onSyslogRcv(uchar *pHostname, uchar __attribute__((unused)) *pIP, uchar *pMsg, s { DEFiRet; parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, MSG_PARSE_HOSTNAME, - NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL); + NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL, 0); RETiRet; } diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index d13314e8..d58a0d8e 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -155,6 +155,7 @@ BEGINrunInput uchar fromHostIP[NI_MAXHOST]; uchar fromHostFQDN[NI_MAXHOST]; ssize_t l; + time_t ttGenTime; struct syslogTime stTime; int iNbrTimeUsed; CODESTARTrunInput @@ -227,10 +228,11 @@ CODESTARTrunInput if(net.isAllowedSender(net.pAllowedSenders_UDP, (struct sockaddr *)&frominet, (char*)fromHostFQDN)) { if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { - datetime.getCurrTime(&stTime); + datetime.getCurrTime(&stTime, &ttGenTime); } parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", &stTime); + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", + &stTime, ttGenTime); } else { dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN); if(glbl.GetOption_DisallowWarning) { diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 77d347d9..efa0365d 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -221,7 +221,7 @@ static rsRetVal readSocket(int fd, int iSock) if (iRcvd > 0) { parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock], (uchar*)"127.0.0.1", pRcv, - iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock", NULL); + iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock", NULL, 0); } else if (iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); diff --git a/runtime/datetime.c b/runtime/datetime.c index 20ca6191..aa1956d7 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -62,9 +62,14 @@ DEFobjCurrIf(errmsg) * most portable and removes the need for additional structures * (but I have to admit it is somewhat "bulky";)). * - * Obviously, all caller-provided pointers must not be NULL... + * Obviously, *t must not be NULL... + * + * rgerhards, 2008-10-07: added ttSeconds to provide a way to + * obtain the second-resolution UNIX timestamp. This is needed + * in some situations to minimize time() calls (namely when doing + * output processing). This can be left NULL if not needed. */ -static void getCurrTime(struct syslogTime *t) +static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) { struct timeval tp; struct tm *tm; @@ -83,6 +88,9 @@ static void getCurrTime(struct syslogTime *t) # else gettimeofday(&tp, NULL); # endif + if(ttSeconds != NULL) + *ttSeconds = tp.tv_sec; + tm = localtime_r((time_t*) &(tp.tv_sec), &tmBuf); t->year = tm->tm_year + 1900; diff --git a/runtime/datetime.h b/runtime/datetime.h index 2210af02..0739588d 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -35,7 +35,7 @@ typedef struct datetime_s { /* interfaces */ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ - void (*getCurrTime)(struct syslogTime *t); + void (*getCurrTime)(struct syslogTime *t, time_t *ttSeconds); rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, char** ppszTS); rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, char** pszTS); int (*formatTimestampToMySQL)(struct syslogTime *ts, char* pDst, size_t iLenDst); diff --git a/runtime/msg.c b/runtime/msg.c index df8c1572..fd838591 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -286,11 +286,12 @@ finalize_it: * udp input). * rgerhards, 2008-10-06 */ -rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime) +rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; CHKiRet(msgBaseConstruct(ppThis)); + (*ppThis)->ttGenTime = ttGenTime; memcpy(&(*ppThis)->tRcvdAt, stTime, sizeof(struct syslogTime)); memcpy(&(*ppThis)->tTIMESTAMP, stTime, sizeof(struct syslogTime)); @@ -316,7 +317,7 @@ rsRetVal msgConstruct(msg_t **ppThis) * especially as I think there is no codepath currently where it would not be * required (after I have cleaned up the pathes ;)). -- rgerhards, 2008-10-02 */ - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); + datetime.getCurrTime(&((*ppThis)->tRcvdAt), &((*ppThis)->ttGenTime)); memcpy(&(*ppThis)->tTIMESTAMP, &(*ppThis)->tRcvdAt, sizeof(struct syslogTime)); finalize_it: @@ -442,7 +443,7 @@ msg_t* MsgDup(msg_t* pOld) assert(pOld != NULL); BEGINfunc - if(msgConstruct(&pNew) != RS_RET_OK) { + if(msgConstructWithTime(&pNew, &pOld->tTIMESTAMP, pOld->ttGenTime) != RS_RET_OK) { return NULL; } @@ -453,8 +454,7 @@ msg_t* MsgDup(msg_t* pOld) pNew->bParseHOSTNAME = pOld->bParseHOSTNAME; pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; - memcpy(&pNew->tRcvdAt, &pOld->tRcvdAt, sizeof(struct syslogTime)); - memcpy(&pNew->tTIMESTAMP, &pOld->tTIMESTAMP, sizeof(struct syslogTime)); + pNew->ttGenTime = pOld->ttGenTime; tmpCOPYSZ(Severity); tmpCOPYSZ(SeverityStr); tmpCOPYSZ(Facility); @@ -508,6 +508,7 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializeSCALAR(pStrm, iSeverity, SHORT); objSerializeSCALAR(pStrm, iFacility, SHORT); objSerializeSCALAR(pStrm, msgFlags, INT); + objSerializeSCALAR(pStrm, ttGenTime, INT); objSerializeSCALAR(pStrm, tRcvdAt, SYSLOGTIME); objSerializeSCALAR(pStrm, tTIMESTAMP, SYSLOGTIME); @@ -1669,7 +1670,7 @@ static uchar *getNOW(eNOWType eNow) return NULL; } - datetime.getCurrTime(&t); + datetime.getCurrTime(&t, NULL); switch(eNow) { case NOW_NOW: snprintf((char*) pBuf, tmpBUFSIZE, "%4.4d-%2.2d-%2.2d", t.year, t.month, t.day); @@ -2477,6 +2478,8 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) MsgSetPROCID(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pCSMSGID")) { MsgSetMSGID(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + } else if(isProp("ttGenTime")) { + pThis->ttGenTime = pProp->val.num; } else if(isProp("tRcvdAt")) { memcpy(&pThis->tRcvdAt, &pProp->val.vSyslogTime, sizeof(struct syslogTime)); } else if(isProp("tTIMESTAMP")) { diff --git a/runtime/msg.h b/runtime/msg.h index d3c0718b..d2fc2f30 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -99,6 +99,13 @@ struct msg { cstr_t *pCSAPPNAME; /* APP-NAME */ cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ + time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. + While this field looks redundant, it is required because a Unix timestamp + is used at later processing stages (namely in the output arena). Thanks to + the subleties of how time is defined, there is no reliable way to reconstruct + the Unix timestamp from the syslogTime fields (in practice, we may be close + enough to reliable, but I prefer to leave the subtle things to the OS, where + it obviously is solved in way or another...). */ struct syslogTime tRcvdAt;/* time the message entered this program */ char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ @@ -119,7 +126,7 @@ struct msg { PROTOTYPEObjClassInit(msg); char* getProgramName(msg_t*); rsRetVal msgConstruct(msg_t **ppThis); -rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime); +rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); diff --git a/runtime/sysvar.c b/runtime/sysvar.c index 5eec8f67..c102d1f5 100644 --- a/runtime/sysvar.c +++ b/runtime/sysvar.c @@ -84,7 +84,7 @@ getNOW(eNOWType eNow, cstr_t **ppStr) uchar szBuf[16]; struct syslogTime t; - datetime.getCurrTime(&t); + datetime.getCurrTime(&t, NULL); switch(eNow) { case NOW_NOW: snprintf((char*) szBuf, sizeof(szBuf)/sizeof(uchar), "%4.4d-%2.2d-%2.2d", t.year, t.month, t.day); diff --git a/tcps_sess.c b/tcps_sess.c index 7ce9f2f8..13644f45 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -230,7 +230,7 @@ PrepareClose(tcps_sess_t *pThis) */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME, - NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ + NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->bAtStrtOfFram = 1; } @@ -313,7 +313,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->iMsg = 0; /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good @@ -324,7 +324,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(c == '\n' && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delemiter? */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } else { @@ -343,7 +343,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL); /* TODO: add real InputName */ + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } diff --git a/tools/syslogd.c b/tools/syslogd.c index a6e17d8f..e794e2d1 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -589,7 +589,7 @@ void untty(void) * If stTime is NULL, the current system time is used. */ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int flags, flowControl_t flowCtlType, - uchar *pszInputName, struct syslogTime *stTime) + uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; register uchar *p; @@ -600,7 +600,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b if(stTime == NULL) { CHKiRet(msgConstruct(&pMsg)); } else { - CHKiRet(msgConstructWithTime(&pMsg, stTime)); + CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } if(pszInputName != NULL) MsgSetInputName(pMsg, (char*) pszInputName); @@ -703,7 +703,7 @@ finalize_it: */ rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlType, - uchar *pszInputName, struct syslogTime *stTime) + uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; register int iMsg; @@ -814,7 +814,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa */ if(iMsg == iMaxLine) { *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ - printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime); + printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime, ttGenTime); } else { /* This case in theory never can happen. If it happens, we have * a logic error. I am checking for it, because if I would not, @@ -866,7 +866,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ /* typically, we should end up here! */ - printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime); + printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime, ttGenTime); finalize_it: if(tmpline != NULL) @@ -1358,13 +1358,6 @@ static int parseRFCSyslogMsg(msg_t *pMsg, int flags) } else { /* we can not parse, so we get the system we * received the data from. - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); - datetime.getCurrTime(&((*ppThis)->tRcvdAt)); */ MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } -- cgit v1.2.3 From 0fa23994669417fff4c4c057ce0c9d1e96f6d56c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Oct 2008 15:10:03 +0200 Subject: cleanup of output timestamp generation --- action.c | 38 ++------------------------------------ runtime/msg.c | 4 ++++ 2 files changed, 6 insertions(+), 36 deletions(-) diff --git a/action.c b/action.c index cc62f028..ec8732bc 100644 --- a/action.c +++ b/action.c @@ -630,42 +630,8 @@ actionWriteToAction(action_t *pAction) FINALIZE; } - - - /* TODO: move this to msg object or some other object. This is just for quick testing! - * ALSO, THIS DOES NOT YET WORK PROPERLY! - * The reason is that we do not know the DST status, which is major pain. I need to - * think about obtaining this information (or the actual Unix timestamp) when I - * create the reception timestamp, but that also means I need to preserve that information - * while in the on-disk queue. Also need to think about a few other implications. - * rgerhards, 2008-09-17 - */ -#if 0 - { - struct tm tTm; - tTm.tm_sec = pAction->f_pMsg->tRcvdAt.second; - tTm.tm_min = pAction->f_pMsg->tRcvdAt.minute; - tTm.tm_hour = pAction->f_pMsg->tRcvdAt.hour; - tTm.tm_mday = pAction->f_pMsg->tRcvdAt.day; - tTm.tm_mon = pAction->f_pMsg->tRcvdAt.month - 1; - tTm.tm_year = pAction->f_pMsg->tRcvdAt.year - 1900; - /********************************************************************************/ - tTm.tm_isdst = 1; /* TODO THIS IS JUST VALID FOR THE NEXT FEW DAYS ;) TODO */ - /********************************************************************************/ - pAction->f_time = mktime(&tTm); -/* note: mktime seems to do a stat(/etc/localtime), so this is also sub-optimal! */ -dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL)); - } -#endif - - pAction->f_time = pAction->f_pMsg->ttGenTime; -dbgprintf("XXXX create our own timestamp: %ld, system time is %ld\n", pAction->f_time, time(NULL)); - //pAction->f_time = getActNow(pAction); /* re-init time flags */ - /* Note: tLastExec could be set in the if block above, but f_time causes us a hard time - * so far, I do not see a solution to getting rid of it. -- rgerhards, 2008-09-16 - */ - - + /* TODO: the time call below may use reception time, not dequeue time - under consideration. -- rgerhards, 2008-09-17 */ + pAction->f_time = pAction->f_pMsg->ttGenTime; /* When we reach this point, we have a valid, non-disabled action. * So let's enqueue our message for execution. -- rgerhards, 2007-07-24 diff --git a/runtime/msg.c b/runtime/msg.c index fd838591..c030fa45 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -778,6 +778,7 @@ int getPRIi(msg_t *pM) char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) { + BEGINfunc if(pM == NULL) return ""; @@ -849,11 +850,13 @@ char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return(pM->pszTIMESTAMP_SecFrac); } + ENDfunc return "INVALID eFmt OPTION!"; } char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) { + BEGINfunc if(pM == NULL) return ""; @@ -925,6 +928,7 @@ char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return(pM->pszRcvdAt_SecFrac); } + ENDfunc return "INVALID eFmt OPTION!"; } -- cgit v1.2.3 From 82b583c4f99dd9beb30360f222c4d2a1152f75e1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Oct 2008 14:56:02 +0200 Subject: restructured imudp receive loop cleaned up previous code and redid it in a way that makes it much easier to extend it also added a new macro DBGPRINTF which is a performance-optimzed version of dbgprintf --- plugins/imudp/imudp.c | 145 +++++++++++++++++++++++++++++--------------------- runtime/debug.h | 1 + 2 files changed, 86 insertions(+), 60 deletions(-) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index d58a0d8e..7f9afb68 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -130,6 +130,85 @@ finalize_it: } +/* This function is a helper to runInput. I have extracted it + * from the main loop just so that we do not have that large amount of code + * in a single place. This function takes a socket and pulls messages from + * it until the socket does not have any more waiting. + * rgerhards, 2008-01-08 + * We try to read from the file descriptor until there + * is no more data. This is done in the hope to get better performance + * out of the system. However, this also means that a descriptor + * monopolizes processing while it contains data. This can lead to + * data loss in other descriptors. However, if the system is incapable of + * handling the workload, we will loss data in any case. So it doesn't really + * matter where the actual loss occurs - it is always random, because we depend + * on scheduling order. -- rgerhards, 2008-10-02 + */ +static inline rsRetVal +processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, + uchar *fromHost, uchar *fromHostFQDN, uchar *fromHostIP) +{ + DEFiRet; + int iNbrTimeUsed; + time_t ttGenTime; + struct syslogTime stTime; + socklen_t socklen; + ssize_t l; + struct sockaddr_storage frominet; + char errStr[1024]; + + iNbrTimeUsed = 0; + while(1) { /* loop is terminated if we have a bad receive, done below in the body */ + socklen = sizeof(struct sockaddr_storage); + l = recvfrom(fd, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); + if(l < 0) { + if(errno != EINTR && errno != EAGAIN) { + rs_strerror_r(errno, errStr, sizeof(errStr)); + DBGPRINTF("INET socket error: %d = %s.\n", errno, errStr); + errmsg.LogError(errno, NO_ERRCODE, "recvfrom inet"); + } + ABORT_FINALIZE(RS_RET_ERR); + } + + /* if we reach this point, we had a good receive and can process the packet received */ + /* check if we have a different sender than before, if so, we need to query some new values */ + if(memcmp(&frominet, frominetPrev, socklen) != 0) { + CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); + memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ + /* Here we check if a host is permitted to send us + * syslog messages. If it isn't, we do not further + * process the message but log a warning (if we are + * configured to do this). + * rgerhards, 2005-09-26 + */ + *pbIsPermitted = net.isAllowedSender(net.pAllowedSenders_UDP, + (struct sockaddr *)&frominet, (char*)fromHostFQDN); + + if(!*pbIsPermitted) { + DBGPRINTF("%s is not an allowed sender\n", (char*)fromHostFQDN); + if(glbl.GetOption_DisallowWarning) { + errmsg.LogError(0, NO_ERRCODE, "UDP message from disallowed sender %s discarded", + (char*)fromHost); + } + } + } + + DBGPRINTF("Message from inetd socket: #%d, host: %s, isPermitted: %d\n", fd, fromHost, *pbIsPermitted); + if(*pbIsPermitted) { + if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { + datetime.getCurrTime(&stTime, &ttGenTime); + } + parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l, + MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", &stTime, ttGenTime); + } + } + + +finalize_it: + RETiRet; +} + + /* This function is called to gather input. * Note that udpLstnSocks must be non-NULL because otherwise we would not have * indicated that we want to run (or we have a programming error ;)). -- rgerhards, 2008-10-02 @@ -148,20 +227,16 @@ BEGINrunInput int nfds; int i; fd_set readfds; - struct sockaddr_storage frominet; struct sockaddr_storage frominetPrev; - socklen_t socklen; + int bIsPermitted; uchar fromHost[NI_MAXHOST]; uchar fromHostIP[NI_MAXHOST]; uchar fromHostFQDN[NI_MAXHOST]; - ssize_t l; - time_t ttGenTime; - struct syslogTime stTime; - int iNbrTimeUsed; CODESTARTrunInput /* start "name caching" algo by making sure the previous system indicator * is invalidated. */ + bIsPermitted = 0; memset(&frominetPrev, 0, sizeof(frominetPrev)); /* this is an endless loop - it is terminated when the thread is * signalled to do so. This, however, is handled by the framework, @@ -198,61 +273,11 @@ CODESTARTrunInput nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL); for (i = 0; nfds && i < *udpLstnSocks; i++) { - if (FD_ISSET(udpLstnSocks[i+1], &readfds)) { - socklen = sizeof(frominet); - iNbrTimeUsed = 0; - do { - /* we now try to read from the file descriptor until there - * is no more data. This is done in the hope to get better performance - * out of the system. However, this also means that a descriptor - * monopolizes processing while it contains data. This can lead to - * data loss in other descriptors. However, if the system is incapable of - * handling the workload, we will loss data in any case. So it doesn't really - * matter where the actual loss occurs - it is always random, because we depend - * on scheduling order. -- rgerhards, 2008-10-02 - */ - l = recvfrom(udpLstnSocks[i+1], (char*) pRcvBuf, iMaxLine, 0, - (struct sockaddr *)&frominet, &socklen); - if(l > 0) { - if(memcmp(&frominet, &frominetPrev, socklen) == 0 || - net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP) == RS_RET_OK) { - memcpy(&frominetPrev, &frominet, socklen); - dbgprintf("Message from inetd socket: #%d, host: %s\n", - udpLstnSocks[i+1], fromHost); - /* Here we check if a host is permitted to send us - * syslog messages. If it isn't, we do not further - * process the message but log a warning (if we are - * configured to do this). - * rgerhards, 2005-09-26 - */ - if(net.isAllowedSender(net.pAllowedSenders_UDP, - (struct sockaddr *)&frominet, (char*)fromHostFQDN)) { - if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { - datetime.getCurrTime(&stTime, &ttGenTime); - } - parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", - &stTime, ttGenTime); - } else { - dbgprintf("%s is not an allowed sender\n", (char*)fromHostFQDN); - if(glbl.GetOption_DisallowWarning) { - errmsg.LogError(0, NO_ERRCODE, "UDP message from disallowed sender %s discarded", - (char*)fromHost); - } - } - } - } else if(l < 0 && errno != EINTR && errno != EAGAIN) { - char errStr[1024]; - rs_strerror_r(errno, errStr, sizeof(errStr)); - dbgprintf("INET socket error: %d = %s.\n", errno, errStr); - errmsg.LogError(errno, NO_ERRCODE, "recvfrom inet"); - /* should be harmless */ - sleep(1); - } - } while(l > 0); /* Warning: do ... while()! */ - - --nfds; /* indicate we have processed one descriptor */ + if (FD_ISSET(udpLstnSocks[i+1], &readfds)) { + processSocket(udpLstnSocks[i+1], &frominetPrev, &bIsPermitted, + fromHost, fromHostFQDN, fromHostIP); } + --nfds; /* indicate we have processed one descriptor */ } /* end of a run, back to loop for next recv() */ } diff --git a/runtime/debug.h b/runtime/debug.h index d9d576b5..7ac29765 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -100,6 +100,7 @@ void dbgSetThrdName(uchar *pszName); void dbgPrintAllDebugInfo(void); /* macros */ +#define DBGPRINTF(...) if(Debug) { dbgprintf(__VA_ARGS__); } #ifdef RTINST # define BEGINfunc static dbgFuncDB_t *pdbgFuncDB; int dbgCALLStaCK_POP_POINT = dbgEntrFunc(&pdbgFuncDB, __FILE__, __func__, __LINE__); # define ENDfunc dbgExitFunc(pdbgFuncDB, dbgCALLStaCK_POP_POINT, RS_RET_NO_IRET); -- cgit v1.2.3 From ace4f2f75202aec39449dac11b9eb1deca7428d7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Oct 2008 18:55:11 +0200 Subject: reordered imudp processing. Message parsing is now done as part of main message queue worker processing (was part of the input thread) This should also improve performance, as potentially more work is done in parallel. --- ChangeLog | 6 + dirty.h | 3 + plugins/imudp/imudp.c | 27 ++++- runtime/Makefile.am | 2 + runtime/msg.h | 7 +- runtime/parser.c | 314 ++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/parser.h | 30 +++++ runtime/queue.c | 2 - runtime/rsyslog.h | 1 + runtime/wti.c | 6 +- tools/syslogd.c | 18 ++- 11 files changed, 396 insertions(+), 20 deletions(-) create mode 100644 runtime/parser.c create mode 100644 runtime/parser.h diff --git a/ChangeLog b/ChangeLog index 0f3d4591..2f9f34a6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,6 +9,12 @@ the output driver. It is recommended to drain queues with the previous version before switching to this one. ********************************* WARNING ********************************* +- reordered imudp processing. Message parsing is now done as part of main + message queue worker processing (was part of the input thread) + This should also improve performance, as potentially more work is + done in parallel. +- bugfix: compressed syslog messages could be slightly mis-uncompressed + if the last byte of the compressed record was a NUL - added $UDPServerTimeRequery option which enables to work with less acurate timestamps in favor of performance. This enables querying of the time only every n-th time if imudp is running in the tight diff --git a/dirty.h b/dirty.h index 8e7ffd84..f5d0415e 100644 --- a/dirty.h +++ b/dirty.h @@ -61,6 +61,9 @@ extern int bReduceRepeatMsgs; #define BACKOFF(f) { if (++(f)->f_repeatcount > MAXREPEAT) \ (f)->f_repeatcount = MAXREPEAT; \ } +extern int bDropTrailingLF; +extern uchar cCCEscapeChar; +extern int bEscapeCCOnRcv; #ifdef USE_NETZIP /* config param: minimum message size to try compression. The smaller * the message, the less likely is any compression gain. We check for diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 7f9afb68..aab201a9 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -40,6 +40,8 @@ #include "srUtils.h" #include "errmsg.h" #include "glbl.h" +#include "msg.h" +#include "parser.h" #include "datetime.h" MODULE_TYPE_INPUT @@ -153,15 +155,16 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, time_t ttGenTime; struct syslogTime stTime; socklen_t socklen; - ssize_t l; + ssize_t lenRcvBuf; struct sockaddr_storage frominet; + msg_t *pMsg; char errStr[1024]; iNbrTimeUsed = 0; while(1) { /* loop is terminated if we have a bad receive, done below in the body */ socklen = sizeof(struct sockaddr_storage); - l = recvfrom(fd, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); - if(l < 0) { + lenRcvBuf = recvfrom(fd, (char*) pRcvBuf, iMaxLine, 0, (struct sockaddr *)&frominet, &socklen); + if(lenRcvBuf < 0) { if(errno != EINTR && errno != EAGAIN) { rs_strerror_r(errno, errStr, sizeof(errStr)); DBGPRINTF("INET socket error: %d = %s.\n", errno, errStr); @@ -193,13 +196,25 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, } } - DBGPRINTF("Message from inetd socket: #%d, host: %s, isPermitted: %d\n", fd, fromHost, *pbIsPermitted); + DBGPRINTF("recv(%d,%d)/%s,acl:%d,msg:%.80s\n", fd, (int) lenRcvBuf, fromHost, *pbIsPermitted, pRcvBuf); + if(*pbIsPermitted) { if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { datetime.getCurrTime(&stTime, &ttGenTime); } - parseAndSubmitMessage(fromHost, fromHostIP, pRcvBuf, l, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_NO_DELAY, (uchar*)"imudp", &stTime, ttGenTime); + /* we now create our own message object and submit it to the queue */ + CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); + /* first trim the buffer to what we have actually received */ + CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); + memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); + pMsg->iLenRawMsg = lenRcvBuf; + MsgSetInputName(pMsg, "imudp"); + MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); + pMsg->bParseHOSTNAME = MSG_PARSE_HOSTNAME; + pMsg->msgFlags = NOFLAG; + MsgSetRcvFrom(pMsg, (char*)fromHost); + CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP)); + CHKiRet(submitMsg(pMsg)); } } diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 81a9d5bd..7c70dd8d 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -16,6 +16,8 @@ librsyslog_la_SOURCES = \ glbl.c \ conf.c \ conf.h \ + parser.h \ + parser.c \ msg.c \ msg.h \ linkedlist.c \ diff --git a/runtime/msg.h b/runtime/msg.h index d2fc2f30..98635f85 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -52,7 +52,10 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; pthread_mutex_t mut; - int iRefCount; /* reference counter (0 = unused) */ + flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because + once data has entered the queue, this property is no longer needed. */ + short iRefCount; /* reference counter (0 = unused) */ + short bIsParsed; /* is message parsed? (0=no, 1=yes), 0 means parser needs to be called */ short bParseHOSTNAME; /* should the hostname be parsed from the message? */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, @@ -60,8 +63,6 @@ struct msg { * sockets. All in all, the parser would need parse templates, that would * resolve all these issues... rgerhards, 2005-10-06 */ - flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because - once data has entered the queue, this property is no longer needed. */ short iSeverity; /* the severity 0..7 */ uchar *pszSeverity; /* severity as string... */ int iLenSeverity; /* ... and its length. */ diff --git a/runtime/parser.c b/runtime/parser.c new file mode 100644 index 00000000..8c4272a0 --- /dev/null +++ b/runtime/parser.c @@ -0,0 +1,314 @@ +/* parser.c + * This module contains functions for message parsers. It still needs to be + * converted into an object (and much extended). + * + * Module begun 2008-10-09 by Rainer Gerhards (based on previous code from syslogd.c) + * + * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" +#include +#include +#include +#ifdef USE_NETZIP +#include +#endif + +#include "rsyslog.h" +#include "dirty.h" +#include "msg.h" +#include "obj.h" +#include "errmsg.h" + +/* some defines */ +#define DEFUPRI (LOG_USER|LOG_NOTICE) + +#warning "msg object must be updated with new property for persisting the queue!" +/* definitions for objects we access */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) +DEFobjCurrIf(errmsg) + +/* static data */ + + +/* this is a dummy class init + */ +rsRetVal parserClassInit(void) +{ + DEFiRet; + + /* request objects we use */ + CHKiRet(objGetObjInterface(&obj)); /* this provides the root pointer for all other queries */ + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); +// TODO: free components! see action.c +finalize_it: + RETiRet; +} + + +/* uncompress a received message if it is compressed. + * pMsg->pszRawMsg buffer is updated. + * rgerhards, 2008-10-09 + */ +static inline rsRetVal uncompressMessage(msg_t *pMsg) +{ + DEFiRet; +# ifdef USE_NETZIP + uchar *deflateBuf = NULL; + uLongf iLenDefBuf; + uchar *pszMsg; + size_t lenMsg; + + assert(pMsg != NULL); + pszMsg = pMsg->pszRawMsg; + lenMsg = pMsg->iLenRawMsg; + + /* we first need to check if we have a compressed record. If so, + * we must decompress it. + */ + if(lenMsg > 0 && *pszMsg == 'z') { /* compressed data present? (do NOT change order if conditions!) */ + /* we have compressed data, so let's deflate it. We support a maximum + * message size of iMaxLine. If it is larger, an error message is logged + * and the message is dropped. We do NOT try to decompress larger messages + * as such might be used for denial of service. It might happen to later + * builds that such functionality be added as an optional, operator-configurable + * feature. + */ + int ret; + iLenDefBuf = glbl.GetMaxLine(); + CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iLenDefBuf + 1))); + ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) pszMsg+1, lenMsg-1); + DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", + ret, (long) iLenDefBuf, (int) (lenMsg-1)); + /* Now check if the uncompression worked. If not, there is not much we can do. In + * that case, we log an error message but ignore the message itself. Storing the + * compressed text is dangerous, as it contains control characters. So we do + * not do this. If someone would like to have a copy, this code here could be + * modified to do a hex-dump of the buffer in question. We do not include + * this functionality right now. + * rgerhards, 2006-12-07 + */ + if(ret != Z_OK) { + errmsg.LogError(0, NO_ERRCODE, "Uncompression of a message failed with return code %d " + "- enable debug logging if you need further information. " + "Message ignored.", ret); + FINALIZE; /* unconditional exit, nothing left to do... */ + } + free(pMsg->pszRawMsg); + pMsg->pszRawMsg = deflateBuf; + pMsg->iLenRawMsg = iLenDefBuf; + deflateBuf = NULL; /* logically "freed" - caller is now responsible */ + } +finalize_it: + if(deflateBuf != NULL) + free(deflateBuf); + +# else /* ifdef USE_NETZIP */ + + /* in this case, we still need to check if the message is compressed. If so, we must + * tell the user we can not accept it. + */ + if(len > 0 && *msg == 'z') { + errmsg.LogError(0, NO_ERRCODE, "Received a compressed message, but rsyslogd does not have compression " + "support enabled. The message will be ignored."); + ABORT_FINALIZE(RS_RET_NO_ZIP); + } + +# endif /* ifdef USE_NETZIP */ + + RETiRet; +} + + +/* sanitize a received message + * if a message gets to large during sanitization, it is truncated. This is + * as specified in the upcoming syslog RFC series. + * rgerhards, 2008-10-09 + * We check if we have a NUL character at the very end of the + * message. This seems to be a frequent problem with a number of senders. + * So I have now decided to drop these NULs. However, if they are intentional, + * that may cause us some problems, e.g. with syslog-sign. On the other hand, + * current code always has problems with intentional NULs (as it needs to escape + * them to prevent problems with the C string libraries), so that does not + * really matter. Just to be on the save side, we'll log destruction of such + * NULs in the debug log. + * rgerhards, 2007-09-14 + */ +static inline rsRetVal +sanitizeMessage(msg_t *pMsg) +{ + DEFiRet; + uchar *pszMsg; + uchar *pDst; /* destination for copy job */ + size_t lenMsg; + size_t iSrc; + size_t iDst; + size_t iMaxLine; + + assert(pMsg != NULL); + +# ifdef USE_NETZIP + CHKiRet(uncompressMessage(pMsg)); +# endif + + pszMsg = pMsg->pszRawMsg; + lenMsg = pMsg->iLenRawMsg; + + /* remove NUL character at end of message (see comment in function header) */ + if(pszMsg[lenMsg-1] == '\0') { + DBGPRINTF("dropped NUL at very end of message\n"); + lenMsg--; + } + + /* then we check if we need to drop trailing LFs, which often make + * their way into syslog messages unintentionally. In order to remain + * compatible to recent IETF developments, we allow the user to + * turn on/off this handling. rgerhards, 2007-07-23 + */ + if(bDropTrailingLF && pszMsg[lenMsg-1] == '\n') { + DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n"); + lenMsg--; + } + + /* now copy over the message and sanitize it */ + /* TODO: can we get cheaper memory alloc? {alloca()?}*/ + iMaxLine = glbl.GetMaxLine(); + CHKmalloc(pDst = malloc(sizeof(uchar) * (iMaxLine + 1))); + iSrc = iDst = 0; + while(iSrc < lenMsg && iDst < iMaxLine) { + if(pszMsg[iSrc] == '\0') { /* guard against \0 characters... */ + /* changed to the sequence (somewhat) proposed in + * draft-ietf-syslog-protocol-19. rgerhards, 2006-11-30 + */ + if(iDst + 3 < iMaxLine) { /* do we have space? */ + pDst[iDst++] = cCCEscapeChar; + pDst[iDst++] = '0'; + pDst[iDst++] = '0'; + pDst[iDst++] = '0'; + } /* if we do not have space, we simply ignore the '\0'... */ + /* log an error? Very questionable... rgerhards, 2006-11-30 */ + /* decided: we do not log an error, it won't help... rger, 2007-06-21 */ + } else if(bEscapeCCOnRcv && iscntrl((int) pszMsg[iSrc])) { + /* we are configured to escape control characters. Please note + * that this most probably break non-western character sets like + * Japanese, Korean or Chinese. rgerhards, 2007-07-17 + * Note: sysklogd logs octal values only for DEL and CCs above 127. + * For others, it logs ^n where n is the control char converted to an + * alphabet character. We like consistency and thus escape it to octal + * in all cases. If someone complains, we may change the mode. At least + * we known now what's going on. + * rgerhards, 2007-07-17 + */ + if(iDst + 3 < iMaxLine) { /* do we have space? */ + pDst[iDst++] = cCCEscapeChar; + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); + } /* again, if we do not have space, we ignore the char - see comment at '\0' */ + } else { + pDst[iDst++] = pszMsg[iSrc]; + } + ++iSrc; + } + pDst[iDst] = '\0'; /* space *is* reserved for this! */ + + /* we have a sanitized string. Let's save it now */ + free(pMsg->pszRawMsg); + if((pMsg->pszRawMsg = malloc((iDst+1) * sizeof(uchar))) == NULL) { + /* when we get no new buffer, we use what we already have ;) */ + pMsg->pszRawMsg = pDst; + } else { + /* trim buffer */ + memcpy(pMsg->pszRawMsg, pDst, iDst+1); + free(pDst); /* too big! */ + pMsg->iLenRawMsg = iDst; + } + +finalize_it: + RETiRet; +} + +/* Parse a received message. The object's rawmsg property is taken and + * parsed according to the relevant standards. This can later be + * extended to support configured parsers. + * rgerhards, 2008-10-09 + */ +rsRetVal parseMsg(msg_t *pMsg) +{ + DEFiRet; + uchar *msg; + int pri; + + CHKiRet(sanitizeMessage(pMsg)); + + /* we needed to sanitize first, because we otherwise do not have a C-string we can print... */ + DBGPRINTF("msg parser: flags %x, from '%s', msg %s\n", pMsg->msgFlags, pMsg->pszRcvFrom, pMsg->pszRawMsg); + + /* pull PRI */ + pri = DEFUPRI; + msg = pMsg->pszRawMsg; + if(*msg == '<') { + pri = 0; + while(isdigit((int) *++msg)) { + pri = 10 * pri + (*msg - '0'); + } + if(*msg == '>') + ++msg; + if(pri & ~(LOG_FACMASK|LOG_PRIMASK)) + pri = DEFUPRI; + } + pMsg->iFacility = LOG_FAC(pri); + pMsg->iSeverity = LOG_PRI(pri); + MsgSetUxTradMsg(pMsg, (char*) msg); + + if(pMsg->bParseHOSTNAME == 0) + MsgSetHOSTNAME(pMsg, (char*) pMsg->pszRcvFrom); + + /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have + * a traditional syslog message or one formatted according to syslog-protocol. + * We need to apply different parsers depending on that. We use the + * -protocol VERSION field for the detection. + */ + if(msg[0] == '1' && msg[1] == ' ') { + dbgprintf("Message has syslog-protocol format.\n"); + setProtocolVersion(pMsg, 1); + if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) { // TODO: parseRFC... should pull flags from pMsg + msgDestruct(&pMsg); + ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! + } + } else { /* we have legacy syslog */ + dbgprintf("Message has legacy syslog format.\n"); + setProtocolVersion(pMsg, 0); + if(parseLegacySyslogMsg(pMsg, pMsg->msgFlags) == 1) { + msgDestruct(&pMsg); + ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! + } + } + + /* finalize message object */ + pMsg->bIsParsed = 1; /* this message is now parsed */ + MsgPrepareEnqueue(pMsg); /* "historical" name - preparese for multi-threading */ + +finalize_it: + RETiRet; +} diff --git a/runtime/parser.h b/runtime/parser.h new file mode 100644 index 00000000..cec9c083 --- /dev/null +++ b/runtime/parser.h @@ -0,0 +1,30 @@ +/* header for parser.c + * This is not yet an object, but contains all those code necessary to + * parse syslog messages. + * + * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_PARSE_H +#define INCLUDED_PARSE_H + +extern rsRetVal parserClassInit(void); +extern rsRetVal parseMsg(msg_t*); + +#endif /* #ifndef INCLUDED_PARSE_H */ diff --git a/runtime/queue.c b/runtime/queue.c index f5f770b3..25c0bd5f 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1513,8 +1513,6 @@ queueRateLimiter(queue_t *pThis) ISOBJ_TYPE_assert(pThis, queue); - dbgoprint((obj_t*) pThis, "entering rate limiter\n"); - iDelay = 0; if(pThis->iDeqtWinToHr != 25) { /* 25 means disabled */ /* time calls are expensive, so only do them when needed */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 361bfb47..619343bd 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -252,6 +252,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_QUEUE_FULL = -2105, /**< queue is full, operation could not be completed */ RS_RET_ACCEPT_ERR = -2106, /**< error during accept() system call */ RS_RET_INVLD_TIME = -2107, /**< invalid timestamp (e.g. could not be parsed) */ + RS_RET_NO_ZIP = -2108, /**< ZIP functionality is not present */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/wti.c b/runtime/wti.c index 365b25d5..4abca090 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -299,7 +299,7 @@ wtiWorkerCancelCleanup(void *arg) pWtp = pThis->pWtp; ISOBJ_TYPE_assert(pWtp, wtp); - dbgprintf("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis)); + DBGPRINTF("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis)); /* call user supplied handler (that one e.g. requeues the element) */ pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->pUsrp); @@ -391,7 +391,7 @@ wtiWorker(wti_t *pThis) /* if we reach this point, we are still protected by the mutex */ if(pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED)) { - dbgprintf("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); + DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); pWtp->pfOnIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED); if(pWtp->toWrkShutdown == -1) { @@ -400,7 +400,7 @@ wtiWorker(wti_t *pThis) } else { timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */ if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) { - dbgprintf("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); + DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); bInactivityTOOccured = 1; /* indicate we had a timeout */ } } diff --git a/tools/syslogd.c b/tools/syslogd.c index e794e2d1..1a26333d 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -128,6 +128,7 @@ #include "vm.h" #include "errmsg.h" #include "datetime.h" +#include "parser.h" #include "sysvar.h" /* definitions for objects we access */ @@ -249,15 +250,15 @@ typedef struct legacyOptsLL_s { legacyOptsLL_t *pLegacyOptsLL = NULL; /* global variables for config file state */ -static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ +int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ int iCompatibilityMode = 0; /* version we should be compatible with; 0 means sysklogd. It is the default, so if no -c option is given, we make ourselvs as compatible to sysklogd as possible. */ static int bDebugPrintTemplateList = 1;/* output template list in debug mode? */ static int bDebugPrintCfSysLineHandlerList = 1;/* output cfsyslinehandler list in debug mode? */ static int bDebugPrintModuleList = 1;/* output module list in debug mode? */ -static uchar cCCEscapeChar = '\\';/* character to be used to start an escape sequence for control chars */ -static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ +uchar cCCEscapeChar = '\\';/* character to be used to start an escape sequence for control chars */ +int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ static int bErrMsgToStderr = 1; /* print error messages to stderr (in addition to everything else)? */ int bReduceRepeatMsgs; /* reduce repeated message - 0 - no, 1 - yes */ int bActExecWhenPrevSusp; /* execute action only when previous one was suspended? */ @@ -596,6 +597,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b int pri; msg_t *pMsg; + pMsg->bIsParsed = 1; /* this is a hack until this function can be removed TODO: do it soon (rgerhards, 2008-10-09)! */ /* Now it is time to create the message object (rgerhards) */ if(stTime == NULL) { CHKiRet(msgConstruct(&pMsg)); @@ -1190,6 +1192,9 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) assert(pMsg != NULL); + if(pMsg->bIsParsed == 0) { + parseMsg(pMsg); + } processMsg(pMsg); msgDestruct(&pMsg); @@ -1311,7 +1316,7 @@ static int parseRFCStructuredData(char **pp2parse, char *pResult) * * rger, 2005-11-24 */ -static int parseRFCSyslogMsg(msg_t *pMsg, int flags) +int parseRFCSyslogMsg(msg_t *pMsg, int flags) { char *p2parse; char *pBuf; @@ -1407,7 +1412,7 @@ static int parseRFCSyslogMsg(msg_t *pMsg, int flags) * but I thought I log it in this comment. * rgerhards, 2006-01-10 */ -static int parseLegacySyslogMsg(msg_t *pMsg, int flags) +int parseLegacySyslogMsg(msg_t *pMsg, int flags) { char *p2parse; char *pBuf; @@ -2908,6 +2913,8 @@ InitGlobalClasses(void) CHKiRet(actionClassInit()); pErrObj = "template"; CHKiRet(templateInit()); + pErrObj = "parser"; + CHKiRet(parserClassInit()); /* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */ pErrObj = "net"; @@ -3526,6 +3533,5 @@ int main(int argc, char **argv) dbgClassInit(); return realMain(argc, argv); } - /* vim:set ai: */ -- cgit v1.2.3 From 5742f9cdd0da18a3ddfb0a51a981637124a6ab25 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 07:48:22 +0200 Subject: fixing segfault caused by all inputs but imudp --- runtime/msg.c | 1 + tools/syslogd.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime/msg.c b/runtime/msg.c index c030fa45..e52c9e3f 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -264,6 +264,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) /* initialize members that are non-zero */ pM->iRefCount = 1; + pM->bIsParsed = 1; /* first we assume this is parsed. If not, input must re-set to 0 */ pM->iSeverity = -1; pM->iFacility = -1; objConstructSetObjInfo(pM); diff --git a/tools/syslogd.c b/tools/syslogd.c index 1a26333d..a45942fa 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -597,13 +597,13 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b int pri; msg_t *pMsg; - pMsg->bIsParsed = 1; /* this is a hack until this function can be removed TODO: do it soon (rgerhards, 2008-10-09)! */ /* Now it is time to create the message object (rgerhards) */ if(stTime == NULL) { CHKiRet(msgConstruct(&pMsg)); } else { CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } + pMsg->bIsParsed = 1; /* this is a hack until this function can be removed TODO: do it soon (rgerhards, 2008-10-09)! */ if(pszInputName != NULL) MsgSetInputName(pMsg, (char*) pszInputName); MsgSetFlowControlType(pMsg, flowCtlType); @@ -1192,6 +1192,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) assert(pMsg != NULL); +RUNLOG_VAR("%d", pMsg->bIsParsed); if(pMsg->bIsParsed == 0) { parseMsg(pMsg); } -- cgit v1.2.3 From 39f2afc8b8a60f09570fcf288e6899cdd42209ba Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 09:16:31 +0200 Subject: bumped interface version number to reflect change to message parsing --- configure.ac | 2 +- runtime/modules.h | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 6b6a699d..e1bc248c 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[3.21.5],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[3.21.6-Test1],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_HEADERS([config.h]) diff --git a/runtime/modules.h b/runtime/modules.h index 7d34bcf7..96016082 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -44,8 +44,11 @@ * rgerhards, 2008-03-04 * version 3 adds modInfo_t ptr to call of modInit -- rgerhards, 2008-03-10 * version 4 removes needUDPSocket OM callback -- rgerhards, 2008-03-22 + * version 5 changes the way parsing works for input modules. This is + * an important change, parseAndSubmitMessage() goes away. Other + * module types are not affected. -- rgerhards, 2008-10-09 */ -#define CURR_MOD_IF_VERSION 4 +#define CURR_MOD_IF_VERSION 5 typedef enum eModType_ { eMOD_IN, /* input module */ -- cgit v1.2.3 From 634cfdbe8ec7182b7fb74b184d269697e5fcab6c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 10:01:30 +0200 Subject: restoring msg parsing for imudp I tried to work too quick this morning. A side-effect of an earlier change was that no UDP messages were parsed, which lead to their loss, because no PRI was set in this case. --- plugins/imudp/imudp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index aab201a9..114b433c 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -207,6 +207,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, /* first trim the buffer to what we have actually received */ CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); + pMsg->bIsParsed = 0; /* indicate message needs to be parsed */ pMsg->iLenRawMsg = lenRcvBuf; MsgSetInputName(pMsg, "imudp"); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); -- cgit v1.2.3 From af3c944563b8651c14b2b9087ea76f7eeacfc065 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 11:21:24 +0200 Subject: added experimental pthread_yield() which so far seems to increase performance. There is also reason for it to do so, see http://kb.monitorware.com/post14216.html#p14216 --- runtime/queue.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runtime/queue.c b/runtime/queue.c index 25c0bd5f..3ef6d5a0 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2185,6 +2185,11 @@ finalize_it: /* and release the mutex */ d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); + /* the following pthread_yield is experimental, but brought us performance + * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 + * rgerhards, 2008-10-09 + */ + pthread_yield(); } RETiRet; -- cgit v1.2.3 From 1229143ca3d194bb0daaa5252809d59ee0c3a0bf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 12:56:28 +0200 Subject: minor: reorder to slightly reduce size of critical section --- runtime/queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/queue.c b/runtime/queue.c index 3ef6d5a0..2d1bf7e3 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2181,10 +2181,10 @@ finalize_it: if(pThis->qType != QUEUETYPE_DIRECT) { /* make sure at least one worker is running. */ queueAdviseMaxWorkers(pThis); - dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n"); /* and release the mutex */ d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); + dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n"); /* the following pthread_yield is experimental, but brought us performance * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 * rgerhards, 2008-10-09 -- cgit v1.2.3 From 6c6e9a0f3f7d454ba9553a750b195d7f99c7299a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Oct 2008 13:45:56 +0200 Subject: moved bParseHostname and bIsParsed to msgFlags This enables us to use more efficient calling conventions and also helps us keep the on-disk structure of a msg object more consistent in future releases. --- dirty.h | 13 +------------ plugins/immark/immark.c | 1 + plugins/imrelp/imrelp.c | 5 +++-- plugins/imudp/imudp.c | 4 +--- plugins/imuxsock/imuxsock.c | 4 +++- runtime/atomic.h | 2 ++ runtime/debug.h | 1 + runtime/msg.c | 1 - runtime/msg.h | 13 ++++++++++++- runtime/parser.c | 3 +-- runtime/queue.c | 5 +++-- tcps_sess.c | 11 ++++++----- tools/syslogd.c | 21 ++++++++++++--------- 13 files changed, 46 insertions(+), 38 deletions(-) diff --git a/dirty.h b/dirty.h index f5d0415e..4b13adcd 100644 --- a/dirty.h +++ b/dirty.h @@ -27,20 +27,9 @@ #ifndef DIRTY_H_INCLUDED #define DIRTY_H_INCLUDED 1 -/* Flags to logmsg(). - */ -#define NOFLAG 0x000 /* no flag is set (to be used when a flag must be specified and none is required) */ -#define INTERNAL_MSG 0x001 /* msg generated by logmsgInternal() --> special handling */ -/* NO LONGER USED: #define SYNC_FILE 0x002 / * do fsync on file after printing */ -#define IGNDATE 0x004 /* ignore, if given, date in message and use date of reception as msg date */ -#define MARK 0x008 /* this message is a mark */ - -#define MSG_PARSE_HOSTNAME 1 -#define MSG_DONT_PARSE_HOSTNAME 0 - rsRetVal submitMsg(msg_t *pMsg); rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); -rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); +rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); /* TODO: the following 2 need to go in conf obj interface... */ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); diff --git a/plugins/immark/immark.c b/plugins/immark/immark.c index 323da3fe..8504f872 100644 --- a/plugins/immark/immark.c +++ b/plugins/immark/immark.c @@ -41,6 +41,7 @@ #include "cfsysline.h" #include "module-template.h" #include "errmsg.h" +#include "msg.h" MODULE_TYPE_INPUT diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c index 4515acd7..2255e643 100644 --- a/plugins/imrelp/imrelp.c +++ b/plugins/imrelp/imrelp.c @@ -42,6 +42,7 @@ #include "cfsysline.h" #include "module-template.h" #include "net.h" +#include "msg.h" MODULE_TYPE_INPUT @@ -83,8 +84,8 @@ static relpRetVal onSyslogRcv(uchar *pHostname, uchar __attribute__((unused)) *pIP, uchar *pMsg, size_t lenMsg) { DEFiRet; - parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, MSG_PARSE_HOSTNAME, - NOFLAG, eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL, 0); + parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, PARSE_HOSTNAME, + eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL, 0); RETiRet; } diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 114b433c..a49378cf 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -207,12 +207,10 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, /* first trim the buffer to what we have actually received */ CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); - pMsg->bIsParsed = 0; /* indicate message needs to be parsed */ pMsg->iLenRawMsg = lenRcvBuf; MsgSetInputName(pMsg, "imudp"); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); - pMsg->bParseHOSTNAME = MSG_PARSE_HOSTNAME; - pMsg->msgFlags = NOFLAG; + pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; MsgSetRcvFrom(pMsg, (char*)fromHost); CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP)); CHKiRet(submitMsg(pMsg)); diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index efa0365d..1d88a2b5 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -42,6 +42,7 @@ #include "errmsg.h" #include "net.h" #include "glbl.h" +#include "msg.h" MODULE_TYPE_INPUT @@ -221,7 +222,8 @@ static rsRetVal readSocket(int fd, int iSock) if (iRcvd > 0) { parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock], (uchar*)"127.0.0.1", pRcv, - iRcvd, funixParseHost[iSock], funixFlags[iSock], funixFlowCtl[iSock], (uchar*)"imuxsock", NULL, 0); + iRcvd, funixParseHost[iSock] ? (funixFlags[iSock] | PARSE_HOSTNAME) : funixFlags[iSock], + funixFlowCtl[iSock], (uchar*)"imuxsock", NULL, 0); } else if (iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); diff --git a/runtime/atomic.h b/runtime/atomic.h index 2dbe7f52..7ad8e2e4 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -42,12 +42,14 @@ */ #ifdef HAVE_ATOMIC_BUILTINS # define ATOMIC_INC(data) ((void) __sync_fetch_and_add(&(data), 1)) +# define ATOMIC_DEC(data) ((void) __sync_sub_and_fetch(&(data), 1)) # define ATOMIC_DEC_AND_FETCH(data) __sync_sub_and_fetch(&(data), 1) # define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff)) # define ATOMIC_STORE_1_TO_32BIT(data) __sync_lock_test_and_set(&(data), 1) #else # warning "atomic builtins not available, using nul operations" # define ATOMIC_INC(data) (++(data)) +# define ATOMIC_DEC(data) (--(data)) # define ATOMIC_DEC_AND_FETCH(data) (--(data)) # define ATOMIC_FETCH_32BIT(data) (data) # define ATOMIC_STORE_1_TO_32BIT(data) (data) = 1 diff --git a/runtime/debug.h b/runtime/debug.h index 7ac29765..59b3e776 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -101,6 +101,7 @@ void dbgPrintAllDebugInfo(void); /* macros */ #define DBGPRINTF(...) if(Debug) { dbgprintf(__VA_ARGS__); } +#define DBGOPRINT(...) if(Debug) { dbgoprint(__VA_ARGS__); } #ifdef RTINST # define BEGINfunc static dbgFuncDB_t *pdbgFuncDB; int dbgCALLStaCK_POP_POINT = dbgEntrFunc(&pdbgFuncDB, __FILE__, __func__, __LINE__); # define ENDfunc dbgExitFunc(pdbgFuncDB, dbgCALLStaCK_POP_POINT, RS_RET_NO_IRET); diff --git a/runtime/msg.c b/runtime/msg.c index e52c9e3f..c030fa45 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -264,7 +264,6 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) /* initialize members that are non-zero */ pM->iRefCount = 1; - pM->bIsParsed = 1; /* first we assume this is parsed. If not, input must re-set to 0 */ pM->iSeverity = -1; pM->iFacility = -1; objConstructSetObjInfo(pM); diff --git a/runtime/msg.h b/runtime/msg.h index 98635f85..d98111a8 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -55,7 +55,6 @@ struct msg { flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ short iRefCount; /* reference counter (0 = unused) */ - short bIsParsed; /* is message parsed? (0=no, 1=yes), 0 means parser needs to be called */ short bParseHOSTNAME; /* should the hostname be parsed from the message? */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, @@ -122,6 +121,18 @@ struct msg { int msgFlags; /* flags associated with this message */ }; + +/* message flags (msgFlags), not an enum for historical reasons + */ +#define NOFLAG 0x000 /* no flag is set (to be used when a flag must be specified and none is required) */ +#define INTERNAL_MSG 0x001 /* msg generated by logmsgInternal() --> special handling */ +/* 0x002 not used because it was previously a known value - rgerhards, 2008-10-09 */ +#define IGNDATE 0x004 /* ignore, if given, date in message and use date of reception as msg date */ +#define MARK 0x008 /* this message is a mark */ +#define NEEDS_PARSING 0x010 /* raw message, must be parsed before processing can be done */ +#define PARSE_HOSTNAME 0x020 /* parse the hostname during message parsing */ + + /* function prototypes */ PROTOTYPEObjClassInit(msg); diff --git a/runtime/parser.c b/runtime/parser.c index 8c4272a0..fbdeebeb 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -41,7 +41,6 @@ /* some defines */ #define DEFUPRI (LOG_USER|LOG_NOTICE) -#warning "msg object must be updated with new property for persisting the queue!" /* definitions for objects we access */ DEFobjStaticHelpers DEFobjCurrIf(glbl) @@ -306,7 +305,7 @@ rsRetVal parseMsg(msg_t *pMsg) } /* finalize message object */ - pMsg->bIsParsed = 1; /* this message is now parsed */ + pMsg->msgFlags &= ~NEEDS_PARSING; /* this message is now parsed */ MsgPrepareEnqueue(pMsg); /* "historical" name - preparese for multi-threading */ finalize_it: diff --git a/runtime/queue.c b/runtime/queue.c index b0043ef5..42b8137d 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -49,6 +49,7 @@ #include "obj.h" #include "wtp.h" #include "wti.h" +#include "atomic.h" /* static data */ DEFobjStaticHelpers @@ -996,7 +997,7 @@ queueAdd(queue_t *pThis, void *pUsr) CHKiRet(pThis->qAdd(pThis, pUsr)); if(pThis->qType != QUEUETYPE_DIRECT) { - ++pThis->iQueueSize; + ATOMIC_INC(pThis->iQueueSize); dbgoprint((obj_t*) pThis, "entry added, size now %d entries\n", pThis->iQueueSize); } @@ -1025,7 +1026,7 @@ queueDel(queue_t *pThis, void *pUsr) iRet = queueGetUngottenObj(pThis, (obj_t**) pUsr); } else { iRet = pThis->qDel(pThis, pUsr); - --pThis->iQueueSize; + ATOMIC_DEC(pThis->iQueueSize); } dbgoprint((obj_t*) pThis, "entry deleted, state %d, size now %d entries\n", diff --git a/tcps_sess.c b/tcps_sess.c index 13644f45..e8bef5b1 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -42,6 +42,7 @@ #include "obj.h" #include "errmsg.h" #include "netstrm.h" +#include "msg.h" /* static data */ @@ -229,8 +230,8 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, MSG_PARSE_HOSTNAME, - NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->bAtStrtOfFram = 1; } @@ -313,7 +314,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->iMsg = 0; /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good @@ -324,7 +325,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(c == '\n' && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delemiter? */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } else { @@ -343,7 +344,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } diff --git a/tools/syslogd.c b/tools/syslogd.c index a45942fa..13696955 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -588,8 +588,11 @@ void untty(void) * a timestamp that is to be used as timegenerated instead of the current system time. * This is meant to facilitate performance optimization. Some inputs support such modes. * If stTime is NULL, the current system time is used. + * + * rgerhards, 2008-10-09: + * interface change: bParseHostname removed, now in flags */ -static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int bParseHost, int flags, flowControl_t flowCtlType, +static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int flags, flowControl_t flowCtlType, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; @@ -603,13 +606,11 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b } else { CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } - pMsg->bIsParsed = 1; /* this is a hack until this function can be removed TODO: do it soon (rgerhards, 2008-10-09)! */ if(pszInputName != NULL) MsgSetInputName(pMsg, (char*) pszInputName); MsgSetFlowControlType(pMsg, flowCtlType); MsgSetRawMsg(pMsg, (char*)msg); - pMsg->bParseHOSTNAME = bParseHost; /* test for special codes */ pri = DEFUPRI; p = msg; @@ -634,7 +635,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int b * the message was received from (that, for obvious reasons, * being the local host). rgerhards 2004-11-16 */ - if(bParseHost == 0) + if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) MsgSetHOSTNAME(pMsg, (char*)hname); MsgSetRcvFrom(pMsg, (char*)hname); CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP)); @@ -702,9 +703,12 @@ finalize_it: * a timestamp that is to be used as timegenerated instead of the current system time. * This is meant to facilitate performance optimization. Some inputs support such modes. * If stTime is NULL, the current system time is used. + * + * rgerhards, 2008-10-09: + * interface change: bParseHostname removed, now in flags */ rsRetVal -parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bParseHost, int flags, flowControl_t flowCtlType, +parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlType, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; @@ -816,7 +820,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa */ if(iMsg == iMaxLine) { *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ - printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime, ttGenTime); + printline(hname, hnameIP, tmpline, flags, flowCtlType, pszInputName, stTime, ttGenTime); } else { /* This case in theory never can happen. If it happens, we have * a logic error. I am checking for it, because if I would not, @@ -868,7 +872,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int bPa *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ /* typically, we should end up here! */ - printline(hname, hnameIP, tmpline, bParseHost, flags, flowCtlType, pszInputName, stTime, ttGenTime); + printline(hname, hnameIP, tmpline, flags, flowCtlType, pszInputName, stTime, ttGenTime); finalize_it: if(tmpline != NULL) @@ -1192,8 +1196,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) assert(pMsg != NULL); -RUNLOG_VAR("%d", pMsg->bIsParsed); - if(pMsg->bIsParsed == 0) { + if((pMsg->msgFlags & NEEDS_PARSING) != 0) { parseMsg(pMsg); } processMsg(pMsg); -- cgit v1.2.3 From a27e249e445deecb1d9ef57fbbb203d71bf061dd Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 21 Oct 2008 10:55:33 +0200 Subject: bugfix: (potentially big) memory leak on HUP This occured if queues could not be drained before timeout. Thanks to David Lang for pointing this out. --- ChangeLog | 2 ++ runtime/queue.c | 45 ++++++++++++++++++++++++++++++++------------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/ChangeLog b/ChangeLog index 2a3c9acf..9a02d675 100644 --- a/ChangeLog +++ b/ChangeLog @@ -25,6 +25,8 @@ version before switching to this one. and consistency - bugfix: subsecond time properties generated by imfile, imklog and internal messages could be slightly inconsistent +- bugfix: (potentially big) memory leak on HUP if queues could not be + drained before timeout - thanks to David Lang for pointing this out --------------------------------------------------------------------------- Version 3.21.5 [DEVEL] (rgerhards), 2008-09-30 - performance optimization: unnecessary time() calls during message diff --git a/runtime/queue.c b/runtime/queue.c index 42b8137d..7fa2df91 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -88,6 +88,30 @@ ENDfunc return pThis->iQueueSize + pThis->iUngottenObjs; } + +/* This function drains the queue in cases where this needs to be done. The most probable + * reason is a HUP which needs to discard data (because the queue is configured to be lossy). + * During a shutdown, this is typically not needed, as the OS frees up ressources and does + * this much quicker than when we clean up ourselvs. -- rgerhards, 2008-10-21 + * This function returns void, as it makes no sense to communicate an error back, even if + * it happens. + */ +static inline void queueDrain(queue_t *pThis) +{ + void *pUsr; + + ASSERT(pThis != NULL); + + /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ + while(pThis->iQueueSize-- > 0) { + pThis->qDel(pThis, &pUsr); + if(pUsr != NULL) { + objDestruct(pUsr); + } + } +} + + /* --------------- code for disk-assisted (DA) queue modes -------------------- */ @@ -196,14 +220,6 @@ queueTurnOffDAMode(queue_t *pThis) } } - /* TODO: we have a *really biiiiig* memory leak here: if the queue could not be persisted, all of - * its data elements are still in memory. That doesn't really matter if we are terminated, but on - * HUP this memory leaks. We MUST add a loop of destructor calls here. However, this takes time - * (possibly a lot), so it is probably best to have a config variable for that. - * Something for 3.11.1! - * rgerhards, 2008-01-30 - */ - RETiRet; } @@ -461,12 +477,15 @@ static rsRetVal qDestructFixedArray(queue_t *pThis) ASSERT(pThis != NULL); + queueDrain(pThis); /* discard any remaining queue entries */ + if(pThis->tVars.farray.pBuf != NULL) free(pThis->tVars.farray.pBuf); RETiRet; } + static rsRetVal qAddFixedArray(queue_t *pThis, void* in) { DEFiRet; @@ -570,11 +589,11 @@ static rsRetVal qConstructLinkedList(queue_t *pThis) static rsRetVal qDestructLinkedList(queue_t __attribute__((unused)) *pThis) { DEFiRet; - - /* with the linked list type, there is nothing to do here. The - * reason is that the Destructor is only called after all entries - * have bene taken off the queue. In this case, there is nothing - * dynamic left with the linked list. + + queueDrain(pThis); /* discard any remaining queue entries */ + + /* with the linked list type, there is nothing left to do here. The + * reason is that there are no dynamic elements for the list itself. */ RETiRet; -- cgit v1.2.3 From cf38fc81759b01af5125b1a05e0d6fe8e2e1bc21 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 22 Oct 2008 13:54:40 +0200 Subject: added a setting "$OptimizeForUniprocessor" ...to enable users to turn off pthread_yield calls which are counter-productive on multiprocessor machines (but have been shown to be useful on uniprocessors) --- ChangeLog | 3 +++ doc/rsyslog_conf.html | 6 ++++-- runtime/glbl.c | 5 +++++ runtime/glbl.h | 1 + runtime/queue.c | 5 ++++- runtime/queue.h | 1 + runtime/wti.c | 15 ++++++++++++++- runtime/wti.h | 1 + runtime/wtp.c | 16 +++++++++++++++- runtime/wtp.h | 1 + 10 files changed, 49 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 41aef49f..6856c3e7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -21,6 +21,9 @@ version before switching to this one. receive loop (aka receiving messsages at a high rate) - doc bugfix: queue doc had wrong parameter name for setting controlling worker thread shutdown period +- added a setting "$OptimizeForUniprocessor" to enable users to turn off + pthread_yield calls which are counter-productive on multiprocessor + machines (but have been shown to be useful on uniprocessors) --------------------------------------------------------------------------- Version 3.21.6 [DEVEL] (rgerhards), 2008-10-22 - consolidated time calls during msg object creation, improves performance diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index 2b773476..be543833 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -236,8 +236,10 @@ supported in order to be compliant to the upcoming new syslog RFC series.

  • $ModLoad
  • $RepeatedMsgReduction
  • $ResetConfigVariables
    • -
  • $WorkDirectory <name> (directory for spool -and other work files)
    • +
  • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better +performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The +default may change as uniprocessor systems become less common.
    • +
  • $WorkDirectory <name> (directory for spool and other work files)
  • $UDPServerAddress <IP> (imudp) -- local IP address (or name) the UDP listens should bind to
  • $UDPServerRun <port> (imudp) -- former diff --git a/runtime/glbl.c b/runtime/glbl.c index 1114fcd3..c752288f 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -51,6 +51,7 @@ DEFobjStaticHelpers * class... */ static uchar *pszWorkDir = NULL; +static int bOptimizeUniProc = 1; /* enable uniprocessor optimizations */ static int iMaxLine = 2048; /* maximum length of a syslog message */ static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */ static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */ @@ -85,6 +86,7 @@ static dataType Get##nameFunc(void) \ return(nameVar); \ } +SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int) SIMP_PROP(MaxLine, iMaxLine, int) SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */ SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int) @@ -173,6 +175,7 @@ CODESTARTobjQueryInterface(glbl) pIf->Get##name = Get##name; \ pIf->Set##name = Set##name; SIMP_PROP(MaxLine); + SIMP_PROP(OptimizeUniProc); SIMP_PROP(DefPFFamily); SIMP_PROP(DropMalPTRMsgs); SIMP_PROP(Option_DisallowWarning); @@ -216,6 +219,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a pszWorkDir = NULL; } bDropMalPTRMsgs = 0; + bOptimizeUniProc = 1; return RS_RET_OK; } @@ -235,6 +239,7 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercafile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCAF, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrKeyFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); ENDObjClassInit(glbl) diff --git a/runtime/glbl.h b/runtime/glbl.h index 0c83bdd5..0b871b3c 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -41,6 +41,7 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ dataType (*Get##name)(void); \ rsRetVal (*Set##name)(dataType); SIMP_PROP(MaxLine, int) + SIMP_PROP(OptimizeUniProc, int) SIMP_PROP(DefPFFamily, int) SIMP_PROP(DropMalPTRMsgs, int) SIMP_PROP(Option_DisallowWarning, int) diff --git a/runtime/queue.c b/runtime/queue.c index 7fa2df91..a3e29a96 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1285,6 +1285,7 @@ rsRetVal queueConstruct(queue_t **ppThis, queueType_t qType, int iWorkerThreads, /* we have an object, so let's fill the properties */ objConstructSetObjInfo(pThis); + pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); if((pThis->pszSpoolDir = (uchar*) strdup((char*)glbl.GetWorkDir())) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -2208,8 +2209,10 @@ finalize_it: /* the following pthread_yield is experimental, but brought us performance * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 * rgerhards, 2008-10-09 + * but this is only true for uniprocessors, so we guard it with an optimize flag -- rgerhards, 2008-10-22 */ - pthread_yield(); + if(pThis->bOptimizeUniProc) + pthread_yield(); } RETiRet; diff --git a/runtime/queue.h b/runtime/queue.h index 9e75b31b..a2dd594f 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -58,6 +58,7 @@ typedef struct qWrkThrd_s { typedef struct queue_s { BEGINobjInstance; queueType_t qType; + int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ int bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ int bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ int bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ diff --git a/runtime/wti.c b/runtime/wti.c index 4abca090..e8a77474 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -45,9 +45,11 @@ #include "wtp.h" #include "wti.h" #include "obj.h" +#include "glbl.h" /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(glbl) /* forward-definitions */ @@ -202,6 +204,7 @@ ENDobjDestruct(wti) /* Standard-Constructor for the wti object */ BEGINobjConstruct(wti) /* be sure to specify the object type also in END macro! */ + pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); pthread_cond_init(&pThis->condExitDone, NULL); pthread_mutex_init(&pThis->mut, NULL); ENDobjConstruct(wti) @@ -367,7 +370,8 @@ wtiWorker(wti_t *pThis) wtpProcessThrdChanges(pWtp); pthread_testcancel(); /* see big comment in function header */ # if !defined(__hpux) /* pthread_yield is missing there! */ - pthread_yield(); /* see big comment in function header */ + if(pThis->bOptimizeUniProc) + pthread_yield(); /* see big comment in function header */ # endif /* if we have a rate-limiter set for this worker pool, let's call it. Please @@ -466,6 +470,14 @@ finalize_it: /* dummy */ rsRetVal wtiQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } +/* exit our class + */ +BEGINObjClassExit(wti, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nsdsel_gtls) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); +ENDObjClassExit(wti) + /* Initialize the wti class. Must be called as the very first method * before anything else is called inside this class. @@ -473,6 +485,7 @@ rsRetVal wtiQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } */ BEGINObjClassInit(wti, 1, OBJ_IS_CORE_MODULE) /* one is the object version (most important for persisting) */ /* request objects we use */ + CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(wti) /* diff --git a/runtime/wti.h b/runtime/wti.h index 0cd6744e..6b60b833 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -31,6 +31,7 @@ /* the worker thread instance class */ typedef struct wti_s { BEGINobjInstance; + int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ pthread_t thrdID; /* thread ID */ qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */ obj_t *pUsrp; /* pointer to an object meaningful for current user pointer (e.g. queue pUsr data elemt) */ diff --git a/runtime/wtp.c b/runtime/wtp.c index 45034fa7..06173e04 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -46,9 +46,11 @@ #include "wtp.h" #include "wti.h" #include "obj.h" +#include "glbl.h" /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(glbl) /* forward-definitions */ @@ -75,6 +77,7 @@ static rsRetVal NotImplementedDummy() { return RS_RET_OK; } /* Standard-Constructor for the wtp object */ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! */ + pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); pthread_mutex_init(&pThis->mut, NULL); pthread_cond_init(&pThis->condThrdTrm, NULL); /* set all function pointers to "not implemented" dummy so that we can safely call them */ @@ -484,7 +487,8 @@ wtpStartWrkr(wtp_t *pThis, int bLockMutex) * hold the queue's mutex, but at least it has a chance to start on a single-CPU system. */ # if !defined(__hpux) /* pthread_yield is missing there! */ - pthread_yield(); + if(pThis->bOptimizeUniProc) + pthread_yield(); # endif /* indicate we just started a worker and would like to see it running */ @@ -617,12 +621,22 @@ finalize_it: /* dummy */ rsRetVal wtpQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } +/* exit our class + */ +BEGINObjClassExit(wtp, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(nsdsel_gtls) + /* release objects we no longer need */ + objRelease(glbl, CORE_COMPONENT); +ENDObjClassExit(wtp) + + /* Initialize the stream class. Must be called as the very first method * before anything else is called inside this class. * rgerhards, 2008-01-09 */ BEGINObjClassInit(wtp, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ + CHKiRet(objUse(glbl, CORE_COMPONENT)); ENDObjClassInit(wtp) /* diff --git a/runtime/wtp.h b/runtime/wtp.h index 13ebe536..88d88afd 100644 --- a/runtime/wtp.h +++ b/runtime/wtp.h @@ -52,6 +52,7 @@ typedef enum { /* the worker thread pool (wtp) object */ typedef struct wtp_s { BEGINobjInstance; + int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ wtpState_t wtpState; int iNumWorkerThreads;/* number of worker threads to use */ int iCurNumWrkThrd;/* current number of active worker threads */ -- cgit v1.2.3 From 6334d335d89ae5df344f833c5095e9dea2abf6fb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 23 Oct 2008 14:46:47 +0200 Subject: added configuration directive "HUPisRestart" ...which enables to configure HUP to be either a full restart or "just" a leightweight way to close open files --- ChangeLog | 12 ++++-- action.c | 33 +++++++++++++++ action.h | 1 + configure.ac | 2 +- doc/manual.html | 2 +- doc/rsyslog_conf.html | 20 +++++---- runtime/glbl.c | 5 +++ runtime/glbl.h | 1 + runtime/module-template.h | 31 +++++++++++++- runtime/modules.c | 5 +++ runtime/modules.h | 1 + tools/omfile.c | 102 +++++++++++++++++++++++++++++----------------- tools/syslogd.c | 59 ++++++++++++++++++++++----- 13 files changed, 212 insertions(+), 62 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6856c3e7..a11ce27d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 3.23.1 [DEVEL] (rgerhards), 2008-10-?? +Version 4.1.0 [DEVEL] (rgerhards), 2008-10-?? ********************************* WARNING ********************************* This version has a slightly different on-disk format for message entries. @@ -9,6 +9,13 @@ the output driver. It is recommended to drain queues with the previous version before switching to this one. ********************************* WARNING ********************************* +- greatly enhanced performance when compared to v3. +- added configuration directive "HUPisRestart" which enables to configure + HUP to be either a full restart or "just" a leightweight way to + close open files. +- added a setting "$OptimizeForUniprocessor" to enable users to turn off + pthread_yield calls which are counter-productive on multiprocessor + machines (but have been shown to be useful on uniprocessors) - reordered imudp processing. Message parsing is now done as part of main message queue worker processing (was part of the input thread) This should also improve performance, as potentially more work is @@ -21,9 +28,6 @@ version before switching to this one. receive loop (aka receiving messsages at a high rate) - doc bugfix: queue doc had wrong parameter name for setting controlling worker thread shutdown period -- added a setting "$OptimizeForUniprocessor" to enable users to turn off - pthread_yield calls which are counter-productive on multiprocessor - machines (but have been shown to be useful on uniprocessors) --------------------------------------------------------------------------- Version 3.21.6 [DEVEL] (rgerhards), 2008-10-22 - consolidated time calls during msg object creation, improves performance diff --git a/action.c b/action.c index ec8732bc..dee46f16 100644 --- a/action.c +++ b/action.c @@ -498,6 +498,39 @@ finalize_it: } #pragma GCC diagnostic warning "-Wempty-body" + +/* call the HUP handler for a given action, if such a handler is defined. The + * action mutex is locked, because the HUP handler most probably needs to modify + * some internal state information. + * rgerhards, 2008-10-22 + */ +#pragma GCC diagnostic ignored "-Wempty-body" +rsRetVal +actionCallHUPHdlr(action_t *pAction) +{ + DEFiRet; + int iCancelStateSave; + + ASSERT(pAction != NULL); + DBGPRINTF("Action %p checks HUP hdlr: %p\n", pAction, pAction->pMod->doHUP); + + if(pAction->pMod->doHUP == NULL) { + FINALIZE; /* no HUP handler, so we are done ;) */ + } + + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + d_pthread_mutex_lock(&pAction->mutActExec); + pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); + pthread_setcancelstate(iCancelStateSave, NULL); + CHKiRet(pAction->pMod->doHUP(pAction->pModData)); + pthread_cleanup_pop(1); /* unlock mutex */ + +finalize_it: + RETiRet; +} +#pragma GCC diagnostic warning "-Wempty-body" + + /* set the action message queue mode * TODO: probably move this into queue object, merge with MainMsgQueue! * rgerhards, 2008-01-28 diff --git a/action.h b/action.h index d26d15b6..8d9d5102 100644 --- a/action.h +++ b/action.h @@ -85,6 +85,7 @@ rsRetVal actionSetGlobalResumeInterval(int iNewVal); rsRetVal actionDoAction(action_t *pAction); rsRetVal actionCallAction(action_t *pAction, msg_t *pMsg); rsRetVal actionWriteToAction(action_t *pAction); +rsRetVal actionCallHUPHdlr(action_t *pAction); rsRetVal actionClassInit(void); rsRetVal addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringRequest_t *pOMSR, int bSuspended); diff --git a/configure.ac b/configure.ac index 2f17e575..981ef193 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[3.23.1],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.1.0],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_HEADERS([config.h]) diff --git a/doc/manual.html b/doc/manual.html index 884c43c0..f1b7d657 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -16,7 +16,7 @@ relay chains while at the same time being very easy to setup for the novice user. And as we know what enterprise users really need, there is also professional rsyslog support available directly from the source!

    -

    This documentation is for version 3.21.6 (devel branch) of rsyslog. +

    This documentation is for version 4.1.0 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index be543833..cd3db405 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -59,17 +59,19 @@ remember, that a modules configuration directive (and functionality) is only available if it has been loaded (using $ModLoad).

    Lines

    Lines can be continued by specifying a backslash ("\") as the last -character of the line.
    +character of the line. There is a hard-coded maximum line length of 4K. +If you need lines larger than that, you need to change compile-time +settings inside rsyslog and recompile.

    Global Directives

    All global directives need to be specified on a line by their own and must start with a dollar-sign. Here is a list in alphabetical order. Follow links for a description.

    +

    Please note that not all directives here are actually global. Some affect +only the next action. This documentation will be changed soon.

    Not all directives have an in-depth description right now. Default values for them are in bold. A more in-depth description will -appear as implementation progresses. Directives may change during that -process, we will NOT try hard to maintain backwards compatibility -(after all, v3 is still very early in development and quite -unstable...). So you have been warned ;)

    +appear as implementation progresses. +

    Be sure to read information about queues in rsyslog - many parameter settings modify queue parameters. If in doubt, use the default, it is usually well-chosen and applicable in most cases.

    @@ -175,12 +177,16 @@ default 60000 (1 minute)]
  • $GssForwardServiceName
  • $GssListenServiceName
  • $GssMode
    • +
  • $HUPisRestart [on/off] - if set to on, a HUP is a full daemon restart. This means any queued messages are discarded (depending +on queue configuration, of course) all modules are unloaded and reloaded. This mode keeps compatible with sysklogd, but is +not recommended for use with rsyslog. To do a full restart, simply stop and start the daemon. The default is "on" for +compatibility reasons. If it is set to "off", a HUP will only close open files. This is a much quicker action and usually +the only one that is needed e.g. for log rotation. It is recommended to set the setting to "off".
  • $IncludeConfig
  • MainMsgQueueCheckpointInterval <number>
  • $MainMsgQueueDequeueSlowdown <number> [number is timeout in microseconds (1000000us is 1sec!), default 0 (no delay). Simple rate-limiting!]
    • -
  • $MainMsgQueueDiscardMark <number> [default -9750]
    • +
  • $MainMsgQueueDiscardMark <number> [default 9750]
  • $MainMsgQueueDiscardSeverity <severity> [either a textual or numerical severity! default 4 (warning)]
  • $MainMsgQueueFileName <name>
    • diff --git a/runtime/glbl.c b/runtime/glbl.c index c752288f..2a6bfb11 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -52,6 +52,7 @@ DEFobjStaticHelpers */ static uchar *pszWorkDir = NULL; static int bOptimizeUniProc = 1; /* enable uniprocessor optimizations */ +static int bHUPisRestart = 1; /* should SIGHUP cause a full system restart? */ static int iMaxLine = 2048; /* maximum length of a syslog message */ static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */ static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */ @@ -87,6 +88,7 @@ static dataType Get##nameFunc(void) \ } SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int) +SIMP_PROP(HUPisRestart, bHUPisRestart, int) SIMP_PROP(MaxLine, iMaxLine, int) SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */ SIMP_PROP(DropMalPTRMsgs, bDropMalPTRMsgs, int) @@ -176,6 +178,7 @@ CODESTARTobjQueryInterface(glbl) pIf->Set##name = Set##name; SIMP_PROP(MaxLine); SIMP_PROP(OptimizeUniProc); + SIMP_PROP(HUPisRestart); SIMP_PROP(DefPFFamily); SIMP_PROP(DropMalPTRMsgs); SIMP_PROP(Option_DisallowWarning); @@ -220,6 +223,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a } bDropMalPTRMsgs = 0; bOptimizeUniProc = 1; + bHUPisRestart = 1; return RS_RET_OK; } @@ -240,6 +244,7 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdriverkeyfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrKeyFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"hupisrestart", 0, eCmdHdlrBinary, NULL, &bHUPisRestart, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); ENDObjClassInit(glbl) diff --git a/runtime/glbl.h b/runtime/glbl.h index 0b871b3c..44e41e3e 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -42,6 +42,7 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Set##name)(dataType); SIMP_PROP(MaxLine, int) SIMP_PROP(OptimizeUniProc, int) + SIMP_PROP(HUPisRestart, int) SIMP_PROP(DefPFFamily, int) SIMP_PROP(DropMalPTRMsgs, int) SIMP_PROP(Option_DisallowWarning, int) diff --git a/runtime/module-template.h b/runtime/module-template.h index eb39b587..6f7d877c 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -481,6 +481,33 @@ static rsRetVal afterRun(void)\ } -/* - * vi:set ai: +/* doHUP() + * This function is optional. Currently, it is available to output plugins + * only, but may be made available to other types of plugins in the future. + * A plugin does not need to define this entry point. If if does, it gets + * called when a non-restart type of HUP is done. A plugin should register + * this function so that it can close files, connection or other ressources + * on HUP - if it can be assume the user wanted to do this as a part of HUP + * processing. Note that the name "HUP" has historical reasons, it stems back + * to the infamous SIGHUP which was sent to restart a syslogd. We still retain + * that legacy, but may move this to a different signal. + * rgerhards, 2008-10-22 + */ +#define CODEqueryEtryPt_doHUP \ + else if(!strcmp((char*) name, "doHUP")) {\ + *pEtryPoint = doHUP;\ + } +#define BEGINdoHUP \ +static rsRetVal doHUP(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTdoHUP + +#define ENDdoHUP \ + RETiRet;\ +} + + +/* vim:set ai: */ diff --git a/runtime/modules.c b/runtime/modules.c index d5730ede..169d234b 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -347,6 +347,7 @@ static rsRetVal doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_t*), uchar *name, void *pModHdlr) { DEFiRet; + rsRetVal localRet; modInfo_t *pNew = NULL; rsRetVal (*modGetType)(eModType_t *pType); @@ -391,6 +392,10 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parseSelectorAct", &pNew->mod.om.parseSelectorAct)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"isCompatibleWithFeature", &pNew->isCompatibleWithFeature)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"tryResume", &pNew->tryResume)); + /* try load optional interfaces */ + localRet = (*pNew->modQueryEtryPt)((uchar*)"doHUP", &pNew->doHUP); + if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + ABORT_FINALIZE(localRet); break; case eMOD_LIB: break; diff --git a/runtime/modules.h b/runtime/modules.h index 96016082..372529ee 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -91,6 +91,7 @@ typedef struct modInfo_s { rsRetVal (*tryResume)(void*);/* called to see if module actin can be resumed now */ rsRetVal (*modExit)(void); /* called before termination or module unload */ rsRetVal (*modGetID)(void **); /* get its unique ID from module */ + rsRetVal (*doHUP)(void *); /* non-restart type HUP handler */ /* below: parse a configuration line - return if processed * or not. If not, must be parsed to next module. */ diff --git a/tools/omfile.c b/tools/omfile.c index 8144386f..00a82933 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -174,7 +174,7 @@ rsRetVal setDynaFileCacheSize(void __attribute__((unused)) *pVal, int iNewVal) } iDynaFileCacheSize = iNewVal; - dbgprintf("DynaFileCacheSize changed to %d.\n", iNewVal); + DBGPRINTF("DynaFileCacheSize changed to %d.\n", iNewVal); RETiRet; } @@ -244,7 +244,6 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR */ pData->f_sizeLimitCmd = (char*) pOch->cmdOnSizeLimit; -RUNLOG_VAR("%p", pszTplName); iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName); @@ -327,7 +326,7 @@ static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int b if(pCache[iEntry] == NULL) FINALIZE; - dbgprintf("Removed entry %d for file '%s' from dynaCache.\n", iEntry, + DBGPRINTF("Removed entry %d for file '%s' from dynaCache.\n", iEntry, pCache[iEntry]->pName == NULL ? "[OPEN FAILED]" : (char*)pCache[iEntry]->pName); /* if the name is NULL, this is an improperly initilized entry which * needs to be discarded. In this case, neither the file is to be closed @@ -349,9 +348,11 @@ finalize_it: } -/* This function frees the dynamic file name cache. +/* This function frees all dynamic file name cache entries and closes the + * relevant files. Part of Shutdown and HUP processing. + * rgerhards, 2008-10-23 */ -static void dynaFileFreeCache(instanceData *pData) +static inline void dynaFileFreeCacheEntries(instanceData *pData) { register int i; ASSERT(pData != NULL); @@ -360,17 +361,36 @@ static void dynaFileFreeCache(instanceData *pData) for(i = 0 ; i < pData->iCurrCacheSize ; ++i) { dynaFileDelCacheEntry(pData->dynCache, i, 1); } + ENDfunc; +} + + +/* This function frees the dynamic file name cache. + */ +static void dynaFileFreeCache(instanceData *pData) +{ + ASSERT(pData != NULL); + BEGINfunc; + dynaFileFreeCacheEntries(pData); if(pData->dynCache != NULL) d_free(pData->dynCache); ENDfunc; } -/* This is a shared code for both static and dynamic files. +/* This is now shared code for all types of files. It simply prepares + * file access, which, among others, means the the file wil be opened + * and any directories in between will be created (based on config, of + * course). -- rgerhards, 2008-10-22 */ static void prepareFile(instanceData *pData, uchar *newFileName) { + if(pData->fileType == eTypePIPE) { + pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK); + FINALIZE; /* we are done in this case */ + } + if(access((char*)newFileName, F_OK) == 0) { /* file already exists */ pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, @@ -392,8 +412,7 @@ static void prepareFile(instanceData *pData, uchar *newFileName) /* check and set uid/gid */ if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { /* we need to set owner/group */ - if(fchown(pData->fd, pData->fileUID, - pData->fileGID) != 0) { + if(fchown(pData->fd, pData->fileUID, pData->fileGID) != 0) { if(pData->bFailOnChown) { int eSave = errno; close(pData->fd); @@ -409,6 +428,12 @@ static void prepareFile(instanceData *pData, uchar *newFileName) } } } +finalize_it: + if((pData->fd) != 0 && isatty(pData->fd)) { + DBGPRINTF("file %d is a tty file\n", pData->fd); + pData->fileType = eTypeTTY; + untty(); + } } @@ -482,7 +507,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg /* we need to allocate memory for the cache structure */ pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry)); if(pCache[iFirstFree] == NULL) { - dbgprintf("prepareDynfile(): could not alloc mem, discarding this request\n"); + DBGPRINTF("prepareDynfile(): could not alloc mem, discarding this request\n"); return -1; } } @@ -496,10 +521,11 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg * message. Otherwise, we could run into a never-ending loop. The bad * news is that we also lose errors on startup messages, but so it is. */ - if(iMsgOpts & INTERNAL_MSG) - dbgprintf("Could not open dynaFile, discarding message\n"); - else + if(iMsgOpts & INTERNAL_MSG) { + DBGPRINTF("Could not open dynaFile, discarding message\n"); + } else { errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", (char*)newFileName); + } dynaFileDelCacheEntry(pCache, iFirstFree, 1); pData->iCurrElt = -1; return -1; @@ -509,8 +535,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg pCache[iFirstFree]->pName = (uchar*)strdup((char*)newFileName); /* TODO: check for NULL (very unlikely) */ pCache[iFirstFree]->lastUsed = time(NULL); pData->iCurrElt = iFirstFree; - dbgprintf("Added new entry %d for file cache, file '%s'.\n", - iFirstFree, newFileName); + DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); return 0; } @@ -533,6 +558,8 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa if(pData->bDynamicName) { if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) ABORT_FINALIZE(RS_RET_ERR); + } else if(pData->fd == -1) { + prepareFile(pData, pData->f_fname); } /* create the message based on format specified */ @@ -584,11 +611,10 @@ again: ABORT_FINALIZE(RS_RET_OK); (void) close(pData->fd); - /* - * Check for EBADF on TTY's due to vhangup() + /* Check for EBADF on TTY's due to vhangup() * Linux uses EIO instead (mrn 12 May 96) */ - if ((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE) + if((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE) #ifdef linux && e == EIO) { #else @@ -637,13 +663,8 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - dbgprintf(" (%s)\n", pData->f_fname); - /* pData->fd == -1 is an indicator that the we couldn't - * open the file at startup. For dynaFiles, this is ok, - * all others are doomed. - */ - if(pData->bDynamicName || (pData->fd != -1)) - iRet = writeFile(ppString, iMsgOpts, pData); + DBGPRINTF(" (%s)\n", pData->f_fname); + iRet = writeFile(ppString, iMsgOpts, pData); ENDdoAction @@ -726,7 +747,7 @@ CODESTARTparseSelectorAct if((pData->dynCache = (dynaFileCacheEntry**) calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))) == NULL) { iRet = RS_RET_OUT_OF_MEMORY; - dbgprintf("Could not allocate memory for dynaFileCache - selector disabled.\n"); + DBGPRINTF("Could not allocate memory for dynaFileCache - selector disabled.\n"); } break; @@ -760,23 +781,15 @@ CODESTARTparseSelectorAct pData->dirUID = dirUID; pData->dirGID = dirGID; - if(pData->fileType == eTypePIPE) { - pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK); - } else { - prepareFile(pData, pData->f_fname); - } + prepareFile(pData, pData->f_fname); - if ( pData->fd < 0 ){ + if(pData->fd < 0 ) { pData->fd = -1; - dbgprintf("Error opening log file: %s\n", pData->f_fname); + DBGPRINTF("Error opening log file: %s\n", pData->f_fname); errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); break; } - if (isatty(pData->fd)) { - pData->fileType = eTypeTTY; - untty(); - } - if (strcmp((char*) p, _PATH_CONSOLE) == 0) + if(strcmp((char*) p, _PATH_CONSOLE) == 0) pData->fileType = eTypeCONSOLE; break; default: @@ -811,6 +824,20 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a } +BEGINdoHUP +CODESTARTdoHUP + if(pData->bDynamicName) { + dynaFileFreeCacheEntries(pData); + pData->iCurrElt = -1; /* invalidate current element */ + } else { + if(pData->fd != -1) { + close(pData->fd); + pData->fd = -1; + } + } +ENDdoHUP + + BEGINmodExit CODESTARTmodExit if(pszTplName != NULL) @@ -821,6 +848,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_doHUP ENDqueryEtryPt diff --git a/tools/syslogd.c b/tools/syslogd.c index 13696955..7145779d 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -220,7 +220,7 @@ static char *PidFile = _PATH_LOGPID; /* read-only after startup */ static pid_t myPid; /* our pid for use in self-generated messages, e.g. on startup */ /* mypid is read-only after the initial fork() */ -static int restart = 0; /* do restart (config read) - multithread safe */ +static int bHadHUP = 0; /* did we have a HUP? */ static int bParseHOSTNAMEandTAG = 1; /* global config var: should the hostname and tag be * parsed inside message - rgerhards, 2006-03-13 */ @@ -2543,20 +2543,18 @@ static rsRetVal setMainMsgQueType(void __attribute__((unused)) *pVal, uchar *psz * The following function is resposible for handling a SIGHUP signal. Since * we are now doing mallocs/free as part of init we had better not being * doing this during a signal handler. Instead this function simply sets - * a flag variable which will tell the main loop to go through a restart. + * a flag variable which will tells the main loop to do "the right thing". */ void sighup_handler() { struct sigaction sigAct; - restart = 1; + bHadHUP = 1; memset(&sigAct, 0, sizeof (sigAct)); sigemptyset(&sigAct.sa_mask); sigAct.sa_handler = sighup_handler; sigaction(SIGHUP, &sigAct, NULL); - - return; } @@ -2578,6 +2576,49 @@ static void processImInternal(void) } +/* helper to doHUP(), this "HUPs" each action. The necessary locking + * is done inside the action class and nothing we need to take care of. + * rgerhards, 2008-10-22 + */ +DEFFUNC_llExecFunc(doHUPActions) +{ + BEGINfunc + actionCallHUPHdlr((action_t*) pData); + ENDfunc + return RS_RET_OK; /* we ignore errors, we can not do anything either way */ +} + + +/* This function processes a HUP after one has been detected. Note that this + * is *NOT* the sighup handler. The signal is recorded by the handler, that record + * detected inside the mainloop and then this function is called to do the + * real work. -- rgerhards, 2008-10-22 + */ +static inline void +doHUP(void) +{ + selector_t *f; + char buf[512]; + + snprintf(buf, sizeof(buf) / sizeof(char), + " [origin software=\"rsyslogd\" " "swVersion=\"" VERSION + "\" x-pid=\"%d\" x-info=\"http://www.rsyslog.com\"] rsyslogd was HUPed, type '%s'.", + (int) myPid, glbl.GetHUPisRestart() ? "restart" : "lightweight"); + errno = 0; + logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*)buf, 0); + + if(glbl.GetHUPisRestart()) { + DBGPRINTF("Received SIGHUP, configured to be restart, reloading rsyslogd.\n"); + init(); /* main queue is stopped as part of init() */ + } else { + DBGPRINTF("Received SIGHUP, configured to be a non-restart type of HUP - notifying actions.\n"); + for(f = Files; f != NULL ; f = f->f_next) { + llExecFunc(&f->llActList, doHUPActions, NULL); + } + } +} + + /* This is the main processing loop. It is called after successful initialization. * When it returns, the syslogd terminates. * Its sole function is to provide some housekeeping things. The real work is done @@ -2634,11 +2675,9 @@ mainloop(void) if(bReduceRepeatMsgs == 1) doFlushRptdMsgs(); - if(restart) { - dbgprintf("\nReceived SIGHUP, reloading rsyslogd.\n"); - /* main queue is stopped as part of init() */ - init(); - restart = 0; + if(bHadHUP) { + doHUP(); + bHadHUP = 0; continue; } } -- cgit v1.2.3 From 4f5ab3342c262309ffaed769a5aa9584b7c0e6a3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 28 Oct 2008 14:42:06 +0100 Subject: updated support options --- doc/professional_support.html | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/professional_support.html b/doc/professional_support.html index 2cb6c1e1..7724ede8 100644 --- a/doc/professional_support.html +++ b/doc/professional_support.html @@ -49,6 +49,30 @@ configuration file, but will place easy to read comments in the places where you need to put them in. The agreement is governed under German law. You may also purchase this service if you would like to have your own configuration file reviewed, e.g. for auditing purposes.

    +

    Local Installation Support

    +

    If you intend to install rsyslog on your system but would like +to do so with minimal effort and according to your specification, you +can ask us to perform the installation for you. You get a perfect +installation, exactly like you needed, but without a need to +touch the system. This is a perfect choice for the busy administrator! +

    In order to perform this work, we just need ssh access to your +system and the proper permissions. +

    We charge a low one-time fee for this service. For details, please +contact info@adiscon.com. +

    Local Installation Maintenance

    +

    If you used our services to set up the system, why not keep it +running perfectly with maintenance support? Under this contract, we +assure you run a recent build that does not interfere with your +environment and we even carry out change requests you may have. So this +is a hassle-free, everything cared about solution. +

    Again, all we need to have is ssh access and the proper permissions +to your machine. Of course, work will only be carried out when you +expect us to do so. You are always in control of what happens. This +is a perfect outsourcing solution for those who would like to run +a great logging system but can not afford the time to keep it +in perfect shape! +

    We charge a low monthly fee for this service. For details, please +contact info@adiscon.com.

    Custom Development

    Do you need an exotic feature that otherwise would not be implemented? Do you need something really quick, quicker than it is available via @@ -85,4 +109,4 @@ Copyright  Gerhards and Adiscon. Released under the GNU GPL version 3 or higher.

    - \ No newline at end of file + -- cgit v1.2.3 From 47bbd838bdaa200d89d8210a6a0ba9c322129bca Mon Sep 17 00:00:00 2001 From: Tom Bergfeld Date: Fri, 31 Oct 2008 14:47:30 +0100 Subject: restructured rsyslog.conf documentation Signed-off-by: Rainer Gerhards --- ChangeLog | 1 + doc/expression.html | 3 +- doc/features.html | 13 +- doc/im3195.html | 2 + doc/imfile.html | 2 + doc/imgssapi.html | 2 + doc/imklog.html | 2 + doc/imrelp.html | 2 + doc/imtcp.html | 2 + doc/imuxsock.html | 2 + doc/log_rotation_fix_size.html | 10 + doc/netstream.html | 4 +- doc/omlibdbi.html | 2 + doc/ommail.html | 3 +- doc/ommysql.html | 2 + doc/omrelp.html | 4 +- doc/omsnmp.html | 1 + doc/property_replacer.html | 10 + doc/queues.html | 9 + ...onf1_actionexeconlywhenpreviousissuspended.html | 2 + doc/rsconf1_actionresumeinterval.html | 4 +- doc/rsconf1_allowedsender.html | 2 + doc/rsconf1_controlcharacterescapeprefix.html | 2 + doc/rsconf1_debugprintcfsyslinehandlerlist.html | 4 +- doc/rsconf1_debugprintmodulelist.html | 3 +- doc/rsconf1_debugprinttemplatelist.html | 4 +- doc/rsconf1_dircreatemode.html | 4 +- doc/rsconf1_dirgroup.html | 4 +- doc/rsconf1_dirowner.html | 4 +- ...rsconf1_dropmsgswithmaliciousdnsptrrecords.html | 4 +- doc/rsconf1_droptrailinglfonreception.html | 2 + doc/rsconf1_dynafilecachesize.html | 4 +- doc/rsconf1_escapecontrolcharactersonreceive.html | 2 + doc/rsconf1_failonchownfailure.html | 4 +- doc/rsconf1_filecreatemode.html | 2 + doc/rsconf1_filegroup.html | 4 +- doc/rsconf1_fileowner.html | 4 +- doc/rsconf1_gssforwardservicename.html | 2 + doc/rsconf1_gsslistenservicename.html | 2 + doc/rsconf1_gssmode.html | 2 + doc/rsconf1_includeconfig.html | 4 +- doc/rsconf1_mainmsgqueuesize.html | 2 + doc/rsconf1_markmessageperiod.html | 4 +- doc/rsconf1_moddir.html | 4 +- doc/rsconf1_modload.html | 2 + doc/rsconf1_repeatedmsgreduction.html | 4 +- doc/rsconf1_resetconfigvariables.html | 4 +- doc/rsconf1_umask.html | 4 +- doc/rsyslog_conf.html | 1235 +------------------- doc/rsyslog_high_database_rate.html | 9 + doc/rsyslog_mysql.html | 11 +- doc/rsyslog_ng_comparison.html | 10 + doc/rsyslog_stunnel.html | 11 +- doc/rsyslog_tls.html | 10 + doc/syslog_protocol.html | 9 + 55 files changed, 225 insertions(+), 1239 deletions(-) diff --git a/ChangeLog b/ChangeLog index a11ce27d..0bf28fc4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -28,6 +28,7 @@ version before switching to this one. receive loop (aka receiving messsages at a high rate) - doc bugfix: queue doc had wrong parameter name for setting controlling worker thread shutdown period +- restructured rsyslog.conf documentation --------------------------------------------------------------------------- Version 3.21.6 [DEVEL] (rgerhards), 2008-10-22 - consolidated time calls during msg object creation, improves performance diff --git a/doc/expression.html b/doc/expression.html index e7eb7842..9e37cb7a 100644 --- a/doc/expression.html +++ b/doc/expression.html @@ -2,6 +2,7 @@ Expressions +back

    Expressions

    Rsyslog supports expressions at a growing number of places. So far, they are supported for filtering messages.

    Expression support is provided by RainerScript. For now, please see the formal expression definition in RainerScript ABNF. It is the "expr" node.

    C-like comments (/* some comment */) are supported inside the expression, but not yet in the rest of the configuration file.

    [rsyslog.conf overview] @@ -13,4 +14,4 @@ Copyright Gerhards and Adiscon. Released under the GNU GPL version 3 or higher.

    - \ No newline at end of file + diff --git a/doc/features.html b/doc/features.html index d221eb77..89796a41 100644 --- a/doc/features.html +++ b/doc/features.html @@ -1,8 +1,8 @@ rsyslog features - +back

    RSyslog - Features

    This page lists both current features as well as those being considered for future versions of rsyslog. If you @@ -134,4 +134,15 @@ future of RFC 3195 in rsyslog.

    To see when each feature was added, see the rsyslog change log (online only).

    + +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + + diff --git a/doc/im3195.html b/doc/im3195.html index d6f2f2ed..aad9f3d1 100644 --- a/doc/im3195.html +++ b/doc/im3195.html @@ -4,6 +4,8 @@ +back +

    RFC3195 Input Module

    Module Name:    im3195

    Author: Rainer Gerhards diff --git a/doc/imfile.html b/doc/imfile.html index 5bdbce5c..af0413dd 100644 --- a/doc/imfile.html +++ b/doc/imfile.html @@ -2,6 +2,8 @@ Text File Input Monitor +back +

    Text File Input Module

    Module Name:    imfile

    Author: Rainer Gerhards diff --git a/doc/imgssapi.html b/doc/imgssapi.html index d644303e..ec183fe7 100644 --- a/doc/imgssapi.html +++ b/doc/imgssapi.html @@ -4,6 +4,8 @@ +back +

    GSSAPI Syslog Input Module

    Module Name:    imgssapi

    Author: varmojfekoj

    diff --git a/doc/imklog.html b/doc/imklog.html index b5b21e84..9166bae6 100644 --- a/doc/imklog.html +++ b/doc/imklog.html @@ -4,6 +4,8 @@ +back +

    Kernel Log Input Module

    Module Name:    imklog

    Author: Rainer Gerhards diff --git a/doc/imrelp.html b/doc/imrelp.html index bfdaad84..53826ac2 100644 --- a/doc/imrelp.html +++ b/doc/imrelp.html @@ -4,6 +4,8 @@ +back +

    RELP Input Module

    Module Name:    imrelp

    Author: Rainer Gerhards

    diff --git a/doc/imtcp.html b/doc/imtcp.html index ecc72748..583cd531 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -2,6 +2,8 @@ TCP Syslog Input Module +back +

    TCP Syslog Input Module

    Module Name:    imtcp

    Author: Rainer Gerhards diff --git a/doc/imuxsock.html b/doc/imuxsock.html index 77491992..472470a0 100644 --- a/doc/imuxsock.html +++ b/doc/imuxsock.html @@ -4,6 +4,8 @@ Unix Socket Input +back +

    Unix Socket Input

    Module Name:    imuxsock

    Author: Rainer Gerhards diff --git a/doc/log_rotation_fix_size.html b/doc/log_rotation_fix_size.html index 0b9a3b2e..190b24cb 100644 --- a/doc/log_rotation_fix_size.html +++ b/doc/log_rotation_fix_size.html @@ -3,6 +3,8 @@ +back +

    Log rotation with rsyslog

    Written by Michael Meckelein

    @@ -54,6 +56,14 @@ file and fill it up with new logs. So the latest logs are always in log_roatatio

    With this approach two files for logging are used, each with a maximum size of 50 MB. So we can say we have successfully configured a log rotation which satisfies our requirement. We keep the logs at a fixed-size level of100 MB.

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    diff --git a/doc/netstream.html b/doc/netstream.html index e7d54c12..cbfa12ae 100644 --- a/doc/netstream.html +++ b/doc/netstream.html @@ -3,6 +3,8 @@ +back +

    Network Stream Drivers

    Network stream drivers are a layer between various parts of rsyslogd (e.g. the imtcp module) and the transport layer. They provide sequenced delivery, authentication and @@ -18,4 +20,4 @@ Copyright Gerhards and Adiscon. Released under the GNU GPL version 3 or higher.

    - \ No newline at end of file + diff --git a/doc/omlibdbi.html b/doc/omlibdbi.html index 8ff74371..ec1d01b6 100644 --- a/doc/omlibdbi.html +++ b/doc/omlibdbi.html @@ -4,6 +4,8 @@ +back +

    Generic Database Output Module (omlibdbi)

    Module Name:    omlibdbi

    Author: Rainer Gerhards diff --git a/doc/ommail.html b/doc/ommail.html index c18cf3f8..0841dc9f 100644 --- a/doc/ommail.html +++ b/doc/ommail.html @@ -1,6 +1,6 @@ mail output module - sending syslog messages via mail - +back

    Mail Output Module (ommail)

    @@ -142,4 +142,5 @@ Copyright © 2008 by Rainer Gerhards and Adiscon. Released under the GNU GPL version 3 or higher.

    + diff --git a/doc/ommysql.html b/doc/ommysql.html index 79d913eb..7a3f5930 100644 --- a/doc/ommysql.html +++ b/doc/ommysql.html @@ -5,6 +5,8 @@ +back +

    MySQL Database Output Module

    Module Name:    ommysql

    Author: Michael Meckelein (Initial Author) / Rainer Gerhards diff --git a/doc/omrelp.html b/doc/omrelp.html index 0952cc71..82a62afc 100644 --- a/doc/omrelp.html +++ b/doc/omrelp.html @@ -4,6 +4,8 @@ +back +

    RELP Output Module (omlibdbi)

    Module Name:    omrelp

    Author: Rainer Gerhards @@ -51,4 +53,4 @@ Copyright Gerhards and Adiscon. Released under the GNU GPL version 3 or higher.

    - \ No newline at end of file + diff --git a/doc/omsnmp.html b/doc/omsnmp.html index 31aaef24..b38a594f 100644 --- a/doc/omsnmp.html +++ b/doc/omsnmp.html @@ -4,6 +4,7 @@ SNMP Output Module +back

    SNMP Output Module

    Module Name:    omsnmp

    diff --git a/doc/property_replacer.html b/doc/property_replacer.html index f666fb76..34e2116c 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -1,6 +1,7 @@ The Rsyslogd Property Replacer +back

    The Property Replacer

    The property replacer is a core component in rsyslogd's output system. A syslog message has a number of @@ -398,4 +399,13 @@ to record severity and facility of a message)

  • Configuration file syntax, this is where you actually use the property replacer.
    • +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + diff --git a/doc/queues.html b/doc/queues.html index 7461121b..727bc26a 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -3,6 +3,7 @@ Understanding rsyslog queues +back

    Understanding rsyslog Queues

    Rsyslog uses queues whenever two activities need to be loosely coupled. With a @@ -357,5 +358,13 @@ environment for the next action).

    parameters, because not all are applicable. For example, in current output module design, actions do not support multi-threading. Consequently, the number of worker threads is fixed to one for action queues and can not be changed.

    +[manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    diff --git a/doc/rsconf1_actionexeconlywhenpreviousissuspended.html b/doc/rsconf1_actionexeconlywhenpreviousissuspended.html index d5cf8b14..1626b4ca 100644 --- a/doc/rsconf1_actionexeconlywhenpreviousissuspended.html +++ b/doc/rsconf1_actionexeconlywhenpreviousissuspended.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $ActionExecOnlyWhenPreviousIsSuspended

    Type: global configuration directive

    Default: off

    diff --git a/doc/rsconf1_actionresumeinterval.html b/doc/rsconf1_actionresumeinterval.html index a854a212..c0365470 100644 --- a/doc/rsconf1_actionresumeinterval.html +++ b/doc/rsconf1_actionresumeinterval.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $ActionResumeInterval

    Type: global configuration directive

    Default: 30

    @@ -27,4 +29,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_allowedsender.html b/doc/rsconf1_allowedsender.html index 4a980b89..ac39e268 100644 --- a/doc/rsconf1_allowedsender.html +++ b/doc/rsconf1_allowedsender.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $AllowedSender

    Type: global configuration directive

    Default: all allowed

    diff --git a/doc/rsconf1_controlcharacterescapeprefix.html b/doc/rsconf1_controlcharacterescapeprefix.html index 6dab1e2e..45cd9230 100644 --- a/doc/rsconf1_controlcharacterescapeprefix.html +++ b/doc/rsconf1_controlcharacterescapeprefix.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $ControlCharacterEscapePrefix

    Type: global configuration directive

    Default: \

    diff --git a/doc/rsconf1_debugprintcfsyslinehandlerlist.html b/doc/rsconf1_debugprintcfsyslinehandlerlist.html index 1aad7552..e158de43 100644 --- a/doc/rsconf1_debugprintcfsyslinehandlerlist.html +++ b/doc/rsconf1_debugprintcfsyslinehandlerlist.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DebugPrintCFSyslineHandlerList

    Type: global configuration directive

    Default: on

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_debugprintmodulelist.html b/doc/rsconf1_debugprintmodulelist.html index 4d8e9bff..f25663fb 100644 --- a/doc/rsconf1_debugprintmodulelist.html +++ b/doc/rsconf1_debugprintmodulelist.html @@ -3,6 +3,7 @@ rsyslog.conf file +back

    $DebugPrintModuleList

    Type: global configuration directive

    Default: on

    @@ -19,4 +20,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_debugprinttemplatelist.html b/doc/rsconf1_debugprinttemplatelist.html index 243530e1..b5f1f28f 100644 --- a/doc/rsconf1_debugprinttemplatelist.html +++ b/doc/rsconf1_debugprinttemplatelist.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DebugPrintTemplateList

    Type: global configuration directive

    Default: on

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_dircreatemode.html b/doc/rsconf1_dircreatemode.html index 66a35e18..9a9c61eb 100644 --- a/doc/rsconf1_dircreatemode.html +++ b/doc/rsconf1_dircreatemode.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DirCreateMode

    Type: global configuration directive

    Default: 0644

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_dirgroup.html b/doc/rsconf1_dirgroup.html index 868e5ecd..de070126 100644 --- a/doc/rsconf1_dirgroup.html +++ b/doc/rsconf1_dirgroup.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DirGroup

    Type: global configuration directive

    Default:

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_dirowner.html b/doc/rsconf1_dirowner.html index e85a5122..da8e252d 100644 --- a/doc/rsconf1_dirowner.html +++ b/doc/rsconf1_dirowner.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DirOwner

    Type: global configuration directive

    Default:

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html b/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html index e0a53ae6..95027a70 100644 --- a/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html +++ b/doc/rsconf1_dropmsgswithmaliciousdnsptrrecords.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DropMsgsWithMaliciousDnsPTRRecords

    Type: global configuration directive

    Default: off

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_droptrailinglfonreception.html b/doc/rsconf1_droptrailinglfonreception.html index 1e3aa8af..fb59b871 100644 --- a/doc/rsconf1_droptrailinglfonreception.html +++ b/doc/rsconf1_droptrailinglfonreception.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DropTrailingLFOnReception

    Type: global configuration directive

    Default: on

    diff --git a/doc/rsconf1_dynafilecachesize.html b/doc/rsconf1_dynafilecachesize.html index 3813f981..cacbf6e5 100644 --- a/doc/rsconf1_dynafilecachesize.html +++ b/doc/rsconf1_dynafilecachesize.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $DynaFileCacheSize

    Type: global configuration directive

    Default: 10

    @@ -20,4 +22,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_escapecontrolcharactersonreceive.html b/doc/rsconf1_escapecontrolcharactersonreceive.html index 26917736..178f9a6f 100644 --- a/doc/rsconf1_escapecontrolcharactersonreceive.html +++ b/doc/rsconf1_escapecontrolcharactersonreceive.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $EscapeControlCharactersOnReceive

    Type: global configuration directive

    Default: on

    diff --git a/doc/rsconf1_failonchownfailure.html b/doc/rsconf1_failonchownfailure.html index 0e646e36..d8bbab82 100644 --- a/doc/rsconf1_failonchownfailure.html +++ b/doc/rsconf1_failonchownfailure.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $FailOnChownFailure

    Type: global configuration directive

    Default: on

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_filecreatemode.html b/doc/rsconf1_filecreatemode.html index c8440864..10b0317b 100644 --- a/doc/rsconf1_filecreatemode.html +++ b/doc/rsconf1_filecreatemode.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $FileCreateMode

    Type: global configuration directive

    Default: 0644

    diff --git a/doc/rsconf1_filegroup.html b/doc/rsconf1_filegroup.html index b9acaab7..dd5b8ad5 100644 --- a/doc/rsconf1_filegroup.html +++ b/doc/rsconf1_filegroup.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $FileGroup

    Type: global configuration directive

    Default:

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_fileowner.html b/doc/rsconf1_fileowner.html index 7a9cbbc7..935cfffd 100644 --- a/doc/rsconf1_fileowner.html +++ b/doc/rsconf1_fileowner.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $FileOwner

    Type: global configuration directive

    Default:

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_gssforwardservicename.html b/doc/rsconf1_gssforwardservicename.html index 9d39dc2a..45d9ba98 100644 --- a/doc/rsconf1_gssforwardservicename.html +++ b/doc/rsconf1_gssforwardservicename.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $GssForwardServiceName

    Type: global configuration directive

    Default: host

    diff --git a/doc/rsconf1_gsslistenservicename.html b/doc/rsconf1_gsslistenservicename.html index cd03dc58..5fdf3edc 100644 --- a/doc/rsconf1_gsslistenservicename.html +++ b/doc/rsconf1_gsslistenservicename.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $GssListenServiceName

    Type: global configuration directive

    Default: host

    diff --git a/doc/rsconf1_gssmode.html b/doc/rsconf1_gssmode.html index 71c50696..2b1d5656 100644 --- a/doc/rsconf1_gssmode.html +++ b/doc/rsconf1_gssmode.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $GssMode

    Type: global configuration directive

    Default: encryption

    diff --git a/doc/rsconf1_includeconfig.html b/doc/rsconf1_includeconfig.html index 24462f77..132cee6f 100644 --- a/doc/rsconf1_includeconfig.html +++ b/doc/rsconf1_includeconfig.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $IncludeConfig

    Type: global configuration directive

    Default:

    @@ -43,4 +45,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_mainmsgqueuesize.html b/doc/rsconf1_mainmsgqueuesize.html index acf88e94..ffed1c09 100644 --- a/doc/rsconf1_mainmsgqueuesize.html +++ b/doc/rsconf1_mainmsgqueuesize.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $MainMsgQueueSize

    Type: global configuration directive

    Default: 10000

    diff --git a/doc/rsconf1_markmessageperiod.html b/doc/rsconf1_markmessageperiod.html index 9b6590cd..2c833339 100644 --- a/doc/rsconf1_markmessageperiod.html +++ b/doc/rsconf1_markmessageperiod.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $MarkMessagePeriod

    Type: specific to immark input module

    Default: 1800 (20 minutes)

    @@ -27,4 +29,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_moddir.html b/doc/rsconf1_moddir.html index ced07dc9..889de05d 100644 --- a/doc/rsconf1_moddir.html +++ b/doc/rsconf1_moddir.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $ModDir

    Type: global configuration directive

    Default: system default for user libraries, e.g. @@ -24,4 +26,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_modload.html b/doc/rsconf1_modload.html index a2b8087a..ce457ea5 100644 --- a/doc/rsconf1_modload.html +++ b/doc/rsconf1_modload.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $ModLoad

    Type: global configuration directive

    Default:

    diff --git a/doc/rsconf1_repeatedmsgreduction.html b/doc/rsconf1_repeatedmsgreduction.html index 20e56f89..248e8343 100644 --- a/doc/rsconf1_repeatedmsgreduction.html +++ b/doc/rsconf1_repeatedmsgreduction.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $RepeatedMsgReduction

    Type: global configuration directive

    Default: depending on -e

    @@ -20,4 +22,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_resetconfigvariables.html b/doc/rsconf1_resetconfigvariables.html index 9794d158..46cf0bdf 100644 --- a/doc/rsconf1_resetconfigvariables.html +++ b/doc/rsconf1_resetconfigvariables.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $ResetConfigVariables

    Type: global configuration directive

    Default:

    @@ -19,4 +21,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsconf1_umask.html b/doc/rsconf1_umask.html index ee47dbad..8e41e672 100644 --- a/doc/rsconf1_umask.html +++ b/doc/rsconf1_umask.html @@ -3,6 +3,8 @@ rsyslog.conf file +back +

    $UMASK

    Type: global configuration directive

    Default:

    @@ -21,4 +23,4 @@ Copyright © 2007 by Rainer Gerhard Adiscon. Released under the GNU GPL version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index cd3db405..852d95b5 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -20,257 +20,13 @@ possible. While, for obvious reasons, enhanced features require a different config file syntax, rsyslogd should be able to work with a standard syslog.conf file. This is especially useful while you are migrating from syslogd to rsyslogd.

    -

    Modules

    -

    Rsyslog has a modular design. Consequently, there is a growing -number of modules. Here is the entry point to their documentation and -what they do (list is currently not complete)

    -
      -
    • omsnmp - SNMP -trap output module
      • -
    • omrelp - RELP -output module
      • -
    • omgssapi - output module for GSS-enabled syslog
      • -
    • ommysql - output module for MySQL
      • -
    • ompgsql - output module for PostgreSQL
      • -
    • omlibdbi - -generic database output module (Firebird/Interbase, MS SQL, Sybase, -SQLLite, Ingres, Oracle, mSQL)
      • -
    • ommail - -permits rsyslog to alert folks by mail if something important happens
      • -
    • imfile --  input module for text files
      • -
    • imrelp - RELP -input module
      • -
    • imudp - udp syslog message input
      • -
    • imtcp - input -plugin for plain tcp syslog
      • -
    • imgssapi - -input plugin for plain tcp and GSS-enabled syslog
      • -
    • immark - support for mark messages
      • -
    • imklog - kernel logging
      • -
    • imuxsock - -unix sockets, including the system log socket
      • -
    • im3195 - -accepts syslog messages via RFC 3195
      • -
    -

    Please note that each module provides configuration -directives, which are NOT necessarily being listed below. Also -remember, that a modules configuration directive (and functionality) is -only available if it has been loaded (using $ModLoad).

    +

    Modules

    Lines

    Lines can be continued by specifying a backslash ("\") as the last character of the line. There is a hard-coded maximum line length of 4K. If you need lines larger than that, you need to change compile-time settings inside rsyslog and recompile. -

    Global Directives

    -

    All global directives need to be specified on a line by their -own and must start with a dollar-sign. Here is a list in alphabetical -order. Follow links for a description.

    -

    Please note that not all directives here are actually global. Some affect -only the next action. This documentation will be changed soon. -

    Not all directives have an in-depth description right now. -Default values for them are in bold. A more in-depth description will -appear as implementation progresses. -

    -

    Be sure to read information about queues in rsyslog - -many parameter settings modify queue parameters. If in doubt, use the -default, it is usually well-chosen and applicable in most cases.

    -
      -
    • $ActionExecOnlyWhenPreviousIsSuspended
      • -
    • $ActionExecOnlyOnceEveryInterval <seconds> - -execute action only if the last execute is at last -<seconds> seconds in the past (more info in ommail, -but may be used with any action)
      • -
    • $ActionExecOnlyEveryNthTime <number> - If configured, the next action will -only be executed every n-th time. For example, if configured to 3, the first two messages -that go into the action will be dropped, the 3rd will actually cause the action to execute, -the 4th and 5th will be dropped, the 6th executed under the action, ... and so on. Note: -this setting is automatically re-set when the actual action is defined.
      • -
    • $ActionExecOnlyEveryNthTimeTimeout <number-of-seconds> - has a meaning only if -$ActionExecOnlyEveryNthTime is also configured for the same action. If so, the timeout -setting specifies after which period the counting of "previous actions" expires and -a new action count is begun. Specify 0 (the default) to disable timeouts. -
      -Why is this option needed? Consider this case: a message comes in at, eg., 10am. That's -count 1. Then, nothing happens for the next 10 hours. At 8pm, the next -one occurs. That's count 2. Another 5 hours later, the next message -occurs, bringing the total count to 3. Thus, this message now triggers -the rule. -
      -The question is if this is desired behavior? Or should the rule only be -triggered if the messages occur within an e.g. 20 minute window? If the -later is the case, you need a -
      -$ActionExecOnlyEveryNthTimeTimeout 1200 -
      -This directive will timeout previous messages seen if they are older -than 20 minutes. In the example above, the count would now be always 1 -and consequently no rule would ever be triggered. - -
    • $ActionFileDefaultTemplate [templateName] - sets a new default template for file actions
      • -
    • $ActionFileEnableSync [on/off] - enables file -syncing capability of omfile
      • -
    • $ActionForwardDefaultTemplate [templateName] - sets a new -default template for UDP and plain TCP forwarding action
      • -
    • $ActionGSSForwardDefaultTemplate [templateName] - sets a -new default template for GSS-API forwarding action
      • -
    • $ActionQueueCheckpointInterval <number>
      • -
    • $ActionQueueDequeueSlowdown <number> [number -is timeout in microseconds (1000000us is 1sec!), -default 0 (no delay). Simple rate-limiting!]
      • -
    • $ActionQueueDiscardMark <number> [default -9750]
      • -
    • $ActionQueueDiscardSeverity <number> -[*numerical* severity! default 4 (warning)]
      • -
    • $ActionQueueFileName <name>
      • -
    • $ActionQueueHighWaterMark <number> [default -8000]
      • -
    • $ActionQueueImmediateShutdown [on/off]
      • -
    • $ActionQueueSize <number>
      • -
    • $ActionQueueLowWaterMark <number> [default -2000]
      • -
    • $ActionQueueMaxFileSize <size_nbr>, default 1m
      • -
    • $ActionQueueTimeoutActionCompletion <number> -[number is timeout in ms (1000ms is 1sec!), default 1000, 0 means -immediate!]
      • -
    • $ActionQueueTimeoutEnqueue <number> [number -is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]
      • -
    • $ActionQueueTimeoutShutdown <number> [number -is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]
      • -
    • $ActionQueueWorkerTimeoutThreadShutdown -<number> [number is timeout in ms (1000ms is 1sec!), -default 60000 (1 minute)]
      • -
    • $ActionQueueType [FixedArray/LinkedList/Direct/Disk]
      • -
    • $ActionQueueSaveOnShutdown  [on/off] -
      • -
    • $ActionQueueWorkerThreads <number>, num worker threads, default 1, recommended 1
      • -
    • $ActionQueueWorkerThreadMinumumMessages <number>, default 100
      • -
    • $ActionResumeInterval
      • -
    • $ActionResumeRetryCount <number> [default 0, -1 means eternal]
      • -
    • $ActionSendResendLastMsgOnReconn <[on/off]> specifies if the last message is to be resend when a connecition broken and has been reconnedcted. May increase reliability, but comes at the risk of message duplication. -
    • $ActionSendStreamDriver <driver basename> just like $DefaultNetstreamDriver, but for the specific action -
    • $ActionSendStreamDriverMode <mode>, default 0, mode to use with the stream driver -(driver-specific)
    • $ActionSendStreamDriverAuthMode <mode>,  authentication mode to use with the stream driver -(driver-specific)
    • $ActionSendStreamDriverPermittedPeer <ID>,  accepted fingerprint (SHA1) or name of remote peer -(driver-specific) - directive may go away!
      • -
    • $AllowedSender
      • -
    • $ControlCharacterEscapePrefix
      • -
    • $DebugPrintCFSyslineHandlerList
      • - -
    • $DebugPrintModuleList
      • -
    • $DebugPrintTemplateList
      • -
    • $DefaultNetstreamDriver <drivername>, the default network stream driver to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem>
      • -
    • $DefaultNetstreamDriverCertFile </path/to/certfile.pem>
      • -
    • $DefaultNetstreamDriverKeyFile </path/to/keyfile.pem>
      • -
    • $DirCreateMode
      • -
    • $DirGroup
      • -
    • $DirOwner
      • -
    • $DropMsgsWithMaliciousDnsPTRRecords
      • -
    • $DropTrailingLFOnReception
      • -
    • $DynaFileCacheSize
      • -
    • $EscapeControlCharactersOnReceive
      • -
    • $ErrorMessagesToStderr [on|off] - direct rsyslogd error message to stderr (in addition to other targets)
      • -
    • $FailOnChownFailure
      • -
    • $FileCreateMode
      • -
    • $FileGroup
      • -
    • $FileOwner
      • -
    • $GssForwardServiceName
      • -
    • $GssListenServiceName
      • -
    • $GssMode
      • -
    • $HUPisRestart [on/off] - if set to on, a HUP is a full daemon restart. This means any queued messages are discarded (depending -on queue configuration, of course) all modules are unloaded and reloaded. This mode keeps compatible with sysklogd, but is -not recommended for use with rsyslog. To do a full restart, simply stop and start the daemon. The default is "on" for -compatibility reasons. If it is set to "off", a HUP will only close open files. This is a much quicker action and usually -the only one that is needed e.g. for log rotation. It is recommended to set the setting to "off".
      • -
    • $IncludeConfig
    • MainMsgQueueCheckpointInterval <number>
      • -
    • $MainMsgQueueDequeueSlowdown <number> [number -is timeout in microseconds (1000000us is 1sec!), -default 0 (no delay). Simple rate-limiting!]
      • -
    • $MainMsgQueueDiscardMark <number> [default 9750]
      • -
    • $MainMsgQueueDiscardSeverity <severity> -[either a textual or numerical severity! default 4 (warning)]
      • -
    • $MainMsgQueueFileName <name>
      • -
    • $MainMsgQueueHighWaterMark <number> [default -8000]
      • -
    • $MainMsgQueueImmediateShutdown [on/off]
      • -
    • $MainMsgQueueSize
      • -
    • $MainMsgQueueLowWaterMark <number> [default -2000]
      • -
    • $MainMsgQueueMaxFileSize <size_nbr>, default -1m
      • -
    • $MainMsgQueueTimeoutActionCompletion -<number> [number is timeout in ms (1000ms is 1sec!), -default -1000, 0 means immediate!]
      • -
    • $MainMsgQueueTimeoutEnqueue <number> [number -is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]
      • -
    • $MainMsgQueueTimeoutShutdown <number> [number -is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]
      • -
    • $MainMsgQueueWorkerTimeoutThreadShutdown -<number> [number is timeout in ms (1000ms is 1sec!), -default 60000 (1 minute)]
      • -
    • $MainMsgQueueType [FixedArray/LinkedList/Direct/Disk]
      • -
    • $MainMsgQueueSaveOnShutdown  [on/off] -
      • -
    • $MainMsgQueueWorkerThreads <number>, num -worker threads, default 1, recommended 1
      • -
    • $MainMsgQueueWorkerThreadMinumumMessages <number>, default 100
      • -
    • $MarkMessagePeriod (immark)
      • -
    • $MaxMessageSize <size_nbr>, default 2k - allows to specify maximum supported message size -(both for sending and receiving). The default -should be sufficient for almost all cases. Do not set this below 1k, as it would cause -interoperability problems with other syslog implementations.
      -Change the setting to e.g. 32768 if you would like to -support large message sizes for IHE (32k is the current maximum -needed for IHE). I was initially tempted to set the default to 32k, -but there is a some memory footprint with the current -implementation in rsyslog. -
      If you intend to receive Windows Event Log data (e.g. via -EventReporter), you might want to -increase this number to an even higher value, as event -log messages can be very lengthy ("$MaxMessageSize 64k" is not a bad idea). -Note: testing showed that 4k seems to be -the typical maximum for UDP based syslog. This is an IP stack -restriction. Not always ... but very often. If you go beyond -that value, be sure to test that rsyslogd actually does what -you think it should do ;) It is highly suggested to use a TCP based transport -instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restrictions. -
      Note that 2k, the current default, is the smallest size that must be -supported in order to be compliant to the upcoming new syslog RFC series. -
      • -
    • $ModDir
      • -
    • $ModLoad
      • -
    • $RepeatedMsgReduction
      • -
    • $ResetConfigVariables
      • -
    • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better -performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The -default may change as uniprocessor systems become less common.
      • -
    • $WorkDirectory <name> (directory for spool and other work files)
      • -
    • $UDPServerAddress <IP> (imudp) -- local IP -address (or name) the UDP listens should bind to
      • -
    • $UDPServerRun <port> (imudp) -- former --r<port> option, default 514, start UDP server on this -port, "*" means all addresses
      • -
    • $UDPServerTimeRequery <nbr-of-times> (imudp) -- this is a performance -optimization. Getting the system time is very costly. With this setting, imudp can -be instructed to obtain the precise time only once every n-times. This logic is -only activated if messages come in at a very fast rate, so doing less frequent -time calls should usually be acceptable. The default value is two, because we have -seen that even without optimization the kernel often returns twice the identical time. -You can set this value as high as you like, but do so at your own risk. The higher -the value, the less precise the timestamp. -
    • $UMASK
      • -
    -

    Where <size_nbr> is specified above, -modifiers can be used after the number part. For example, 1k means -1024. Supported are k(ilo), m(ega), g(iga), t(era), p(eta) and e(xa). -Lower case letters refer to the traditional binary defintion (e.g. 1m -equals 1,048,576) whereas upper case letters refer to their new -1000-based definition (e.g 1M equals 1,000,000).

    -

    Numbers may include '.' and ',' for readability. So you can -for example specify either "1000" or "1,000" with the same result. -Please note that rsyslogd simply ignores the punctuation. Form it's -point of view, "1,,0.0.,.,0" also has the value 1000.

    +

    Global Directives

    Basic Structure

    Rsyslog supports standard sysklogd's configuration file format and extends it. So in general, you can take a "normal" syslog.conf and @@ -289,975 +45,15 @@ priorities belonging to the specified action.

    Lines starting with a hash mark ("#'') and empty lines are ignored.

    -

    Templates

    -

    Templates are a key feature of rsyslog. They allow to specify -any -format a user might want. They are also used for dynamic file name -generation. Every output in rsyslog uses templates - this holds true -for files, user messages and so on. The database writer expects its -template to be a proper SQL statement - so this is highly customizable -too. You might ask how does all of this work when no templates at all -are specified. Good question ;) The answer is simple, though. Templates -compatible with the stock syslogd formats are hardcoded into rsyslogd. -So if no template is specified, we use one of these hardcoded -templates. Search for "template_" in syslogd.c and you will find the -hardcoded ones.

    -

    A template consists of a template directive, a name, the -actual template text and optional options. A sample is:

    -
    $template MyTemplateName,"\7Text -%property% some more text\n",<options>
    -

    The "$template" is the template directive. It tells rsyslog -that this line contains a template. "MyTemplateName" is the template -name. All -other config lines refer to this name. The text within quotes is the -actual template text. The backslash is an escape character, much as it -is in C. It does all these "cool" things. For example, \7 rings the -bell (this is an ASCII value), \n is a new line. C programmers and perl -coders have the advantage of knowing this, but the set in rsyslog is a -bit restricted currently. -

    -

    All text in the template is used literally, except for things -within percent signs. These are properties and allow you access to the -contents of the syslog message. Properties are accessed via the -property replacer (nice name, huh) and it can do cool things, too. For -example, it can pick a substring or do date-specific formatting. More -on this is below, on some lines of the property replacer.
    -
    -The <options> part is optional. It carries options -influencing the template as whole. See details below. Be sure NOT to -mistake template options with property options - the later ones are -processed by the property replacer and apply to a SINGLE property, only -(and not the whole template).
    -
    -Template options are case-insensitive. Currently defined are:

    -

    sql - format the string suitable for a SQL -statement in MySQL format. This will replace single quotes ("'") and -the backslash character by their backslash-escaped counterpart ("\'" -and "\\") inside each field. Please note that in MySQL configuration, -the NO_BACKSLASH_ESCAPES -mode must be turned off for this format to work (this is the default).

    -

    stdsql - format the string suitable for a -SQL statement that is to be sent to a standards-compliant sql server. -This will replace single quotes ("'") by two single quotes ("''") -inside each field. You must use stdsql together with MySQL if in MySQL -configuration the -NO_BACKSLASH_ESCAPES is -turned on.

    -

    Either the sql or stdsql  -option must be specified when a template is used -for writing to a database, otherwise injection might occur. Please note -that due to the unfortunate fact that several vendors have violated the -sql standard and introduced their own escape methods, it is impossible -to have a single option doing all the work.  So you yourself -must make sure you are using the right format. If you choose -the wrong one, you are still vulnerable to sql injection.
    -
    -Please note that the database writer *checks* that the sql option is -present in the template. If it is not present, the write database -action is disabled. This is to guard you against accidental forgetting -it and then becoming vulnerable to SQL injection. The sql option can -also be useful with files - especially if you want to import them into -a database on another machine for performance reasons. However, do NOT -use it if you do not have a real need for it - among others, it takes -some toll on the processing time. Not much, but on a really busy system -you might notice it ;)

    -

    The default template for the write to database action has the -sql option set. As we currently support only MySQL and the sql option -matches the default MySQL configuration, this is a good choice. -However, if you have turned on -NO_BACKSLASH_ESCAPES in -your MySQL config, you need to supply a template with the stdsql -option. Otherwise you will become vulnerable to SQL injection.
    -
    -To escape:
    -% = \%
    -\ = \\ --> '\' is used to escape (as in C)
    -$template TraditionalFormat,%timegenerated% %HOSTNAME% -%syslogtag%%msg%\n"
    -
    -Properties can be accessed by the property -replacer (see there for details).

    -

    Please note that templates can also by -used to generate selector lines with dynamic file names. For -example, if you would like to split syslog messages from different -hosts to different files (one per host), you can define the following -template:

    -
    $template -DynFile,"/var/log/system-%HOSTNAME%.log"
    -

    This template can then be used when defining an output -selector line. It will result in something like -"/var/log/system-localhost.log"

    -

    Template -names beginning with "RSYSLOG_" are reserved for rsyslog use. Do NOT -use them if, otherwise you may receive a conflict in the future (and -quite unpredictable behaviour). There is a small set of pre-defined -templates that you can use without the need to define it:

    -
      -
    • RSYSLOG_TraditionalFileFormat -- the "old style" default log file format with low-precision timestamps
      • -
    • RSYSLOG_FileFormat -- a modern-style logfile format similar to TraditionalFileFormat, buth -with high-precision timestamps and timezone information
      • -
    • RSYSLOG_TraditionalForwardFormat -- the traditional forwarding format with low-precision timestamps. Most -useful if you send messages to other syslogd's or rsyslogd -below -version 3.12.5.
      • -
    • RSYSLOG_ForwardFormat -- a new high-precision forwarding format very similar to the -traditional one, but with high-precision timestamps and timezone -information. Recommended to be used when sending messages to rsyslog -3.12.5 or above.
      • -
    • RSYSLOG_SyslogProtocol23Format -- the format specified in IETF's internet-draft -ietf-syslog-protocol-23, which is assumed to be come the new syslog -standard RFC. This format includes several improvements. The rsyslog -message parser understands this format, so you can use it together with -all relatively recent versions of rsyslog. Other syslogd's may get -hopelessly confused if receiving that format, so check before you use -it. Note that the format is unlikely to change when the final RFC comes -out, but this may happen.
      • -
    • RSYSLOG_DebugFormat -- a special format used for troubleshooting property problems. This format -is meant to be written to a log file. Do not use for production or remote -forwarding.
      • -
    -

    Output Channels

    -

    Output Channels are a new concept first introduced in rsyslog -0.9.0. As of this writing, it is most likely that they will -be replaced by something different in the future. So if you -use them, be prepared to change you configuration file syntax when you -upgrade to a later release.
    -
    -The idea behind output channel definitions is that it shall provide an -umbrella for any type of output that the user might want. In essence,
    -this is the "file" part of selector lines (and this is why we are not -sure output channel syntax will stay after the next review). There is a
    -difference, though: selector channels both have filter conditions -(currently facility and severity) as well as the output destination. -Output channels define the output definition, only. As of this build, -they can only be used to write to files - not pipes, ttys or whatever -else. If we stick with output channels, this will change over time.

    -

    In concept, an output channel includes everything needed to -know about an output actions. In practice, the current implementation -only carries
    -a filename, a maximum file size and a command to be issued when this -file size is reached. More things might be present in future version, -which might also change the syntax of the directive.

    -

    Output channels are defined via an $outchannel directive. It's -syntax is as follows:
    -
    -$outchannel name,file-name,max-size,action-on-max-size
    -
    -name is the name of the output channel (not the file), file-name is the -file name to be written to, max-size the maximum allowed size and -action-on-max-size a command to be issued when the max size is reached. -This command always has exactly one parameter. The binary is that part -of action-on-max-size before the first space, its parameter is -everything behind that space.
    -
    -Please note that max-size is queried BEFORE writing the log message to -the file. So be sure to set this limit reasonably low so that any -message might fit. For the current release, setting it 1k lower than -you expected is helpful. The max-size must always be specified in bytes -- there are no special symbols (like 1k, 1m,...) at this point of -development.
    -
    -Keep in mind that $outchannel just defines a channel with "name". It -does not activate it. To do so, you must use a selector line (see -below). That selector line includes the channel name plus an $ sign in -front of it. A sample might be:
    -
    -*.* $mychannel
    -
    -In its current form, output channels primarily provide the ability to -size-limit an output file. To do so, specify a maximum size. When this -size is reached, rsyslogd will execute the action-on-max-size command -and then reopen the file and retry. The command should be something -like a log rotation -script or a similar thing.

    -

    If there is no action-on-max-size command or the command did -not resolve the situation, the file is closed and never reopened by -rsyslogd (except, of course, by huping it). This logic was integrated -when we first experienced severe issues with files larger 2gb, which -could lead to rsyslogd dumping core. In such cases, it is more -appropriate to stop writing to a single file. Meanwhile, rsyslogd has -been fixed to support files larger 2gb, but obviously only on file -systems and operating system versions that do so. So it can still make -sense to enforce a 2gb file size limit.

    -

    Filter Conditions

    -

    Rsyslog offers four different types "filter conditions":

    -
      -
    • BSD-style blocks
      • -
    • "traditional" severity and facility based selectors
      • -
    • property-based filters
      • -
    • expression-based filters
      • -
    -

    Blocks

    -

    Rsyslogd supports BSD-style blocks inside rsyslog.conf. Each -block of lines is separated from the previous block by a program or -hostname specification. A block will only log messages corresponding to -the most recent program and hostname specifications given. Thus, a -block which selects ‘ppp’ as the program, directly followed by a block -that selects messages from the hostname ‘dialhost’, then the second -block will only log messages from the ppp program on dialhost. -

    -

    A program specification is a line beginning with ‘!prog’ and -the following blocks will be associated with calls to syslog from that -specific program. A program specification for ‘foo’ will also match any -message logged by the kernel with the prefix ‘foo: ’. Alternatively, a -program specification ‘-foo’ causes the following blocks to be applied -to messages from any program but the one specified. A hostname -specification of the form ‘+hostname’ and the following blocks will be -applied to messages received from the specified hostname. -Alternatively, a hostname specification ‘-hostname’ causes the -following blocks to be applied to messages from any host but the one -specified. If the hostname is given as ‘@’, the local hostname will be -used. (NOT YET IMPLEMENTED) A program or hostname specification may be -reset by giving the program or hostname as ‘*’.

    -

    Please note that the "#!prog", "#+hostname" and "#-hostname" -syntax available in BSD syslogd is not supported by rsyslogd. By -default, no hostname or program is set.

    -

    Selectors

    -

    Selectors are the traditional way of filtering syslog -messages. They have been kept in rsyslog with their original -syntax, because it is well-known, highly effective and also needed for -compatibility with stock syslogd configuration files. If you just need -to filter based on priority and facility, you should do this with -selector lines. They are not second-class citizens -in rsyslog and offer the best performance for this job.

    -

    The selector field itself again consists of two parts, a -facility and a priority, separated by a period (".''). Both parts are -case insensitive and can also be specified as decimal numbers, but -don't do that, you have been warned. Both facilities and priorities are -described in rsyslog(3). The names mentioned below correspond to the -similar LOG_-values in /usr/include/rsyslog.h.
    -
    -The facility is one of the following keywords: auth, authpriv, cron, -daemon, kern, lpr, mail, mark, news, security (same as auth), syslog, -user, uucp and local0 through local7. The keyword security should not -be used anymore and mark is only for internal use and therefore should -not be used in applications. Anyway, you may want to specify and -redirect these messages here. The facility specifies the subsystem that -produced the message, i.e. all mail programs log with the mail facility -(LOG_MAIL) if they log using syslog.
    -
    -The priority is one of the following keywords, in ascending order: -debug, info, notice, warning, warn (same as warning), err, error (same -as err), crit, alert, emerg, panic (same as emerg). The keywords error, -warn and panic are deprecated and should not be used anymore. The -priority defines the severity of the message.
    -
    -The behavior of the original BSD syslogd is that all messages of the -specified priority and higher are logged according to the given action. -Rsyslogd behaves the same, but has some extensions.
    -
    -In addition to the above mentioned names the rsyslogd(8) understands -the following extensions: An asterisk ("*'') stands for all facilities -or all priorities, depending on where it is used (before or after the -period). The keyword none stands for no priority of the given facility.
    -
    -You can specify multiple facilities with the same priority pattern in -one statement using the comma (",'') operator. You may specify as much -facilities as you want. Remember that only the facility part from such -a statement is taken, a priority part would be skipped.

    -

    Multiple selectors may be specified for a single action using -the semicolon (";'') separator. Remember that each selector in the -selector field is capable to overwrite the preceding ones. Using this -behavior you can exclude some priorities from the pattern.

    -

    Rsyslogd has a syntax extension to the original BSD source, -that makes its use more intuitively. You may precede every priority -with an equation sign ("='') to specify only this single priority and -not any of the above. You may also (both is valid, too) precede the -priority with an exclamation mark ("!'') to ignore all that -priorities, either exact this one or this and any higher priority. If -you use both extensions than the exclamation mark must occur before the -equation sign, just use it intuitively.

    -

    Property-Based Filters

    -

    Property-based filters are unique to rsyslogd. They allow to -filter on any property, like HOSTNAME, syslogtag and msg. A list of all -currently-supported properties can be found in the property replacer documentation -(but keep in mind that only the properties, not the replacer is -supported). With this filter, each properties can be checked against a -specified value, using a specified compare operation. Currently, there -is only a single compare operation (contains) available, but additional -operations will be added in the future.

    -

    A property-based filter must start with a colon in column 0. -This tells rsyslogd that it is the new filter type. The colon must be -followed by the property name, a comma, the name of the compare -operation to carry out, another comma and then the value to compare -against. This value must be quoted. There can be spaces and tabs -between the commas. Property names and compare operations are -case-sensitive, so "msg" works, while "MSG" is an invalid property -name. In brief, the syntax is as follows:

    -

    :property, [!]compare-operation, "value"

    -

    The following compare-operations are -currently supported:

    - - - - - - - - - - - - - - - - - - - -
    containsChecks if the string provided in value is contained in -the property. There must be an exact match, wildcards are not supported.
    isequalCompares the "value" string provided and the property -contents. These two values must be exactly equal to match. The -difference to contains is that contains searches for the value anywhere -inside the property value, whereas all characters must be identical for -isequal. As such, isequal is most useful for fields like syslogtag or -FROMHOST, where you probably know the exact contents.
    startswithChecks if the value is found exactly at the beginning -of the property value. For example, if you search for "val" with -

    :msg, startswith, "val"

    -

    it will be a match if msg contains "values are in this -message" but it won't match if the msg contains "There are values in -this message" (in the later case, contains would match). Please note -that "startswith" is by far faster than regular expressions. So even -once they are implemented, it can make very much sense -(performance-wise) to use "startswith".

    -
    regexCompares the property against the provided POSIX -regular -expression.
    -

    You can use the bang-character (!) immediately in front of a -compare-operation, the outcome of this operation is negated. For -example, if msg contains "This is an informative message", the -following sample would not match:

    -

    :msg, contains, "error"

    -

    but this one matches:

    -

    :msg, !contains, "error"

    -

    Using negation can be useful if you would like to do some -generic processing but exclude some specific events. You can use the -discard action in conjunction with that. A sample would be:

    -

    *.* -/var/log/allmsgs-including-informational.log
    -:msg, contains, "informational"  ~ -
    -*.* /var/log/allmsgs-but-informational.log

    -

    Do not overlook the red tilde in line 2! In this sample, all -messages are written to the file allmsgs-including-informational.log. -Then, all messages containing the string "informational" are discarded. -That means the config file lines below the "discard line" (number 2 in -our sample) will not be applied to this message. Then, all remaining -lines will also be written to the file allmsgs-but-informational.log.

    -

    Value is a quoted string. It supports some -escape sequences:

    -

    \" - the quote character (e.g. "String with \"Quotes\"")
    -\\ - the backslash character (e.g. "C:\\tmp")

    -

    Escape sequences always start with a backslash. Additional -escape sequences might be added in the future. Backslash characters must -be escaped. Any other sequence then those outlined above is invalid and -may lead to unpredictable results.

    -

    Probably, "msg" is the most prominent use case of property -based filters. It is the actual message text. If you would like to -filter based on some message content (e.g. the presence of a specific -code), this can be done easily by:

    -

    :msg, contains, "ID-4711"

    -

    This filter will match when the message contains the string -"ID-4711". Please note that the comparison is case-sensitive, so it -would not match if "id-4711" would be contained in the message.

    -

    :msg, regex, "fatal .* error"

    -

    This filter uses a POSIX regular expression. It matches when -the -string contains the words "fatal" and "error" with anything in between -(e.g. "fatal net error" and "fatal lib error" but not "fatal error" as -two spaces are required by the regular expression!).

    -

    Getting property-based filters right can sometimes be -challenging. In order to help you do it with as minimal effort as -possible, rsyslogd spits out debug information for all property-based -filters during their evaluation. To enable this, run rsyslogd in -foreground and specify the "-d" option.

    -

    Boolean operations inside property based filters (like -'message contains "ID17" or message contains "ID18"') are currently not -supported (except for "not" as outlined above). Please note that while -it is possible to query facility and severity via property-based -filters, it is far more advisable to use classic selectors (see above) -for those cases.

    -

    Expression-Based Filters

    -Expression based filters allow -filtering on arbitrary complex expressions, which can include boolean, -arithmetic and string operations. Expression filters will evolve into a -full configuration scripting language. Unfortunately, their syntax will -slightly change during that process. So if you use them now, you need -to be prepared to change your configuration files some time later. -However, we try to implement the scripting facility as soon as possible -(also in respect to stage work needed). So the window of exposure is -probably not too long.
    -
    -Expression based filters are indicated by the keyword "if" in column 1 -of a new line. They have this format:
    -
    -if expr then action-part-of-selector-line
    -
    -"If" and "then" are fixed keywords that mus be present. "expr" is a -(potentially quite complex) expression. So the expression documentation for -details. "action-part-of-selector-line" is an action, just as you know -it (e.g. "/var/log/logfile" to write to that file).
    -
    -A few quick samples:
    -
    - -*.* /var/log/file1 # the traditional way
    -if $msg contains 'error' /var/log/errlog # the expression-based way
    -     -
    -Right now, you need to specify numerical values if you would like to -check for facilities and severity. These can be found in RFC 3164. -If you don't like that, you can of course also use the textual property -- just be sure to use the right one. As expression support is enhanced, -this will change. For example, if you would like to filter on message -that have facility local0, start with "DEVNAME" and have either -"error1" or "error0" in their message content, you could use the -following filter:
    -
    - -if $syslogfacility-text == 'local0' and $msg -startswith 'DEVNAME' and ($msg contains 'error1' or $msg contains -'error0') then /var/log/somelog
    -     -
    -Please note that the above must -all be on one line! And if you would like to store all -messages except those that contain "error1" or "error0", you just need -to add a "not":
    -
    - -if $syslogfacility-text == 'local0' and $msg -startswith 'DEVNAME' and not -($msg contains 'error1' or $msg contains -'error0') then /var/log/somelog
    -     -
    -If you would like to do case-insensitive comparisons, use -"contains_i" instead of "contains" and "startswith_i" instead of -"startswith".
    -
    -Note that regular expressions are currently NOT -supported in expression-based filters. These will be added later when -function support is added to the expression engine (the reason is that -regular expressions will be a separate loadable module, which requires -some more prequisites before it can be implemented).
    -

    ACTIONS

    -

    The action field of a rule describes what to do with the -message. In general, message content is written to a kind of "logfile". -But also other actions might be done, like writing to a database table -or forwarding to another host.
    -
    -Templates can be used with all actions. If used, the specified template -is used to generate the message content (instead of the default -template). To specify a template, write a semicolon after the action -value immediately followed by the template name.
    -
    -Beware: templates MUST be defined BEFORE they are used. It is OK to -define some templates, then use them in selector lines, define more -templates and use use them in the following selector lines. But it is -NOT permitted to use a template in a selector line that is above its -definition. If you do this, the action will be ignored.

    -

    You can have multiple actions for a single selector  (or -more precisely a single filter of such a selector line). Each action -must be on its own line and the line must start with an ampersand -('&') character and have no filters. An example would be

    -

    *.=crit rger
    -& root
    -& /var/log/critmsgs

    -

    These three lines send critical messages to the user rger and -root and also store them in /var/log/critmsgs. Using multiple -actions per selector is convenient and also offers -a performance benefit. As the filter needs to be evaluated -only once, there is less computation required to process the directive -compared to the otherwise-equal config directives below:

    -

    *.=crit rger
    -*.=crit root
    -*.=crit /var/log/critmsgs

    -

     

    -

    Regular File

    -

    Typically messages are logged to real files. The file has to -be specified with full pathname, beginning with a slash "/''.
    -
    -You may prefix each entry with the minus "-'' sign to omit syncing the -file after every logging. Note that you might lose information if the -system crashes right behind a write attempt. Nevertheless this might -give you back some performance, especially if you run programs that use -logging in a very verbose manner.

    -

    If your system is connected to a reliable UPS and you receive -lots of log data (e.g. firewall logs), it might be a very good idea to -turn of -syncing by specifying the "-" in front of the file name.

    -

    The filename can be either static (always -the same) or dynamic (different based on message -received). The later is useful if you would automatically split -messages into different files based on some message criteria. For -example, dynamic file name selectors allow you to split messages into -different files based on the host that sent them. With dynamic file -names, everything is automatic and you do not need any filters.

    -

    It works via the template system. First, you define a template -for the file name. An example can be seen above in the description of -template. We will use the "DynFile" template defined there. Dynamic -filenames are indicated by specifying a questions mark "?" instead of a -slash, followed by the template name. Thus, the selector line for our -dynamic file name would look as follows:

    -
    -*.* ?DynFile -
    -

    That's all you need to do. Rsyslog will now automatically -generate file names for you and store the right messages into the right -files. Please note that the minus sign also works with dynamic file -name selectors. Thus, to avoid syncing, you may use

    -
    -*.* -?DynFile
    -

    And of course you can use templates to specify the output -format:

    -
    -*.* ?DynFile;MyTemplate
    -

    A word of caution: rsyslog creates files as -needed. So if a new host is using your syslog server, rsyslog will -automatically create a new file for it.

    -

    Creating directories is also supported. For -example you can use the hostname as directory and the program name as -file name:

    -
    -$template DynFile,"/var/log/%HOSTNAME%/%programname%.log"
    -

    Named Pipes

    -

    This version of rsyslogd(8) has support for logging output to -named pipes (fifos). A fifo or named pipe can be used as a destination -for log messages by prepending a pipe symbol ("|'') to the name of the -file. This is handy for debugging. Note that the fifo must be created -with the mkfifo(1) command before rsyslogd(8) is started.

    -

    Terminal and Console

    -

    If the file you specified is a tty, special tty-handling is -done, same with /dev/console.

    -

    Remote Machine

    -

    Rsyslogd provides full remote logging, i.e. is able to send -messages to a remote host running rsyslogd(8) and to receive messages -from remote hosts. Using this feature you're able to control all syslog -messages on one host, if all other machines will log remotely to that. -This tears down
    -administration needs.
    -
    -Please note that this version of rsyslogd by default does NOT -forward messages it has received from the network to another host. -Specify the "-h" option to enable this.

    -

    To forward messages to another host, prepend the hostname with -the at sign ("@"). A single at sign means that messages will -be forwarded via UDP protocol (the standard for syslog). If you prepend -two at signs ("@@"), the messages will be transmitted via TCP. Please -note that plain TCP based syslog is not officially standardized, but -most major syslogds support it (e.g. syslog-ng or WinSyslog). The -forwarding action indicator (at-sign) can be followed by one or more -options. If they are given, they must be immediately (without a space) -following the final at sign and be enclosed in parenthesis. The -individual options must be separated by commas. The following options -are right now defined:

    - - - - - - - - - - - -
    -

    z<number>

    -    		Enable zlib-compression for the message. The -<number> is the compression level. It can be 1 (lowest -gain, lowest CPU overhead) to 9 (maximum compression, highest CPU -overhead). The level can also be 0, which means "no compression". If -given, the "z" option is ignored. So this does not make an awful lot of -sense. There is hardly a difference between level 1 and 9 for typical -syslog messages. You can expect a compression gain between 0% and 30% -for typical messages. Very chatty messages may compress up to 50%, but -this is seldom seen with typically traffic. Please note that rsyslogd -checks the compression gain. Messages with 60 bytes or less will never -be compressed. This is because compression gain is pretty unlikely and -we prefer to save CPU cycles. Messages over that size are always -compressed. However, it is checked if there is a gain in compression -and only if there is, the compressed message is transmitted. Otherwise, -the uncompressed messages is transmitted. This saves the receiver CPU -cycles for decompression. It also prevents small message to actually -become larger in compressed form. -

    Please note that when a TCP transport is used, -compression will also turn on syslog-transport-tls framing. See the "o" -option for important information on the implications.

    -

    Compressed messages are automatically detected and -decompressed by the receiver. There is nothing that needs to be -configured on the receiver side.

    -
    -

    o

    -    		This option is experimental. Use at your own -risk and only if you know why you need it! If in doubt, do NOT turn it -on. -

    This option is only valid for plain TCP based -transports. It selects a different framing based on IETF internet draft -syslog-transport-tls-06. This framing offers some benefits over -traditional LF-based framing. However, the standardization effort is -not yet complete. There may be changes in upcoming versions of this -standard. Rsyslog will be kept in line with the standard. There is some -chance that upcoming changes will be incompatible to the current -specification. In this case, all systems using -transport-tls framing -must be upgraded. There will be no effort made to retain compatibility -between different versions of rsyslog. The primary reason for that is -that it seems technically impossible to provide compatibility between -some of those changes. So you should take this note very serious. It is -not something we do not *like* to do (and may change our mind if enough -people beg...), it is something we most probably *can not* do for -technical reasons (aka: you can beg as much as you like, it won't -change anything...).

    -

    The most important implication is that compressed syslog -messages via TCP must be considered with care. Unfortunately, it is -technically impossible to transfer compressed records over traditional -syslog plain tcp transports, so you are left with two evil choices...

    -
    -


    -The hostname may be followed by a colon and the destination port.

    -

    The following is an example selector line with forwarding:

    -

    *.*    @@(o,z9)192.168.0.1:1470

    -

    In this example, messages are forwarded via plain TCP with -experimental framing and maximum compression to the host 192.168.0.1 at -port 1470.

    -

    *.* @192.168.0.1

    -

    In the example above, messages are forwarded via UDP to the -machine 192.168.0.1, the destination port defaults to 514. Messages -will not be compressed.

    -

    Note that IPv6 addresses contain colons. So if an IPv6 address is specified -in the hostname part, rsyslogd could not detect where the IP address ends -and where the port starts. There is a syntax extension to support this: -put squary brackets around the address (e.g. "[2001::1]"). Square -brackets also work with real host names and IPv4 addresses, too. -

    A valid sample to send messages to the IPv6 host 2001::1 at port 515 -is as follows: -

    *.* @[2001::1]:515 -

    This works with TCP, too. -

    Note to sysklogd users: sysklogd does not -support RFC 3164 format, which is the default forwarding template in -rsyslog. As such, you will experience duplicate hostnames if rsyslog is -the sender and sysklogd is the receiver. The fix is simple: you need to -use a different template. Use that one:

    -

    $template -sysklogd,"<%PRI%>%TIMESTAMP% %syslogtag%%msg%\""
    -*.* @192.168.0.1;sysklogd

    -

    List of Users

    -

    Usually critical messages are also directed to "root'' on -that machine. You can specify a list of users that shall get the -message by simply writing the login. You may specify more than one user -by separating them with commas (",''). If they're logged in they get -the message. Don't think a mail would be sent, that might be too late.

    -

    Everyone logged on

    -

    Emergency messages often go to all users currently online to -notify them that something strange is happening with the system. To -specify this wall(1)-feature use an asterisk ("*'').

    -

    Call Plugin

    -

    This is a generic way to call an output plugin. The plugin -must support this functionality. Actual parameters depend on the -module, so see the module's doc on what to supply. The general syntax -is as follows:

    -

    :modname:params;template

    -

    Currently, the ommysql database output module supports this -syntax (in addtion to the ">" syntax it traditionally -supported). For ommysql, the module name is "ommysql" and the params -are the traditional ones. The ;template part is not module specific, it -is generic rsyslog functionality available to all modules.

    -

    As an example, the ommysql module may be called as follows:

    -

    :ommysql:dbhost,dbname,dbuser,dbpassword;dbtemplate

    -

    For details, please see the "Database Table" section of this -documentation.

    -

    Note: as of this writing, the ":modname:" part is hardcoded -into the module. So the name to use is not necessarily the name the -module's plugin file is called.

    -

    Database Table

    -

    This allows logging of the message to a database table. -Currently, only MySQL databases are supported. However, other database -drivers will most probably be developed as plugins. By default, a MonitorWare-compatible -schema is required for this to work. You can create that schema with -the createDB.SQL file that came with the rsyslog package. You can also
    -use any other schema of your liking - you just need to define a proper -template and assign this template to the action.
    -
    -The database writer is called by specifying a greater-then sign -(">") in front of the database connect information. Immediately -after that
    -sign the database host name must be given, a comma, the database name, -another comma, the database user, a comma and then the user's password. -If a specific template is to be used, a semicolon followed by the -template name can follow the connect information. This is as follows:
    -
    ->dbhost,dbname,dbuser,dbpassword;dbtemplate

    -

    Important: to use the database functionality, the -MySQL output module must be loaded in the config file BEFORE -the first database table action is used. This is done by placing the

    -

    $ModLoad ommysql

    -

    directive some place above the first use of the database write -(we recommend doing at the the beginning of the config file).

    -

    Discard

    -

    If the discard action is carried out, the received message is -immediately discarded. No further processing of it occurs. Discard has -primarily been added to filter out messages before carrying on any -further processing. For obvious reasons, the results of "discard" are -depending on where in the configuration file it is being used. Please -note that once a message has been discarded there is no way to retrieve -it in later configuration file lines.

    -

    Discard can be highly effective if you want to filter out some -annoying messages that otherwise would fill your log files. To do that, -place the discard actions early in your log files. This often plays -well with property-based filters, giving you great freedom in -specifying what you do not want.

    -

    Discard is just the single tilde character with no further -parameters:

    -

    ~

    -

    For example,

    -

    *.*   ~

    -

    discards everything (ok, you can achive the same by not -running rsyslogd at all...).

    -

    Output Channel

    -

    Binds an output channel definition (see there for details) to -this action. Output channel actions must start with a $-sign, e.g. if -you would like to bind your output channel definition "mychannel" to -the action, use "$mychannel". Output channels support template -definitions like all all other actions.

    -

    Shell Execute

    -

    This executes a program in a subshell. The program is passed -the template-generated message as the only command line parameter. -Rsyslog waits until the program terminates and only then continues to -run.

    -

    ^program-to-execute;template

    -

    The program-to-execute can be any valid executable. It -receives the template string as a single parameter (argv[1]).

    -

    WARNING: The Shell Execute action was added -to serve an urgent need. While it is considered reasonable save when -used with some thinking, its implications must be considered. The -current implementation uses a system() call to execute the command. -This is not the best way to do it (and will hopefully changed in -further releases). Also, proper escaping of special characters is done -to prevent command injection. However, attackers always find smart ways -to circumvent escaping, so we can not say if the escaping applied will -really safe you from all hassles. Lastly, rsyslog will wait until the -shell command terminates. Thus, a program error in it (e.g. an infinite -loop) can actually disable rsyslog. Even without that, during the -programs run-time no messages are processed by rsyslog. As the IP -stacks buffers are quickly overflowed, this bears an increased risk of -message loss. You must be aware of these implications. Even though they -are severe, there are several cases where the "shell execute" action is -very useful. This is the reason why we have included it in its current -form. To mitigate its risks, always a) test your program thoroughly, b) -make sure its runtime is as short as possible (if it requires a longer -run-time, you might want to spawn your own sub-shell asynchronously), -c) apply proper firewalling so that only known senders can send syslog -messages to rsyslog. Point c) is especially important: if rsyslog is -accepting message from any hosts, chances are much higher that an -attacker might try to exploit the "shell execute" action.

    -

    TEMPLATE NAME

    -

    Every ACTION can be followed by a template name. If so, that -template is used for message formatting. If no name is given, a -hard-coded default template is used for the action. There can only be -one template name for each given action. The default template is -specific to each action. For a description of what a template is and -what you can do with it, see "TEMPLATES" at the top of this document.

    -

    EXAMPLES

    -

    Below are example for templates and selector lines. I hope +

    Templates

    +

    Output Channels

    +

    Filter Conditions

    +

    Actions

    +

    Examples

    +

    Here you will find examples for templates and selector lines. I hope they are self-explanatory. If not, please see www.monitorware.com/rsyslog/ for advise.

    -

    TEMPLATES

    -

    Please note that the samples are split across multiple lines. -A template MUST NOT actually be split across multiple lines.
    -
    -A template that resembles traditional syslogd file output:
    -$template TraditionalFormat,"%timegenerated% %HOSTNAME%
    -%syslogtag%%msg:::drop-last-lf%\n"
    -
    -A template that tells you a little more about the message:
    -$template -precise,"%syslogpriority%,%syslogfacility%,%timegenerated%,%HOSTNAME%,
    -%syslogtag%,%msg%\n"
    -
    -A template for RFC 3164 format:
    -$template RFC3164fmt,"<%PRI%>%TIMESTAMP% %HOSTNAME% -%syslogtag%%msg%"
    -
    -A template for the format traditonally used for user messages:
    -$template usermsg," XXXX%syslogtag%%msg%\n\r"
    -
    -And a template with the traditonal wall-message format:
    -$template wallmsg,"\r\n\7Message from syslogd@%HOSTNAME% at -%timegenerated%
    -
    -A template that can be used for the database write (please note the SQL
    -template option)
    -$template MySQLInsert,"insert iut, message, receivedat values
    -('%iut%', '%msg:::UPPERCASE%', '%timegenerated:::date-mysql%')
    -into systemevents\r\n", SQL
    -
    -The following template emulates WinSyslog -format (it's an Adiscon -format, you do not feel bad if you don't know it ;)). It's interesting -to see how it takes different parts out of the date stamps. What -happens is that the date stamp is split into the actual date and time -and the these two are combined with just a comma in between them.
    -
    -$template WinSyslogFmt,"%HOSTNAME%,%timegenerated:1:10:date-rfc3339%,
    -%timegenerated:12:19:date-rfc3339%,%timegenerated:1:10:date-rfc3339%,
    -%timegenerated:12:19:date-rfc3339%,%syslogfacility%,%syslogpriority%,
    -%syslogtag%%msg%\n"

    -

    SELECTOR LINES

    -

    # Store critical stuff in critical
    -#
    -*.=crit;kern.none /var/adm/critical
    -
    -This will store all messages with the priority crit in the file -/var/adm/critical, except for any kernel message.
    -
    -
    -# Kernel messages are first, stored in the kernel
    -# file, critical messages and higher ones also go
    -# to another host and to the console. Messages to
    -# the host finlandia are forwarded in RFC 3164
    -# format (using the template defined above).
    -#
    -kern.* /var/adm/kernel
    -kern.crit @finlandia;RFC3164fmt
    -kern.crit /dev/console
    -kern.info;kern.!err /var/adm/kernel-info
    -
    -The first rule direct any message that has the kernel facility to the -file /var/adm/kernel.
    -
    -The second statement directs all kernel messages of the priority crit -and higher to the remote host finlandia. This is useful, because if the -host crashes and the disks get irreparable errors you might not be able -to read the stored messages. If they're on a remote host, too, you -still can try to find out the reason for the crash.
    -
    -The third rule directs these messages to the actual console, so the -person who works on the machine will get them, too.
    -
    -The fourth line tells rsyslogd to save all kernel messages that come -with priorities from info up to warning in the file -/var/adm/kernel-info. Everything from err and higher is excluded.
    -
    -
    -# The tcp wrapper loggs with mail.info, we display
    -# all the connections on tty12
    -#
    -mail.=info /dev/tty12
    -
    -This directs all messages that uses mail.info (in source LOG_MAIL | -LOG_INFO) to /dev/tty12, the 12th console. For example the tcpwrapper -tcpd(8) uses this as it's default.
    -
    -
    -# Store all mail concerning stuff in a file
    -#
    -mail.*;mail.!=info /var/adm/mail
    -
    -This pattern matches all messages that come with the mail facility, -except for the info priority. These will be stored in the file -/var/adm/mail.
    -
    -
    -# Log all mail.info and news.info messages to info
    -#
    -mail,news.=info /var/adm/info
    -
    -This will extract all messages that come either with mail.info or with -news.info and store them in the file /var/adm/info.
    -
    -
    -# Log info and notice messages to messages file
    -#
    -*.=info;*.=notice;\
    -mail.none /var/log/messages
    -
    -This lets rsyslogd log all messages that come with either the info or -the notice facility into the file /var/log/messages, except for all
    -messages that use the mail facility.
    -
    -
    -# Log info messages to messages file
    -#
    -*.=info;\
    -mail,news.none /var/log/messages
    -
    -This statement causes rsyslogd to log all messages that come with the -info priority to the file /var/log/messages. But any message coming -either with the mail or the news facility will not be stored.
    -
    -
    -# Emergency messages will be displayed using wall
    -#
    -*.=emerg *
    -
    -This rule tells rsyslogd to write all emergency messages to all -currently logged in users. This is the wall action.
    -
    -
    -# Messages of the priority alert will be directed
    -# to the operator
    -#
    -*.alert root,rgerhards
    -
    -This rule directs all messages with a priority of alert or higher to -the terminals of the operator, i.e. of the users "root'' and -"rgerhards'' if they're logged in.
    -
    -
    -*.* @finlandia
    -
    -This rule would redirect all messages to a remote host called -finlandia. This is useful especially in a cluster of machines where all -syslog messages will be stored on only one machine.
    -
    -In the format shown above, UDP is used for transmitting the message. -The destination port is set to the default auf 514. Rsyslog is also -capable of using much more secure and reliable TCP sessions for message -forwarding. Also, the destination port can be specified. To select TCP, -simply add one additional @ in front of the host name (that is, @host -is UPD, @@host is TCP). For example:
    -
    -
    -*.* @@finlandia
    -
    -To specify the destination port on the remote machine, use a colon -followed by the port number after the machine name. The following -forwards to port 1514 on finlandia:
    -
    -
    -*.* @@finlandia:1514
    -
    -This syntax works both with TCP and UDP based syslog. However, you will -probably primarily need it for TCP, as there is no well-accepted port -for this transport (it is non-standard). For UDP, you can usually stick -with the default auf 514, but might want to modify it for security rea-
    -sons. If you would like to do that, it's quite easy:
    -
    -
    -*.* @finlandia:1514
    -
    -
    -
    -*.* >dbhost,dbname,dbuser,dbpassword;dbtemplate
    -
    -This rule writes all message to the database "dbname" hosted on -"dbhost". The login is done with user "dbuser" and password -"dbpassword". The actual table that is updated is specified within the -template (which contains the insert statement). The template is called -"dbtemplate" in this case.

    -

    :msg,contains,"error" @errorServer

    -

    This rule forwards all messages that contain the word "error" -in the msg part to the server "errorServer". Forwarding is via UDP. -Please note the colon in fron

    -

    CONFIGURATION FILE SYNTAX DIFFERENCES

    +

    Configuration File Syntax Differences

    Rsyslogd uses a slightly different syntax for its configuration file than the original BSD sources. Originally all messages of a specific priority and above were forwarded to the log @@ -1272,4 +68,15 @@ additional features (like template and database support). For obvious reasons, the syntax for defining such features is available in rsyslogd, only.

    - + +

    [back to top] +[manual index] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + + +> diff --git a/doc/rsyslog_high_database_rate.html b/doc/rsyslog_high_database_rate.html index 158a4df6..2bae58c6 100644 --- a/doc/rsyslog_high_database_rate.html +++ b/doc/rsyslog_high_database_rate.html @@ -7,6 +7,7 @@ +back

    Handling a massive syslog database insert rate with Rsyslog

    @@ -171,6 +172,14 @@ comments or find bugs (I *do* bugs - no way... ;)), please http://www.gnu.org/copyleft/fdl.html.

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    diff --git a/doc/rsyslog_mysql.html b/doc/rsyslog_mysql.html index 753c86ec..a27bd59e 100644 --- a/doc/rsyslog_mysql.html +++ b/doc/rsyslog_mysql.html @@ -1,6 +1,6 @@ Writing syslog Data to MySQL - +back

    Writing syslog messages to MySQL

    @@ -259,4 +259,13 @@ document under the terms of the GNU Free Documentation License, Version with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be viewed at http://www.gnu.org/copyleft/fdl.html.

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + diff --git a/doc/rsyslog_ng_comparison.html b/doc/rsyslog_ng_comparison.html index 2f383f78..8e121a8d 100644 --- a/doc/rsyslog_ng_comparison.html +++ b/doc/rsyslog_ng_comparison.html @@ -1,6 +1,7 @@ rsyslog vs. syslog-ng - a comparison +back

    rsyslog vs. syslog-ng

    Written by Rainer Gerhards (2008-05-06)

    @@ -584,4 +585,13 @@ feature sheet. I have not yet been able to fully work through it. In the mean time, you may want to read it in parallel. It is available at Balabit's site.

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + diff --git a/doc/rsyslog_stunnel.html b/doc/rsyslog_stunnel.html index 104a672e..f4f82cd0 100644 --- a/doc/rsyslog_stunnel.html +++ b/doc/rsyslog_stunnel.html @@ -1,5 +1,6 @@ +back SSL Encrypting syslog with stunnel

    SSL Encrypting Syslog with Stunnel

    @@ -236,5 +237,13 @@ comments or find bugs (I *do* bugs - no way... ;)), please Texts. A copy of the license can be viewed at http://www.gnu.org/copyleft/fdl.html.

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    - \ No newline at end of file + diff --git a/doc/rsyslog_tls.html b/doc/rsyslog_tls.html index 7d156c3a..ebb08ebe 100644 --- a/doc/rsyslog_tls.html +++ b/doc/rsyslog_tls.html @@ -1,5 +1,6 @@ TLS (SSL) Encrypting syslog +back @@ -304,4 +305,13 @@ document under the terms of the GNU Free Documentation License, Version with no Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be viewed at http://www.gnu.org/copyleft/fdl.html.

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + diff --git a/doc/syslog_protocol.html b/doc/syslog_protocol.html index 72de5c27..57eb9ffe 100644 --- a/doc/syslog_protocol.html +++ b/doc/syslog_protocol.html @@ -3,6 +3,7 @@ syslog-protocol support in rsyslog +back

    syslog-protocol support in rsyslog

    Rsyslog  provides a trial implementation of the proposed @@ -191,6 +192,14 @@ discussed ;)

    syslog-protocol should be further evaluated and be fully understood

     

    +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    -- cgit v1.2.3 From 3c615c60beb851f3a42cea3fcc31f4a2243cedaa Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 7 Nov 2008 14:49:24 +0100 Subject: doc: added link to regular expression checker online tool --- doc/manual.html | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual.html b/doc/manual.html index f1b7d657..1d09d460 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -33,6 +33,7 @@ the links below for the

    • troubleshooting rsyslog problems
    • configuration file syntax (rsyslog.conf)
      • +
    • a regular expression checker/generator tool for rsyslog
    • property replacer, an important core component
    • a commented sample rsyslog.conf
    • rsyslog bug list
      • -- cgit v1.2.3 From 170d0d6f375241e0d0ca85a1327df82165fec439 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 11 Nov 2008 11:00:37 +0100 Subject: added forgotten files they were new after restructuring the doc... --- doc/Makefile.am | 7 + doc/rsyslog_conf_actions.html | 336 ++++++++++++++++++++++++++++++++++++++++ doc/rsyslog_conf_examples.html | 209 +++++++++++++++++++++++++ doc/rsyslog_conf_filter.html | 280 +++++++++++++++++++++++++++++++++ doc/rsyslog_conf_global.html | 227 +++++++++++++++++++++++++++ doc/rsyslog_conf_modules.html | 53 +++++++ doc/rsyslog_conf_output.html | 81 ++++++++++ doc/rsyslog_conf_templates.html | 150 ++++++++++++++++++ 8 files changed, 1343 insertions(+) create mode 100644 doc/rsyslog_conf_actions.html create mode 100644 doc/rsyslog_conf_examples.html create mode 100644 doc/rsyslog_conf_filter.html create mode 100644 doc/rsyslog_conf_global.html create mode 100644 doc/rsyslog_conf_modules.html create mode 100644 doc/rsyslog_conf_output.html create mode 100644 doc/rsyslog_conf_templates.html diff --git a/doc/Makefile.am b/doc/Makefile.am index 22d368e0..fef1e44c 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -98,6 +98,13 @@ html_files = \ omrelp.html \ status.html \ troubleshoot.html \ + rsyslog_conf_actions.html \ + rsyslog_conf_examples.html \ + rsyslog_conf_filter.html \ + rsyslog_conf_global.html \ + rsyslog_conf_modules.html \ + rsyslog_conf_output.html \ + rsyslog_conf_templates.html \ src/classes.dia EXTRA_DIST = $(html_files) diff --git a/doc/rsyslog_conf_actions.html b/doc/rsyslog_conf_actions.html new file mode 100644 index 00000000..2ef3f4b0 --- /dev/null +++ b/doc/rsyslog_conf_actions.html @@ -0,0 +1,336 @@ + +Actions - rsyslog.conf + +

      This is a part of the rsyslog.conf documentation.

      +back +

      Actions

      +

      The action field of a rule describes what to do with the +message. In general, message content is written to a kind of "logfile". +But also other actions might be done, like writing to a database table +or forwarding to another host.
      +
      +Templates can be used with all actions. If used, the specified template +is used to generate the message content (instead of the default +template). To specify a template, write a semicolon after the action +value immediately followed by the template name.
      +
      +Beware: templates MUST be defined BEFORE they are used. It is OK to +define some templates, then use them in selector lines, define more +templates and use use them in the following selector lines. But it is +NOT permitted to use a template in a selector line that is above its +definition. If you do this, the action will be ignored.

      +

      You can have multiple actions for a single selector  (or +more precisely a single filter of such a selector line). Each action +must be on its own line and the line must start with an ampersand +('&') character and have no filters. An example would be

      +

      *.=crit rger
      +& root
      +& /var/log/critmsgs

      +

      These three lines send critical messages to the user rger and +root and also store them in /var/log/critmsgs. Using multiple +actions per selector is convenient and also offers +a performance benefit. As the filter needs to be evaluated +only once, there is less computation required to process the directive +compared to the otherwise-equal config directives below:

      +

      *.=crit rger
      +*.=crit root
      +*.=crit /var/log/critmsgs

      +

       

      +

      Regular File

      +

      Typically messages are logged to real files. The file has to +be specified with full pathname, beginning with a slash "/''.
      +
      +
      +You may prefix each entry with the minus "-'' sign to omit syncing the +file after every logging. Note that you might lose information if the +system crashes right behind a write attempt. Nevertheless this might +give you back some performance, especially if you run programs that use +logging in a very verbose manner.

      +

      If your system is connected to a reliable UPS and you receive +lots of log data (e.g. firewall logs), it might be a very good idea to +turn of +syncing by specifying the "-" in front of the file name.

      +

      The filename can be either static (always +the same) or dynamic (different based on message +received). The later is useful if you would automatically split +messages into different files based on some message criteria. For +example, dynamic file name selectors allow you to split messages into +different files based on the host that sent them. With dynamic file +names, everything is automatic and you do not need any filters.

      +

      It works via the template system. First, you define a template +for the file name. An example can be seen above in the description of +template. We will use the "DynFile" template defined there. Dynamic +filenames are indicated by specifying a questions mark "?" instead of a +slash, followed by the template name. Thus, the selector line for our +dynamic file name would look as follows:

      +
      +*.* ?DynFile +
      +

      That's all you need to do. Rsyslog will now automatically +generate file names for you and store the right messages into the right +files. Please note that the minus sign also works with dynamic file +name selectors. Thus, to avoid syncing, you may use

      +
      +*.* -?DynFile
      +

      And of course you can use templates to specify the output +format:

      +
      +*.* ?DynFile;MyTemplate
      +

      A word of caution: rsyslog creates files as +needed. So if a new host is using your syslog server, rsyslog will +automatically create a new file for it.

      +

      Creating directories is also supported. For +example you can use the hostname as directory and the program name as +file name:

      +
      +$template DynFile,"/var/log/%HOSTNAME%/%programname%.log"
      +

      Named Pipes

      +

      This version of rsyslogd(8) has support for logging output to +named pipes (fifos). A fifo or named pipe can be used as a destination +for log messages by prepending a pipe symbol ("|'') to the name of the +file. This is handy for debugging. Note that the fifo must be created +with the mkfifo(1) command before rsyslogd(8) is started.

      +

      Terminal and Console

      +

      If the file you specified is a tty, special tty-handling is +done, same with /dev/console.

      +

      Remote Machine

      +

      Rsyslogd provides full remote logging, i.e. is able to send +messages to a remote host running rsyslogd(8) and to receive messages +from remote hosts. Using this feature you're able to control all syslog +messages on one host, if all other machines will log remotely to that. +This tears down
      +administration needs.
      +
      +Please note that this version of rsyslogd by default does NOT +forward messages it has received from the network to another host. +Specify the "-h" option to enable this.

      +

      To forward messages to another host, prepend the hostname with +the at sign ("@"). A single at sign means that messages will +be forwarded via UDP protocol (the standard for syslog). If you prepend +two at signs ("@@"), the messages will be transmitted via TCP. Please +note that plain TCP based syslog is not officially standardized, but +most major syslogds support it (e.g. syslog-ng or WinSyslog). The +forwarding action indicator (at-sign) can be followed by one or more +options. If they are given, they must be immediately (without a space) +following the final at sign and be enclosed in parenthesis. The +individual options must be separated by commas. The following options +are right now defined:

      + + + + + + + + + + + +
      +

      z<number>

      +      		Enable zlib-compression for the message. The +<number> is the compression level. It can be 1 (lowest +gain, lowest CPU overhead) to 9 (maximum compression, highest CPU +overhead). The level can also be 0, which means "no compression". If +given, the "z" option is ignored. So this does not make an awful lot of +sense. There is hardly a difference between level 1 and 9 for typical +syslog messages. You can expect a compression gain between 0% and 30% +for typical messages. Very chatty messages may compress up to 50%, but +this is seldom seen with typically traffic. Please note that rsyslogd +checks the compression gain. Messages with 60 bytes or less will never +be compressed. This is because compression gain is pretty unlikely and +we prefer to save CPU cycles. Messages over that size are always +compressed. However, it is checked if there is a gain in compression +and only if there is, the compressed message is transmitted. Otherwise, +the uncompressed messages is transmitted. This saves the receiver CPU +cycles for decompression. It also prevents small message to actually +become larger in compressed form. +

      Please note that when a TCP transport is used, +compression will also turn on syslog-transport-tls framing. See the "o" +option for important information on the implications.

      +

      Compressed messages are automatically detected and +decompressed by the receiver. There is nothing that needs to be +configured on the receiver side.

      +
      +

      o

      +      		This option is experimental. Use at your own +risk and only if you know why you need it! If in doubt, do NOT turn it +on. +

      This option is only valid for plain TCP based +transports. It selects a different framing based on IETF internet draft +syslog-transport-tls-06. This framing offers some benefits over +traditional LF-based framing. However, the standardization effort is +not yet complete. There may be changes in upcoming versions of this +standard. Rsyslog will be kept in line with the standard. There is some +chance that upcoming changes will be incompatible to the current +specification. In this case, all systems using -transport-tls framing +must be upgraded. There will be no effort made to retain compatibility +between different versions of rsyslog. The primary reason for that is +that it seems technically impossible to provide compatibility between +some of those changes. So you should take this note very serious. It is +not something we do not *like* to do (and may change our mind if enough +people beg...), it is something we most probably *can not* do for +technical reasons (aka: you can beg as much as you like, it won't +change anything...).

      +

      The most important implication is that compressed syslog +messages via TCP must be considered with care. Unfortunately, it is +technically impossible to transfer compressed records over traditional +syslog plain tcp transports, so you are left with two evil choices...

      +
      +


      +The hostname may be followed by a colon and the destination port.

      +

      The following is an example selector line with forwarding:

      +

      *.*    @@(o,z9)192.168.0.1:1470

      +

      In this example, messages are forwarded via plain TCP with +experimental framing and maximum compression to the host 192.168.0.1 at +port 1470.

      +

      *.* @192.168.0.1

      +

      In the example above, messages are forwarded via UDP to the +machine 192.168.0.1, the destination port defaults to 514. Messages +will not be compressed.

      +

      Note that IPv6 addresses contain colons. So if an IPv6 address is specified +in the hostname part, rsyslogd could not detect where the IP address ends +and where the port starts. There is a syntax extension to support this: +put squary brackets around the address (e.g. "[2001::1]"). Square +brackets also work with real host names and IPv4 addresses, too. +

      A valid sample to send messages to the IPv6 host 2001::1 at port 515 +is as follows: +

      *.* @[2001::1]:515 +

      This works with TCP, too. +

      Note to sysklogd users: sysklogd does not +support RFC 3164 format, which is the default forwarding template in +rsyslog. As such, you will experience duplicate hostnames if rsyslog is +the sender and sysklogd is the receiver. The fix is simple: you need to +use a different template. Use that one:

      +

      $template +sysklogd,"<%PRI%>%TIMESTAMP% %syslogtag%%msg%\""
      +*.* @192.168.0.1;sysklogd

      +

      List of Users

      +

      Usually critical messages are also directed to "root'' on +that machine. You can specify a list of users that shall get the +message by simply writing the login. You may specify more than one user +by separating them with commas (",''). If they're logged in they get +the message. Don't think a mail would be sent, that might be too late.

      +

      Everyone logged on

      +

      Emergency messages often go to all users currently online to +notify them that something strange is happening with the system. To +specify this wall(1)-feature use an asterisk ("*'').

      +

      Call Plugin

      +

      This is a generic way to call an output plugin. The plugin +must support this functionality. Actual parameters depend on the +module, so see the module's doc on what to supply. The general syntax +is as follows:

      +

      :modname:params;template

      +

      Currently, the ommysql database output module supports this +syntax (in addtion to the ">" syntax it traditionally +supported). For ommysql, the module name is "ommysql" and the params +are the traditional ones. The ;template part is not module specific, it +is generic rsyslog functionality available to all modules.

      +

      As an example, the ommysql module may be called as follows:

      +

      :ommysql:dbhost,dbname,dbuser,dbpassword;dbtemplate

      +

      For details, please see the "Database Table" section of this +documentation.

      +

      Note: as of this writing, the ":modname:" part is hardcoded +into the module. So the name to use is not necessarily the name the +module's plugin file is called.

      +

      Database Table

      +

      This allows logging of the message to a database table. +Currently, only MySQL databases are supported. However, other database +drivers will most probably be developed as plugins. By default, a MonitorWare-compatible +schema is required for this to work. You can create that schema with +the createDB.SQL file that came with the rsyslog package. You can also
      +use any other schema of your liking - you just need to define a proper +template and assign this template to the action.
      +
      +The database writer is called by specifying a greater-then sign +(">") in front of the database connect information. Immediately +after that
      +sign the database host name must be given, a comma, the database name, +another comma, the database user, a comma and then the user's password. +If a specific template is to be used, a semicolon followed by the +template name can follow the connect information. This is as follows:
      +
      +>dbhost,dbname,dbuser,dbpassword;dbtemplate

      +

      Important: to use the database functionality, the +MySQL output module must be loaded in the config file BEFORE +the first database table action is used. This is done by placing the

      +

      $ModLoad ommysql

      +

      directive some place above the first use of the database write +(we recommend doing at the the beginning of the config file).

      +

      Discard

      +

      If the discard action is carried out, the received message is +immediately discarded. No further processing of it occurs. Discard has +primarily been added to filter out messages before carrying on any +further processing. For obvious reasons, the results of "discard" are +depending on where in the configuration file it is being used. Please +note that once a message has been discarded there is no way to retrieve +it in later configuration file lines.

      +

      Discard can be highly effective if you want to filter out some +annoying messages that otherwise would fill your log files. To do that, +place the discard actions early in your log files. This often plays +well with property-based filters, giving you great freedom in +specifying what you do not want.

      +

      Discard is just the single tilde character with no further +parameters:

      +

      ~

      +

      For example,

      +

      *.*   ~

      +

      discards everything (ok, you can achive the same by not +running rsyslogd at all...).

      +

      Output Channel

      +

      Binds an output channel definition (see there for details) to +this action. Output channel actions must start with a $-sign, e.g. if +you would like to bind your output channel definition "mychannel" to +the action, use "$mychannel". Output channels support template +definitions like all all other actions.

      +

      Shell Execute

      +

      This executes a program in a subshell. The program is passed +the template-generated message as the only command line parameter. +Rsyslog waits until the program terminates and only then continues to +run.

      +

      ^program-to-execute;template

      +

      The program-to-execute can be any valid executable. It +receives the template string as a single parameter (argv[1]).

      +

      WARNING: The Shell Execute action was added +to serve an urgent need. While it is considered reasonable save when +used with some thinking, its implications must be considered. The +current implementation uses a system() call to execute the command. +This is not the best way to do it (and will hopefully changed in +further releases). Also, proper escaping of special characters is done +to prevent command injection. However, attackers always find smart ways +to circumvent escaping, so we can not say if the escaping applied will +really safe you from all hassles. Lastly, rsyslog will wait until the +shell command terminates. Thus, a program error in it (e.g. an infinite +loop) can actually disable rsyslog. Even without that, during the +programs run-time no messages are processed by rsyslog. As the IP +stacks buffers are quickly overflowed, this bears an increased risk of +message loss. You must be aware of these implications. Even though they +are severe, there are several cases where the "shell execute" action is +very useful. This is the reason why we have included it in its current +form. To mitigate its risks, always a) test your program thoroughly, b) +make sure its runtime is as short as possible (if it requires a longer +run-time, you might want to spawn your own sub-shell asynchronously), +c) apply proper firewalling so that only known senders can send syslog +messages to rsyslog. Point c) is especially important: if rsyslog is +accepting message from any hosts, chances are much higher that an +attacker might try to exploit the "shell execute" action.

      +

      Template Name

      +

      Every ACTION can be followed by a template name. If so, that +template is used for message formatting. If no name is given, a +hard-coded default template is used for the action. There can only be +one template name for each given action. The default template is +specific to each action. For a description of what a template is and +what you can do with it, see "TEMPLATES" at the top of this document.

      + + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + diff --git a/doc/rsyslog_conf_examples.html b/doc/rsyslog_conf_examples.html new file mode 100644 index 00000000..b46460e5 --- /dev/null +++ b/doc/rsyslog_conf_examples.html @@ -0,0 +1,209 @@ + +Examples - rsyslog.conf + +

      This is a part of the rsyslog.conf documentation.

      +back +

      Examples

      +

      Below are example for templates and selector lines. I hope +they are self-explanatory. If not, please see +www.monitorware.com/rsyslog/ for advise.

      +

      TEMPLATES

      +

      Please note that the samples are split across multiple lines. +A template MUST NOT actually be split across multiple lines.
      +
      +A template that resembles traditional syslogd file output:
      +$template TraditionalFormat,"%timegenerated% %HOSTNAME%
      +%syslogtag%%msg:::drop-last-lf%\n"
      +
      +A template that tells you a little more about the message:
      +$template +precise,"%syslogpriority%,%syslogfacility%,%timegenerated%,%HOSTNAME%,
      +%syslogtag%,%msg%\n"
      +
      +A template for RFC 3164 format:
      +$template RFC3164fmt,"<%PRI%>%TIMESTAMP% %HOSTNAME% +%syslogtag%%msg%"
      +
      +A template for the format traditonally used for user messages:
      +$template usermsg," XXXX%syslogtag%%msg%\n\r"
      +
      +And a template with the traditonal wall-message format:
      +$template wallmsg,"\r\n\7Message from syslogd@%HOSTNAME% at +%timegenerated%
      +
      +A template that can be used for the database write (please note the SQL
      +template option)
      +$template MySQLInsert,"insert iut, message, receivedat values
      +('%iut%', '%msg:::UPPERCASE%', '%timegenerated:::date-mysql%')
      +into systemevents\r\n", SQL
      +
      +The following template emulates WinSyslog +format (it's an Adiscon +format, you do not feel bad if you don't know it ;)). It's interesting +to see how it takes different parts out of the date stamps. What +happens is that the date stamp is split into the actual date and time +and the these two are combined with just a comma in between them.
      +
      +$template WinSyslogFmt,"%HOSTNAME%,%timegenerated:1:10:date-rfc3339%,
      +%timegenerated:12:19:date-rfc3339%,%timegenerated:1:10:date-rfc3339%,
      +%timegenerated:12:19:date-rfc3339%,%syslogfacility%,%syslogpriority%,
      +%syslogtag%%msg%\n"

      +

      SELECTOR LINES

      +

      # Store critical stuff in critical
      +#
      +*.=crit;kern.none /var/adm/critical
      +
      +This will store all messages with the priority crit in the file +/var/adm/critical, except for any kernel message.
      +
      +
      +# Kernel messages are first, stored in the kernel
      +# file, critical messages and higher ones also go
      +# to another host and to the console. Messages to
      +# the host finlandia are forwarded in RFC 3164
      +# format (using the template defined above).
      +#
      +kern.* /var/adm/kernel
      +kern.crit @finlandia;RFC3164fmt
      +kern.crit /dev/console
      +kern.info;kern.!err /var/adm/kernel-info
      +
      +The first rule direct any message that has the kernel facility to the +file /var/adm/kernel.
      +
      +The second statement directs all kernel messages of the priority crit +and higher to the remote host finlandia. This is useful, because if the +host crashes and the disks get irreparable errors you might not be able +to read the stored messages. If they're on a remote host, too, you +still can try to find out the reason for the crash.
      +
      +The third rule directs these messages to the actual console, so the +person who works on the machine will get them, too.
      +
      +The fourth line tells rsyslogd to save all kernel messages that come +with priorities from info up to warning in the file +/var/adm/kernel-info. Everything from err and higher is excluded.
      +
      +
      +# The tcp wrapper loggs with mail.info, we display
      +# all the connections on tty12
      +#
      +mail.=info /dev/tty12
      +
      +This directs all messages that uses mail.info (in source LOG_MAIL | +LOG_INFO) to /dev/tty12, the 12th console. For example the tcpwrapper +tcpd(8) uses this as it's default.
      +
      +
      +# Store all mail concerning stuff in a file
      +#
      +mail.*;mail.!=info /var/adm/mail
      +
      +This pattern matches all messages that come with the mail facility, +except for the info priority. These will be stored in the file +/var/adm/mail.
      +
      +
      +# Log all mail.info and news.info messages to info
      +#
      +mail,news.=info /var/adm/info
      +
      +This will extract all messages that come either with mail.info or with +news.info and store them in the file /var/adm/info.
      +
      +
      +# Log info and notice messages to messages file
      +#
      +*.=info;*.=notice;\
      +mail.none /var/log/messages
      +
      +This lets rsyslogd log all messages that come with either the info or +the notice facility into the file /var/log/messages, except for all
      +messages that use the mail facility.
      +
      +
      +# Log info messages to messages file
      +#
      +*.=info;\
      +mail,news.none /var/log/messages
      +
      +This statement causes rsyslogd to log all messages that come with the +info priority to the file /var/log/messages. But any message coming +either with the mail or the news facility will not be stored.
      +
      +
      +# Emergency messages will be displayed using wall
      +#
      +*.=emerg *
      +
      +This rule tells rsyslogd to write all emergency messages to all +currently logged in users. This is the wall action.
      +
      +
      +# Messages of the priority alert will be directed
      +# to the operator
      +#
      +*.alert root,rgerhards
      +
      +This rule directs all messages with a priority of alert or higher to +the terminals of the operator, i.e. of the users "root'' and +"rgerhards'' if they're logged in.
      +
      +
      +*.* @finlandia
      +
      +This rule would redirect all messages to a remote host called +finlandia. This is useful especially in a cluster of machines where all +syslog messages will be stored on only one machine.
      +
      +In the format shown above, UDP is used for transmitting the message. +The destination port is set to the default auf 514. Rsyslog is also +capable of using much more secure and reliable TCP sessions for message +forwarding. Also, the destination port can be specified. To select TCP, +simply add one additional @ in front of the host name (that is, @host +is UPD, @@host is TCP). For example:
      +
      +
      +*.* @@finlandia
      +
      +To specify the destination port on the remote machine, use a colon +followed by the port number after the machine name. The following +forwards to port 1514 on finlandia:
      +
      +
      +*.* @@finlandia:1514
      +
      +This syntax works both with TCP and UDP based syslog. However, you will +probably primarily need it for TCP, as there is no well-accepted port +for this transport (it is non-standard). For UDP, you can usually stick +with the default auf 514, but might want to modify it for security rea-
      +sons. If you would like to do that, it's quite easy:
      +
      +
      +*.* @finlandia:1514
      +
      +
      +
      +*.* >dbhost,dbname,dbuser,dbpassword;dbtemplate
      +
      +This rule writes all message to the database "dbname" hosted on +"dbhost". The login is done with user "dbuser" and password +"dbpassword". The actual table that is updated is specified within the +template (which contains the insert statement). The template is called +"dbtemplate" in this case.

      +

      :msg,contains,"error" @errorServer

      +

      This rule forwards all messages that contain the word "error" +in the msg part to the server "errorServer". Forwarding is via UDP. +Please note the colon in fron

      + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + diff --git a/doc/rsyslog_conf_filter.html b/doc/rsyslog_conf_filter.html new file mode 100644 index 00000000..55244c15 --- /dev/null +++ b/doc/rsyslog_conf_filter.html @@ -0,0 +1,280 @@ + +Filter Conditions - rsyslog.conf + +

      This is a part of the rsyslog.conf documentation.

      +back +

      Filter Conditions

      +

      Rsyslog offers four different types "filter conditions":

      +
        +
      • BSD-style blocks
        • +
      • "traditional" severity and facility based selectors
        • +
      • property-based filters
        • +
      • expression-based filters
        • +
      +

      Blocks

      +

      Rsyslogd supports BSD-style blocks inside rsyslog.conf. Each +block of lines is separated from the previous block by a program or +hostname specification. A block will only log messages corresponding to +the most recent program and hostname specifications given. Thus, a +block which selects ‘ppp’ as the program, directly followed by a block +that selects messages from the hostname ‘dialhost’, then the second +block will only log messages from the ppp program on dialhost. +

      +

      A program specification is a line beginning with ‘!prog’ and +the following blocks will be associated with calls to syslog from that +specific program. A program specification for ‘foo’ will also match any +message logged by the kernel with the prefix ‘foo: ’. Alternatively, a +program specification ‘-foo’ causes the following blocks to be applied +to messages from any program but the one specified. A hostname +specification of the form ‘+hostname’ and the following blocks will be +applied to messages received from the specified hostname. +Alternatively, a hostname specification ‘-hostname’ causes the +following blocks to be applied to messages from any host but the one +specified. If the hostname is given as ‘@’, the local hostname will be +used. (NOT YET IMPLEMENTED) A program or hostname specification may be +reset by giving the program or hostname as ‘*’.

      +

      Please note that the "#!prog", "#+hostname" and "#-hostname" +syntax available in BSD syslogd is not supported by rsyslogd. By +default, no hostname or program is set.

      +

      Selectors

      +

      Selectors are the traditional way of filtering syslog +messages. They have been kept in rsyslog with their original +syntax, because it is well-known, highly effective and also needed for +compatibility with stock syslogd configuration files. If you just need +to filter based on priority and facility, you should do this with +selector lines. They are not second-class citizens +in rsyslog and offer the best performance for this job.

      +

      The selector field itself again consists of two parts, a +facility and a priority, separated by a period (".''). Both parts are +case insensitive and can also be specified as decimal numbers, but +don't do that, you have been warned. Both facilities and priorities are +described in rsyslog(3). The names mentioned below correspond to the +similar LOG_-values in /usr/include/rsyslog.h.
      +
      +The facility is one of the following keywords: auth, authpriv, cron, +daemon, kern, lpr, mail, mark, news, security (same as auth), syslog, +user, uucp and local0 through local7. The keyword security should not +be used anymore and mark is only for internal use and therefore should +not be used in applications. Anyway, you may want to specify and +redirect these messages here. The facility specifies the subsystem that +produced the message, i.e. all mail programs log with the mail facility +(LOG_MAIL) if they log using syslog.
      +
      +The priority is one of the following keywords, in ascending order: +debug, info, notice, warning, warn (same as warning), err, error (same +as err), crit, alert, emerg, panic (same as emerg). The keywords error, +warn and panic are deprecated and should not be used anymore. The +priority defines the severity of the message.
      +
      +The behavior of the original BSD syslogd is that all messages of the +specified priority and higher are logged according to the given action. +Rsyslogd behaves the same, but has some extensions.
      +
      +In addition to the above mentioned names the rsyslogd(8) understands +the following extensions: An asterisk ("*'') stands for all facilities +or all priorities, depending on where it is used (before or after the +period). The keyword none stands for no priority of the given facility.
      +
      +You can specify multiple facilities with the same priority pattern in +one statement using the comma (",'') operator. You may specify as much +facilities as you want. Remember that only the facility part from such +a statement is taken, a priority part would be skipped.

      +

      Multiple selectors may be specified for a single action using +the semicolon (";'') separator. Remember that each selector in the +selector field is capable to overwrite the preceding ones. Using this +behavior you can exclude some priorities from the pattern.

      +

      Rsyslogd has a syntax extension to the original BSD source, +that makes its use more intuitively. You may precede every priority +with an equation sign ("='') to specify only this single priority and +not any of the above. You may also (both is valid, too) precede the +priority with an exclamation mark ("!'') to ignore all that +priorities, either exact this one or this and any higher priority. If +you use both extensions than the exclamation mark must occur before the +equation sign, just use it intuitively.

      +

      Property-Based Filters

      +

      Property-based filters are unique to rsyslogd. They allow to +filter on any property, like HOSTNAME, syslogtag and msg. A list of all +currently-supported properties can be found in the property replacer documentation +(but keep in mind that only the properties, not the replacer is +supported). With this filter, each properties can be checked against a +specified value, using a specified compare operation. Currently, there +is only a single compare operation (contains) available, but additional +operations will be added in the future.

      +

      A property-based filter must start with a colon in column 0. +This tells rsyslogd that it is the new filter type. The colon must be +followed by the property name, a comma, the name of the compare +operation to carry out, another comma and then the value to compare +against. This value must be quoted. There can be spaces and tabs +between the commas. Property names and compare operations are +case-sensitive, so "msg" works, while "MSG" is an invalid property +name. In brief, the syntax is as follows:

      +

      :property, [!]compare-operation, "value"

      +

      The following compare-operations are +currently supported:

      + + + + + + + + + + + + + + + + + + + +
      containsChecks if the string provided in value is contained in +the property. There must be an exact match, wildcards are not supported.
      isequalCompares the "value" string provided and the property +contents. These two values must be exactly equal to match. The +difference to contains is that contains searches for the value anywhere +inside the property value, whereas all characters must be identical for +isequal. As such, isequal is most useful for fields like syslogtag or +FROMHOST, where you probably know the exact contents.
      startswithChecks if the value is found exactly at the beginning +of the property value. For example, if you search for "val" with +

      :msg, startswith, "val"

      +

      it will be a match if msg contains "values are in this +message" but it won't match if the msg contains "There are values in +this message" (in the later case, contains would match). Please note +that "startswith" is by far faster than regular expressions. So even +once they are implemented, it can make very much sense +(performance-wise) to use "startswith".

      +
      regexCompares the property against the provided POSIX +regular +expression.
      +

      You can use the bang-character (!) immediately in front of a +compare-operation, the outcome of this operation is negated. For +example, if msg contains "This is an informative message", the +following sample would not match:

      +

      :msg, contains, "error"

      +

      but this one matches:

      +

      :msg, !contains, "error"

      +

      Using negation can be useful if you would like to do some +generic processing but exclude some specific events. You can use the +discard action in conjunction with that. A sample would be:

      +

      *.* +/var/log/allmsgs-including-informational.log
      +:msg, contains, "informational"  ~ +
      +*.* /var/log/allmsgs-but-informational.log

      +

      Do not overlook the red tilde in line 2! In this sample, all +messages are written to the file allmsgs-including-informational.log. +Then, all messages containing the string "informational" are discarded. +That means the config file lines below the "discard line" (number 2 in +our sample) will not be applied to this message. Then, all remaining +lines will also be written to the file allmsgs-but-informational.log.

      +

      Value is a quoted string. It supports some +escape sequences:

      +

      \" - the quote character (e.g. "String with \"Quotes\"")
      +\\ - the backslash character (e.g. "C:\\tmp")

      +

      Escape sequences always start with a backslash. Additional +escape sequences might be added in the future. Backslash characters must +be escaped. Any other sequence then those outlined above is invalid and +may lead to unpredictable results.

      +

      Probably, "msg" is the most prominent use case of property +based filters. It is the actual message text. If you would like to +filter based on some message content (e.g. the presence of a specific +code), this can be done easily by:

      +

      :msg, contains, "ID-4711"

      +

      This filter will match when the message contains the string +"ID-4711". Please note that the comparison is case-sensitive, so it +would not match if "id-4711" would be contained in the message.

      +

      :msg, regex, "fatal .* error"

      +

      This filter uses a POSIX regular expression. It matches when +the +string contains the words "fatal" and "error" with anything in between +(e.g. "fatal net error" and "fatal lib error" but not "fatal error" as +two spaces are required by the regular expression!).

      +

      Getting property-based filters right can sometimes be +challenging. In order to help you do it with as minimal effort as +possible, rsyslogd spits out debug information for all property-based +filters during their evaluation. To enable this, run rsyslogd in +foreground and specify the "-d" option.

      +

      Boolean operations inside property based filters (like +'message contains "ID17" or message contains "ID18"') are currently not +supported (except for "not" as outlined above). Please note that while +it is possible to query facility and severity via property-based +filters, it is far more advisable to use classic selectors (see above) +for those cases.

      +

      Expression-Based Filters

      +Expression based filters allow +filtering on arbitrary complex expressions, which can include boolean, +arithmetic and string operations. Expression filters will evolve into a +full configuration scripting language. Unfortunately, their syntax will +slightly change during that process. So if you use them now, you need +to be prepared to change your configuration files some time later. +However, we try to implement the scripting facility as soon as possible +(also in respect to stage work needed). So the window of exposure is +probably not too long.
      +
      +Expression based filters are indicated by the keyword "if" in column 1 +of a new line. They have this format:
      +
      +if expr then action-part-of-selector-line
      +
      +"If" and "then" are fixed keywords that mus be present. "expr" is a +(potentially quite complex) expression. So the expression documentation for +details. "action-part-of-selector-line" is an action, just as you know +it (e.g. "/var/log/logfile" to write to that file).
      +
      +A few quick samples:
      +
      + +*.* /var/log/file1 # the traditional way
      +if $msg contains 'error' /var/log/errlog # the expression-based way
      +       +
      +Right now, you need to specify numerical values if you would like to +check for facilities and severity. These can be found in RFC 3164. +If you don't like that, you can of course also use the textual property +- just be sure to use the right one. As expression support is enhanced, +this will change. For example, if you would like to filter on message +that have facility local0, start with "DEVNAME" and have either +"error1" or "error0" in their message content, you could use the +following filter:
      +
      + +if $syslogfacility-text == 'local0' and $msg +startswith 'DEVNAME' and ($msg contains 'error1' or $msg contains +'error0') then /var/log/somelog
      +       +
      +Please note that the above must +all be on one line! And if you would like to store all +messages except those that contain "error1" or "error0", you just need +to add a "not":
      +
      + +if $syslogfacility-text == 'local0' and $msg +startswith 'DEVNAME' and not +($msg contains 'error1' or $msg contains +'error0') then /var/log/somelog
      +       +
      +If you would like to do case-insensitive comparisons, use +"contains_i" instead of "contains" and "startswith_i" instead of +"startswith".
      +
      +Note that regular expressions are currently NOT +supported in expression-based filters. These will be added later when +function support is added to the expression engine (the reason is that +regular expressions will be a separate loadable module, which requires +some more prequisites before it can be implemented).
      + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html new file mode 100644 index 00000000..bc618dd0 --- /dev/null +++ b/doc/rsyslog_conf_global.html @@ -0,0 +1,227 @@ + +Global Directives - rsyslog.conf + +

      This is a part of the rsyslog.conf documentation.

      +back +

      Global Directives

      +

      All global directives need to be specified on a line by their +own and must start with a dollar-sign. Here is a list in alphabetical +order. Follow links for a description.

      +

      Please note that not all directives here are actually global. Some affect +only the next action. This documentation will be changed soon. +

      Not all directives have an in-depth description right now. +Default values for them are in bold. A more in-depth description will +appear as implementation progresses. +

      +

      Be sure to read information about queues in rsyslog - +many parameter settings modify queue parameters. If in doubt, use the +default, it is usually well-chosen and applicable in most cases.

      +
        +
      • $ActionExecOnlyWhenPreviousIsSuspended
        • +
      • $ActionExecOnlyOnceEveryInterval <seconds> - +execute action only if the last execute is at last +<seconds> seconds in the past (more info in ommail, +but may be used with any action)
        • +
      • $ActionExecOnlyEveryNthTime <number> - If configured, the next action will +only be executed every n-th time. For example, if configured to 3, the first two messages +that go into the action will be dropped, the 3rd will actually cause the action to execute, +the 4th and 5th will be dropped, the 6th executed under the action, ... and so on. Note: +this setting is automatically re-set when the actual action is defined.
        • +
      • $ActionExecOnlyEveryNthTimeTimeout <number-of-seconds> - has a meaning only if +$ActionExecOnlyEveryNthTime is also configured for the same action. If so, the timeout +setting specifies after which period the counting of "previous actions" expires and +a new action count is begun. Specify 0 (the default) to disable timeouts. +
        +Why is this option needed? Consider this case: a message comes in at, eg., 10am. That's +count 1. Then, nothing happens for the next 10 hours. At 8pm, the next +one occurs. That's count 2. Another 5 hours later, the next message +occurs, bringing the total count to 3. Thus, this message now triggers +the rule. +
        +The question is if this is desired behavior? Or should the rule only be +triggered if the messages occur within an e.g. 20 minute window? If the +later is the case, you need a +
        +$ActionExecOnlyEveryNthTimeTimeout 1200 +
        +This directive will timeout previous messages seen if they are older +than 20 minutes. In the example above, the count would now be always 1 +and consequently no rule would ever be triggered. + +
      • $ActionFileDefaultTemplate [templateName] - sets a new default template for file actions
        • +
      • $ActionFileEnableSync [on/off] - enables file +syncing capability of omfile
        • +
      • $ActionForwardDefaultTemplate [templateName] - sets a new +default template for UDP and plain TCP forwarding action
        • +
      • $ActionGSSForwardDefaultTemplate [templateName] - sets a +new default template for GSS-API forwarding action
        • +
      • $ActionQueueCheckpointInterval <number>
        • +
      • $ActionQueueDequeueSlowdown <number> [number +is timeout in microseconds (1000000us is 1sec!), +default 0 (no delay). Simple rate-limiting!]
        • +
      • $ActionQueueDiscardMark <number> [default +9750]
        • +
      • $ActionQueueDiscardSeverity <number> +[*numerical* severity! default 4 (warning)]
        • +
      • $ActionQueueFileName <name>
        • +
      • $ActionQueueHighWaterMark <number> [default +8000]
        • +
      • $ActionQueueImmediateShutdown [on/off]
        • +
      • $ActionQueueSize <number>
        • +
      • $ActionQueueLowWaterMark <number> [default +2000]
        • +
      • $ActionQueueMaxFileSize <size_nbr>, default 1m
        • +
      • $ActionQueueTimeoutActionCompletion <number> +[number is timeout in ms (1000ms is 1sec!), default 1000, 0 means +immediate!]
        • +
      • $ActionQueueTimeoutEnqueue <number> [number +is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]
        • +
      • $ActionQueueTimeoutShutdown <number> [number +is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]
        • +
      • $ActionQueueWorkerTimeoutThreadShutdown +<number> [number is timeout in ms (1000ms is 1sec!), +default 60000 (1 minute)]
        • +
      • $ActionQueueType [FixedArray/LinkedList/Direct/Disk]
        • +
      • $ActionQueueSaveOnShutdown  [on/off] +
        • +
      • $ActionQueueWorkerThreads <number>, num worker threads, default 1, recommended 1
        • +
      • $ActionQueueWorkerThreadMinumumMessages <number>, default 100
        • +
      • $ActionResumeInterval
        • +
      • $ActionResumeRetryCount <number> [default 0, -1 means eternal]
        • +
      • $ActionSendResendLastMsgOnReconn <[on/off]> specifies if the last message is to be resend when a connecition broken and has been reconnedcted. May increase reliability, but comes at the risk of message duplication. +
      • $ActionSendStreamDriver <driver basename> just like $DefaultNetstreamDriver, but for the specific action +
      • $ActionSendStreamDriverMode <mode>, default 0, mode to use with the stream driver +(driver-specific)
      • $ActionSendStreamDriverAuthMode <mode>,  authentication mode to use with the stream driver +(driver-specific)
      • $ActionSendStreamDriverPermittedPeer <ID>,  accepted fingerprint (SHA1) or name of remote peer +(driver-specific) - directive may go away!
        • +
      • $AllowedSender
        • +
      • $ControlCharacterEscapePrefix
        • +
      • $DebugPrintCFSyslineHandlerList
        • + +
      • $DebugPrintModuleList
        • +
      • $DebugPrintTemplateList
        • +
      • $DefaultNetstreamDriver <drivername>, the default network stream driver to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem>
        • +
      • $DefaultNetstreamDriverCertFile </path/to/certfile.pem>
        • +
      • $DefaultNetstreamDriverKeyFile </path/to/keyfile.pem>
        • +
      • $DirCreateMode
        • +
      • $DirGroup
        • +
      • $DirOwner
        • +
      • $DropMsgsWithMaliciousDnsPTRRecords
        • +
      • $DropTrailingLFOnReception
        • +
      • $DynaFileCacheSize
        • +
      • $EscapeControlCharactersOnReceive
        • +
      • $ErrorMessagesToStderr [on|off] - direct rsyslogd error message to stderr (in addition to other targets)
        • +
      • $FailOnChownFailure
        • +
      • $FileCreateMode
        • +
      • $FileGroup
        • +
      • $FileOwner
        • +
      • $GssForwardServiceName
        • +
      • $GssListenServiceName
        • +
      • $GssMode
        • +
      • $HUPisRestart [on/off] - if set to on, a HUP is a full daemon restart. This means any queued messages are discarded (depending +on queue configuration, of course) all modules are unloaded and reloaded. This mode keeps compatible with sysklogd, but is +not recommended for use with rsyslog. To do a full restart, simply stop and start the daemon. The default is "on" for +compatibility reasons. If it is set to "off", a HUP will only close open files. This is a much quicker action and usually +the only one that is needed e.g. for log rotation. It is recommended to set the setting to "off".
        • +
      • $IncludeConfig
      • MainMsgQueueCheckpointInterval <number>
        • +
      • $MainMsgQueueDequeueSlowdown <number> [number +is timeout in microseconds (1000000us is 1sec!), +default 0 (no delay). Simple rate-limiting!]
        • +
      • $MainMsgQueueDiscardMark <number> [default 9750]
        • +
      • $MainMsgQueueDiscardSeverity <severity> +[either a textual or numerical severity! default 4 (warning)]
        • +
      • $MainMsgQueueFileName <name>
        • +
      • $MainMsgQueueHighWaterMark <number> [default +8000]
        • +
      • $MainMsgQueueImmediateShutdown [on/off]
        • +
      • $MainMsgQueueSize
        • +
      • $MainMsgQueueLowWaterMark <number> [default +2000]
        • +
      • $MainMsgQueueMaxFileSize <size_nbr>, default +1m
        • +
      • $MainMsgQueueTimeoutActionCompletion +<number> [number is timeout in ms (1000ms is 1sec!), +default +1000, 0 means immediate!]
        • +
      • $MainMsgQueueTimeoutEnqueue <number> [number +is timeout in ms (1000ms is 1sec!), default 2000, 0 means indefinite]
        • +
      • $MainMsgQueueTimeoutShutdown <number> [number +is timeout in ms (1000ms is 1sec!), default 0 (indefinite)]
        • +
      • $MainMsgQueueWorkerTimeoutThreadShutdown +<number> [number is timeout in ms (1000ms is 1sec!), +default 60000 (1 minute)]
        • +
      • $MainMsgQueueType [FixedArray/LinkedList/Direct/Disk]
        • +
      • $MainMsgQueueSaveOnShutdown  [on/off] +
        • +
      • $MainMsgQueueWorkerThreads <number>, num +worker threads, default 1, recommended 1
        • +
      • $MainMsgQueueWorkerThreadMinumumMessages <number>, default 100
        • +
      • $MarkMessagePeriod (immark)
        • +
      • $MaxMessageSize <size_nbr>, default 2k - allows to specify maximum supported message size +(both for sending and receiving). The default +should be sufficient for almost all cases. Do not set this below 1k, as it would cause +interoperability problems with other syslog implementations.
        +Change the setting to e.g. 32768 if you would like to +support large message sizes for IHE (32k is the current maximum +needed for IHE). I was initially tempted to set the default to 32k, +but there is a some memory footprint with the current +implementation in rsyslog. +
        If you intend to receive Windows Event Log data (e.g. via +EventReporter), you might want to +increase this number to an even higher value, as event +log messages can be very lengthy ("$MaxMessageSize 64k" is not a bad idea). +Note: testing showed that 4k seems to be +the typical maximum for UDP based syslog. This is an IP stack +restriction. Not always ... but very often. If you go beyond +that value, be sure to test that rsyslogd actually does what +you think it should do ;) It is highly suggested to use a TCP based transport +instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restrictions. +
        Note that 2k, the current default, is the smallest size that must be +supported in order to be compliant to the upcoming new syslog RFC series. +
        • +
      • $ModDir
        • +
      • $ModLoad
        • +
      • $RepeatedMsgReduction
        • +
      • $ResetConfigVariables
        • +
      • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better +performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The +default may change as uniprocessor systems become less common.
        • +
      • $WorkDirectory <name> (directory for spool and other work files)
        • +
      • $UDPServerAddress <IP> (imudp) -- local IP +address (or name) the UDP listens should bind to
        • +
      • $UDPServerRun <port> (imudp) -- former +-r<port> option, default 514, start UDP server on this +port, "*" means all addresses
        • +
      • $UDPServerTimeRequery <nbr-of-times> (imudp) -- this is a performance +optimization. Getting the system time is very costly. With this setting, imudp can +be instructed to obtain the precise time only once every n-times. This logic is +only activated if messages come in at a very fast rate, so doing less frequent +time calls should usually be acceptable. The default value is two, because we have +seen that even without optimization the kernel often returns twice the identical time. +You can set this value as high as you like, but do so at your own risk. The higher +the value, the less precise the timestamp. +
      • $UMASK
        • +
      +

      Where <size_nbr> is specified above, +modifiers can be used after the number part. For example, 1k means +1024. Supported are k(ilo), m(ega), g(iga), t(era), p(eta) and e(xa). +Lower case letters refer to the traditional binary defintion (e.g. 1m +equals 1,048,576) whereas upper case letters refer to their new +1000-based definition (e.g 1M equals 1,000,000).

      +

      Numbers may include '.' and ',' for readability. So you can +for example specify either "1000" or "1,000" with the same result. +Please note that rsyslogd simply ignores the punctuation. Form it's +point of view, "1,,0.0.,.,0" also has the value 1000.

      + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + + diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html new file mode 100644 index 00000000..890a55c8 --- /dev/null +++ b/doc/rsyslog_conf_modules.html @@ -0,0 +1,53 @@ + +Modules - rsyslog.conf + +

      This is a part of the rsyslog.conf documentation.

      +back +

      Modules

      +

      Rsyslog has a modular design. Consequently, there is a growing +number of modules. Here is the entry point to their documentation and +what they do (list is currently not complete)

      +
        +
      • omsnmp - SNMP +trap output module
        • +
      • omrelp - RELP +output module
        • +
      • omgssapi - output module for GSS-enabled syslog
        • +
      • ommysql - output module for MySQL
        • +
      • ompgsql - output module for PostgreSQL
        • +
      • omlibdbi - +generic database output module (Firebird/Interbase, MS SQL, Sybase, +SQLLite, Ingres, Oracle, mSQL)
        • +
      • ommail - +permits rsyslog to alert folks by mail if something important happens
        • +
      • imfile +-  input module for text files
        • +
      • imrelp - RELP +input module
        • +
      • imudp - udp syslog message input
        • +
      • imtcp - input +plugin for plain tcp syslog
        • +
      • imgssapi - +input plugin for plain tcp and GSS-enabled syslog
        • +
      • immark - support for mark messages
        • +
      • imklog - kernel logging
        • +
      • imuxsock - +unix sockets, including the system log socket
        • +
      • im3195 - +accepts syslog messages via RFC 3195
        • +
      +

      Please note that each module provides configuration +directives, which are NOT necessarily being listed below. Also +remember, that a modules configuration directive (and functionality) is +only available if it has been loaded (using $ModLoad).

      +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + diff --git a/doc/rsyslog_conf_output.html b/doc/rsyslog_conf_output.html new file mode 100644 index 00000000..c52aaa5e --- /dev/null +++ b/doc/rsyslog_conf_output.html @@ -0,0 +1,81 @@ + +Output Channels - rsyslog.conf + +

      This is a part of the rsyslog.conf documentation.

      +back +

      Output Channels

      +

      Output Channels are a new concept first introduced in rsyslog +0.9.0. As of this writing, it is most likely that they will +be replaced by something different in the future. So if you +use them, be prepared to change you configuration file syntax when you +upgrade to a later release.
      +
      +The idea behind output channel definitions is that it shall provide an +umbrella for any type of output that the user might want. In essence,
      +this is the "file" part of selector lines (and this is why we are not +sure output channel syntax will stay after the next review). There is a
      +difference, though: selector channels both have filter conditions +(currently facility and severity) as well as the output destination. +they can only be used to write to files - not pipes, ttys or whatever +Output channels define the output definition, only. As of this build, +else. If we stick with output channels, this will change over time.

      +

      In concept, an output channel includes everything needed to +know about an output actions. In practice, the current implementation +only carries
      +a filename, a maximum file size and a command to be issued when this +file size is reached. More things might be present in future version, +which might also change the syntax of the directive.

      +

      Output channels are defined via an $outchannel directive. It's +syntax is as follows:
      +
      +$outchannel name,file-name,max-size,action-on-max-size
      +
      +name is the name of the output channel (not the file), file-name is the +file name to be written to, max-size the maximum allowed size and +action-on-max-size a command to be issued when the max size is reached. +This command always has exactly one parameter. The binary is that part +of action-on-max-size before the first space, its parameter is +everything behind that space.
      +
      +Please note that max-size is queried BEFORE writing the log message to +the file. So be sure to set this limit reasonably low so that any +message might fit. For the current release, setting it 1k lower than +you expected is helpful. The max-size must always be specified in bytes +- there are no special symbols (like 1k, 1m,...) at this point of +development.
      +
      +Keep in mind that $outchannel just defines a channel with "name". It +does not activate it. To do so, you must use a selector line (see +below). That selector line includes the channel name plus an $ sign in +front of it. A sample might be:
      +
      +*.* $mychannel
      +
      +In its current form, output channels primarily provide the ability to +size-limit an output file. To do so, specify a maximum size. When this +size is reached, rsyslogd will execute the action-on-max-size command +and then reopen the file and retry. The command should be something +like a log rotation +script or a similar thing.

      +

      If there is no action-on-max-size command or the command did +not resolve the situation, the file is closed and never reopened by +rsyslogd (except, of course, by huping it). This logic was integrated +when we first experienced severe issues with files larger 2gb, which +could lead to rsyslogd dumping core. In such cases, it is more +appropriate to stop writing to a single file. Meanwhile, rsyslogd has +been fixed to support files larger 2gb, but obviously only on file +systems and operating system versions that do so. So it can still make +sense to enforce a 2gb file size limit.

      + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + + diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html new file mode 100644 index 00000000..90b5fafe --- /dev/null +++ b/doc/rsyslog_conf_templates.html @@ -0,0 +1,150 @@ + +Templates - rsyslog.conf + +

      This is a part of the rsyslog.conf - documentation.

      +back +

      Templates

      +

      Templates are a key feature of rsyslog. They allow to specify +any +format a user might want. They are also used for dynamic file name +generation. Every output in rsyslog uses templates - this holds true +for files, user messages and so on. The database writer expects its +template to be a proper SQL statement - so this is highly customizable +too. You might ask how does all of this work when no templates at all +are specified. Good question ;) The answer is simple, though. Templates +compatible with the stock syslogd formats are hardcoded into rsyslogd. +So if no template is specified, we use one of these hardcoded +templates. Search for "template_" in syslogd.c and you will find the +hardcoded ones.

      +

      A template consists of a template directive, a name, the +actual template text and optional options. A sample is:

      +
      $template MyTemplateName,"\7Text +%property% some more text\n",<options>
      +

      The "$template" is the template directive. It tells rsyslog +that this line contains a template. "MyTemplateName" is the template +name. All +other config lines refer to this name. The text within quotes is the +actual template text. The backslash is an escape character, much as it +is in C. It does all these "cool" things. For example, \7 rings the +bell (this is an ASCII value), \n is a new line. C programmers and perl +coders have the advantage of knowing this, but the set in rsyslog is a +bit restricted currently. +

      +

      All text in the template is used literally, except for things +within percent signs. These are properties and allow you access to the +contents of the syslog message. Properties are accessed via the +property replacer (nice name, huh) and it can do cool things, too. For +example, it can pick a substring or do date-specific formatting. More +on this is below, on some lines of the property replacer.
      +
      +The <options> part is optional. It carries options +influencing the template as whole. See details below. Be sure NOT to +mistake template options with property options - the later ones are +processed by the property replacer and apply to a SINGLE property, only +(and not the whole template).
      +
      +Template options are case-insensitive. Currently defined are:

      +

      sql - format the string suitable for a SQL +statement in MySQL format. This will replace single quotes ("'") and +the backslash character by their backslash-escaped counterpart ("\'" +and "\\") inside each field. Please note that in MySQL configuration, +the NO_BACKSLASH_ESCAPES +mode must be turned off for this format to work (this is the default).

      +

      stdsql - format the string suitable for a +SQL statement that is to be sent to a standards-compliant sql server. +This will replace single quotes ("'") by two single quotes ("''") +inside each field. You must use stdsql together with MySQL if in MySQL +configuration the +NO_BACKSLASH_ESCAPES is +turned on.

      +

      Either the sql or stdsql  +option must be specified when a template is used +for writing to a database, otherwise injection might occur. Please note +that due to the unfortunate fact that several vendors have violated the +sql standard and introduced their own escape methods, it is impossible +to have a single option doing all the work.  So you yourself +must make sure you are using the right format. If you choose +the wrong one, you are still vulnerable to sql injection.
      +
      +Please note that the database writer *checks* that the sql option is +present in the template. If it is not present, the write database +action is disabled. This is to guard you against accidental forgetting +it and then becoming vulnerable to SQL injection. The sql option can +also be useful with files - especially if you want to import them into +a database on another machine for performance reasons. However, do NOT +use it if you do not have a real need for it - among others, it takes +some toll on the processing time. Not much, but on a really busy system +you might notice it ;)

      +

      The default template for the write to database action has the +sql option set. As we currently support only MySQL and the sql option +matches the default MySQL configuration, this is a good choice. +However, if you have turned on +NO_BACKSLASH_ESCAPES in +your MySQL config, you need to supply a template with the stdsql +option. Otherwise you will become vulnerable to SQL injection.
      +
      +To escape:
      +% = \%
      +\ = \\ --> '\' is used to escape (as in C)
      +$template TraditionalFormat,%timegenerated% %HOSTNAME% +%syslogtag%%msg%\n"
      +
      +Properties can be accessed by the property +replacer (see there for details).

      +

      Please note that templates can also by +used to generate selector lines with dynamic file names. For +example, if you would like to split syslog messages from different +hosts to different files (one per host), you can define the following +template:

      +
      $template +DynFile,"/var/log/system-%HOSTNAME%.log"
      +

      This template can then be used when defining an output +selector line. It will result in something like +"/var/log/system-localhost.log"

      +

      Template +names beginning with "RSYSLOG_" are reserved for rsyslog use. Do NOT +use them if, otherwise you may receive a conflict in the future (and +quite unpredictable behaviour). There is a small set of pre-defined +templates that you can use without the need to define it:

      +
        +
      • RSYSLOG_TraditionalFileFormat +- the "old style" default log file format with low-precision timestamps
        • +
      • RSYSLOG_FileFormat +- a modern-style logfile format similar to TraditionalFileFormat, buth +with high-precision timestamps and timezone information
        • +
      • RSYSLOG_TraditionalForwardFormat +- the traditional forwarding format with low-precision timestamps. Most +useful if you send messages to other syslogd's or rsyslogd +below +version 3.12.5.
        • +
      • RSYSLOG_ForwardFormat +- a new high-precision forwarding format very similar to the +traditional one, but with high-precision timestamps and timezone +information. Recommended to be used when sending messages to rsyslog +3.12.5 or above.
        • +
      • RSYSLOG_SyslogProtocol23Format +- the format specified in IETF's internet-draft +ietf-syslog-protocol-23, which is assumed to be come the new syslog +standard RFC. This format includes several improvements. The rsyslog +message parser understands this format, so you can use it together with +all relatively recent versions of rsyslog. Other syslogd's may get +hopelessly confused if receiving that format, so check before you use +it. Note that the format is unlikely to change when the final RFC comes +out, but this may happen.
        • +
      • RSYSLOG_DebugFormat +- a special format used for troubleshooting property problems. This format +is meant to be written to a log file. Do not use for production or remote +forwarding.
        • +
      + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + -- cgit v1.2.3 From 249b27952a9faea95662eb230f4c86a0db874fe5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 11 Nov 2008 11:38:37 +0100 Subject: improved doc on property replacer regular expressions --- doc/Makefile.am | 1 + doc/property_replacer.html | 10 +++------- doc/rsyslog_conf_nomatch.html | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 doc/rsyslog_conf_nomatch.html diff --git a/doc/Makefile.am b/doc/Makefile.am index fef1e44c..5c2f5313 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -105,6 +105,7 @@ html_files = \ rsyslog_conf_modules.html \ rsyslog_conf_output.html \ rsyslog_conf_templates.html \ + rsyslog_conf_nomatch.html \ src/classes.dia EXTRA_DIST = $(html_files) diff --git a/doc/property_replacer.html b/doc/property_replacer.html index 34e2116c..9ea41aed 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -229,7 +229,7 @@ sequence with a regular expression is: "%msg:R:.*Sev:. \(.*\) \[.*--end%"

      It is possible to specify some parametes after the "R". These are comma-separated. They are: -

      R,<regexp-type>,<submatch>,<nomatch>,<match-number> +

      R,<regexp-type>,<submatch>,<nomatch>,<match-number>

      regexp-type is either "BRE" for Posix basic regular expressions or "ERE" for extended ones. The string must be given in upper case. The default is "BRE" to be consistent with earlier versions of rsyslog that @@ -241,12 +241,8 @@ that the first match is number 0, the second 1 and so on. Up to 10 matches (up to number 9) are supported. Please note that it would be more natural to have the match-number in front of submatch, but this would break backward-compatibility. So the match-number must be specified after "nomatch". -

      nomatch is either "DFLT", "BLANK" or "FIELD" (all upper case!). It tells -what to use if no match is found. With "DFLT", the strig "**NO MATCH**" is -used. This was the only supported value up to rsyslog 3.19.5. With "BLANK" -a blank text is used (""). Finally, "FIELD" uses the full property text -instead of the expression. Some folks have requested that, so it seems -to be useful. +

      nomatch specifies what should +be used in case no match is found.

      The following is a sample of an ERE expression that takes the first submatch from the message string and replaces the expression with the full field if no match is found: diff --git a/doc/rsyslog_conf_nomatch.html b/doc/rsyslog_conf_nomatch.html new file mode 100644 index 00000000..5c4f3f90 --- /dev/null +++ b/doc/rsyslog_conf_nomatch.html @@ -0,0 +1,37 @@ + +nomatch mode - property replacer - rsyslog.conf + +

      nomatch mode - property replacer - rsyslog.con

      +

      This is a part of the rsyslog.conf documentation +of the property replacer.

      +

      The "nomatch-Mode" specifies which string the property replacer +shall return if a regular expression did not find the search string.. Traditionally, +the string "**NO MATCH**" was returned, but many people complained this was almost never useful. +Still, this mode is support as "DFLT" for legacy configurations. +

      Two additional and potentially useful modes exist: in one (BLANK) a blank string +is returned. This is probably useful for inserting values into databases where no +value shall be inserted if the expression could not be found. A use case may be +that you record a traffic log based on firewall rules and the "bytes transmitted" counter +is extracted via a regular expression. If no "bytes transmitted" counter is available +in the current message, it is probably a good idea to return an empty string, which the +database layer can turn into a zero. +

      The other mode is "FIELD", in which the complete field is returned. This may be useful +in cases where absense of a match is considered a failure and the message that triggered +it shall be logged. +

      If in doubt, it is highly suggested to use the +rsyslog online regular expression +checker and generator to see these options in action. With that online tool, +you can craft regular expressions based on samples and try out the different modes. + +

      [manual index] +[rsyslog.conf] +[rsyslog site]

      +

      This documentation is part of the +rsyslog project.
      +Copyright © 2008 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

      + + + + -- cgit v1.2.3 From 704a1145d64532df36624a3c9850b0c86f38f76f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 11 Nov 2008 12:22:06 +0100 Subject: doc: added new nomatch mode Note: the actual nomatch mode is not yet available in this code, this needs to be merged from v3-stable first. This happens soon, but I wanted to make sure the doc is right. --- doc/rsyslog_conf_nomatch.html | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/rsyslog_conf_nomatch.html b/doc/rsyslog_conf_nomatch.html index 5c4f3f90..5f25f3e4 100644 --- a/doc/rsyslog_conf_nomatch.html +++ b/doc/rsyslog_conf_nomatch.html @@ -8,9 +8,11 @@ of the property replacer.

      shall return if a regular expression did not find the search string.      . Traditionally, the string "**NO MATCH**" was returned, but many people complained this was almost never useful. Still, this mode is support as "DFLT" for legacy configurations. -

      Two additional and potentially useful modes exist: in one (BLANK) a blank string +

      Three additional and potentially useful modes exist: in one (BLANK) a blank string is returned. This is probably useful for inserting values into databases where no -value shall be inserted if the expression could not be found. A use case may be +value shall be inserted if the expression could not be found. +

      A similar mode is "ZERO" where the string "0" is returned. This is suitable +for numerical values. A use case may be that you record a traffic log based on firewall rules and the "bytes transmitted" counter is extracted via a regular expression. If no "bytes transmitted" counter is available in the current message, it is probably a good idea to return an empty string, which the @@ -23,6 +25,15 @@ it shall be logged. checker and generator to see these options in action. With that online tool, you can craft regular expressions based on samples and try out the different modes. +

      Summary of nomatch Modes

      + + + + + + + +
      ModeReturned
      DFLT"**NO MATCH**"
      BLANK"" (empty string)
      ZERO"0"
      FIELDfull content of original field
       Interactive Tool

      [manual index] [rsyslog.conf] [rsyslog site]

      -- cgit v1.2.3 From afd425232f3198e7bb6f3dbcf0aa3cb4c24d5f7e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 18 Nov 2008 13:46:38 +0100 Subject: enhanced legacy syslog parser to detect year if part of the timestamp The format is based on what Cisco devices seem to emit. --- ChangeLog | 5 +++++ runtime/datetime.c | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/ChangeLog b/ChangeLog index f758b7ce..70ad7e1f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -13,6 +13,8 @@ version before switching to this one. - added configuration directive "HUPisRestart" which enables to configure HUP to be either a full restart or "just" a leightweight way to close open files. +- enhanced legacy syslog parser to detect year if part of the timestamp + the format is based on what Cisco devices seem to emit. - added a setting "$OptimizeForUniprocessor" to enable users to turn off pthread_yield calls which are counter-productive on multiprocessor machines (but have been shown to be useful on uniprocessors) @@ -29,6 +31,8 @@ version before switching to this one. - doc bugfix: queue doc had wrong parameter name for setting controlling worker thread shutdown period - restructured rsyslog.conf documentation +- bugfix: memory leak in ompgsql + Thanks to Ken for providing the patch --------------------------------------------------------------------------- Version 3.21.7 [BETA] (rgerhards), 2008-11-11 - this is the new beta branch, based on the former 3.21.6 devel @@ -45,6 +49,7 @@ Version 3.21.6 [DEVEL] (rgerhards), 2008-10-22 - added capability to support multiple module search pathes. Thank to Marius Tomaschewski for providing the patch. - bugfix: im3195 did no longer compile +- improved "make distcheck" by ensuring everything relevant is recompiled --------------------------------------------------------------------------- Version 3.21.5 [DEVEL] (rgerhards), 2008-09-30 - performance optimization: unnecessary time() calls during message diff --git a/runtime/datetime.c b/runtime/datetime.c index aa1956d7..676f76d5 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -312,6 +312,7 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) /* variables to temporarily hold time information while we parse */ int month; int day; + int year = 0; /* 0 means no year provided */ int hour; /* 24 hour clock */ int minute; int second; @@ -472,7 +473,23 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) if(*pszTS++ != ' ') ABORT_FINALIZE(RS_RET_INVLD_TIME); + + /* time part */ hour = srSLMGParseInt32(&pszTS); + if(hour > 1970 && hour < 2100) { + /* if so, we assume this actually is a year. This is a format found + * e.g. in Cisco devices. + * (if you read this 2100+ trying to fix a bug, congratulate myself + * to how long the code survived - me no longer ;)) -- rgerhards, 2008-11-18 + */ + year = hour; + + /* re-query the hour, this time it must be valid */ + if(*pszTS++ != ' ') + ABORT_FINALIZE(RS_RET_INVLD_TIME); + hour = srSLMGParseInt32(&pszTS); + } + if(hour < 0 || hour > 23) ABORT_FINALIZE(RS_RET_INVLD_TIME); @@ -502,6 +519,8 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) *ppszTS = pszTS; /* provide updated parse position back to caller */ pTime->timeType = 1; pTime->month = month; + if(year > 0) + pTime->year = year; /* persist year if detected */ pTime->day = day; pTime->hour = hour; pTime->minute = minute; -- cgit v1.2.3 From 57c9a3accee3a3e9b46d984c76c9aae7e2ec9c27 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 19 Nov 2008 13:20:31 +0100 Subject: exprimental implementaiton of $PrivDropToUser directive ... which permits to drop root privileges. This is not a completely secure way of dropping permissions, e.g. the group permissions need to be dropped, too. Also, there are several vulnerability Windows (see code comments). Finally, at least the imklog module on linux does not work when privileges are dropped. This code may still be a valuable addition, and so I have created an experimental branch so that people can check it out. --- tools/syslogd.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/tools/syslogd.c b/tools/syslogd.c index 7145779d..51ce1830 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -272,6 +272,7 @@ static int bHaveMainQueue = 0;/* set to 1 if the main queue - in queueing mode - * If the main queue is either not yet ready or not running in * queueing mode (mode DIRECT!), then this is set to 0. */ +static int uidDropPriv = 0; /* user-id to which priveleges should be dropped to (AFTER init()!) */ extern int errno; @@ -2063,6 +2064,30 @@ static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) } +/* drop to specified user + * if something goes wrong, the function never returns + * Note that such an abort can cause damage to on-disk structures, so we should + * re-design the "interface" in the long term. -- rgerhards, 2008-11-19 + */ +static void doDropPrivUid(int iUid) +{ + int res; + uchar szBuf[1024]; + +dbgprintf("userid before drop: %d\n", getuid()); + res = setuid(iUid); +dbgprintf("userid after drop: %d\n", getuid()); + if(res) { + /* if we can not set the userid, this is fatal, so let's unconditionally abort */ + perror("could not set requested userid"); + exit(1); + } + DBGPRINTF("setuid(%d): %d\n", iUid, res); + snprintf((char*)szBuf, sizeof(szBuf)/sizeof(uchar), "rsyslogd's userid changed to %d", iUid); + logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, szBuf, 0); +} + + /* helper to freeSelectors(), used with llExecFunc() to flush * pending output. -- rgerhards, 2007-08-02 * We do not need to lock the action object here as the processing @@ -2807,6 +2832,8 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"errormessagestostderr", 0, eCmdHdlrBinary, NULL, &bErrMsgToStderr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"maxmessagesize", 0, eCmdHdlrSize, setMaxMsgSize, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"privdroptouser", 0, eCmdHdlrUID, NULL, &uidDropPriv, NULL)); +// CHKiRet(regCfSysLineHdlr((uchar *)"privdroptogroup", 0, eCmdHdlrGID, doDropPrivGroup, NULL, NULL)); /* now add other modules handlers (we should work on that to be able to do it in ClassInit(), but so far * that is not possible). -- rgerhards, 2008-01-28 @@ -2900,8 +2927,20 @@ static rsRetVal mainThread() } /* Send a signal to the parent so it can terminate. */ - if (myPid != ppid) - kill (ppid, SIGTERM); + if(myPid != ppid) + kill(ppid, SIGTERM); + + + /* If instructed to do so, we now drop privileges. Note that this is not 100% secure, + * because inputs and outputs are already running at this time. However, we can implement + * dropping of privileges rather quickly and it will work in many cases. While it is not + * the ultimate solution, the current one is still much better than not being able to + * drop privileges at all. Doing it correctly, requires a change in architecture, which + * we should do over time. TODO -- rgerhards, 2008-11-19 + */ + if(uidDropPriv != 0) { + doDropPrivUid(uidDropPriv); + } /* END OF INTIALIZATION * ... but keep in mind that we might do a restart and thus init() might @@ -3520,11 +3559,14 @@ int realMain(int argc, char **argv) /* process compatibility mode settings */ - if(iCompatibilityMode < 3) { + if(iCompatibilityMode < 4) { errmsg.LogError(0, NO_ERRCODE, "WARNING: rsyslogd is running in compatibility mode. Automatically " "generated config directives may interfer with your rsyslog.conf settings. " - "We suggest upgrading your config and adding -c3 as the first " + "We suggest upgrading your config and adding -c4 as the first " "rsyslogd option."); + } + + if(iCompatibilityMode < 3) { if(MarkInterval > 0) { legacyOptsEnq((uchar *) "ModLoad immark"); snprintf((char *) legacyConfLine, sizeof(legacyConfLine), "MarkMessagePeriod %d", MarkInterval); -- cgit v1.2.3 From 48480e833488c8bc571c62ad84331038edddf3c3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 21 Nov 2008 15:38:34 +0100 Subject: improved doc a big --- doc/manual.html | 3 +++ doc/queues.html | 79 +++++++++++++++++++++++++-------------------------------- 2 files changed, 38 insertions(+), 44 deletions(-) diff --git a/doc/manual.html b/doc/manual.html index e208755d..1762fcca 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -16,6 +16,9 @@ relay chains while at the same time being very easy to setup for the novice user. And as we know what enterprise users really need, there is also professional rsyslog support available directly from the source!

      +

      Please visit the rsyslog sponsor's page +to honor the project sponsor or become one yourself! We are very grateful for any help towards the +project goals.

      This documentation is for version 4.1.0 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status. diff --git a/doc/queues.html b/doc/queues.html index 727bc26a..41c5865f 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -17,6 +17,36 @@ rule processor, which then evaluates which actions are to be carried out. In front of each action, there is also a queue, which potentially de-couples the filter processing from the actual action (e.g. writing to file, database or forwarding to another host).

      +

      Where are Queues Used?

      +

       Currently, queues are used for the main message queue and for the +actions.

      +

      There is a single main message queue inside rsyslog. Each input module +delivers messages to it. The main message queue worker filters messages based on +rules specified in rsyslog.conf and dispatches them to the individual action +queues. Once a message is in an action queue, it is deleted from the main +message queue.

      +

      There are multiple action queues, one for each configured action. By default, +these queues operate in direct (non-queueing) mode. Action queues are fully +configurable and thus can be changed to whatever is best for the given use case.

      +

      Future versions of rsyslog will most probably utilize queues at other places, +too.

      +

      Wherever "<object>"  is used in the config file +statements, substitute "<object>" with either "MainMsg" or "Action". The +former will set main message queue +parameters, the later parameters for the next action that will be +created. Action queue parameters can not be modified once the action has been +specified. For example, to tell the main message queue to save its content on +shutdown, use $MainMsgQueueSaveOnShutdown on".

      +

      If the same parameter is specified multiple times before a queue is created, +the last one specified takes precedence. The main message queue is created after +parsing the config file and all of its potential includes. An action queue is +created each time an action selector is specified. Action queue parameters are +reset to default after an action queue has been created (to provide a clean +environment for the next action).

      +

      Not all queues necessarily support the full set of queue configuration +parameters, because not all are applicable. For example, in current output +module design, actions do not support multi-threading. Consequently, the number +of worker threads is fixed to one for action queues and can not be changed.

      Queue Modes

      Rsyslog supports different queue modes, some with submodes. Each of them has specific advantages and disadvantages. Selecting the right queue mode is quite @@ -114,8 +144,7 @@ only memory if in use. A FixedArray queue may have a too large static memory footprint in such cases.

      In general, it is advised to use LinkedList mode if in doubt. The processing overhead compared to FixedArray is low and may be - -outweigh by the reduction in memory use. Paging in most-often-unused +outweigh by the reduction in memory use. Paging in most-often-unused pointer array pages can be much slower than dynamically allocating them.

      To create an in-memory queue, use the "$<object>QueueType LinkedList" or  "$<object>QueueType FixedArray" config directive.

      @@ -260,19 +289,15 @@ unavoidable and you prefer to discard less important messages first.

      disk space, it is finally full. If so, rsyslogd throttles the data element submitter. If that, for example, is a reliable input (TCP, local log socket), that will slow down the message originator which is a good - -resolution for this scenario.

      -

      During - -throtteling, a disk-assisted queue continues to write to disk and +resolution for this scenario.

      +

      During throtteling, a disk-assisted queue continues to write to disk and messages are also discarded based on severity as well as regular dequeuing and processing continues. So chances are good the situation will be resolved by simply throttling. Note, though, that throtteling is highly undesirable for unreliable sources, like UDP message reception. So it is not a good thing to run into throtteling mode at all.

      We can not hold processing - -infinitely, not even when throtteling. For example, throtteling the local +infinitely, not even when throtteling. For example, throtteling the local log socket too long would cause the system at whole come to a standstill. To prevent this, rsyslogd times out after a configured period ("$<object>QueueTimeoutEnqueue", specified in milliseconds) if no space becomes available. As a last resort, it @@ -303,8 +328,7 @@ There are two configuration directives, both should be used together or results are unpredictable:" $<object>QueueDequeueTimeBegin <hour>" and "$<object>QueueDequeueTimeEnd <hour>". The hour parameter must be specified in 24-hour format (so 10pm is 22). A use case for this parameter can be found in the rsyslog wiki.

      Terminating Queues

      Terminating a process sounds easy, but can be complex. - -Terminating a running queue is in fact the most complex operation a queue +Terminating a running queue is in fact the most complex operation a queue object can perform. You don't see that from a user's point of view, but its quite hard work for the developer to do everything in the right order.

      The complexity arises when the queue has still data enqueued when it @@ -325,39 +349,6 @@ it terminates. This includes data elements there were begun being processed by workers that needed to be cancelled due to too-long processing. For a large queue, this operation may be lengthy. No timeout applies to a required shutdown save.

      -

      Where are Queues Used?

      -

       Currently, queues are used for the main message queue and for the -actions.

      -

      There is a single main message queue inside rsyslog. Each input module -delivers messages to it. The main message queue worker filters messages based on -rules specified in rsyslog.conf and dispatches them to the individual action -queues. Once a message is in an action queue, it is deleted from the main -message queue.

      -

      There are multiple action queues, one for each configured action. By default, -these queues operate in direct (non-queueing) mode. Action queues are fully -configurable and thus can be changed to whatever is best for the given use case.

      -

      Future versions of rsyslog will most probably utilize queues at other places, -too.

      -

      - -Wherever "<object>"  was used above in the config file -statements, substitute "<object>" with either "MainMsg" or "Action". The -former will set main message queue - -parameters, the later parameters for the next action that will be -created. Action queue parameters can not be modified once the action has been -specified. For example, to tell the main message queue to save its content on -shutdown, use $MainMsgQueueSaveOnShutdown on".

      -

      If the same parameter is specified multiple times before a queue is created, -the last one specified takes precedence. The main message queue is created after -parsing the config file and all of its potential includes. An action queue is -created each time an action selector is specified. Action queue parameters are -reset to default after an action queue has been created (to provide a clean -environment for the next action).

      -

      Not all queues necessarily support the full set of queue configuration -parameters, because not all are applicable. For example, in current output -module design, actions do not support multi-threading. Consequently, the number -of worker threads is fixed to one for action queues and can not be changed.

      [manual index] [rsyslog.conf] [rsyslog site]

      -- cgit v1.2.3 From 588f4f98d8b484ea6b2cba3449db9c9b035b562f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 21 Nov 2008 15:41:34 +0100 Subject: doc bugfix: property-based filter had inconsistent description Thank to Peter Matulis for pointing this out. Actually, all compare operations described exist. --- doc/rsyslog_conf_filter.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/rsyslog_conf_filter.html b/doc/rsyslog_conf_filter.html index 55244c15..cf21ff95 100644 --- a/doc/rsyslog_conf_filter.html +++ b/doc/rsyslog_conf_filter.html @@ -97,9 +97,7 @@ filter on any property, like HOSTNAME, syslogtag and msg. A list of all currently-supported properties can be found in the property replacer documentation (but keep in mind that only the properties, not the replacer is supported). With this filter, each properties can be checked against a -specified value, using a specified compare operation. Currently, there -is only a single compare operation (contains) available, but additional -operations will be added in the future.

      +specified value, using a specified compare operation.

      A property-based filter must start with a colon in column 0. This tells rsyslogd that it is the new filter type. The colon must be followed by the property name, a comma, the name of the compare -- cgit v1.2.3 From 5f44f8592761161921ceefffeedadbec2c308453 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 26 Nov 2008 11:21:29 +0100 Subject: improved doc a bit --- doc/features.html | 7 +++++++ doc/manual.html | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/doc/features.html b/doc/features.html index 89796a41..7a31a03b 100644 --- a/doc/features.html +++ b/doc/features.html @@ -116,6 +116,13 @@ submit feature requests there (or via our forums). If we like them but they look quite long-lived (aka "not soon to be implemented"), they will possibly be migrated to this list here and at some time moved back to the bugzilla tracker.

      +

      Note that we also maintain a +list of features that are looking for sponsors. +If you are interested in any of these features, or any other feature, you may consider sponsoring +the implementation. This is also a great way to show your commitment to the open source +community. Plus, it can be financially attractive: just think about how much less it may +be to sponsor a feature instead of purchasing a commercial implementation. Also, the benefit +of being recognised as a sponsor may even drive new customers to your business!

      • port it to more *nix variants (eg AIX and HP UX) - this needs volunteers with access to those machines and knowledge
        • diff --git a/doc/manual.html b/doc/manual.html index 1762fcca..dee076b8 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -16,8 +16,8 @@ relay chains while at the same time being very easy to setup for the novice user. And as we know what enterprise users really need, there is also professional rsyslog support available directly from the source!

        -

        Please visit the rsyslog sponsor's page -to honor the project sponsor or become one yourself! We are very grateful for any help towards the +

        Please visit the rsyslog sponsor's page +to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

        This documentation is for version 4.1.0 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current -- cgit v1.2.3 From dc478db1ca80ef222f83985b539dfec1c66063e2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 26 Nov 2008 14:17:36 +0100 Subject: added ability to drop privileges Added $PrivDropToGroup, $PrivDropToUser, $PrivDropToGroupID, $PrivDropToUserID config directives to enable dropping privileges. This is an effort to provide a security enhancement. For the limits of this approach, see http://wiki.rsyslog.com/index.php/Security --- ChangeLog | 6 +++++ configure.ac | 2 +- doc/Makefile.am | 1 + doc/droppriv.html | 60 ++++++++++++++++++++++++++++++++++++++++++++ doc/manual.html | 2 +- doc/rsyslog_conf_global.html | 5 ++++ tools/syslogd.c | 43 ++++++++++++++++++++++++++++--- 7 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 doc/droppriv.html diff --git a/ChangeLog b/ChangeLog index 70ad7e1f..fe8df1fc 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,10 @@ --------------------------------------------------------------------------- +Version 4.1.1 [DEVEL] (rgerhards), 2008-11-?? +- added $PrivDropToGroup, $PrivDropToUser, $PrivDropToGroupID, + $PrivDropToUserID config directives to enable dropping privileges. + This is an effort to provide a security enhancement. For the limits of this + approach, see http://wiki.rsyslog.com/index.php/Security +--------------------------------------------------------------------------- Version 4.1.0 [DEVEL] (rgerhards), 2008-11-18 ********************************* WARNING ********************************* diff --git a/configure.ac b/configure.ac index 981ef193..8eecf2ab 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.0],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.1.1],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_HEADERS([config.h]) diff --git a/doc/Makefile.am b/doc/Makefile.am index 5c2f5313..b58f813b 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -5,6 +5,7 @@ html_files = \ features.html \ generic_design.html \ expression.html \ + droppriv.html \ history.html \ how2help.html \ install.html \ diff --git a/doc/droppriv.html b/doc/droppriv.html new file mode 100644 index 00000000..7293e872 --- /dev/null +++ b/doc/droppriv.html @@ -0,0 +1,60 @@ + +dropping privileges in rsyslog + + +

        Dropping privileges in rsyslog

        +

        Available since:    4.1.1

        +

        Description:

        +

        +Rsyslogd provides the ability to drop privileges by +impersonating as another user and/or group after startup. + +

        Please note that due to POSIX standards, rsyslogd always needs to start +up as root if there is a listener who must bind to a network port below 1024. +For example, the UDP listener usually needs to listen to 514 and as such +rsyslogd needs to start up as root. + +

        If you do not need this functionality, you can start rsyslog directly as an ordinary +user. That is probably the safest way of operations. However, if a startup as +root is required, you can use the $PrivDropToGroup and $PrivDropToUser config +directives to specify a group and/or user that rsyslogd should drop to after initialization. +Once this happend, the daemon runs without high privileges (depending, of +course, on the permissions of the user account you specified). +

        There is some additional information available in the +rsyslog wiki. +

        Configuration Directives:

        +
          +
        • $PrivDropToUser
          +Name of the user rsyslog should run under after startup. Please note that +this user is looked up in the system tables. If the lookup fails, privileges are +NOT dropped. Thus it is advisable to use the less convenient $PrivDropToUserID directive. +If the user id can be looked up, but can not be set, rsyslog aborts. +
          +
          • +
        • $PrivDropToUserID
          +Much the same as $PrivDropToUser, except that a numerical user id instead of a name +is specified.Thus, privilege drop will always happen. +rsyslogd aborts. +
        • $PrivDropToGroup
          +Name of the group rsyslog should run under after startup. Please note that +this user is looked up in the system tables. If the lookup fails, privileges are +NOT dropped. Thus it is advisable to use the less convenient $PrivDropToGroupID directive. +Note that all supplementary groups are removed from the process if $PrivDropToGroup is +specified. +If the group id can be looked up, but can not be set, rsyslog aborts. +
          +
          • +
        • $PrivDropToGroupID
          +Much the same as $PrivDropToGroup, except that a numerical group id instead of a name +is specified. Thus, privilege drop will always happen. +
        +

        [rsyslog.conf overview] +[manual index] [rsyslog site]

        +

        This documentation is part of the rsyslog +project.
        +Copyright © 2008 by Rainer +Gerhards and +Adiscon. +Released under the GNU GPL version 3 or higher.

        + + diff --git a/doc/manual.html b/doc/manual.html index e208755d..8210165f 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -16,7 +16,7 @@ relay chains while at the same time being very easy to setup for the novice user. And as we know what enterprise users really need, there is also professional rsyslog support available directly from the source!

        -

        This documentation is for version 4.1.0 (devel branch) of rsyslog. +

        This documentation is for version 4.1.1 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

        If you like rsyslog, you might diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index bc618dd0..d02245e3 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -200,6 +200,11 @@ time calls should usually be acceptable. The default value is two, because we ha seen that even without optimization the kernel often returns twice the identical time. You can set this value as high as you like, but do so at your own risk. The higher the value, the less precise the timestamp. +

      • $PrivDropToGroup
        • +
      • $PrivDropToGroupID
        • +
      • $PrivDropToUser
        • +
      • $PrivDropToUserID
        • +
    • $UMASK

    Where <size_nbr> is specified above, diff --git a/tools/syslogd.c b/tools/syslogd.c index 51ce1830..d132d139 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -82,6 +82,7 @@ #include #include #include +#include #if HAVE_SYS_TIMESPEC_H # include @@ -273,6 +274,7 @@ static int bHaveMainQueue = 0;/* set to 1 if the main queue - in queueing mode - * queueing mode (mode DIRECT!), then this is set to 0. */ static int uidDropPriv = 0; /* user-id to which priveleges should be dropped to (AFTER init()!) */ +static int gidDropPriv = 0; /* group-id to which priveleges should be dropped to (AFTER init()!) */ extern int errno; @@ -2064,6 +2066,34 @@ static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) } +/* drop to specified group + * if something goes wrong, the function never returns + * Note that such an abort can cause damage to on-disk structures, so we should + * re-design the "interface" in the long term. -- rgerhards, 2008-11-26 + */ +static void doDropPrivGid(int iGid) +{ + int res; + uchar szBuf[1024]; + + res = setgroups(0, NULL); /* remove all supplementary group IDs */ + if(res) { + perror("could not remove supplemental group IDs"); + exit(1); + } + DBGPRINTF("setgroups(0, NULL): %d\n", res); + res = setgid(iGid); + if(res) { + /* if we can not set the userid, this is fatal, so let's unconditionally abort */ + perror("could not set requested group id"); + exit(1); + } + DBGPRINTF("setgid(%d): %d\n", iGid, res); + snprintf((char*)szBuf, sizeof(szBuf)/sizeof(uchar), "rsyslogd's groupid changed to %d", iGid); + logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, szBuf, 0); +} + + /* drop to specified user * if something goes wrong, the function never returns * Note that such an abort can cause damage to on-disk structures, so we should @@ -2074,9 +2104,7 @@ static void doDropPrivUid(int iUid) int res; uchar szBuf[1024]; -dbgprintf("userid before drop: %d\n", getuid()); res = setuid(iUid); -dbgprintf("userid after drop: %d\n", getuid()); if(res) { /* if we can not set the userid, this is fatal, so let's unconditionally abort */ perror("could not set requested userid"); @@ -2833,7 +2861,9 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"errormessagestostderr", 0, eCmdHdlrBinary, NULL, &bErrMsgToStderr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"maxmessagesize", 0, eCmdHdlrSize, setMaxMsgSize, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"privdroptouser", 0, eCmdHdlrUID, NULL, &uidDropPriv, NULL)); -// CHKiRet(regCfSysLineHdlr((uchar *)"privdroptogroup", 0, eCmdHdlrGID, doDropPrivGroup, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"privdroptouserid", 0, eCmdHdlrInt, NULL, &uidDropPriv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"privdroptogroup", 0, eCmdHdlrGID, NULL, &gidDropPriv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"privdroptogroupid", 0, eCmdHdlrGID, NULL, &gidDropPriv, NULL)); /* now add other modules handlers (we should work on that to be able to do it in ClassInit(), but so far * that is not possible). -- rgerhards, 2008-01-28 @@ -2938,10 +2968,17 @@ static rsRetVal mainThread() * drop privileges at all. Doing it correctly, requires a change in architecture, which * we should do over time. TODO -- rgerhards, 2008-11-19 */ + if(gidDropPriv != 0) { + doDropPrivGid(gidDropPriv); + glbl.SetHUPisRestart(0); /* we can not do restart-type HUPs with dropped privs */ + } + if(uidDropPriv != 0) { doDropPrivUid(uidDropPriv); + glbl.SetHUPisRestart(0); /* we can not do restart-type HUPs with dropped privs */ } + /* END OF INTIALIZATION * ... but keep in mind that we might do a restart and thus init() might * be called again. If that happens, we must shut down the worker thread, -- cgit v1.2.3 From d44ae01df3cd41a25523e82f3acc76f048d20aa7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 26 Nov 2008 16:55:05 +0100 Subject: fixing issue with test suite which was not yet adapted to v4 --- ChangeLog | 2 +- tests/cfg.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index 66af15d1..22c6006d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.1.1 [DEVEL] (rgerhards), 2008-11-?? +Version 4.1.1 [DEVEL] (rgerhards), 2008-11-26 - added $PrivDropToGroup, $PrivDropToUser, $PrivDropToGroupID, $PrivDropToUserID config directives to enable dropping privileges. This is an effort to provide a security enhancement. For the limits of this diff --git a/tests/cfg.sh b/tests/cfg.sh index 99729823..fb22fbf3 100755 --- a/tests/cfg.sh +++ b/tests/cfg.sh @@ -36,7 +36,7 @@ echo "local directory" # # check empty config file # -../tools/rsyslogd -c3 -N1 -f/dev/null 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -N1 -f/dev/null 2>&1 |tail --lines=+2 > tmp cmp tmp $srcdir/DevNull.cfgtest if [ ! $? -eq 0 ]; then echo "DevNull.cfgtest failed" @@ -51,7 +51,7 @@ fi; # # check missing config file # -../tools/rsyslogd -c3 -N1 -f/This/does/not/exist 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -N1 -f/This/does/not/exist 2>&1 |tail --lines=+2 > tmp cmp tmp $srcdir/NoExistFile.cfgtest if [ ! $? -eq 0 ]; then echo "NoExistFile.cfgtest failed" @@ -74,7 +74,7 @@ exit 0 # # check config with invalid directive # -../tools/rsyslogd -c3 -u2 -N1 -f$srcdir/cfg1.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg1.testin 2>&1 |tail --lines=+2 > tmp cmp tmp $srcdir/cfg1.cfgtest if [ ! $? -eq 0 ]; then echo "cfg1.cfgtest failed" @@ -91,7 +91,7 @@ fi; # the one with the invalid config directive, so that we may see # an effect of the included config ;) # -../tools/rsyslogd -c3 -u2 -N1 -f$srcdir/cfg2.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg2.testin 2>&1 |tail --lines=+2 > tmp cmp tmp $srcdir/cfg2.cfgtest if [ ! $? -eq 0 ]; then echo "cfg2.cfgtest failed" @@ -106,7 +106,7 @@ fi; # # check included config file, where included file does not exist # -../tools/rsyslogd -c3 -u2 -N1 -f$srcdir/cfg3.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg3.testin 2>&1 |tail --lines=+2 > tmp cmp tmp $srcdir/cfg3.cfgtest if [ ! $? -eq 0 ]; then echo "cfg3.cfgtest failed" @@ -121,7 +121,7 @@ fi; # # check a reasonable complex, but correct, log file # -../tools/rsyslogd -c3 -u2 -N1 -f$srcdir/cfg4.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg4.testin 2>&1 |tail --lines=+2 > tmp cmp tmp $srcdir/cfg4.cfgtest if [ ! $? -eq 0 ]; then echo "cfg4.cfgtest failed" -- cgit v1.2.3 From 1f1e3254924aff524ed209042e70d3af333b092d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 26 Nov 2008 17:18:22 +0100 Subject: updated project status --- doc/status.html | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/status.html b/doc/status.html index 5c8409ec..b814d0bc 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,19 +2,19 @@ rsyslog status page

    rsyslog status page

    -

    This page reflects the status as of 2008-10-22.

    +

    This page reflects the status as of 2008-11-26.

    Current Releases

    -

    development: 3.21.6 [2008-10-22] - -change log - -download +

    development: 4.1.1 [2008-11-26] - +change log - +download -
    beta: 3.19.12 [2008-10-16] - -change log - -download

    +
    beta: 3.21.7 [2008-11-11] - +change log - +download

    -

    v3 stable: 3.18.5 [2008-10-09] - change log - -download +

    v3 stable: 3.20.0 [2008-11-05] - change log - +download
    v2 stable: 2.0.6 [2008-08-07] - change log - download -- cgit v1.2.3 From 6b905b511b685f2ae28ef94d2e0ba14d1a3f4df3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 3 Dec 2008 10:45:11 +0100 Subject: bugfix: code did not compile without zlib --- ChangeLog | 3 +++ runtime/parser.c | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 22c6006d..3ac558e3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,7 @@ --------------------------------------------------------------------------- +Version 4.1.2 [DEVEL] (rgerhards), 2008-11-?? +- bugfix: code did not compile without zlib +--------------------------------------------------------------------------- Version 4.1.1 [DEVEL] (rgerhards), 2008-11-26 - added $PrivDropToGroup, $PrivDropToUser, $PrivDropToGroupID, $PrivDropToUserID config directives to enable dropping privileges. diff --git a/runtime/parser.c b/runtime/parser.c index fbdeebeb..15dfd4e0 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -127,12 +127,16 @@ finalize_it: /* in this case, we still need to check if the message is compressed. If so, we must * tell the user we can not accept it. */ - if(len > 0 && *msg == 'z') { + //pszMsg = pMsg->pszRawMsg; + //lenMsg = pMsg->iLenRawMsg; + //if(lenMsg > 0 && *msg == 'z') { + if(pMsg->iLenRawMsg > 0 && *pMsg->pszRawMsg == 'z') { errmsg.LogError(0, NO_ERRCODE, "Received a compressed message, but rsyslogd does not have compression " "support enabled. The message will be ignored."); ABORT_FINALIZE(RS_RET_NO_ZIP); } +finalize_it: # endif /* ifdef USE_NETZIP */ RETiRet; -- cgit v1.2.3 From c5bfd2b24ca8c490401a0835ec741c05acf0ed3e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 3 Dec 2008 10:46:27 +0100 Subject: some cleanup (forgotten...) --- runtime/parser.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/runtime/parser.c b/runtime/parser.c index 15dfd4e0..ec2a28c7 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -127,9 +127,6 @@ finalize_it: /* in this case, we still need to check if the message is compressed. If so, we must * tell the user we can not accept it. */ - //pszMsg = pMsg->pszRawMsg; - //lenMsg = pMsg->iLenRawMsg; - //if(lenMsg > 0 && *msg == 'z') { if(pMsg->iLenRawMsg > 0 && *pMsg->pszRawMsg == 'z') { errmsg.LogError(0, NO_ERRCODE, "Received a compressed message, but rsyslogd does not have compression " "support enabled. The message will be ignored."); @@ -295,7 +292,7 @@ rsRetVal parseMsg(msg_t *pMsg) if(msg[0] == '1' && msg[1] == ' ') { dbgprintf("Message has syslog-protocol format.\n"); setProtocolVersion(pMsg, 1); - if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) { // TODO: parseRFC... should pull flags from pMsg + if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) { msgDestruct(&pMsg); ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! } -- cgit v1.2.3 From 3e1220f434533b5e91de51f5de17cc76eaa8af45 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Dec 2008 01:10:06 +0100 Subject: fixed some compiler warnings --- dirty.h | 2 ++ runtime/parser.c | 1 + 2 files changed, 3 insertions(+) diff --git a/dirty.h b/dirty.h index 4b13adcd..db9bc31b 100644 --- a/dirty.h +++ b/dirty.h @@ -30,6 +30,8 @@ rsRetVal submitMsg(msg_t *pMsg); rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); +int parseRFCSyslogMsg(msg_t *pMsg, int flags); +int parseLegacySyslogMsg(msg_t *pMsg, int flags); /* TODO: the following 2 need to go in conf obj interface... */ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); diff --git a/runtime/parser.c b/runtime/parser.c index ec2a28c7..b549cd19 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -27,6 +27,7 @@ #include "config.h" #include #include +#include #include #ifdef USE_NETZIP #include -- cgit v1.2.3 From 85c85c1496856307a0080150176842e9a480a5a3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Dec 2008 02:14:20 +0100 Subject: doc bugfix: typo in v3 compatibility document thanks to Andrej for reporting --- ChangeLog | 2 ++ doc/v3compatibility.html | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 43dcbff8..d60e6f76 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +- doc bugfix: typo in v3 compatibility document + thanks to Andrej for reporting --------------------------------------------------------------------------- Version 4.1.2 [DEVEL] (rgerhards), 2008-12-04 - bugfix: code did not compile without zlib diff --git a/doc/v3compatibility.html b/doc/v3compatibility.html index 51619947..ad8776bb 100644 --- a/doc/v3compatibility.html +++ b/doc/v3compatibility.html @@ -95,7 +95,7 @@ set the local address the server should listen to via $UDPServerAddress

    The following example configures an UDP syslog server at the local address 192.0.2.1 on port 514:

    $ModLoad imudp
    -$UDPSeverAddress 192.0.2.1 # this MUST be before the $UDPServerRun +$UDPServerAddress 192.0.2.1 # this MUST be before the $UDPServerRun directive!
    $UDPServerRun 514

    "$UDPServerAddress *" means listen on all local interfaces. @@ -103,10 +103,10 @@ This is the default if no directive is specified.

    Please note that now multiple listeners are supported. For example, you can do the following:

    $ModLoad imudp
    -$UDPSeverAddress 192.0.2.1 # this MUST be before the $UDPServerRun +$UDPServerAddress 192.0.2.1 # this MUST be before the $UDPServerRun directive!
    $UDPServerRun 514
    -$UDPSeverAddress * # all local interfaces
    +$UDPServerAddress * # all local interfaces
    $UDPServerRun 1514

    These config file settings run two listeners: one at 192.0.2.1:514 and one on port 1514, which listens on all local -- cgit v1.2.3 From 8b7649fcda9eb15951dcad7728220228b7e29303 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Dec 2008 14:39:22 +0100 Subject: preparing for 4.1.2 release --- ChangeLog | 4 ++-- doc/status.html | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index d60e6f76..c6bc46d0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,3 @@ -- doc bugfix: typo in v3 compatibility document - thanks to Andrej for reporting --------------------------------------------------------------------------- Version 4.1.2 [DEVEL] (rgerhards), 2008-12-04 - bugfix: code did not compile without zlib @@ -12,6 +10,8 @@ Version 4.1.2 [DEVEL] (rgerhards), 2008-12-04 emitted only once in a minute (this currently is a hard-coded limit, if someone comes up with a good reason to make it configurable, we will probably do that). +- doc bugfix: typo in v3 compatibility document directive syntax + thanks to Andrej for reporting - imported other changes from 3.21.8 and 3.20.1 (see there) --------------------------------------------------------------------------- Version 4.1.1 [DEVEL] (rgerhards), 2008-11-26 diff --git a/doc/status.html b/doc/status.html index b814d0bc..74694280 100644 --- a/doc/status.html +++ b/doc/status.html @@ -5,16 +5,16 @@

    This page reflects the status as of 2008-11-26.

    Current Releases

    -

    development: 4.1.1 [2008-11-26] - -change log - -download +

    development: 4.1.2 [2008-12-05] - +change log - +download -
    beta: 3.21.7 [2008-11-11] - -change log - -download

    +
    beta: 3.21.9 [2008-12-04] - +change log - +download

    -

    v3 stable: 3.20.0 [2008-11-05] - change log - -download +

    v3 stable: 3.20.2 [2008-12-04] - change log - +download
    v2 stable: 2.0.6 [2008-08-07] - change log - download -- cgit v1.2.3 From 128edc1598a13c894fe3853673d1231b9feafc39 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Dec 2008 13:31:55 +0100 Subject: bugfix: imudp went into an endless loop under some circumstances (but could also leave it under some other circumstances...) Thanks to David Lang and speedfox for reporting this issue. --- ChangeLog | 3 +++ plugins/imudp/imudp.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index c6bc46d0..0ebc2eef 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +- bugfix: imudp went into an endless loop under some circumstances + (but could also leave it under some other circumstances...) + Thanks to David Lang and speedfox for reporting this issue. --------------------------------------------------------------------------- Version 4.1.2 [DEVEL] (rgerhards), 2008-12-04 - bugfix: code did not compile without zlib diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 037da56d..0193ac06 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -297,12 +297,12 @@ CODESTARTrunInput /* wait for io to become ready */ nfds = select(maxfds+1, (fd_set *) &readfds, NULL, NULL, NULL); - for (i = 0; nfds && i < *udpLstnSocks; i++) { - if (FD_ISSET(udpLstnSocks[i+1], &readfds)) { + for(i = 0; nfds && i < *udpLstnSocks; i++) { + if(FD_ISSET(udpLstnSocks[i+1], &readfds)) { processSocket(udpLstnSocks[i+1], &frominetPrev, &bIsPermitted, fromHost, fromHostFQDN, fromHostIP); - } --nfds; /* indicate we have processed one descriptor */ + } } /* end of a run, back to loop for next recv() */ } -- cgit v1.2.3 From 60b8ce14bf33e76237cf82dd1f68acc750e64316 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Dec 2008 15:42:47 +0100 Subject: added $PreserveFQDN config file directive Enables to use FQDNs in sender names where the legacy default --- ChangeLog | 3 +++ doc/rsyslog_conf_global.html | 3 +++ plugins/imudp/imudp.c | 1 + runtime/glbl.c | 5 +++++ runtime/glbl.h | 4 +++- runtime/net.c | 4 +++- runtime/parser.c | 2 +- 7 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0ebc2eef..aa53fff2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +- added $PreserveFQDN config file directive + Enables to use FQDNs in sender names where the legacy default + would have stripped the domain part. - bugfix: imudp went into an endless loop under some circumstances (but could also leave it under some other circumstances...) Thanks to David Lang and speedfox for reporting this issue. diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index d02245e3..6e03e571 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -186,6 +186,9 @@ supported in order to be compliant to the upcoming new syslog RFC series.

  • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The default may change as uniprocessor systems become less common.
    • +
  • $PreserveFQDN [on/off) - if set to off (legacy default to remain compatible +to sysklogd), the domain part from a name that is within the same domain as the receiving +system is stripped. If set to on, full names are always used.
  • $WorkDirectory <name> (directory for spool and other work files)
  • $UDPServerAddress <IP> (imudp) -- local IP address (or name) the UDP listens should bind to
    • diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 0193ac06..72450513 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -181,6 +181,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, /* check if we have a different sender than before, if so, we need to query some new values */ if(memcmp(&frominet, frominetPrev, socklen) != 0) { CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); +DBGPRINTF("returned: fromHost '%s', FQDN: '%s'\n", fromHost, fromHostFQDN); memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ /* Here we check if a host is permitted to send us * syslog messages. If it isn't, we do not further diff --git a/runtime/glbl.c b/runtime/glbl.c index 2a6bfb11..d06c88ff 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -53,6 +53,7 @@ DEFobjStaticHelpers static uchar *pszWorkDir = NULL; static int bOptimizeUniProc = 1; /* enable uniprocessor optimizations */ static int bHUPisRestart = 1; /* should SIGHUP cause a full system restart? */ +static int bPreserveFQDN = 0; /* should FQDNs always be preserved? */ static int iMaxLine = 2048; /* maximum length of a syslog message */ static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */ static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */ @@ -88,6 +89,7 @@ static dataType Get##nameFunc(void) \ } SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int) +SIMP_PROP(PreserveFQDN, bPreserveFQDN, int) SIMP_PROP(HUPisRestart, bHUPisRestart, int) SIMP_PROP(MaxLine, iMaxLine, int) SIMP_PROP(DefPFFamily, iDefPFFamily, int) /* note that in the future we may check the family argument */ @@ -178,6 +180,7 @@ CODESTARTobjQueryInterface(glbl) pIf->Set##name = Set##name; SIMP_PROP(MaxLine); SIMP_PROP(OptimizeUniProc); + SIMP_PROP(PreserveFQDN); SIMP_PROP(HUPisRestart); SIMP_PROP(DefPFFamily); SIMP_PROP(DropMalPTRMsgs); @@ -224,6 +227,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bDropMalPTRMsgs = 0; bOptimizeUniProc = 1; bHUPisRestart = 1; + bPreserveFQDN = 0; return RS_RET_OK; } @@ -245,6 +249,7 @@ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ CHKiRet(regCfSysLineHdlr((uchar *)"defaultnetstreamdrivercertfile", 0, eCmdHdlrGetWord, NULL, &pszDfltNetstrmDrvrCertFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"optimizeforuniprocessor", 0, eCmdHdlrBinary, NULL, &bOptimizeUniProc, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"hupisrestart", 0, eCmdHdlrBinary, NULL, &bHUPisRestart, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"preservefqdn", 0, eCmdHdlrBinary, NULL, &bPreserveFQDN, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); ENDObjClassInit(glbl) diff --git a/runtime/glbl.h b/runtime/glbl.h index 44e41e3e..205a5212 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -43,6 +43,7 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ SIMP_PROP(MaxLine, int) SIMP_PROP(OptimizeUniProc, int) SIMP_PROP(HUPisRestart, int) + SIMP_PROP(PreserveFQDN, int) SIMP_PROP(DefPFFamily, int) SIMP_PROP(DropMalPTRMsgs, int) SIMP_PROP(Option_DisallowWarning, int) @@ -57,7 +58,8 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ SIMP_PROP(DfltNetstrmDrvrCertFile, uchar*) #undef SIMP_PROP ENDinterface(glbl) -#define glblCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define glblCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +/* version 2 had PreserveFQDN added - rgerhards, 2008-12-08 */ /* the remaining prototypes */ PROTOTYPEObj(glbl); diff --git a/runtime/net.c b/runtime/net.c index 1472b4db..30c923fe 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -1204,7 +1204,9 @@ rsRetVal cvthname(struct sockaddr_storage *f, uchar *pszHost, uchar *pszHostFQDN * make this in option in the long term. (rgerhards, 2007-09-11) */ strcpy((char*)pszHost, (char*)pszHostFQDN); - if ((p = (uchar*) strchr((char*)pszHost, '.'))) { /* find start of domain name "machine.example.com" */ + if( (glbl.GetPreserveFQDN() == 0) + && (p = (uchar*) strchr((char*)pszHost, '.'))) { /* find start of domain name "machine.example.com" */ + strcmp((char*)(p + 1), (char*)glbl.GetLocalDomain())); if(strcmp((char*)(p + 1), (char*)glbl.GetLocalDomain()) == 0) { *p = '\0'; /* simply terminate the string */ } else { diff --git a/runtime/parser.c b/runtime/parser.c index b549cd19..b4ab0a3e 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -263,7 +263,7 @@ rsRetVal parseMsg(msg_t *pMsg) CHKiRet(sanitizeMessage(pMsg)); /* we needed to sanitize first, because we otherwise do not have a C-string we can print... */ - DBGPRINTF("msg parser: flags %x, from '%s', msg %s\n", pMsg->msgFlags, pMsg->pszRcvFrom, pMsg->pszRawMsg); + DBGPRINTF("msg parser: flags %x, from '%s', msg '%s'\n", pMsg->msgFlags, pMsg->pszRcvFrom, pMsg->pszRawMsg); /* pull PRI */ pri = DEFUPRI; -- cgit v1.2.3 From 6ce531106bc26c117da8e9700d489b462e41eb8f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Dec 2008 19:18:55 +0100 Subject: added sponsor information - thanks to BlinkMind for sponsoring $PreserveFQDN --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index aa53fff2..efbd6afb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,8 @@ - added $PreserveFQDN config file directive Enables to use FQDNs in sender names where the legacy default would have stripped the domain part. + Thanks to BlinkMind, Inc. http://www.blinkmind.com for sponsoring this + development. - bugfix: imudp went into an endless loop under some circumstances (but could also leave it under some other circumstances...) Thanks to David Lang and speedfox for reporting this issue. -- cgit v1.2.3 From a10bc421fffbeaa872ae0cdcb651f0a7e613ee7f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Dec 2008 19:55:27 +0100 Subject: resolved compile problem, e.g. on FreeBSD I commented out some debug code that is only useful in some testing scenarios and re-enabled the old code. This solved a FreeBSD compile issue. Also, I fixed some other syntax error, which somehow went into the tree (I am still puzzled about that, especially as some have already and successfully build from that tree... anyhow ;)). --- runtime/debug.c | 4 ++-- runtime/net.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/debug.c b/runtime/debug.c index 102d5d0f..53291fc2 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -819,8 +819,8 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) if(altdbg != -1) write(altdbg, pszWriteBuf, lenWriteBuf); } - // old, reenable TODO lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf), "%s: ", pszThrdName); - lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf), "{%ld}%s: ", (long) syscall(SYS_gettid), pszThrdName); + lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf), "%s: ", pszThrdName); + // use for testing: lenWriteBuf = snprintf(pszWriteBuf, sizeof(pszWriteBuf), "{%ld}%s: ", (long) syscall(SYS_gettid), pszThrdName); if(stddbg != -1) write(stddbg, pszWriteBuf, lenWriteBuf); if(altdbg != -1) write(altdbg, pszWriteBuf, lenWriteBuf); /* print object name header if we have an object */ diff --git a/runtime/net.c b/runtime/net.c index 30c923fe..6fa27658 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -1206,7 +1206,7 @@ rsRetVal cvthname(struct sockaddr_storage *f, uchar *pszHost, uchar *pszHostFQDN strcpy((char*)pszHost, (char*)pszHostFQDN); if( (glbl.GetPreserveFQDN() == 0) && (p = (uchar*) strchr((char*)pszHost, '.'))) { /* find start of domain name "machine.example.com" */ - strcmp((char*)(p + 1), (char*)glbl.GetLocalDomain())); + strcmp((char*)(p + 1), (char*)glbl.GetLocalDomain()); if(strcmp((char*)(p + 1), (char*)glbl.GetLocalDomain()) == 0) { *p = '\0'; /* simply terminate the string */ } else { -- cgit v1.2.3 From 483be404716d8e3e55f200955e1904219eb97a9b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Dec 2008 14:26:19 +0100 Subject: enhanced imtcp, among others to handel invalid NetScreen framing - added $InputTCPServerAddtlFrameDelimiter config directive, which enabeles to specify an additional, non-standard message delimiter for processing plain tcp syslog. This is primarily a fix for the invalid framing used in Juniper's NetScreen products. Credit to forum user Arv for suggesting this solution. - added $InputTCPServerInputName property, which enables a name to be specified that will be available during message processing in the inputname property. This is considered useful for logic that treats messages differently depending on which input received them. --- ChangeLog | 9 +++++++++ doc/imtcp.html | 27 ++++++++++++++++++++++++++- plugins/imtcp/imtcp.c | 17 +++++++++++++++++ tcps_sess.c | 12 +++++++----- tcpsrv.c | 35 +++++++++++++++++++++++++++++++++++ tcpsrv.h | 19 ++++++++++++++++++- 6 files changed, 112 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index efbd6afb..f95989e2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ +- added $InputTCPServerAddtlFrameDelimiter config directive, which + enabeles to specify an additional, non-standard message delimiter + for processing plain tcp syslog. This is primarily a fix for the invalid + framing used in Juniper's NetScreen products. Credit to forum user + Arv for suggesting this solution. +- added $InputTCPServerInputName property, which enables a name to be + specified that will be available during message processing in the + inputname property. This is considered useful for logic that treats + messages differently depending on which input received them. - added $PreserveFQDN config file directive Enables to use FQDNs in sender names where the legacy default would have stripped the domain part. diff --git a/doc/imtcp.html b/doc/imtcp.html index 583cd531..0ee0f96a 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -20,11 +20,36 @@ $InputTCPServerRun multiple times. This is not currently supported.

    Configuration Directives:

      +
    • $InputTCPServerAddtlFrameDelimiter <Delimiter>
      +This directive permits to specify an additional frame delimiter for plain tcp syslog. +The industry-standard specifies using the LF character as frame delimiter. Some vendors, +notable Juniper in their NetScreen products, use an invalid frame delimiter, in Juniper's +case the NUL character. This directive permits to specify the ASCII value of the delimiter +in question. Please note that this does not guarantee that all wrong implementations can +be cured with this directive. It is not even a sure fix with all versions of NetScreen, +as I suggest the NUL character is the effect of a (common) coding error and thus will +probably go away at some time in the future. But for the time being, the value 0 can +probably be used to make rsyslog handle NetScreen's invalid syslog/tcp framing. +For additional information, see this +forum thread. +
      If this doesn't work for you, please do not blame the rsyslog team. Instead file +a bug report with Juniper! +
      Note that a similar, but worse, issue exists with Cisco's IOS implementation. They do +not use any framing at all. This is confirmed from Cisco's side, but there seems to be +very limited interest in fixing this issue. This directive can not fix the Cisco bug. +That would require much more code changes, which I was unable to do so far. Full details +can be found at the Cisco tcp syslog anomaly +page.
    • $InputTCPServerRun <port>
      Starts a TCP server on selected port
      • $InputTCPMaxSessions <number>
      Sets the maximum number of sessions supported
    • $InputTCPServerStreamDriverMode <number>
      -Sets the driver mode for the currently selected network stream driver. <number> is driver specifc.
    • $InputTCPServerStreamDriverAuthMode <mode-string>
      +Sets the driver mode for the currently selected network stream driver. <number> is driver specifc.
      • +
    • $InputTCPServerInputName <name>
      +Sets a name for the inputname property. If no name is set "imtcp" is used by default. Setting a +name is not strictly necessary, but can be useful to apply filtering based on which input +the message was received from. +
    • $InputTCPServerStreamDriverAuthMode <mode-string>
      Sets the authentication mode for the currently selected network stream driver. <mode-string> is driver specifc.
    • $InputTCPServerStreamDriverPermittedPeer <id-string>
      Sets permitted peer IDs. Only these peers are able to connect to the listener. <id-string> semantics depend on the currently selected diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 89f1dbcf..19138d94 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -80,7 +80,9 @@ static permittedPeers_t *pPermPeersRoot = NULL; /* config settings */ static int iTCPSessMax = 200; /* max number of sessions */ static int iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */ +static int iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; /* addtl frame delimiter, e.g. for netscreen, default none */ static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ +static uchar *pszInputName = NULL; /* value for inputname property, NULL is OK and handled by core engine */ /* callbacks */ @@ -166,6 +168,8 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa CHKiRet(tcpsrv.SetCBOnRegularClose(pOurTcpsrv, onRegularClose)); CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); CHKiRet(tcpsrv.SetDrvrMode(pOurTcpsrv, iStrmDrvrMode)); + CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? (uchar*)"imtcp" : pszInputName)); + CHKiRet(tcpsrv.SetAddtlFrameDelim(pOurTcpsrv, iAddtlFrameDelim)); /* now set optional params, but only if they were actually configured */ if(pszStrmDrvrAuthMode != NULL) { CHKiRet(tcpsrv.SetDrvrAuthMode(pOurTcpsrv, pszStrmDrvrAuthMode)); @@ -239,6 +243,15 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus { iTCPSessMax = 200; iStrmDrvrMode = 0; + iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; + if(pszInputName != NULL) { + free(pszInputName); + pszInputName = NULL; + } + if(pszStrmDrvrAuthMode != NULL) { + free(pszStrmDrvrAuthMode); + pszStrmDrvrAuthMode = NULL; + } return RS_RET_OK; } @@ -273,6 +286,10 @@ CODEmodInit_QueryRegCFSLineHdlr eCmdHdlrGetWord, NULL, &pszStrmDrvrAuthMode, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdriverpermittedpeer", 0, eCmdHdlrGetWord, setPermittedPeer, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserveraddtlframedelimiter", 0, eCmdHdlrInt, + NULL, &iAddtlFrameDelim, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverinputname", 0, + eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/tcps_sess.c b/tcps_sess.c index e8bef5b1..d0edc018 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -231,7 +231,7 @@ PrepareClose(tcps_sess_t *pThis) */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); pThis->bAtStrtOfFram = 1; } @@ -314,7 +314,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); pThis->iMsg = 0; /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good @@ -323,9 +323,11 @@ processDataRcvd(tcps_sess_t *pThis, char c) */ } - if(c == '\n' && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delemiter? */ + if(( (c == '\n') + || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) + ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } else { @@ -344,7 +346,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, NULL, NULL, 0); /* TODO: add real InputName */ + PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); pThis->iMsg = 0; pThis->inputState = eAtStrtFram; } diff --git a/tcpsrv.c b/tcpsrv.c index 885edba3..bb81a281 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -513,6 +513,7 @@ finalize_it: /* this is a very special case - this time only we do not exit the /* Standard-Constructor */ BEGINobjConstruct(tcpsrv) /* be sure to specify the object type also in END macro! */ pThis->iSessMax = TCPSESS_MAX_DEFAULT; /* TODO: useful default ;) */ + pThis->addtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; ENDobjConstruct(tcpsrv) @@ -560,6 +561,8 @@ CODESTARTobjDestruct(tcpsrv) free(pThis->pszDrvrAuthMode); if(pThis->ppLstn != NULL) free(pThis->ppLstn); + if(pThis->pszInputName != NULL) + free(pThis->pszInputName); ENDobjDestruct(tcpsrv) @@ -658,6 +661,36 @@ SetUsrP(tcpsrv_t *pThis, void *pUsr) } +/* Set additional framing to use (if any) -- rgerhards, 2008-12-10 */ +static rsRetVal +SetAddtlFrameDelim(tcpsrv_t *pThis, int iDelim) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcpsrv); + pThis->addtlFrameDelim = iDelim; + RETiRet; +} + + +/* Set the input name to use -- rgerhards, 2008-12-10 */ +static rsRetVal +SetInputName(tcpsrv_t *pThis, uchar *name) +{ + uchar *pszName; + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcpsrv); + if(name == NULL) + pszName = NULL; + else + CHKmalloc(pszName = (uchar*)strdup((char*)name)); + if(pThis->pszInputName != NULL) + free(pThis->pszInputName); + pThis->pszInputName = pszName; +finalize_it: + RETiRet; +} + + /* here follows a number of methods that shuffle authentication settings down * to the drivers. Drivers not supporting these settings may return an error * state. @@ -727,6 +760,8 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->Run = Run; pIf->SetUsrP = SetUsrP; + pIf->SetInputName = SetInputName; + pIf->SetAddtlFrameDelim = SetAddtlFrameDelim; pIf->SetDrvrMode = SetDrvrMode; pIf->SetDrvrAuthMode = SetDrvrAuthMode; pIf->SetDrvrPermPeers = SetDrvrPermPeers; diff --git a/tcpsrv.h b/tcpsrv.h index 01110866..2924bafa 100644 --- a/tcpsrv.h +++ b/tcpsrv.h @@ -25,17 +25,28 @@ #include "obj.h" #include "tcps_sess.h" +/* support for framing anomalies */ +typedef enum ETCPsyslogFramingAnomaly { + frame_normal = 0, + frame_NetScreen = 1, + frame_CiscoIOS = 2 +} eTCPsyslogFramingAnomaly; + +#define TCPSRV_NO_ADDTL_DELIMITER -1 /* specifies that no additional delimiter is to be used in TCP framing */ + /* the tcpsrv object */ struct tcpsrv_s { BEGINobjInstance; /**< Data to implement generic object - MUST be the first data element! */ netstrms_t *pNS; /**< pointer to network stream subsystem */ int iDrvrMode; /**< mode of the stream driver to use */ uchar *pszDrvrAuthMode; /**< auth mode of the stream driver to use */ + uchar *pszInputName; /**< value to be used as input name */ permittedPeers_t *pPermPeers;/**< driver's permitted peers */ int iLstnMax; /**< max nbr of listeners currently supported */ netstrm_t **ppLstn; /**< our netstream listners */ int iSessMax; /**< max number of sessions supported */ char *TCPLstnPort; /**< the port the listener shall listen on */ + int addtlFrameDelim; /**< additional frame delimiter for plain TCP syslog framing (e.g. to handle NetScreen) */ tcps_sess_t **pSessions;/**< array of all of our sessions */ void *pUsr; /**< a user-settable pointer (provides extensibility for "derived classes")*/ /* callbacks */ @@ -64,6 +75,8 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*create_tcp_socket)(tcpsrv_t *pThis); rsRetVal (*Run)(tcpsrv_t *pThis); /* set methods */ + rsRetVal (*SetAddtlFrameDelim)(tcpsrv_t*, int); + rsRetVal (*SetInputName)(tcpsrv_t*, uchar*); rsRetVal (*SetUsrP)(tcpsrv_t*, void*); rsRetVal (*SetCBIsPermittedHost)(tcpsrv_t*, int (*) (struct sockaddr *addr, char*, void*, void*)); rsRetVal (*SetCBOpenLstnSocks)(tcpsrv_t *, rsRetVal (*)(tcpsrv_t*)); @@ -80,7 +93,11 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetCBOnSessDestruct)(tcpsrv_t*, rsRetVal (*) (void*)); rsRetVal (*SetCBOnSessConstructFinalize)(tcpsrv_t*, rsRetVal (*) (void*)); ENDinterface(tcpsrv) -#define tcpsrvCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ +#define tcpsrvCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ +/* change for v4: + * - SetAddtlFrameDelim() added -- rgerhards, 2008-12-10 + * - SetInputName() added -- rgerhards, 2008-12-10 + */ /* prototypes */ -- cgit v1.2.3 From 246962be65941c7994aa18e4f9327f239c62eb9e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Dec 2008 11:35:58 +0100 Subject: preparing for 4.1.3 release --- ChangeLog | 4 +++- configure.ac | 2 +- doc/features.html | 2 +- doc/manual.html | 2 +- doc/status.html | 6 +++--- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ChangeLog b/ChangeLog index e82169fe..5056f68d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ +--------------------------------------------------------------------------- +Version 4.1.3 [DEVEL] (rgerhards), 2008-12-17 - added $InputTCPServerAddtlFrameDelimiter config directive, which - enabeles to specify an additional, non-standard message delimiter + enables to specify an additional, non-standard message delimiter for processing plain tcp syslog. This is primarily a fix for the invalid framing used in Juniper's NetScreen products. Credit to forum user Arv for suggesting this solution. diff --git a/configure.ac b/configure.ac index e8aa644a..d204b7fd 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.2],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.1.3],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_HEADERS([config.h]) diff --git a/doc/features.html b/doc/features.html index 7a31a03b..336b31cc 100644 --- a/doc/features.html +++ b/doc/features.html @@ -101,7 +101,7 @@ arithmetic expressions in message filters

      • World's first

      Rsyslog has an interesting number of "world's firsts" - things that were implemented for the first time ever in rsyslog. Some of them are still features not available elsewhere.
        -
      • world's first implementation of IETF I-D syslog-protocol (February 2006, version 1.12.2 and above)
      • world's first implementation of dynamic syslog on-the-wire compression (December 2006, version 1.13.0 and above)
      • world's first open-source implementation of a disk-queueing syslogd (January 2008, version 3.11.0 and above)
        • +
      • world's first implementation of IETF I-D syslog-protocol (February 2006, version 1.12.2 and above), now RFC5424
      • world's first implementation of dynamic syslog on-the-wire compression (December 2006, version 1.13.0 and above)
      • world's first open-source implementation of a disk-queueing syslogd (January 2008, version 3.11.0 and above)
      • world's first implementation of IETF I-D syslog-transport-tls (May 2008, version 3.19.0 and above)
      diff --git a/doc/manual.html b/doc/manual.html index 63a68b4f..e8842de6 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

      Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

      -

      This documentation is for version 4.1.2 (devel branch) of rsyslog. +

      This documentation is for version 4.1.3 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

      If you like rsyslog, you might diff --git a/doc/status.html b/doc/status.html index 74694280..c1a42963 100644 --- a/doc/status.html +++ b/doc/status.html @@ -5,9 +5,9 @@

      This page reflects the status as of 2008-11-26.

      Current Releases

      -

      development: 4.1.2 [2008-12-05] - -change log - -download +

      development: 4.1.3 [2008-12-17] - +change log - +download
      beta: 3.21.9 [2008-12-04] - change log - -- cgit v1.2.3 From a185665be4cf699752589d81ef6e396dd61f68b6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Dec 2008 12:52:49 +0100 Subject: experimentally altered "last message repeated n times" to include msg This was suggested by David Lang, to help identify the message that was repeated. A problem is that I do not have the expanded template at hand when the "last ... times" message is generated. Spending much time on this functionality is also probably not a good thing, as the whole functionality will be overhauled (and once this is done we will not at all have the output template at hand). So the approach is to use a single field - here msg - and inlcude it as a notation of what was repeated. This is far from being perfect, but eventually good enough. I will now wait for feedback before going any further. --- action.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/action.c b/action.c index dee46f16..ab4d2f4d 100644 --- a/action.c +++ b/action.c @@ -621,9 +621,15 @@ actionWriteToAction(action_t *pAction) */ if(pAction->f_prevcount > 1) { msg_t *pMsg; +#if 0 /* old */ uchar szRepMsg[64]; snprintf((char*)szRepMsg, sizeof(szRepMsg), "last message repeated %d times", pAction->f_prevcount); +#else + uchar szRepMsg[1024]; + snprintf((char*)szRepMsg, sizeof(szRepMsg), "message repeated %d times: [%.800]", + pAction->f_prevcount, getMSG(pAction->f_pMsg)); +#endif if((pMsg = MsgDup(pAction->f_pMsg)) == NULL) { /* it failed - nothing we can do against it... */ @@ -658,12 +664,11 @@ actionWriteToAction(action_t *pAction) dbgprintf("action not yet ready again to be executed, onceInterval %d, tCurr %d, tNext %d\n", (int) pAction->iSecsExecOnceInterval, (int) getActNow(pAction), (int) (pAction->iSecsExecOnceInterval + pAction->tLastExec)); - /* TODO: the time call below may use reception time, not dequeue time - under consideration. -- rgerhards, 2008-09-17 */ pAction->tLastExec = getActNow(pAction); /* re-init time flags */ FINALIZE; } - /* TODO: the time call below may use reception time, not dequeue time - under consideration. -- rgerhards, 2008-09-17 */ + /* we use reception time, not dequeue time - this is considered more appropriate and also faster ;) -- rgerhards, 2008-09-17 */ pAction->f_time = pAction->f_pMsg->ttGenTime; /* When we reach this point, we have a valid, non-disabled action. -- cgit v1.2.3 From ec2019abe255949d5adcafc8f81cb949b45885e3 Mon Sep 17 00:00:00 2001 From: varmojfekoj Date: Mon, 12 Jan 2009 12:37:41 +0100 Subject: fixed type in format string (s as string indicator missing) Signed-off-by: Rainer Gerhards --- action.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.c b/action.c index ab4d2f4d..c95c0def 100644 --- a/action.c +++ b/action.c @@ -627,7 +627,7 @@ actionWriteToAction(action_t *pAction) pAction->f_prevcount); #else uchar szRepMsg[1024]; - snprintf((char*)szRepMsg, sizeof(szRepMsg), "message repeated %d times: [%.800]", + snprintf((char*)szRepMsg, sizeof(szRepMsg), "message repeated %d times: [%.800s]", pAction->f_prevcount, getMSG(pAction->f_pMsg)); #endif -- cgit v1.2.3 From d5360aa57436bb931c513677bc2cbdb1733a4c6b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 19 Jan 2009 11:18:39 +0100 Subject: updated project status after 3.20.3 release --- doc/status.html | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/status.html b/doc/status.html index c1a42963..01bd4479 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,7 +2,7 @@ rsyslog status page

      rsyslog status page

      -

      This page reflects the status as of 2008-11-26.

      +

      This page reflects the status as of 2009-01-19.

      Current Releases

      development: 4.1.3 [2008-12-17] - @@ -13,8 +13,8 @@ change log - download

      -

      v3 stable: 3.20.2 [2008-12-04] - change log - -download +

      v3 stable: 3.20.3 [2009-01-19] - change log - +download
      v2 stable: 2.0.6 [2008-08-07] - change log - download @@ -24,7 +24,8 @@ upgrade, you may consider purchasing a out that it is really not a good idea to still run a v0 version.

      If you updgrade from version 2, be sure to read the rsyslog v3 -compatibility document.

      +compatibility document. There are no additional compatibility concerns at this time for +upgrading from v3 to v4. If some occur, we will post an additional compatiblity document.

      (How are versions named?)

      Platforms

      -- cgit v1.2.3 From ead2c355e3261f98817ccd52bc3644103140e824 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 26 Jan 2009 22:30:55 +0100 Subject: bugfix: unitialized mutex was used in msg.c:getPRI This was subtle, because getPRI is called as part of the debugging code (always executed) in syslogd.c:logmsg. --- runtime/msg.c | 9 +++++++-- runtime/msg.h | 1 + tools/syslogd.c | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 2e2d41ad..02a4cd8a 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -190,6 +190,7 @@ static void MsgPrepareEnqueueLockingCase(msg_t *pThis) * rgerhards, 2008-07-14 */ pthread_mutexattr_destroy(&pThis->mutAttr); + pThis->bDoLock = 1; ENDfunc } @@ -199,14 +200,16 @@ static void MsgLockLockingCase(msg_t *pThis) { /* DEV debug only! dbgprintf("MsgLock(0x%lx)\n", (unsigned long) pThis); */ assert(pThis != NULL); - pthread_mutex_lock(&pThis->mut); + if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ + pthread_mutex_lock(&pThis->mut); } static void MsgUnlockLockingCase(msg_t *pThis) { /* DEV debug only! dbgprintf("MsgUnlock(0x%lx)\n", (unsigned long) pThis); */ assert(pThis != NULL); - pthread_mutex_unlock(&pThis->mut); + if(pThis->bDoLock == 1) /* TODO: this is a testing hack, we should find a way with better performance! -- rgerhards, 2009-01-27 */ + pthread_mutex_unlock(&pThis->mut); } /* delete the mutex object on message destruction (locking case) @@ -745,6 +748,7 @@ char *getMSG(msg_t *pM) char *getPRI(msg_t *pM) { int pri; + BEGINfunc if(pM == NULL) return ""; @@ -764,6 +768,7 @@ char *getPRI(msg_t *pM) } MsgUnlock(pM); + ENDfunc return (char*)pM->pszPRI; } diff --git a/runtime/msg.h b/runtime/msg.h index d98111a8..c8350626 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,6 +51,7 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; +short bDoLock; /* use the mutex? */ pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 2cac8fe4..f0d63932 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2431,6 +2431,7 @@ init(void) ABORT_FINALIZE(RS_RET_VALIDATION_RUN); /* switch the message object to threaded operation, if necessary */ +/* TODO:XXX: I think we must do this also if we have action queues! -- rgerhards, 2009-01-26 */ if(MainMsgQueType == QUEUETYPE_DIRECT || iMainMsgQueueNumWorkers > 1) { MsgEnableThreadSafety(); } -- cgit v1.2.3 From 14d5cc7f55ffc7980c0bb73f50b53da175271358 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 29 Jan 2009 11:58:26 +0100 Subject: fixed atomic operations --- runtime/atomic.h | 2 +- runtime/msg.c | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/runtime/atomic.h b/runtime/atomic.h index 7ad8e2e4..ec7acb8c 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -47,7 +47,7 @@ # define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff)) # define ATOMIC_STORE_1_TO_32BIT(data) __sync_lock_test_and_set(&(data), 1) #else -# warning "atomic builtins not available, using nul operations" +# warning "atomic builtins not available, using nul operations - rsyslogd will probably be racy!" # define ATOMIC_INC(data) (++(data)) # define ATOMIC_DEC(data) (--(data)) # define ATOMIC_DEC_AND_FETCH(data) (--(data)) diff --git a/runtime/msg.c b/runtime/msg.c index 2e2d41ad..cf291b5d 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -328,14 +328,13 @@ finalize_it: BEGINobjDestruct(msg) /* be sure to specify the object type also in END and CODESTART macros! */ int currRefCount; CODESTARTobjDestruct(msg) - /* DEV Debugging only ! dbgprintf("msgDestruct\t0x%lx, Ref now: %d\n", (unsigned long)pM, pM->iRefCount - 1); */ -//# ifdef DO_HAVE_ATOMICS -// currRefCount = ATOMIC_DEC_AND_FETCH(pThis->iRefCount); -//# else + /* DEV Debugging only ! dbgprintf("msgDestruct\t0x%lx, Ref now: %d\n", (unsigned long)pThis, pThis->iRefCount - 1); */ +# ifdef HAVE_ATOMIC_BUILTINS + currRefCount = ATOMIC_DEC_AND_FETCH(pThis->iRefCount); +# else MsgLock(pThis); currRefCount = --pThis->iRefCount; -//# endif -// we need a mutex, because we may be suspended after getting the refcount but before +# endif if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ -- cgit v1.2.3 From 51d3552866c6b81b0860d32f8142f3bb83d0054f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 29 Jan 2009 15:20:05 +0100 Subject: preparing for 4.1.4 release --- ChangeLog | 8 ++++++++ configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1bd552b9..0d6f651d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,11 @@ +--------------------------------------------------------------------------- +Version 4.1.4 [DEVEL] (rgerhards), 2009-01-29 +- bugfix: inconsistent use of mutex/atomic operations could cause segfault + details are too many, for full analysis see blog post at: + http://blog.gerhards.net/2009/01/rsyslog-data-race-analysis.html +- bugfix: unitialized mutex was used in msg.c:getPRI + This was subtle, because getPRI is called as part of the debugging code + (always executed) in syslogd.c:logmsg. - bufgix: $PreserveFQDN was not properly handled for locally emitted messages --------------------------------------------------------------------------- diff --git a/configure.ac b/configure.ac index d204b7fd..1bf460ad 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.3],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.1.4],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_HEADERS([config.h]) diff --git a/doc/manual.html b/doc/manual.html index bc4c0bc5..bd927d78 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

      Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

      -

      This documentation is for version 4.1.3 (devel branch) of rsyslog. +

      This documentation is for version 4.1.4 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

      If you like rsyslog, you might -- cgit v1.2.3 From c5ece5504314797027c3742891db99759707c6b2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 29 Jan 2009 16:47:09 +0100 Subject: minor doc: updated project status --- doc/status.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/status.html b/doc/status.html index 01bd4479..9a6677b3 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,12 +2,12 @@ rsyslog status page

      rsyslog status page

      -

      This page reflects the status as of 2009-01-19.

      +

      This page reflects the status as of 2009-01-29.

      Current Releases

      -

      development: 4.1.3 [2008-12-17] - -change log - -download +

      development: 4.1.4 [2009-01-29] - +change log - +download
      beta: 3.21.9 [2008-12-04] - change log - -- cgit v1.2.3 From 94f8ca5e2cc12dd2ea9fed6027b531b182cfe73d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 30 Jan 2009 13:28:10 +0100 Subject: added useful internal doc link --- doc/rsyslog_conf_templates.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/rsyslog_conf_templates.html b/doc/rsyslog_conf_templates.html index 90b5fafe..6c68b801 100644 --- a/doc/rsyslog_conf_templates.html +++ b/doc/rsyslog_conf_templates.html @@ -33,7 +33,8 @@ bit restricted currently.

      All text in the template is used literally, except for things within percent signs. These are properties and allow you access to the contents of the syslog message. Properties are accessed via the -property replacer (nice name, huh) and it can do cool things, too. For +property replacer +(nice name, huh) and it can do cool things, too. For example, it can pick a substring or do date-specific formatting. More on this is below, on some lines of the property replacer.

      -- cgit v1.2.3 From f826c284d2fc1b68ca85a237d5c6ac92a5233534 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 2 Feb 2009 14:45:03 +0100 Subject: updated project status after 3.21.10 release --- doc/status.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/status.html b/doc/status.html index 9a6677b3..f9e5852c 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,16 +2,16 @@ rsyslog status page

      rsyslog status page

      -

      This page reflects the status as of 2009-01-29.

      +

      This page reflects the status as of 2009-02-02.

      Current Releases

      development: 4.1.4 [2009-01-29] - change log - download -
      beta: 3.21.9 [2008-12-04] - -change log - -download

      +
      beta: 3.21.10 [2009-02-02] - +change log - +download

      v3 stable: 3.20.3 [2009-01-19] - change log - download -- cgit v1.2.3 From 6ce4a9d605e62b32ed62b5d6e498de5202565cee Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 2 Feb 2009 15:28:24 +0100 Subject: added new config directive $RepeatedMsgContainsOriginalMsg so that the "last message repeated n times" messages, if generated, may have an alternate format that contains the message that is being repeated. Note that this is on an action-by-action basis. --- ChangeLog | 5 +++++ action.c | 19 +++++++++++-------- action.h | 3 ++- doc/rsyslog_conf_global.html | 8 +++++++- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index 5056f68d..acd3df4d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,9 @@ --------------------------------------------------------------------------- +Version 4.1.5 [DEVEL] (rgerhards), 2009-02-?? +- added new config directive $RepeatedMsgContainsOriginalMsg so that the + "last message repeated n times" messages, if generated, may + have an alternate format that contains the message that is being repeated +--------------------------------------------------------------------------- Version 4.1.3 [DEVEL] (rgerhards), 2008-12-17 - added $InputTCPServerAddtlFrameDelimiter config directive, which enables to specify an additional, non-standard message delimiter diff --git a/action.c b/action.c index c95c0def..a41f976c 100644 --- a/action.c +++ b/action.c @@ -58,6 +58,7 @@ static int iActExecEveryNthOccur = 0; /* execute action every n-th occurence (0, static time_t iActExecEveryNthOccurTO = 0; /* timeout for n-occurence setting (in seconds, 0=never) */ static int glbliActionResumeInterval = 30; int glbliActionResumeRetryCount = 0; /* how often should suspended actions be retried? */ +static int bActionRepMsgHasMsg = 0; /* last messsage repeated... has msg fragment in it */ /* main message queue and its configuration parameters */ static queueType_t ActionQueType = QUEUETYPE_DIRECT; /* type of the main message queue above */ @@ -621,15 +622,7 @@ actionWriteToAction(action_t *pAction) */ if(pAction->f_prevcount > 1) { msg_t *pMsg; -#if 0 /* old */ - uchar szRepMsg[64]; - snprintf((char*)szRepMsg, sizeof(szRepMsg), "last message repeated %d times", - pAction->f_prevcount); -#else uchar szRepMsg[1024]; - snprintf((char*)szRepMsg, sizeof(szRepMsg), "message repeated %d times: [%.800s]", - pAction->f_prevcount, getMSG(pAction->f_pMsg)); -#endif if((pMsg = MsgDup(pAction->f_pMsg)) == NULL) { /* it failed - nothing we can do against it... */ @@ -637,6 +630,14 @@ actionWriteToAction(action_t *pAction) ABORT_FINALIZE(RS_RET_ERR); } + if(pAction->bRepMsgHasMsg == 0) { /* old format repeat message? */ + snprintf((char*)szRepMsg, sizeof(szRepMsg), "last message repeated %d times", + pAction->f_prevcount); + } else { + snprintf((char*)szRepMsg, sizeof(szRepMsg), "message repeated %d times: [%.800s]", + pAction->f_prevcount, getMSG(pAction->f_pMsg)); + } + /* We now need to update the other message properties. * ... RAWMSG is a problem ... Please note that digital * signatures inside the message are also invalidated. @@ -823,6 +824,7 @@ actionAddCfSysLineHdrl(void) CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuetimeend", 0, eCmdHdlrInt, NULL, &iActionQueueDeqtWinToHr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyeverynthtime", 0, eCmdHdlrInt, NULL, &iActExecEveryNthOccur, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionexeconlyeverynthtimetimeout", 0, eCmdHdlrInt, NULL, &iActExecEveryNthOccurTO, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"repeatedmsgcontainsoriginalmsg", 0, eCmdHdlrBinary, NULL, &bActionRepMsgHasMsg, NULL)); finalize_it: RETiRet; @@ -856,6 +858,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques pAction->iSecsExecOnceInterval = iActExecOnceInterval; pAction->iExecEveryNthOccur = iActExecEveryNthOccur; pAction->iExecEveryNthOccurTO = iActExecEveryNthOccurTO; + pAction->bRepMsgHasMsg = bActionRepMsgHasMsg; iActExecEveryNthOccur = 0; /* auto-reset */ iActExecEveryNthOccurTO = 0; /* auto-reset */ diff --git a/action.h b/action.h index 8d9d5102..e35e634c 100644 --- a/action.h +++ b/action.h @@ -57,7 +57,8 @@ struct action_s { time_t tLastOccur; /* time last occurence was seen (for timing them out) */ struct modInfo_s *pMod;/* pointer to output module handling this selector */ void *pModData; /* pointer to module data - content is module-specific */ - int f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ + short bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ + short f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ int f_prevcount; /* repetition cnt of prevline */ int f_repeatcount; /* number of "repeated" msgs */ int iNumTpls; /* number of array entries for template element below */ diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 6e03e571..b0c1e400 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -181,6 +181,12 @@ supported in order to be compliant to the upcoming new syslog RFC series.

    • $ModDir
    • $ModLoad
      • +
    • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, +have a different format that contains the message that is being repeated. +Note that only the first "n" characters are included, with n to be at least 80 characters, most +probably more (this may change from version to version, thus no specific limit is given). The bottom +line is that n is large enough to get a good idea which message was repeated but it is not necessarily +large enough for the whole message. (Introduced with 4.1.5). Once set, it affects all following actions.
    • $RepeatedMsgReduction
    • $ResetConfigVariables
    • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better @@ -226,7 +232,7 @@ point of view, "1,,0.0.,.,0" also has the value 1000.

      [rsyslog site]

      This documentation is part of the rsyslog project.
      -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL version 2 or higher.

      -- cgit v1.2.3 From 5005bce38763051b5b12e48ac60c3ff17097a952 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 4 Mar 2009 18:22:48 +0100 Subject: added ERE support in filter conditions new comparison operation "ereregex" --- ChangeLog | 2 ++ doc/rsyslog_conf_filter.html | 8 +++++++- runtime/conf.c | 2 ++ runtime/stringbuf.c | 44 ++++++++++++++++++++++++++++++++++---------- runtime/stringbuf.h | 2 +- tools/syslogd.c | 7 ++++++- tools/syslogd.h | 3 ++- 7 files changed, 54 insertions(+), 14 deletions(-) diff --git a/ChangeLog b/ChangeLog index ba2a6c13..f1bc354c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-02-?? +- added ERE support in filter conditions + new comparison operation "ereregex" - added new config directive $RepeatedMsgContainsOriginalMsg so that the "last message repeated n times" messages, if generated, may have an alternate format that contains the message that is being repeated diff --git a/doc/rsyslog_conf_filter.html b/doc/rsyslog_conf_filter.html index cf21ff95..1d30d8ae 100644 --- a/doc/rsyslog_conf_filter.html +++ b/doc/rsyslog_conf_filter.html @@ -141,7 +141,13 @@ once they are implemented, it can make very much sense regex Compares the property against the provided POSIX -regular +BRE regular +expression. + + +ereregex +Compares the property against the provided POSIX +ERE regular expression. diff --git a/runtime/conf.c b/runtime/conf.c index f71d5669..c5208d86 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -873,6 +873,8 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) f->f_filterData.prop.operation = FIOP_STARTSWITH; } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (unsigned char*) "regex", 5)) { f->f_filterData.prop.operation = FIOP_REGEX; + } else if(!rsCStrOffsetSzStrCmp(pCSCompOp, iOffset, (unsigned char*) "ereregex", 8)) { + f->f_filterData.prop.operation = FIOP_EREREGEX; } else { errmsg.LogError(0, NO_ERRCODE, "error: invalid compare operation '%s' - ignoring selector", (char*) rsCStrGetSzStrNoNULL(pCSCompOp)); diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 93d1e1ef..150439a1 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -694,6 +694,32 @@ int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz) return -1; /* pCS1 is less then psz */ } +#if 0 +/* check if a CStr object matches a POSIX ERE regex. + * added 2009-03-04 by rgerhards + * TODO: we should merge this with rsCStrSzStrMatchReg + */ +rsRetVal rsCStrSzStrMatchRegexERE(cstr_t *pCS1, uchar *psz) +{ + regex_t preq; + DEFiRet; + + BEGINfunc + + if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { + regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), REG_EXTENDED); + ret = regexp.regexec(&preq, (char*) psz, 0, NULL, 0); + regexp.regfree(&preq); + } else { + ret = 1; /* simulate "not found" */ + } + + ENDfunc + RETiRet; +} +#endif + + /* check if a CStr object matches a regex. * msamia@redhat.com 2007-07-12 * @return returns 0 if matched @@ -701,25 +727,23 @@ int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz) * rgerhards, 2007-07-16: bug is no real bug, because rsyslogd ensures there * never is a \0 *inside* a property string. * Note that the function returns -1 if regexp functionality is not available. - * TODO: change calling interface! -- rgerhards, 2008-03-07 + * rgerhards: 2009-03-04: ERE support added, via parameter iType: 0 - BRE, 1 - ERE */ -int rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz) +rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType) { regex_t preq; - int ret; - - BEGINfunc + DEFiRet; if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { - regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), 0); - ret = regexp.regexec(&preq, (char*) psz, 0, NULL, 0); + regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), iType == 1 ? REG_EXTENDED : 0); + CHKiRet(regexp.regexec(&preq, (char*) psz, 0, NULL, 0)); regexp.regfree(&preq); } else { - ret = 1; /* simulate "not found" */ + ABORT_FINALIZE(RS_RET_NOT_FOUND); } - ENDfunc - return ret; +finalize_it: + RETiRet; } diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index c1966449..f3e08439 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -136,7 +136,7 @@ int rsCStrCaseInsensitiveLocateInSzStr(cstr_t *pThis, uchar *sz); int rsCStrStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrSzStrStartsWithCStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); -int rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz); +rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); diff --git a/tools/syslogd.c b/tools/syslogd.c index 9ced4562..6b8ce82f 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1071,7 +1071,12 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce break; case FIOP_REGEX: if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal) == 0) + (unsigned char*) pszPropVal, 0) == RS_RET_OK) + bRet = 1; + break; + case FIOP_EREREGEX: + if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, 1) == RS_RET_OK) bRet = 1; break; default: diff --git a/tools/syslogd.h b/tools/syslogd.h index e866a16b..f1b11a91 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -70,7 +70,8 @@ struct filed { FIOP_CONTAINS = 1, /* contains string? */ FIOP_ISEQUAL = 2, /* is (exactly) equal? */ FIOP_STARTSWITH = 3, /* starts with a string? */ - FIOP_REGEX = 4 /* matches a regular expression? */ + FIOP_REGEX = 4, /* matches a (BRE) regular expression? */ + FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ } operation; cstr_t *pCSCompValue; /* value to "compare" against */ char isNegated; /* actually a boolean ;) */ -- cgit v1.2.3 From bbcbd87ebdafe8a344c375104264a9ebdd3948d5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 4 Mar 2009 18:34:36 +0100 Subject: some optimization on regex code also some commented-out leftover removed --- runtime/stringbuf.c | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 150439a1..8d52834d 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -694,31 +694,6 @@ int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz) return -1; /* pCS1 is less then psz */ } -#if 0 -/* check if a CStr object matches a POSIX ERE regex. - * added 2009-03-04 by rgerhards - * TODO: we should merge this with rsCStrSzStrMatchReg - */ -rsRetVal rsCStrSzStrMatchRegexERE(cstr_t *pCS1, uchar *psz) -{ - regex_t preq; - DEFiRet; - - BEGINfunc - - if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { - regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), REG_EXTENDED); - ret = regexp.regexec(&preq, (char*) psz, 0, NULL, 0); - regexp.regfree(&preq); - } else { - ret = 1; /* simulate "not found" */ - } - - ENDfunc - RETiRet; -} -#endif - /* check if a CStr object matches a regex. * msamia@redhat.com 2007-07-12 @@ -735,7 +710,7 @@ rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType) DEFiRet; if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { - regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), iType == 1 ? REG_EXTENDED : 0); + regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), (iType == 1 ? REG_EXTENDED : 0) | REG_NOSUB); CHKiRet(regexp.regexec(&preq, (char*) psz, 0, NULL, 0)); regexp.regfree(&preq); } else { -- cgit v1.2.3 From 42db7de5968d2db0fa855a9f029f6bccc0a30650 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 4 Mar 2009 18:46:06 +0100 Subject: fixed newly introduced memory leak (bug created 30 minutes ago or so) --- runtime/stringbuf.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 8d52834d..a5dc625a 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -707,12 +707,15 @@ int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz) rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType) { regex_t preq; + int ret; DEFiRet; if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), (iType == 1 ? REG_EXTENDED : 0) | REG_NOSUB); - CHKiRet(regexp.regexec(&preq, (char*) psz, 0, NULL, 0)); + ret = regexp.regexec(&preq, (char*) psz, 0, NULL, 0); regexp.regfree(&preq); + if(ret != 0) + ABORT_FINALIZE(RS_RET_NOT_FOUND); } else { ABORT_FINALIZE(RS_RET_NOT_FOUND); } -- cgit v1.2.3 From 2e388db9ac91eae35ac836b329c8bcadd319a409 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 5 Mar 2009 11:10:43 +0100 Subject: integrated various patches for solaris Unfortunatley, I do not have the full list of contributors available. The patch set was compiled by Ben Taylor, and I made some further changes to adopt it to the news rsyslog branch. Others provided much of the base work, but I can not find the names of the original authors. If you happen to be one of them, please let me know so that I can give proper credits. --- action.c | 44 +++--- action.h | 2 +- configure.ac | 12 +- runtime/conf.c | 7 +- runtime/modules.c | 4 + runtime/net.c | 5 + runtime/queue.c | 420 +++++++++++++++++++++++++++--------------------------- runtime/queue.h | 50 +++---- runtime/rsyslog.c | 2 +- runtime/srutils.c | 2 +- runtime/wti.c | 5 + runtime/wtp.c | 5 + tools/Makefile.am | 2 +- tools/omfile.c | 4 + tools/syslogd.c | 63 ++++---- 15 files changed, 335 insertions(+), 292 deletions(-) diff --git a/action.c b/action.c index a41f976c..cd4ba240 100644 --- a/action.c +++ b/action.c @@ -180,7 +180,7 @@ rsRetVal actionDestruct(action_t *pThis) ASSERT(pThis != NULL); if(pThis->pQueue != NULL) { - queueDestruct(&pThis->pQueue); + qqueueDestruct(&pThis->pQueue); } if(pThis->pMod != NULL) @@ -255,7 +255,7 @@ actionConstructFinalize(action_t *pThis) * to be run on multiple threads. So far, this is forbidden by the interface * spec. -- rgerhards, 2008-01-30 */ - CHKiRet(queueConstruct(&pThis->pQueue, ActionQueType, 1, iActionQueueSize, (rsRetVal (*)(void*,void*))actionCallDoAction)); + CHKiRet(qqueueConstruct(&pThis->pQueue, ActionQueType, 1, iActionQueueSize, (rsRetVal (*)(void*,void*))actionCallDoAction)); obj.SetName((obj_t*) pThis->pQueue, pszQName); /* ... set some properties ... */ @@ -268,24 +268,24 @@ actionConstructFinalize(action_t *pThis) errmsg.LogError(0, NO_ERRCODE, "Invalid " #directive ", error %d. Ignored, running with default setting", iRet); \ } - queueSetpUsr(pThis->pQueue, pThis); - setQPROP(queueSetsizeOnDiskMax, "$ActionQueueMaxDiskSpace", iActionQueMaxDiskSpace); - setQPROP(queueSetMaxFileSize, "$ActionQueueFileSize", iActionQueMaxFileSize); - setQPROPstr(queueSetFilePrefix, "$ActionQueueFileName", pszActionQFName); - setQPROP(queueSetiPersistUpdCnt, "$ActionQueueCheckpointInterval", iActionQPersistUpdCnt); - setQPROP(queueSettoQShutdown, "$ActionQueueTimeoutShutdown", iActionQtoQShutdown ); - setQPROP(queueSettoActShutdown, "$ActionQueueTimeoutActionCompletion", iActionQtoActShutdown); - setQPROP(queueSettoWrkShutdown, "$ActionQueueWorkerTimeoutThreadShutdown", iActionQtoWrkShutdown); - setQPROP(queueSettoEnq, "$ActionQueueTimeoutEnqueue", iActionQtoEnq); - setQPROP(queueSetiHighWtrMrk, "$ActionQueueHighWaterMark", iActionQHighWtrMark); - setQPROP(queueSetiLowWtrMrk, "$ActionQueueLowWaterMark", iActionQLowWtrMark); - setQPROP(queueSetiDiscardMrk, "$ActionQueueDiscardMark", iActionQDiscardMark); - setQPROP(queueSetiDiscardSeverity, "$ActionQueueDiscardSeverity", iActionQDiscardSeverity); - setQPROP(queueSetiMinMsgsPerWrkr, "$ActionQueueWorkerThreadMinimumMessages", iActionQWrkMinMsgs); - setQPROP(queueSetbSaveOnShutdown, "$ActionQueueSaveOnShutdown", bActionQSaveOnShutdown); - setQPROP(queueSetiDeqSlowdown, "$ActionQueueDequeueSlowdown", iActionQueueDeqSlowdown); - setQPROP(queueSetiDeqtWinFromHr, "$ActionQueueDequeueTimeBegin", iActionQueueDeqtWinFromHr); - setQPROP(queueSetiDeqtWinToHr, "$ActionQueueDequeueTimeEnd", iActionQueueDeqtWinToHr); + qqueueSetpUsr(pThis->pQueue, pThis); + setQPROP(qqueueSetsizeOnDiskMax, "$ActionQueueMaxDiskSpace", iActionQueMaxDiskSpace); + setQPROP(qqueueSetMaxFileSize, "$ActionQueueFileSize", iActionQueMaxFileSize); + setQPROPstr(qqueueSetFilePrefix, "$ActionQueueFileName", pszActionQFName); + setQPROP(qqueueSetiPersistUpdCnt, "$ActionQueueCheckpointInterval", iActionQPersistUpdCnt); + setQPROP(qqueueSettoQShutdown, "$ActionQueueTimeoutShutdown", iActionQtoQShutdown ); + setQPROP(qqueueSettoActShutdown, "$ActionQueueTimeoutActionCompletion", iActionQtoActShutdown); + setQPROP(qqueueSettoWrkShutdown, "$ActionQueueWorkerTimeoutThreadShutdown", iActionQtoWrkShutdown); + setQPROP(qqueueSettoEnq, "$ActionQueueTimeoutEnqueue", iActionQtoEnq); + setQPROP(qqueueSetiHighWtrMrk, "$ActionQueueHighWaterMark", iActionQHighWtrMark); + setQPROP(qqueueSetiLowWtrMrk, "$ActionQueueLowWaterMark", iActionQLowWtrMark); + setQPROP(qqueueSetiDiscardMrk, "$ActionQueueDiscardMark", iActionQDiscardMark); + setQPROP(qqueueSetiDiscardSeverity, "$ActionQueueDiscardSeverity", iActionQDiscardSeverity); + setQPROP(qqueueSetiMinMsgsPerWrkr, "$ActionQueueWorkerThreadMinimumMessages", iActionQWrkMinMsgs); + setQPROP(qqueueSetbSaveOnShutdown, "$ActionQueueSaveOnShutdown", bActionQSaveOnShutdown); + setQPROP(qqueueSetiDeqSlowdown, "$ActionQueueDequeueSlowdown", iActionQueueDeqSlowdown); + setQPROP(qqueueSetiDeqtWinFromHr, "$ActionQueueDequeueTimeBegin", iActionQueueDeqtWinFromHr); + setQPROP(qqueueSetiDeqtWinToHr, "$ActionQueueDequeueTimeEnd", iActionQueueDeqtWinToHr); # undef setQPROP # undef setQPROPstr @@ -294,7 +294,7 @@ actionConstructFinalize(action_t *pThis) bActionQSaveOnShutdown, iActionQueMaxDiskSpace); - CHKiRet(queueStart(pThis->pQueue)); + CHKiRet(qqueueStart(pThis->pQueue)); dbgprintf("Action %p: queue %p created\n", pThis, pThis->pQueue); /* and now reset the queue params (see comment in its function header!) */ @@ -675,7 +675,7 @@ actionWriteToAction(action_t *pAction) /* When we reach this point, we have a valid, non-disabled action. * So let's enqueue our message for execution. -- rgerhards, 2007-07-24 */ - iRet = queueEnqObj(pAction->pQueue, pAction->f_pMsg->flowCtlType, (void*) MsgAddRef(pAction->f_pMsg)); + iRet = qqueueEnqObj(pAction->pQueue, pAction->f_pMsg->flowCtlType, (void*) MsgAddRef(pAction->f_pMsg)); if(iRet == RS_RET_OK) pAction->f_prevcount = 0; /* message processed, so we start a new cycle */ diff --git a/action.h b/action.h index e35e634c..dc9bbd74 100644 --- a/action.h +++ b/action.h @@ -68,7 +68,7 @@ struct action_s { * content later). This is preserved after the message has been * processed - it is also used to detect duplicates. */ - queue_t *pQueue; /* action queue */ + qqueue_t *pQueue; /* action queue */ SYNC_OBJ_TOOL; /* required for mutex support */ pthread_mutex_t mutActExec; /* mutex to guard actual execution of doAction for single-threaded modules */ }; diff --git a/configure.ac b/configure.ac index 0c924754..ff790817 100644 --- a/configure.ac +++ b/configure.ac @@ -35,6 +35,13 @@ case "${host}" in # do not DEFINE OS_BSD os_type="bsd" ;; + *-*-solaris*) + os_type="solaris" + AC_DEFINE([OS_SOLARIS], [1], [Indicator for a Solaris OS]) + AC_DEFINE([_POSIX_PTHREAD_SEMANTICS], [1], [Use POSIX pthread semantics]) + SOL_LIBS="-lsocket -lnsl" + AC_SUBST(SOL_LIBS) + ;; esac AC_DEFINE_UNQUOTED([HOSTENV], "$host", [the host environment, can be queried via a system variable]) @@ -233,7 +240,10 @@ if test "x$enable_pthreads" != "xno"; then [ AC_DEFINE([USE_PTHREADS], [1], [Multithreading support enabled.]) PTHREADS_LIBS="-lpthread" - PTHREADS_CFLAGS="-pthread" + case "${os_type}" in + solaris) PTHREADS_CFLAGS="-pthreads" ;; + *) PTHREADS_CFLAGS="-pthread" ;; + esac AC_SUBST(PTHREADS_LIBS) AC_SUBST(PTHREADS_CFLAGS) ], diff --git a/runtime/conf.c b/runtime/conf.c index c5208d86..a670c65b 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -46,7 +46,9 @@ #include #include #ifdef HAVE_LIBGEN_H -# include +# ifndef OS_SOLARIS +# include +# endif #endif #include "rsyslog.h" @@ -68,6 +70,9 @@ #include "ctok.h" #include "ctok_token.h" +#ifdef OS_SOLARIS +# define NAME_MAX MAXNAMELEN +#endif /* forward definitions */ static rsRetVal cfline(uchar *line, selector_t **pfCurr); diff --git a/runtime/modules.c b/runtime/modules.c index 169d234b..d548a949 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -49,6 +49,10 @@ #include #include +#ifdef OS_SOLARIS +# define PATH_MAX MAXPATHLEN +#endif + #include "cfsysline.h" #include "modules.h" #include "errmsg.h" diff --git a/runtime/net.c b/runtime/net.c index 4e6d54a1..db2d7e37 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -63,6 +63,11 @@ #include "errmsg.h" #include "net.h" +#ifdef OS_SOLARIS +# define s6_addr32 _S6_un._S6_u32 + typedef unsigned int u_int32_t; +#endif + MODULE_TYPE_LIB /* static data */ diff --git a/runtime/queue.c b/runtime/queue.c index a3e29a96..7d78460c 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -56,14 +56,14 @@ DEFobjStaticHelpers DEFobjCurrIf(glbl) /* forward-definitions */ -rsRetVal queueChkPersist(queue_t *pThis); -static rsRetVal queueSetEnqOnly(queue_t *pThis, int bEnqOnly, int bLockMutex); -static rsRetVal queueRateLimiter(queue_t *pThis); -static int queueChkStopWrkrDA(queue_t *pThis); -static int queueIsIdleDA(queue_t *pThis); -static rsRetVal queueConsumerDA(queue_t *pThis, wti_t *pWti, int iCancelStateSave); -static rsRetVal queueConsumerCancelCleanup(void *arg1, void *arg2); -static rsRetVal queueUngetObj(queue_t *pThis, obj_t *pUsr, int bLockMutex); +rsRetVal qqueueChkPersist(qqueue_t *pThis); +static rsRetVal qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); +static rsRetVal qqueueRateLimiter(qqueue_t *pThis); +static int qqueueChkStopWrkrDA(qqueue_t *pThis); +static int qqueueIsIdleDA(qqueue_t *pThis); +static rsRetVal qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave); +static rsRetVal qqueueConsumerCancelCleanup(void *arg1, void *arg2); +static rsRetVal qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex); /* some constants for queuePersist () */ #define QUEUE_CHECKPOINT 1 @@ -77,7 +77,7 @@ static rsRetVal queueUngetObj(queue_t *pThis, obj_t *pUsr, int bLockMutex); * rgerhards, 2008-01-29 */ static inline int -queueGetOverallQueueSize(queue_t *pThis) +qqueueGetOverallQueueSize(qqueue_t *pThis) { #if 0 /* leave a bit in for debugging -- rgerhards, 2008-01-30 */ BEGINfunc @@ -96,7 +96,7 @@ ENDfunc * This function returns void, as it makes no sense to communicate an error back, even if * it happens. */ -static inline void queueDrain(queue_t *pThis) +static inline void queueDrain(qqueue_t *pThis) { void *pUsr; @@ -119,26 +119,26 @@ static inline void queueDrain(queue_t *pThis) * this point in time. The mutex must be locked when * ths function is called. -- rgerhards, 2008-01-25 */ -static inline rsRetVal queueAdviseMaxWorkers(queue_t *pThis) +static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) { DEFiRet; int iMaxWorkers; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(!pThis->bEnqOnly) { if(pThis->bRunsDA) { /* if we have not yet reached the high water mark, there is no need to start a * worker. -- rgerhards, 2008-01-26 */ - if(queueGetOverallQueueSize(pThis) >= pThis->iHighWtrMrk || pThis->bQueueStarted == 0) { + if(qqueueGetOverallQueueSize(pThis) >= pThis->iHighWtrMrk || pThis->bQueueStarted == 0) { wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ } } else { if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { iMaxWorkers = 1; } else { - iMaxWorkers = queueGetOverallQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; + iMaxWorkers = qqueueGetOverallQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; } wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); /* disk queues have always one worker */ } @@ -153,11 +153,11 @@ static inline rsRetVal queueAdviseMaxWorkers(queue_t *pThis) * rgerhards, 2008-02-27 */ static rsRetVal -queueWaitDAModeInitialized(queue_t *pThis) +qqueueWaitDAModeInitialized(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->bRunsDA); while(pThis->bRunsDA != 2) { @@ -179,17 +179,17 @@ queueWaitDAModeInitialized(queue_t *pThis) * rgerhards, 2008-01-15 */ static rsRetVal -queueTurnOffDAMode(queue_t *pThis) +qqueueTurnOffDAMode(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->bRunsDA); /* at this point, we need a fully initialized DA queue. So if it isn't, we finally need * to wait for its startup... -- rgerhards, 2008-01-25 */ - queueWaitDAModeInitialized(pThis); + qqueueWaitDAModeInitialized(pThis); /* if we need to pull any data that we still need from the (child) disk queue, * now would be the time to do so. At present, we do not need this, but I'd like to @@ -208,15 +208,15 @@ queueTurnOffDAMode(queue_t *pThis) /* we destruct the queue object, which will also shutdown the queue worker. As the queue is empty, * this will be quick. */ - queueDestruct(&pThis->pqDA); /* and now we are ready to destruct the DA queue */ + qqueueDestruct(&pThis->pqDA); /* and now we are ready to destruct the DA queue */ dbgoprint((obj_t*) pThis, "disk-assistance has been turned off, disk queue was empty (iRet %d)\n", iRet); /* now we need to check if the regular queue has some messages. This may be the case * when it is waiting that the high water mark is reached again. If so, we need to start up * a regular worker. -- rgerhards, 2008-01-26 */ - if(queueGetOverallQueueSize(pThis) > 0) { - queueAdviseMaxWorkers(pThis); + if(qqueueGetOverallQueueSize(pThis) > 0) { + qqueueAdviseMaxWorkers(pThis); } } @@ -232,11 +232,11 @@ queueTurnOffDAMode(queue_t *pThis) * rgerhards, 2008-01-14 */ static rsRetVal -queueChkIsDA(queue_t *pThis) +qqueueChkIsDA(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->pszFilePrefix != NULL) { pThis->bIsDA = 1; dbgoprint((obj_t*) pThis, "is disk-assisted, disk will be used on demand\n"); @@ -260,18 +260,18 @@ queueChkIsDA(queue_t *pThis) * rgerhards, 2008-01-15 */ static rsRetVal -queueStartDA(queue_t *pThis) +qqueueStartDA(qqueue_t *pThis) { DEFiRet; uchar pszDAQName[128]; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->bRunsDA == 2) /* check if already in (fully initialized) DA mode... */ FINALIZE; /* ... then we are already done! */ /* create message queue */ - CHKiRet(queueConstruct(&pThis->pqDA, QUEUETYPE_DISK , 1, 0, pThis->pConsumer)); + CHKiRet(qqueueConstruct(&pThis->pqDA, QUEUETYPE_DISK , 1, 0, pThis->pConsumer)); /* give it a name */ snprintf((char*) pszDAQName, sizeof(pszDAQName)/sizeof(uchar), "%s[DA]", obj.GetName((obj_t*) pThis)); @@ -282,30 +282,30 @@ queueStartDA(queue_t *pThis) */ pThis->pqDA->pqParent = pThis; - CHKiRet(queueSetpUsr(pThis->pqDA, pThis->pUsr)); - CHKiRet(queueSetsizeOnDiskMax(pThis->pqDA, pThis->sizeOnDiskMax)); - CHKiRet(queueSetiDeqSlowdown(pThis->pqDA, pThis->iDeqSlowdown)); - CHKiRet(queueSetMaxFileSize(pThis->pqDA, pThis->iMaxFileSize)); - CHKiRet(queueSetFilePrefix(pThis->pqDA, pThis->pszFilePrefix, pThis->lenFilePrefix)); - CHKiRet(queueSetiPersistUpdCnt(pThis->pqDA, pThis->iPersistUpdCnt)); - CHKiRet(queueSettoActShutdown(pThis->pqDA, pThis->toActShutdown)); - CHKiRet(queueSettoEnq(pThis->pqDA, pThis->toEnq)); - CHKiRet(queueSetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED)); - CHKiRet(queueSetiDeqtWinFromHr(pThis->pqDA, pThis->iDeqtWinFromHr)); - CHKiRet(queueSetiDeqtWinToHr(pThis->pqDA, pThis->iDeqtWinToHr)); - CHKiRet(queueSetiHighWtrMrk(pThis->pqDA, 0)); - CHKiRet(queueSetiDiscardMrk(pThis->pqDA, 0)); + CHKiRet(qqueueSetpUsr(pThis->pqDA, pThis->pUsr)); + CHKiRet(qqueueSetsizeOnDiskMax(pThis->pqDA, pThis->sizeOnDiskMax)); + CHKiRet(qqueueSetiDeqSlowdown(pThis->pqDA, pThis->iDeqSlowdown)); + CHKiRet(qqueueSetMaxFileSize(pThis->pqDA, pThis->iMaxFileSize)); + CHKiRet(qqueueSetFilePrefix(pThis->pqDA, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(qqueueSetiPersistUpdCnt(pThis->pqDA, pThis->iPersistUpdCnt)); + CHKiRet(qqueueSettoActShutdown(pThis->pqDA, pThis->toActShutdown)); + CHKiRet(qqueueSettoEnq(pThis->pqDA, pThis->toEnq)); + CHKiRet(qqueueSetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED)); + CHKiRet(qqueueSetiDeqtWinFromHr(pThis->pqDA, pThis->iDeqtWinFromHr)); + CHKiRet(qqueueSetiDeqtWinToHr(pThis->pqDA, pThis->iDeqtWinToHr)); + CHKiRet(qqueueSetiHighWtrMrk(pThis->pqDA, 0)); + CHKiRet(qqueueSetiDiscardMrk(pThis->pqDA, 0)); if(pThis->toQShutdown == 0) { - CHKiRet(queueSettoQShutdown(pThis->pqDA, 0)); /* if the user really wants... */ + CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 0)); /* if the user really wants... */ } else { /* we use the shortest possible shutdown (0 is endless!) because when we run on disk AND * have an obviously large backlog, we can't finish it in any case. So there is no point * in holding shutdown longer than necessary. -- rgerhards, 2008-01-15 */ - CHKiRet(queueSettoQShutdown(pThis->pqDA, 1)); + CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 1)); } - iRet = queueStart(pThis->pqDA); + iRet = qqueueStart(pThis->pqDA); /* file not found is expected, that means it is no previous QIF available */ if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) FINALIZE; /* something is wrong */ @@ -323,12 +323,12 @@ queueStartDA(queue_t *pThis) pthread_cond_broadcast(&pThis->condDAReady); /* signal we are now initialized and ready to go ;) */ dbgoprint((obj_t*) pThis, "is now running in disk assisted mode, disk queue 0x%lx\n", - queueGetID(pThis->pqDA)); + qqueueGetID(pThis->pqDA)); finalize_it: if(iRet != RS_RET_OK) { if(pThis->pqDA != NULL) { - queueDestruct(&pThis->pqDA); + qqueueDestruct(&pThis->pqDA); } dbgoprint((obj_t*) pThis, "error %d creating disk queue - giving up.\n", iRet); pThis->bIsDA = 0; @@ -345,7 +345,7 @@ finalize_it: * rgerhards, 2008-01-16 */ static inline rsRetVal -queueInitDA(queue_t *pThis, int bEnqOnly, int bLockMutex) +qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; @@ -363,12 +363,12 @@ queueInitDA(queue_t *pThis, int bEnqOnly, int bLockMutex) lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DA", obj.GetName((obj_t*) pThis)); CHKiRet(wtpConstruct (&pThis->pWtpDA)); CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) queueChkStopWrkrDA)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) queueIsIdleDA)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) queueConsumerDA)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) queueConsumerCancelCleanup)); - CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) queueStartDA)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) queueTurnOffDAMode)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); + CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleDA)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerDA)); + CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) qqueueConsumerCancelCleanup)); + CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueStartDA)); + CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueTurnOffDAMode)); CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty)); CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1)); @@ -401,14 +401,14 @@ finalize_it: * rgerhards, 2008-01-14 */ static inline rsRetVal -queueChkStrtDA(queue_t *pThis) +qqueueChkStrtDA(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); /* if we do not hit the high water mark, we have nothing to do */ - if(queueGetOverallQueueSize(pThis) != pThis->iHighWtrMrk) + if(qqueueGetOverallQueueSize(pThis) != pThis->iHighWtrMrk) ABORT_FINALIZE(RS_RET_OK); if(pThis->bRunsDA) { @@ -422,15 +422,15 @@ queueChkStrtDA(queue_t *pThis) * we need at least one). */ dbgoprint((obj_t*) pThis, "%d entries - passed high water mark in DA mode, send notify\n", - queueGetOverallQueueSize(pThis)); - queueAdviseMaxWorkers(pThis); + qqueueGetOverallQueueSize(pThis)); + qqueueAdviseMaxWorkers(pThis); } else { /* this is the case when we are currently not running in DA mode. So it is time * to turn it back on. */ dbgoprint((obj_t*) pThis, "%d entries - passed high water mark for disk-assisted mode, initiating...\n", - queueGetOverallQueueSize(pThis)); - queueInitDA(pThis, QUEUE_MODE_ENQDEQ, MUTEX_ALREADY_LOCKED); /* initiate DA mode */ + qqueueGetOverallQueueSize(pThis)); + qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, MUTEX_ALREADY_LOCKED); /* initiate DA mode */ } finalize_it: @@ -448,7 +448,7 @@ finalize_it: */ /* -------------------- fixed array -------------------- */ -static rsRetVal qConstructFixedArray(queue_t *pThis) +static rsRetVal qConstructFixedArray(qqueue_t *pThis) { DEFiRet; @@ -464,14 +464,14 @@ static rsRetVal qConstructFixedArray(queue_t *pThis) pThis->tVars.farray.head = 0; pThis->tVars.farray.tail = 0; - queueChkIsDA(pThis); + qqueueChkIsDA(pThis); finalize_it: RETiRet; } -static rsRetVal qDestructFixedArray(queue_t *pThis) +static rsRetVal qDestructFixedArray(qqueue_t *pThis) { DEFiRet; @@ -486,7 +486,7 @@ static rsRetVal qDestructFixedArray(queue_t *pThis) } -static rsRetVal qAddFixedArray(queue_t *pThis, void* in) +static rsRetVal qAddFixedArray(qqueue_t *pThis, void* in) { DEFiRet; @@ -499,7 +499,7 @@ static rsRetVal qAddFixedArray(queue_t *pThis, void* in) RETiRet; } -static rsRetVal qDelFixedArray(queue_t *pThis, void **out) +static rsRetVal qDelFixedArray(qqueue_t *pThis, void **out) { DEFiRet; @@ -518,7 +518,7 @@ static rsRetVal qDelFixedArray(queue_t *pThis, void **out) /* first some generic functions which are also used for the unget linked list */ -static inline rsRetVal queueAddLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, void* pUsr) +static inline rsRetVal qqueueAddLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, void* pUsr) { DEFiRet; qLinkedList_t *pEntry; @@ -544,7 +544,7 @@ finalize_it: RETiRet; } -static inline rsRetVal queueDelLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, obj_t **ppUsr) +static inline rsRetVal qqueueDelLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, obj_t **ppUsr) { DEFiRet; qLinkedList_t *pEntry; @@ -571,7 +571,7 @@ static inline rsRetVal queueDelLinkedList(qLinkedList_t **ppRoot, qLinkedList_t /* end generic functions which are also used for the unget linked list */ -static rsRetVal qConstructLinkedList(queue_t *pThis) +static rsRetVal qConstructLinkedList(qqueue_t *pThis) { DEFiRet; @@ -580,13 +580,13 @@ static rsRetVal qConstructLinkedList(queue_t *pThis) pThis->tVars.linklist.pRoot = 0; pThis->tVars.linklist.pLast = 0; - queueChkIsDA(pThis); + qqueueChkIsDA(pThis); RETiRet; } -static rsRetVal qDestructLinkedList(queue_t __attribute__((unused)) *pThis) +static rsRetVal qDestructLinkedList(qqueue_t __attribute__((unused)) *pThis) { DEFiRet; @@ -599,11 +599,11 @@ static rsRetVal qDestructLinkedList(queue_t __attribute__((unused)) *pThis) RETiRet; } -static rsRetVal qAddLinkedList(queue_t *pThis, void* pUsr) +static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr) { DEFiRet; - iRet = queueAddLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, pUsr); + iRet = qqueueAddLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, pUsr); #if 0 qLinkedList_t *pEntry; @@ -627,10 +627,10 @@ finalize_it: RETiRet; } -static rsRetVal qDelLinkedList(queue_t *pThis, obj_t **ppUsr) +static rsRetVal qDelLinkedList(qqueue_t *pThis, obj_t **ppUsr) { DEFiRet; - iRet = queueDelLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, ppUsr); + iRet = qqueueDelLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, ppUsr); #if 0 qLinkedList_t *pEntry; @@ -657,11 +657,11 @@ static rsRetVal qDelLinkedList(queue_t *pThis, obj_t **ppUsr) static rsRetVal -queueLoadPersStrmInfoFixup(strm_t *pStrm, queue_t __attribute__((unused)) *pThis) +qqueueLoadPersStrmInfoFixup(strm_t *pStrm, qqueue_t __attribute__((unused)) *pThis) { DEFiRet; ISOBJ_TYPE_assert(pStrm, strm); - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); CHKiRet(strmSetDir(pStrm, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); finalize_it: RETiRet; @@ -673,14 +673,14 @@ finalize_it: * rgerhards, 2008-01-15 */ static rsRetVal -queueHaveQIF(queue_t *pThis) +qqueueHaveQIF(qqueue_t *pThis) { DEFiRet; uchar pszQIFNam[MAXFNAME]; size_t lenQIFNam; struct stat stat_buf; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->pszFilePrefix == NULL) ABORT_FINALIZE(RS_RET_NO_FILEPREFIX); @@ -710,7 +710,7 @@ finalize_it: * rgerhards, 2008-01-11 */ static rsRetVal -queueTryLoadPersistedInfo(queue_t *pThis) +qqueueTryLoadPersistedInfo(qqueue_t *pThis) { DEFiRet; strm_t *psQIF = NULL; @@ -720,7 +720,7 @@ queueTryLoadPersistedInfo(queue_t *pThis) int iUngottenObjs; obj_t *pUsr; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); /* Construct file name */ lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", @@ -755,15 +755,15 @@ queueTryLoadPersistedInfo(queue_t *pThis) while(iUngottenObjs > 0) { /* fill the queue from disk */ CHKiRet(obj.Deserialize((void*) &pUsr, (uchar*)"msg", psQIF, NULL, NULL)); - queueUngetObj(pThis, pUsr, MUTEX_ALREADY_LOCKED); + qqueueUngetObj(pThis, pUsr, MUTEX_ALREADY_LOCKED); --iUngottenObjs; /* one less */ } /* and now the stream objects (some order as when persisted!) */ CHKiRet(obj.Deserialize(&pThis->tVars.disk.pWrite, (uchar*) "strm", psQIF, - (rsRetVal(*)(obj_t*,void*))queueLoadPersStrmInfoFixup, pThis)); + (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); CHKiRet(obj.Deserialize(&pThis->tVars.disk.pRead, (uchar*) "strm", psQIF, - (rsRetVal(*)(obj_t*,void*))queueLoadPersStrmInfoFixup, pThis)); + (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pWrite)); CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pRead)); @@ -793,7 +793,7 @@ finalize_it: * allowed file size at this point - that should be a config setting... * rgerhards, 2008-01-10 */ -static rsRetVal qConstructDisk(queue_t *pThis) +static rsRetVal qConstructDisk(qqueue_t *pThis) { DEFiRet; int bRestarted = 0; @@ -801,7 +801,7 @@ static rsRetVal qConstructDisk(queue_t *pThis) ASSERT(pThis != NULL); /* and now check if there is some persistent information that needs to be read in */ - iRet = queueTryLoadPersistedInfo(pThis); + iRet = qqueueTryLoadPersistedInfo(pThis); if(iRet == RS_RET_OK) bRestarted = 1; else if(iRet != RS_RET_FILE_NOT_FOUND) @@ -843,7 +843,7 @@ finalize_it: } -static rsRetVal qDestructDisk(queue_t *pThis) +static rsRetVal qDestructDisk(qqueue_t *pThis) { DEFiRet; @@ -855,7 +855,7 @@ static rsRetVal qDestructDisk(queue_t *pThis) RETiRet; } -static rsRetVal qAddDisk(queue_t *pThis, void* pUsr) +static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr) { DEFiRet; number_t nWriteCount; @@ -882,7 +882,7 @@ finalize_it: RETiRet; } -static rsRetVal qDelDisk(queue_t *pThis, void **ppUsr) +static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr) { DEFiRet; @@ -913,18 +913,18 @@ finalize_it: } /* -------------------- direct (no queueing) -------------------- */ -static rsRetVal qConstructDirect(queue_t __attribute__((unused)) *pThis) +static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis) { return RS_RET_OK; } -static rsRetVal qDestructDirect(queue_t __attribute__((unused)) *pThis) +static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis) { return RS_RET_OK; } -static rsRetVal qAddDirect(queue_t *pThis, void* pUsr) +static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) { DEFiRet; @@ -941,7 +941,7 @@ static rsRetVal qAddDirect(queue_t *pThis, void* pUsr) RETiRet; } -static rsRetVal qDelDirect(queue_t __attribute__((unused)) *pThis, __attribute__((unused)) void **out) +static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis, __attribute__((unused)) void **out) { return RS_RET_OK; } @@ -956,12 +956,12 @@ static rsRetVal qDelDirect(queue_t __attribute__((unused)) *pThis, __attribute__ * rgerhards, 2008-01-20 */ static rsRetVal -queueUngetObj(queue_t *pThis, obj_t *pUsr, int bLockMutex) +qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_assert(pUsr); /* TODO: we aborted right at this place at least 3 times -- race? 2008-02-28, -03-10, -03-15 The second time I noticed it the queue was in destruction with NO worker threads running. The pUsr ptr was totally off and provided no clue what it may be pointing @@ -970,7 +970,7 @@ queueUngetObj(queue_t *pThis, obj_t *pUsr, int bLockMutex) dbgoprint((obj_t*) pThis, "ungetting user object %s\n", obj.GetName(pUsr)); BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex); - iRet = queueAddLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, pUsr); + iRet = qqueueAddLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, pUsr); ++pThis->iUngottenObjs; /* indicate one more */ END_MTX_PROTECTED_OPERATIONS(pThis->mut); @@ -986,14 +986,14 @@ queueUngetObj(queue_t *pThis, obj_t *pUsr, int bLockMutex) * rgerhards, 2008-01-29 */ static rsRetVal -queueGetUngottenObj(queue_t *pThis, obj_t **ppUsr) +qqueueGetUngottenObj(qqueue_t *pThis, obj_t **ppUsr) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(ppUsr != NULL); - iRet = queueDelLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, ppUsr); + iRet = qqueueDelLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, ppUsr); --pThis->iUngottenObjs; /* indicate one less */ dbgoprint((obj_t*) pThis, "dequeued ungotten user object %s\n", obj.GetName(*ppUsr)); @@ -1007,7 +1007,7 @@ queueGetUngottenObj(queue_t *pThis, obj_t **ppUsr) * things truely different. -- rgerhards, 2008-02-12 */ static rsRetVal -queueAdd(queue_t *pThis, void *pUsr) +qqueueAdd(qqueue_t *pThis, void *pUsr) { DEFiRet; @@ -1030,7 +1030,7 @@ finalize_it: * ungotten list and, if so, dequeue it first. */ static rsRetVal -queueDel(queue_t *pThis, void *pUsr) +qqueueDel(qqueue_t *pThis, void *pUsr) { DEFiRet; @@ -1042,7 +1042,7 @@ queueDel(queue_t *pThis, void *pUsr) * losing the whole process because it loops... -- rgerhards, 2008-01-03 */ if(pThis->iUngottenObjs > 0) { - iRet = queueGetUngottenObj(pThis, (obj_t**) pUsr); + iRet = qqueueGetUngottenObj(pThis, (obj_t**) pUsr); } else { iRet = pThis->qDel(pThis, pUsr); ATOMIC_DEC(pThis->iQueueSize); @@ -1066,14 +1066,14 @@ queueDel(queue_t *pThis, void *pUsr) * complex) if each would have its own shutdown. The function does not self check * this condition - the caller must make sure it is not called with a parent. */ -static rsRetVal queueShutdownWorkers(queue_t *pThis) +static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) { DEFiRet; DEFVARS_mutexProtection; struct timespec tTimeout; rsRetVal iRetLocal; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ dbgoprint((obj_t*) pThis, "initiating worker thread shutdown sequence\n"); @@ -1087,7 +1087,7 @@ static rsRetVal queueShutdownWorkers(queue_t *pThis) /* first try to shutdown the queue within the regular shutdown period */ BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(queueGetOverallQueueSize(pThis) > 0) { + if(qqueueGetOverallQueueSize(pThis) > 0) { if(pThis->bRunsDA) { /* We may have waited on the low water mark. As it may have changed, we * see if we reactivate the worker. @@ -1125,7 +1125,7 @@ static rsRetVal queueShutdownWorkers(queue_t *pThis) if(pThis->bRunsDA) { END_MTX_PROTECTED_OPERATIONS(pThis->mut); dbgoprint((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n", - queueGetID(pThis->pqDA)); + qqueueGetID(pThis->pqDA)); /* we use the same absolute timeout as above, so we do not use more than the configured * timeout interval! */ @@ -1154,19 +1154,19 @@ static rsRetVal queueShutdownWorkers(queue_t *pThis) /* at this stage, we need to have the DA worker properly initialized and running (if there is one) */ if(pThis->bRunsDA) - queueWaitDAModeInitialized(pThis); + qqueueWaitDAModeInitialized(pThis); BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ /* optimize parameters for shutdown of DA-enabled queues */ - if(pThis->bIsDA && queueGetOverallQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { + if(pThis->bIsDA && qqueueGetOverallQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { /* switch to enqueue-only mode so that no more actions happen */ if(pThis->bRunsDA == 0) { - queueInitDA(pThis, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to DA mode */ + qqueueInitDA(pThis, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to DA mode */ } else { /* TODO: RACE: we may reach this point when the DA worker has been initialized (state 1) * but is not yet running (state 2). In this case, pThis->pqDA is NULL! rgerhards, 2008-02-27 */ - queueSetEnqOnly(pThis->pqDA, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to enqueue-only mode */ + qqueueSetEnqOnly(pThis->pqDA, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to enqueue-only mode */ } END_MTX_PROTECTED_OPERATIONS(pThis->mut); /* make sure we do not timeout before we are done */ @@ -1188,7 +1188,7 @@ static rsRetVal queueShutdownWorkers(queue_t *pThis) * they will automatically terminate as there no longer is any message left to process. */ BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(queueGetOverallQueueSize(pThis) > 0) { + if(qqueueGetOverallQueueSize(pThis) > 0) { timeoutComp(&tTimeout, pThis->toActShutdown); if(wtpGetCurNumWrkr(pThis->pWtpReg, LOCK_MUTEX) > 0) { END_MTX_PROTECTED_OPERATIONS(pThis->mut); @@ -1257,7 +1257,7 @@ static rsRetVal queueShutdownWorkers(queue_t *pThis) * Well, more precisely, they *are in termination*. Some cancel cleanup handlers * may still be running. */ - dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size %d.\n", queueGetOverallQueueSize(pThis)); + dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size %d.\n", qqueueGetOverallQueueSize(pThis)); RETiRet; } @@ -1269,17 +1269,17 @@ static rsRetVal queueShutdownWorkers(queue_t *pThis) * is done by queueStart(). The reason is that we want to give the caller a chance * to modify some parameters before the queue is actually started. */ -rsRetVal queueConstruct(queue_t **ppThis, queueType_t qType, int iWorkerThreads, +rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)) { DEFiRet; - queue_t *pThis; + qqueue_t *pThis; ASSERT(ppThis != NULL); ASSERT(pConsumer != NULL); ASSERT(iWorkerThreads >= 0); - if((pThis = (queue_t *)calloc(1, sizeof(queue_t))) == NULL) { + if((pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t))) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -1316,7 +1316,7 @@ rsRetVal queueConstruct(queue_t **ppThis, queueType_t qType, int iWorkerThreads, pThis->qConstruct = qConstructLinkedList; pThis->qDestruct = qDestructLinkedList; pThis->qAdd = qAddLinkedList; - pThis->qDel = (rsRetVal (*)(queue_t*,void**)) qDelLinkedList; + pThis->qDel = (rsRetVal (*)(qqueue_t*,void**)) qDelLinkedList; break; case QUEUETYPE_DISK: pThis->qConstruct = qConstructDisk; @@ -1343,25 +1343,25 @@ finalize_it: /* cancellation cleanup handler for queueWorker () * Updates admin structure and frees ressources. * Params: - * arg1 - user pointer (in this case a queue_t) + * arg1 - user pointer (in this case a qqueue_t) * arg2 - user data pointer (in this case a queue data element, any object [queue's pUsr ptr!]) * Note that arg2 may be NULL, in which case no dequeued but unprocessed pUsr exists! * rgerhards, 2008-01-16 */ static rsRetVal -queueConsumerCancelCleanup(void *arg1, void *arg2) +qqueueConsumerCancelCleanup(void *arg1, void *arg2) { DEFiRet; - queue_t *pThis = (queue_t*) arg1; + qqueue_t *pThis = (qqueue_t*) arg1; obj_t *pUsr = (obj_t*) arg2; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pUsr != NULL) { /* make sure the data element is not lost */ dbgoprint((obj_t*) pThis, "cancelation cleanup handler consumer called, we need to unget one user data element\n"); - CHKiRet(queueUngetObj(pThis, pUsr, LOCK_MUTEX)); + CHKiRet(qqueueUngetObj(pThis, pUsr, LOCK_MUTEX)); } finalize_it: @@ -1383,13 +1383,13 @@ finalize_it: * the return state! * rgerhards, 2008-01-24 */ -static int queueChkDiscardMsg(queue_t *pThis, int iQueueSize, int bRunsDA, void *pUsr) +static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, int bRunsDA, void *pUsr) { DEFiRet; rsRetVal iRetLocal; int iSeverity; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_assert(pUsr); if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk && bRunsDA == 0) { @@ -1414,7 +1414,7 @@ finalize_it: * rgerhards, 2008-10-21 */ static rsRetVal -queueDequeueConsumable(queue_t *pThis, wti_t *pWti, int iCancelStateSave) +qqueueDequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) { DEFiRet; void *pUsr; @@ -1422,9 +1422,9 @@ queueDequeueConsumable(queue_t *pThis, wti_t *pWti, int iCancelStateSave) int bRunsDA; /* cache for early mutex release */ /* dequeue element (still protected from mutex) */ - iRet = queueDel(pThis, &pUsr); - queueChkPersist(pThis); - iQueueSize = queueGetOverallQueueSize(pThis); /* cache this for after mutex release */ + iRet = qqueueDel(pThis, &pUsr); + qqueueChkPersist(pThis); + iQueueSize = qqueueGetOverallQueueSize(pThis); /* cache this for after mutex release */ bRunsDA = pThis->bRunsDA; /* cache this for after mutex release */ /* We now need to save the user pointer for the cancel cleanup handler, BUT ONLY @@ -1475,7 +1475,7 @@ queueDequeueConsumable(queue_t *pThis, wti_t *pWti, int iCancelStateSave) * provide real-time creation of spool files. * Note: It is OK to use the cached iQueueSize here, because it does not hurt if it is slightly wrong. */ - CHKiRet(queueChkDiscardMsg(pThis, iQueueSize, bRunsDA, pUsr)); + CHKiRet(qqueueChkDiscardMsg(pThis, iQueueSize, bRunsDA, pUsr)); finalize_it: if(iRet != RS_RET_OK && iRet != RS_RET_DISCARDMSG) { @@ -1524,7 +1524,7 @@ finalize_it: * but you get the idea from the code above. */ static rsRetVal -queueRateLimiter(queue_t *pThis) +qqueueRateLimiter(qqueue_t *pThis) { DEFiRet; int iDelay; @@ -1532,7 +1532,7 @@ queueRateLimiter(queue_t *pThis) time_t tCurr; struct tm m; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); iDelay = 0; if(pThis->iDeqtWinToHr != 25) { /* 25 means disabled */ @@ -1587,14 +1587,14 @@ queueRateLimiter(queue_t *pThis) * rgerhards, 2008-01-21 */ static rsRetVal -queueConsumerReg(queue_t *pThis, wti_t *pWti, int iCancelStateSave) +qqueueConsumerReg(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(queueDequeueConsumable(pThis, pWti, iCancelStateSave)); + CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave)); CHKiRet(pThis->pConsumer(pThis->pUsr, pWti->pUsrp)); /* we now need to check if we should deliberately delay processing a bit @@ -1621,15 +1621,15 @@ finalize_it: * rgerhards, 2008-01-14 */ static rsRetVal -queueConsumerDA(queue_t *pThis, wti_t *pWti, int iCancelStateSave) +qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(queueDequeueConsumable(pThis, pWti, iCancelStateSave)); - CHKiRet(queueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->pUsrp)); + CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave)); + CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->pUsrp)); finalize_it: dbgoprint((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); @@ -1645,7 +1645,7 @@ finalize_it: * the DA queue */ static int -queueChkStopWrkrDA(queue_t *pThis) +qqueueChkStopWrkrDA(qqueue_t *pThis) { /* if our queue is in destruction, we drain to the DA queue and so we shall not terminate * until we have done so. @@ -1664,7 +1664,7 @@ queueChkStopWrkrDA(queue_t *pThis) && pThis->pqDA->tVars.disk.sizeOnDisk > pThis->pqDA->sizeOnDiskMax) { /* this queue can never grow, so we can give up... */ bStopWrkr = 1; - } else if(queueGetOverallQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { + } else if(qqueueGetOverallQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { bStopWrkr = 1; } else { bStopWrkr = 0; @@ -1687,9 +1687,9 @@ queueChkStopWrkrDA(queue_t *pThis) * the DA queue */ static int -queueChkStopWrkrReg(queue_t *pThis) +qqueueChkStopWrkrReg(qqueue_t *pThis) { - return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && queueGetOverallQueueSize(pThis) == 0); + return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && qqueueGetOverallQueueSize(pThis) == 0); } @@ -1697,26 +1697,26 @@ queueChkStopWrkrReg(queue_t *pThis) * are not stable! DA queue version */ static int -queueIsIdleDA(queue_t *pThis) +qqueueIsIdleDA(qqueue_t *pThis) { /* remember: iQueueSize is the DA queue size, not the main queue! */ /* TODO: I think we need just a single function for DA and non-DA mode - but I leave it for now as is */ - return(queueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && queueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); + return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); } /* must only be called when the queue mutex is locked, else results * are not stable! Regular queue version */ static int -queueIsIdleReg(queue_t *pThis) +qqueueIsIdleReg(qqueue_t *pThis) { #if 0 /* enable for performance testing */ int ret; - ret = queueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && queueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk); + ret = qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk); if(ret) fprintf(stderr, "queue is idle\n"); return ret; #else /* regular code! */ - return(queueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && queueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); + return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); #endif } @@ -1735,11 +1735,11 @@ queueIsIdleReg(queue_t *pThis) * I am telling this, because I, too, always get confused by those... */ static rsRetVal -queueRegOnWrkrShutdown(queue_t *pThis) +qqueueRegOnWrkrShutdown(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->pqParent != NULL) { pThis->pqParent->bChildIsDone = 1; /* indicate we are done */ @@ -1756,11 +1756,11 @@ queueRegOnWrkrShutdown(queue_t *pThis) * hook to indicate in the parent queue (if we are a child) that we are not done yet. */ static rsRetVal -queueRegOnWrkrStartup(queue_t *pThis) +qqueueRegOnWrkrStartup(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->pqParent != NULL) { pThis->pqParent->bChildIsDone = 0; @@ -1773,7 +1773,7 @@ queueRegOnWrkrStartup(queue_t *pThis) /* start up the queue - it must have been constructed and parameters defined * before. */ -rsRetVal queueStart(queue_t *pThis) /* this is the ConstructionFinalizer */ +rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; rsRetVal iRetLocal; @@ -1811,7 +1811,7 @@ rsRetVal queueStart(queue_t *pThis) /* this is the ConstructionFinalizer */ dbgoprint((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, qsize %d, child %d starting\n", pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize, - queueGetOverallQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1); + qqueueGetOverallQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1); if(pThis->qType == QUEUETYPE_DIRECT) FINALIZE; /* with direct queues, we are already finished... */ @@ -1822,13 +1822,13 @@ rsRetVal queueStart(queue_t *pThis) /* this is the ConstructionFinalizer */ lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:Reg", obj.GetName((obj_t*) pThis)); CHKiRet(wtpConstruct (&pThis->pWtpReg)); CHKiRet(wtpSetDbgHdr (pThis->pWtpReg, pszBuf, lenBuf)); - CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) queueRateLimiter)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) queueChkStopWrkrReg)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) queueIsIdleReg)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) queueConsumerReg)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void*pWti))queueConsumerCancelCleanup)); - CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) queueRegOnWrkrStartup)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) queueRegOnWrkrShutdown)); + CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRateLimiter)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrReg)); + CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleReg)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerReg)); + CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void*pWti))qqueueConsumerCancelCleanup)); + CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrStartup)); + CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrShutdown)); CHKiRet(wtpSetpmutUsr (pThis->pWtpReg, pThis->mut)); CHKiRet(wtpSetpcondBusy (pThis->pWtpReg, &pThis->notEmpty)); CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpReg, pThis->iNumWorkerThreads)); @@ -1841,10 +1841,10 @@ rsRetVal queueStart(queue_t *pThis) /* this is the ConstructionFinalizer */ /* If we are disk-assisted, we need to check if there is a QIF file * which we need to load. -- rgerhards, 2008-01-15 */ - iRetLocal = queueHaveQIF(pThis); + iRetLocal = qqueueHaveQIF(pThis); if(iRetLocal == RS_RET_OK) { dbgoprint((obj_t*) pThis, "on-disk queue present, needs to be reloaded\n"); - queueInitDA(pThis, QUEUE_MODE_ENQDEQ, LOCK_MUTEX); /* initiate DA mode */ + qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, LOCK_MUTEX); /* initiate DA mode */ bInitialized = 1; /* we are done */ } else { /* TODO: use logerror? -- rgerhards, 2008-01-16 */ @@ -1861,7 +1861,7 @@ rsRetVal queueStart(queue_t *pThis) /* this is the ConstructionFinalizer */ /* if the queue already contains data, we need to start the correct number of worker threads. This can be * the case when a disk queue has been loaded. If we did not start it here, it would never start. */ - queueAdviseMaxWorkers(pThis); + qqueueAdviseMaxWorkers(pThis); pThis->bQueueStarted = 1; finalize_it: @@ -1876,7 +1876,7 @@ finalize_it: * and 0 otherwise. * rgerhards, 2008-01-10 */ -static rsRetVal queuePersist(queue_t *pThis, int bIsCheckpoint) +static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) { DEFiRet; strm_t *psQIF = NULL; /* Queue Info File */ @@ -1887,7 +1887,7 @@ static rsRetVal queuePersist(queue_t *pThis, int bIsCheckpoint) ASSERT(pThis != NULL); if(pThis->qType != QUEUETYPE_DISK) { - if(queueGetOverallQueueSize(pThis) > 0) { + if(qqueueGetOverallQueueSize(pThis) > 0) { /* This error code is OK, but we will probably not implement this any time * The reason is that persistence happens via DA queues. But I would like to * leave the code as is, as we so have a hook in case we need one. @@ -1898,13 +1898,13 @@ static rsRetVal queuePersist(queue_t *pThis, int bIsCheckpoint) FINALIZE; /* if the queue is empty, we are happy and done... */ } - dbgoprint((obj_t*) pThis, "persisting queue to disk, %d entries...\n", queueGetOverallQueueSize(pThis)); + dbgoprint((obj_t*) pThis, "persisting queue to disk, %d entries...\n", qqueueGetOverallQueueSize(pThis)); /* Construct file name */ lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); - if((bIsCheckpoint != QUEUE_CHECKPOINT) && (queueGetOverallQueueSize(pThis) == 0)) { + if((bIsCheckpoint != QUEUE_CHECKPOINT) && (qqueueGetOverallQueueSize(pThis) == 0)) { if(pThis->bNeedDelQIF) { unlink((char*)pszQIFNam); pThis->bNeedDelQIF = 0; @@ -1938,7 +1938,7 @@ static rsRetVal queuePersist(queue_t *pThis, int bIsCheckpoint) * to the regular files. -- rgerhards, 2008-01-29 */ while(pThis->iUngottenObjs > 0) { - CHKiRet(queueGetUngottenObj(pThis, &pUsr)); + CHKiRet(qqueueGetUngottenObj(pThis, &pUsr)); CHKiRet((objSerialize(pUsr))(pUsr, psQIF)); objDestruct(pUsr); } @@ -1972,14 +1972,14 @@ finalize_it: * abide to our regular call interface)... * rgerhards, 2008-01-13 */ -rsRetVal queueChkPersist(queue_t *pThis) +rsRetVal qqueueChkPersist(qqueue_t *pThis) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->iPersistUpdCnt && ++pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { - queuePersist(pThis, QUEUE_CHECKPOINT); + qqueuePersist(pThis, QUEUE_CHECKPOINT); pThis->iUpdsSincePersist = 0; } @@ -1988,8 +1988,8 @@ rsRetVal queueChkPersist(queue_t *pThis) /* destructor for the queue object */ -BEGINobjDestruct(queue) /* be sure to specify the object type also in END and CODESTART macros! */ -CODESTARTobjDestruct(queue) +BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(qqueue) pThis->bQueueInDestruction = 1; /* indicate we are in destruction (modifies some behaviour) */ /* shut down all workers (handles *all* of the persistence logic) @@ -1999,7 +1999,7 @@ CODESTARTobjDestruct(queue) * with a child! -- rgerhards, 2008-01-28 */ if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL) - queueShutdownWorkers(pThis); + qqueueShutdownWorkers(pThis); /* finally destruct our (regular) worker thread pool * Note: currently pWtpReg is never NULL, but if we optimize our logic, this may happen, @@ -2024,7 +2024,7 @@ CODESTARTobjDestruct(queue) wtpDestruct(&pThis->pWtpDA); } if(pThis->pqDA != NULL) { - queueDestruct(&pThis->pqDA); + qqueueDestruct(&pThis->pqDA); } /* persist the queue (we always do that - queuePersits() does cleanup if the queue is empty) @@ -2034,7 +2034,7 @@ CODESTARTobjDestruct(queue) * disk queues and DA mode. Anyhow, it doesn't hurt to know that we could extend it here * if need arises (what I doubt...) -- rgerhards, 2008-01-25 */ - CHKiRet_Hdlr(queuePersist(pThis, QUEUE_NO_CHECKPOINT)) { + CHKiRet_Hdlr(qqueuePersist(pThis, QUEUE_NO_CHECKPOINT)) { dbgoprint((obj_t*) pThis, "error %d persisting queue - data lost!\n", iRet); } @@ -2059,7 +2059,7 @@ CODESTARTobjDestruct(queue) if(pThis->pszSpoolDir != NULL) free(pThis->pszSpoolDir); -ENDobjDestruct(queue) +ENDobjDestruct(qqueue) /* set the queue's file prefix @@ -2068,7 +2068,7 @@ ENDobjDestruct(queue) * rgerhards, 2008-01-09 */ rsRetVal -queueSetFilePrefix(queue_t *pThis, uchar *pszPrefix, size_t iLenPrefix) +qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix) { DEFiRet; @@ -2091,11 +2091,11 @@ finalize_it: * rgerhards, 2008-01-09 */ rsRetVal -queueSetMaxFileSize(queue_t *pThis, size_t iMaxFileSize) +qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); if(iMaxFileSize < 1024) { ABORT_FINALIZE(RS_RET_VALUE_TOO_LOW); @@ -2112,13 +2112,13 @@ finalize_it: * Enqueues the new element and awakes worker thread. */ rsRetVal -queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr) +qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) { DEFiRet; int iCancelStateSave; struct timespec t; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); /* first check if we need to discard this message (which will cause CHKiRet() to exit) * rgerhards, 2008-10-07: It is OK to do this outside of mutex protection. The iQueueSize @@ -2127,7 +2127,7 @@ queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr) * threading tools may point this access to be an error, but this is done * intentional. I do not see this causes problems to us. */ - CHKiRet(queueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); + CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); /* Please note that this function is not cancel-safe and consequently * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE @@ -2142,7 +2142,7 @@ queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* then check if we need to add an assistance disk queue */ if(pThis->bIsDA) - CHKiRet(queueChkStrtDA(pThis)); + CHKiRet(qqueueChkStrtDA(pThis)); /* handle flow control * There are two different flow control mechanisms: basic and advanced flow control. @@ -2195,13 +2195,13 @@ queueEnqObj(queue_t *pThis, flowControl_t flowCtlType, void *pUsr) } /* and finally enqueue the message */ - CHKiRet(queueAdd(pThis, pUsr)); - queueChkPersist(pThis); + CHKiRet(qqueueAdd(pThis, pUsr)); + qqueueChkPersist(pThis); finalize_it: if(pThis->qType != QUEUETYPE_DIRECT) { /* make sure at least one worker is running. */ - queueAdviseMaxWorkers(pThis); + qqueueAdviseMaxWorkers(pThis); /* and release the mutex */ d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); @@ -2228,12 +2228,12 @@ finalize_it: * rgerhards, 2008-01-16 */ static rsRetVal -queueSetEnqOnly(queue_t *pThis, int bEnqOnly, int bLockMutex) +qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); /* for simplicity, we do one big mutex lock. This method is extremely seldom * called, so that doesn't matter... -- rgerhards, 2008-01-16 @@ -2272,24 +2272,24 @@ finalize_it: /* some simple object access methods */ -DEFpropSetMeth(queue, iPersistUpdCnt, int) -DEFpropSetMeth(queue, iDeqtWinFromHr, int) -DEFpropSetMeth(queue, iDeqtWinToHr, int) -DEFpropSetMeth(queue, toQShutdown, long) -DEFpropSetMeth(queue, toActShutdown, long) -DEFpropSetMeth(queue, toWrkShutdown, long) -DEFpropSetMeth(queue, toEnq, long) -DEFpropSetMeth(queue, iHighWtrMrk, int) -DEFpropSetMeth(queue, iLowWtrMrk, int) -DEFpropSetMeth(queue, iDiscardMrk, int) -DEFpropSetMeth(queue, iFullDlyMrk, int) -DEFpropSetMeth(queue, iDiscardSeverity, int) -DEFpropSetMeth(queue, bIsDA, int) -DEFpropSetMeth(queue, iMinMsgsPerWrkr, int) -DEFpropSetMeth(queue, bSaveOnShutdown, int) -DEFpropSetMeth(queue, pUsr, void*) -DEFpropSetMeth(queue, iDeqSlowdown, int) -DEFpropSetMeth(queue, sizeOnDiskMax, int64) +DEFpropSetMeth(qqueue, iPersistUpdCnt, int) +DEFpropSetMeth(qqueue, iDeqtWinFromHr, int) +DEFpropSetMeth(qqueue, iDeqtWinToHr, int) +DEFpropSetMeth(qqueue, toQShutdown, long) +DEFpropSetMeth(qqueue, toActShutdown, long) +DEFpropSetMeth(qqueue, toWrkShutdown, long) +DEFpropSetMeth(qqueue, toEnq, long) +DEFpropSetMeth(qqueue, iHighWtrMrk, int) +DEFpropSetMeth(qqueue, iLowWtrMrk, int) +DEFpropSetMeth(qqueue, iDiscardMrk, int) +DEFpropSetMeth(qqueue, iFullDlyMrk, int) +DEFpropSetMeth(qqueue, iDiscardSeverity, int) +DEFpropSetMeth(qqueue, bIsDA, int) +DEFpropSetMeth(qqueue, iMinMsgsPerWrkr, int) +DEFpropSetMeth(qqueue, bSaveOnShutdown, int) +DEFpropSetMeth(qqueue, pUsr, void*) +DEFpropSetMeth(qqueue, iDeqSlowdown, int) +DEFpropSetMeth(qqueue, sizeOnDiskMax, int64) /* This function can be used as a generic way to set properties. Only the subset @@ -2298,11 +2298,11 @@ DEFpropSetMeth(queue, sizeOnDiskMax, int64) * rgerhards, 2008-01-11 */ #define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, (uchar*) name, sizeof(name) - 1) -static rsRetVal queueSetProperty(queue_t *pThis, var_t *pProp) +static rsRetVal qqueueSetProperty(qqueue_t *pThis, var_t *pProp) { DEFiRet; - ISOBJ_TYPE_assert(pThis, queue); + ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pProp != NULL); if(isProp("iQueueSize")) { @@ -2324,19 +2324,19 @@ finalize_it: #undef isProp /* dummy */ -rsRetVal queueQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } +rsRetVal qqueueQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } /* Initialize the stream class. Must be called as the very first method * before anything else is called inside this class. * rgerhards, 2008-01-09 */ -BEGINObjClassInit(queue, 1, OBJ_IS_CORE_MODULE) +BEGINObjClassInit(qqueue, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ CHKiRet(objUse(glbl, CORE_COMPONENT)); /* now set our own handlers */ - OBJSetMethodHandler(objMethod_SETPROPERTY, queueSetProperty); -ENDObjClassInit(queue) + OBJSetMethodHandler(objMethod_SETPROPERTY, qqueueSetProperty); +ENDObjClassInit(qqueue) /* vi:set ai: */ diff --git a/runtime/queue.h b/runtime/queue.h index a2dd594f..a267862d 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -160,7 +160,7 @@ typedef struct queue_s { strm_t *pRead; /* current file to be read */ } disk; } tVars; -} queue_t; +} qqueue_t; /* some symbolic constants for easier reference */ #define QUEUE_MODE_ENQDEQ 0 @@ -177,30 +177,30 @@ typedef struct queue_s { #define QUEUE_TIMEOUT_ETERNAL 24 * 60 * 60 * 1000 /* prototypes */ -rsRetVal queueDestruct(queue_t **ppThis); -rsRetVal queueEnqObj(queue_t *pThis, flowControl_t flwCtlType, void *pUsr); -rsRetVal queueStart(queue_t *pThis); -rsRetVal queueSetMaxFileSize(queue_t *pThis, size_t iMaxFileSize); -rsRetVal queueSetFilePrefix(queue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); -rsRetVal queueConstruct(queue_t **ppThis, queueType_t qType, int iWorkerThreads, +rsRetVal qqueueDestruct(qqueue_t **ppThis); +rsRetVal qqueueEnqObj(qqueue_t *pThis, flowControl_t flwCtlType, void *pUsr); +rsRetVal qqueueStart(qqueue_t *pThis); +rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); +rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); +rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)); -PROTOTYPEObjClassInit(queue); -PROTOTYPEpropSetMeth(queue, iPersistUpdCnt, int); -PROTOTYPEpropSetMeth(queue, iDeqtWinFromHr, int); -PROTOTYPEpropSetMeth(queue, iDeqtWinToHr, int); -PROTOTYPEpropSetMeth(queue, toQShutdown, long); -PROTOTYPEpropSetMeth(queue, toActShutdown, long); -PROTOTYPEpropSetMeth(queue, toWrkShutdown, long); -PROTOTYPEpropSetMeth(queue, toEnq, long); -PROTOTYPEpropSetMeth(queue, iHighWtrMrk, int); -PROTOTYPEpropSetMeth(queue, iLowWtrMrk, int); -PROTOTYPEpropSetMeth(queue, iDiscardMrk, int); -PROTOTYPEpropSetMeth(queue, iDiscardSeverity, int); -PROTOTYPEpropSetMeth(queue, iMinMsgsPerWrkr, int); -PROTOTYPEpropSetMeth(queue, bSaveOnShutdown, int); -PROTOTYPEpropSetMeth(queue, pUsr, void*); -PROTOTYPEpropSetMeth(queue, iDeqSlowdown, int); -PROTOTYPEpropSetMeth(queue, sizeOnDiskMax, int64); -#define queueGetID(pThis) ((unsigned long) pThis) +PROTOTYPEObjClassInit(qqueue); +PROTOTYPEpropSetMeth(qqueue, iPersistUpdCnt, int); +PROTOTYPEpropSetMeth(qqueue, iDeqtWinFromHr, int); +PROTOTYPEpropSetMeth(qqueue, iDeqtWinToHr, int); +PROTOTYPEpropSetMeth(qqueue, toQShutdown, long); +PROTOTYPEpropSetMeth(qqueue, toActShutdown, long); +PROTOTYPEpropSetMeth(qqueue, toWrkShutdown, long); +PROTOTYPEpropSetMeth(qqueue, toEnq, long); +PROTOTYPEpropSetMeth(qqueue, iHighWtrMrk, int); +PROTOTYPEpropSetMeth(qqueue, iLowWtrMrk, int); +PROTOTYPEpropSetMeth(qqueue, iDiscardMrk, int); +PROTOTYPEpropSetMeth(qqueue, iDiscardSeverity, int); +PROTOTYPEpropSetMeth(qqueue, iMinMsgsPerWrkr, int); +PROTOTYPEpropSetMeth(qqueue, bSaveOnShutdown, int); +PROTOTYPEpropSetMeth(qqueue, pUsr, void*); +PROTOTYPEpropSetMeth(qqueue, iDeqSlowdown, int); +PROTOTYPEpropSetMeth(qqueue, sizeOnDiskMax, int64); +#define qqueueGetID(pThis) ((unsigned long) pThis) #endif /* #ifndef QUEUE_H_INCLUDED */ diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 54db12c2..8df100a1 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -157,7 +157,7 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) if(ppErrObj != NULL) *ppErrObj = "wtp"; CHKiRet(wtpClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "queue"; - CHKiRet(queueClassInit(NULL)); + CHKiRet(qqueueClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "vmstk"; CHKiRet(vmstkClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "sysvar"; diff --git a/runtime/srutils.c b/runtime/srutils.c index 1280e40d..d01ca20d 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -458,7 +458,7 @@ srSleep(int iSeconds, int iuSeconds) * Added 2008-01-30 */ char *rs_strerror_r(int errnum, char *buf, size_t buflen) { -#ifdef __hpux +#ifndef HAVE_STRERROR_R char *pszErr; pszErr = strerror(errnum); snprintf(buf, buflen, "%s", pszErr); diff --git a/runtime/wti.c b/runtime/wti.c index e8a77474..9b3450e6 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -39,6 +39,11 @@ #include #include +#ifdef OS_SOLARIS +# include +# define pthread_yield() sched_yield() +#endif + #include "rsyslog.h" #include "stringbuf.h" #include "srUtils.h" diff --git a/runtime/wtp.c b/runtime/wtp.c index 06173e04..41903576 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -40,6 +40,11 @@ #include #include +#ifdef OS_SOLARIS +# include +# define pthread_yield() sched_yield() +#endif + #include "rsyslog.h" #include "stringbuf.h" #include "srUtils.h" diff --git a/tools/Makefile.am b/tools/Makefile.am index 776279a5..e523b854 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -22,7 +22,7 @@ rsyslogd_SOURCES = \ \ ../dirty.h rsyslogd_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) +rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) rsyslogd_LDFLAGS = -export-dynamic if ENABLE_DIAGTOOLS diff --git a/tools/omfile.c b/tools/omfile.c index 1539ae19..ea91d6ef 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -44,6 +44,10 @@ #include #include +#ifdef OS_SOLARIS +# include +#endif + #include "syslogd.h" #include "syslogd-types.h" #include "srUtils.h" diff --git a/tools/syslogd.c b/tools/syslogd.c index 6b8ce82f..63c4b249 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -72,13 +72,18 @@ #include #include #include -#include -#ifdef __sun +#ifdef OS_SOLARIS # include +# include +# include +# include +# include #else +# include # include #endif + #include #include #include @@ -279,7 +284,7 @@ static int gidDropPriv = 0; /* group-id to which priveleges should be dropped to extern int errno; /* main message queue and its configuration parameters */ -static queue_t *pMsgQueue = NULL; /* the main message queue */ +static qqueue_t *pMsgQueue = NULL; /* the main message queue */ static int iMainMsgQueueSize = 10000; /* size of the main message queue above */ static int iMainMsgQHighWtrMark = 8000; /* high water mark for disk-assisted queues */ static int iMainMsgQLowWtrMark = 2000; /* low water mark for disk-assisted queues */ @@ -1620,7 +1625,7 @@ submitMsg(msg_t *pMsg) ISOBJ_TYPE_assert(pMsg, msg); MsgPrepareEnqueue(pMsg); - queueEnqObj(pMsgQueue, pMsg->flowCtlType, (void*) pMsg); + qqueueEnqObj(pMsgQueue, pMsg->flowCtlType, (void*) pMsg); RETiRet; } @@ -1681,7 +1686,7 @@ logmsg(msg_t *pMsg, int flags) /* now submit the message to the main queue - then we are done */ pMsg->msgFlags = flags; MsgPrepareEnqueue(pMsg); - queueEnqObj(pMsgQueue, pMsg->flowCtlType, (void*) pMsg); + qqueueEnqObj(pMsgQueue, pMsg->flowCtlType, (void*) pMsg); ENDfunc } @@ -1979,7 +1984,7 @@ die(int sig) /* drain queue (if configured so) and stop main queue worker thread pool */ dbgprintf("Terminating main queue...\n"); - queueDestruct(&pMsgQueue); + qqueueDestruct(&pMsgQueue); pMsgQueue = NULL; /* Free ressources and close connections. This includes flushing any remaining @@ -2269,8 +2274,8 @@ static void dbgPrintInitInfo(void) static int iMainMsgQtoWrkMinMsgs = 100; static int iMainMsgQbSaveOnShutdown = 1; iMainMsgQueMaxDiskSpace = 0; - setQPROP(queueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", 100); - setQPROP(queueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", 1); + setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", 100); + setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", 1); */ dbgprintf("Work Directory: '%s'.\n", glbl.GetWorkDir()); } @@ -2332,7 +2337,7 @@ init(void) /* delete the message queue, which also flushes all messages left over */ if(pMsgQueue != NULL) { dbgprintf("deleting main message queue\n"); - queueDestruct(&pMsgQueue); /* delete pThis here! */ + qqueueDestruct(&pMsgQueue); /* delete pThis here! */ pMsgQueue = NULL; } @@ -2444,7 +2449,7 @@ init(void) } /* create message queue */ - CHKiRet_Hdlr(queueConstruct(&pMsgQueue, MainMsgQueType, iMainMsgQueueNumWorkers, iMainMsgQueueSize, msgConsumer)) { + CHKiRet_Hdlr(qqueueConstruct(&pMsgQueue, MainMsgQueType, iMainMsgQueueNumWorkers, iMainMsgQueueSize, msgConsumer)) { /* no queue is fatal, we need to give up in that case... */ fprintf(stderr, "fatal error %d: could not create message queue - rsyslogd can not run!\n", iRet); exit(1); @@ -2462,29 +2467,29 @@ init(void) errmsg.LogError(0, NO_ERRCODE, "Invalid " #directive ", error %d. Ignored, running with default setting", iRet); \ } - setQPROP(queueSetMaxFileSize, "$MainMsgQueueFileSize", iMainMsgQueMaxFileSize); - setQPROP(queueSetsizeOnDiskMax, "$MainMsgQueueMaxDiskSpace", iMainMsgQueMaxDiskSpace); - setQPROPstr(queueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName); - setQPROP(queueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt); - setQPROP(queueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown ); - setQPROP(queueSettoActShutdown, "$MainMsgQueueTimeoutActionCompletion", iMainMsgQtoActShutdown); - setQPROP(queueSettoWrkShutdown, "$MainMsgQueueWorkerTimeoutThreadShutdown", iMainMsgQtoWrkShutdown); - setQPROP(queueSettoEnq, "$MainMsgQueueTimeoutEnqueue", iMainMsgQtoEnq); - setQPROP(queueSetiHighWtrMrk, "$MainMsgQueueHighWaterMark", iMainMsgQHighWtrMark); - setQPROP(queueSetiLowWtrMrk, "$MainMsgQueueLowWaterMark", iMainMsgQLowWtrMark); - setQPROP(queueSetiDiscardMrk, "$MainMsgQueueDiscardMark", iMainMsgQDiscardMark); - setQPROP(queueSetiDiscardSeverity, "$MainMsgQueueDiscardSeverity", iMainMsgQDiscardSeverity); - setQPROP(queueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", iMainMsgQWrkMinMsgs); - setQPROP(queueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", bMainMsgQSaveOnShutdown); - setQPROP(queueSetiDeqSlowdown, "$MainMsgQueueDequeueSlowdown", iMainMsgQDeqSlowdown); - setQPROP(queueSetiDeqtWinFromHr, "$MainMsgQueueDequeueTimeBegin", iMainMsgQueueDeqtWinFromHr); - setQPROP(queueSetiDeqtWinToHr, "$MainMsgQueueDequeueTimeEnd", iMainMsgQueueDeqtWinToHr); + setQPROP(qqueueSetMaxFileSize, "$MainMsgQueueFileSize", iMainMsgQueMaxFileSize); + setQPROP(qqueueSetsizeOnDiskMax, "$MainMsgQueueMaxDiskSpace", iMainMsgQueMaxDiskSpace); + setQPROPstr(qqueueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName); + setQPROP(qqueueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt); + setQPROP(qqueueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown ); + setQPROP(qqueueSettoActShutdown, "$MainMsgQueueTimeoutActionCompletion", iMainMsgQtoActShutdown); + setQPROP(qqueueSettoWrkShutdown, "$MainMsgQueueWorkerTimeoutThreadShutdown", iMainMsgQtoWrkShutdown); + setQPROP(qqueueSettoEnq, "$MainMsgQueueTimeoutEnqueue", iMainMsgQtoEnq); + setQPROP(qqueueSetiHighWtrMrk, "$MainMsgQueueHighWaterMark", iMainMsgQHighWtrMark); + setQPROP(qqueueSetiLowWtrMrk, "$MainMsgQueueLowWaterMark", iMainMsgQLowWtrMark); + setQPROP(qqueueSetiDiscardMrk, "$MainMsgQueueDiscardMark", iMainMsgQDiscardMark); + setQPROP(qqueueSetiDiscardSeverity, "$MainMsgQueueDiscardSeverity", iMainMsgQDiscardSeverity); + setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", iMainMsgQWrkMinMsgs); + setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", bMainMsgQSaveOnShutdown); + setQPROP(qqueueSetiDeqSlowdown, "$MainMsgQueueDequeueSlowdown", iMainMsgQDeqSlowdown); + setQPROP(qqueueSetiDeqtWinFromHr, "$MainMsgQueueDequeueTimeBegin", iMainMsgQueueDeqtWinFromHr); + setQPROP(qqueueSetiDeqtWinToHr, "$MainMsgQueueDequeueTimeEnd", iMainMsgQueueDeqtWinToHr); # undef setQPROP # undef setQPROPstr /* ... and finally start the queue! */ - CHKiRet_Hdlr(queueStart(pMsgQueue)) { + CHKiRet_Hdlr(qqueueStart(pMsgQueue)) { /* no queue is fatal, we need to give up in that case... */ fprintf(stderr, "fatal error %d: could not start message queue - rsyslogd can not run!\n", iRet); exit(1); @@ -3093,7 +3098,7 @@ GlobalClassExit(void) CHKiRet(strmClassInit(NULL)); CHKiRet(wtiClassInit(NULL)); CHKiRet(wtpClassInit(NULL)); - CHKiRet(queueClassInit(NULL)); + CHKiRet(qqueueClassInit(NULL)); CHKiRet(vmstkClassInit(NULL)); CHKiRet(sysvarClassInit(NULL)); CHKiRet(vmClassInit(NULL)); -- cgit v1.2.3 From f67cf99ee5cd88bda499aa52d6008bb7d4afe483 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 6 Mar 2009 14:27:15 +0100 Subject: fixed a platform issue the prevented building on solaris --- runtime/queue.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/runtime/queue.c b/runtime/queue.c index 7d78460c..c4a0fad2 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -51,6 +51,11 @@ #include "wti.h" #include "atomic.h" +#ifdef OS_SOLARIS +# include +# define pthread_yield() sched_yield() +#endif + /* static data */ DEFobjStaticHelpers DEFobjCurrIf(glbl) -- cgit v1.2.3 From e8499c6d33d09f6d8b42df72da1661be0ef0f088 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 10 Mar 2009 17:37:13 +0100 Subject: initial implementation of RainerScript functions & strlen() - implemented function support in RainerScript. That means the engine parses and compile functions, as well as executes a few build-in ones. Dynamic loading and registration of functions is not yet supported - but we now have a good foundation to do that later on. NOTE: nested function calls are not yet supported due to a design issue with the function call VM instruction set design. - implemented the strlen() RainerScript function --- ChangeLog | 7 +++++++ runtime/conf.c | 4 ++++ runtime/ctok.c | 17 +++++++++++---- runtime/expr.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- runtime/rsyslog.h | 4 ++++ runtime/vm.c | 28 +++++++++++++++++++++++++ runtime/vmop.c | 3 +++ runtime/vmop.h | 33 ++++++++++++++++++++++++++--- 8 files changed, 149 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index f1bc354c..7602bd41 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +- implemented function support in RainerScript. That means the engine + parses and compile functions, as well as executes a few build-in + ones. Dynamic loading and registration of functions is not yet + supported - but we now have a good foundation to do that later on. + NOTE: nested function calls are not yet supported due to a design + issue with the function call VM instruction set design. +- implemented the strlen() RainerScript function --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-02-?? - added ERE support in filter conditions diff --git a/runtime/conf.c b/runtime/conf.c index a670c65b..ede15cc7 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -794,6 +794,10 @@ dbgprintf("calling expression parser, pp %p ('%s')\n", *pline, *pline); CHKiRet(ctok.Getpp(tok, pline)); CHKiRet(ctok.Destruct(&tok)); + /* debug support - print vmprg after construction (uncomment to use) */ + /* vmprgDebugPrint(f->f_filterData.f_expr->pVmprg); */ + vmprgDebugPrint(f->f_filterData.f_expr->pVmprg); + /* we now need to skip whitespace to the action part, else we confuse * the legacy rsyslog conf parser. -- rgerhards, 2008-02-25 */ diff --git a/runtime/ctok.c b/runtime/ctok.c index ceab15bd..d2cd8bbd 100644 --- a/runtime/ctok.c +++ b/runtime/ctok.c @@ -259,12 +259,17 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) } CHKiRet(rsCStrConstruct(&pstrVal)); - /* this loop is quite simple, a variable name is terminated by whitespace. */ - while(!isspace(c)) { + /* this loop is quite simple, a variable name is terminated when a non-supported + * character is detected. Note that we currently permit a numerical digit as the + * first char, which is not permitted by ABNF. -- rgerhards, 2009-03-10 + */ + while(isalpha(c) || isdigit(c) || (c == '_') || (c == '-')) { CHKiRet(rsCStrAppendChar(pstrVal, tolower(c))); CHKiRet(ctokGetCharFromStream(pThis, &c)); } - CHKiRet(rsCStrFinish(pStrB)); + CHKiRet(ctokUngetCharFromStream(pThis, c)); /* put not processed char back */ + + CHKiRet(rsCStrFinish(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); pstrVal = NULL; @@ -389,6 +394,7 @@ ctokGetToken(ctok_t *pThis, ctok_token_t **ppToken) uchar c; uchar szWord[128]; int bRetry = 0; /* retry parse? Only needed for inline comments... */ + cstr_t *pstrVal; ISOBJ_TYPE_assert(pThis, ctok); ASSERT(ppToken != NULL); @@ -512,7 +518,10 @@ ctokGetToken(ctok_t *pThis, ctok_token_t **ppToken) /* push c back, higher level parser needs it */ CHKiRet(ctokUngetCharFromStream(pThis, c)); pToken->tok = ctok_FUNCTION; - /* TODO: fill function name */ + /* fill function name */ + CHKiRet(rsCStrConstruct(&pstrVal)); + CHKiRet(rsCStrSetSzStr(pstrVal, szWord)); + CHKiRet(var.SetString(pToken->pVar, pstrVal)); } else { /* give up... */ dbgprintf("parser has an invalid word (token) '%s'\n", szWord); pToken->tok = ctok_INVALID; diff --git a/runtime/expr.c b/runtime/expr.c index ee5b9e2c..38ed1c68 100644 --- a/runtime/expr.c +++ b/runtime/expr.c @@ -55,10 +55,62 @@ DEFobjCurrIf(ctok) * rgerhards, 2008-02-19 */ -/* forward definiton - thanks to recursive ABNF, we can not avoid at least one ;) */ +/* forward definition - thanks to recursive ABNF, we can not avoid at least one ;) */ static rsRetVal expr(expr_t *pThis, ctok_t *tok); +static rsRetVal +function(expr_t *pThis, ctok_t *tok) +{ + DEFiRet; + ctok_token_t *pToken = NULL; + int iNumArgs = 0; + var_t *pVar; + + ISOBJ_TYPE_assert(pThis, expr); + ISOBJ_TYPE_assert(tok, ctok); + + CHKiRet(ctok.GetToken(tok, &pToken)); + /* note: pToken is destructed in finalize_it */ + + if(pToken->tok == ctok_LPAREN) { + CHKiRet(ctok_token.Destruct(&pToken)); /* token processed, "eat" it */ + CHKiRet(ctok.GetToken(tok, &pToken)); /* get next one */ + } else + ABORT_FINALIZE(RS_RET_FUNC_NO_LPAREN); + + /* we first push all the params on the stack. Then we call the function */ + while(pToken->tok != ctok_RPAREN) { + ++iNumArgs; + CHKiRet(ctok.UngetToken(tok, pToken)); /* not for us, so let others process it */ + CHKiRet(expr(pThis, tok)); + CHKiRet(ctok.GetToken(tok, &pToken)); /* get next one, needed for while() check */ + if(pToken->tok == ctok_COMMA) { + CHKiRet(ctok_token.Destruct(&pToken)); /* token processed, "eat" it */ + CHKiRet(ctok.GetToken(tok, &pToken)); /* get next one */ + if(pToken->tok == ctok_RPAREN) { + ABORT_FINALIZE(RS_RET_FUNC_MISSING_EXPR); + } + } + } + + + /* now push number of arguments - this must be on top of the stack */ + CHKiRet(var.Construct(&pVar)); + CHKiRet(var.ConstructFinalize(pVar)); + CHKiRet(var.SetNumber(pVar, iNumArgs)); + CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_PUSHCONSTANT, pVar)); /* add to program */ + + +finalize_it: + if(pToken != NULL) { + ctok_token.Destruct(&pToken); /* "eat" processed token */ + } + + RETiRet; +} + + static rsRetVal terminal(expr_t *pThis, ctok_t *tok) { @@ -85,8 +137,12 @@ terminal(expr_t *pThis, ctok_t *tok) break; case ctok_FUNCTION: dbgoprint((obj_t*) pThis, "function\n"); - /* TODO: vm: call - well, need to implement that first */ - ABORT_FINALIZE(RS_RET_NOT_IMPLEMENTED); + CHKiRet(function(pThis, tok)); /* this creates the stack call frame */ + /* ... but we place the call instruction onto the stack ourselfs (because + * we have all relevant information) + */ + CHKiRet(ctok_token.UnlinkVar(pToken, &pVar)); + CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_FUNC_CALL, pVar)); /* add to program */ break; case ctok_MSGVAR: dbgoprint((obj_t*) pThis, "MSGVAR\n"); @@ -406,6 +462,7 @@ ENDobjQueryInterface(expr) */ BEGINObjClassInit(expr, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ + CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(vmprg, CORE_COMPONENT)); CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(ctok_token, CORE_COMPONENT)); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 00290ee5..899f5e13 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -254,6 +254,10 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INVLD_TIME = -2107, /**< invalid timestamp (e.g. could not be parsed) */ RS_RET_NO_ZIP = -2108, /**< ZIP functionality is not present */ RS_RET_CODE_ERR = -2109, /**< program code (internal) error */ + RS_RET_FUNC_NO_LPAREN = -2110, /**< left parenthesis missing after function call (rainerscript) */ + RS_RET_FUNC_MISSING_EXPR = -2111, /**< no expression after comma in function call (rainerscript) */ + RS_RET_INVLD_NBR_ARGUMENTS = -2112, /**< invalid number of arguments for function call (rainerscript) */ + RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/vm.c b/runtime/vm.c index bc6c3dd2..113a9d59 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -331,6 +331,33 @@ finalize_it: ENDop(PUSHSYSVAR) +/* The function call operation is only very roughly implemented. While the plumbing + * to reach this instruction is fine, the instruction itself currently supports only + * functions with a single argument AND with a name that we know. + * TODO: later, we can add here the real logic, that involves looking up function + * names, loading them dynamically ... and all that... + * implementation begun 2009-03-10 by rgerhards + */ +BEGINop(FUNC_CALL) /* remember to set the instruction also in the ENDop macro! */ + var_t *numOperands; + var_t *operand1; + int iStrlen; +CODESTARTop(FUNC_CALL) + vmstk.PopNumber(pThis->pStk, &numOperands); + if(numOperands->val.num != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + vmstk.PopString(pThis->pStk, &operand1); /* guess there's just one ;) */ + if(!rsCStrSzStrCmp(pOp->operand.pVar->val.pStr, (uchar*) "strlen", 6)) { /* only one supported so far ;) */ +RUNLOG_VAR("%s", rsCStrGetSzStr(operand1->val.pStr)); + iStrlen = strlen((char*) rsCStrGetSzStr(operand1->val.pStr)); +RUNLOG_VAR("%d", iStrlen); + } else + ABORT_FINALIZE(RS_RET_INVLD_FUNC); + PUSHRESULTop(operand1, iStrlen); // TODO: dummy, FIXME +finalize_it: +ENDop(FUNC_CALL) + + /* ------------------------------ end instruction set implementation ------------------------------ */ @@ -412,6 +439,7 @@ execProg(vm_t *pThis, vmprg_t *pProg) doOP(DIV); doOP(MOD); doOP(UNARY_MINUS); + doOP(FUNC_CALL); default: ABORT_FINALIZE(RS_RET_INVALID_VMOP); dbgoprint((obj_t*) pThis, "invalid instruction %d in vmprg\n", pCurrOp->opcode); diff --git a/runtime/vmop.c b/runtime/vmop.c index fcacb15b..705acdf2 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -217,6 +217,9 @@ vmopOpcode2Str(vmop_t *pThis, uchar **ppName) case opcode_STRADD: *ppName = (uchar*) "STRADD"; break; + case opcode_FUNC_CALL: + *ppName = (uchar*) "FUNC_CALL"; + break; default: *ppName = (uchar*) "INVALID opcode"; break; diff --git a/runtime/vmop.h b/runtime/vmop.h index c3d5d5f4..938b08fd 100644 --- a/runtime/vmop.h +++ b/runtime/vmop.h @@ -59,17 +59,44 @@ typedef enum { /* do NOT start at 0 to detect uninitialized types after calloc( opcode_PUSHMSGVAR = 1002, /* requires var operand */ opcode_PUSHCONSTANT = 1003, /* requires var operand */ opcode_UNARY_MINUS = 1010, - opcode_END_PROG = 1011 + opcode_FUNC_CALL = 1012, + opcode_END_PROG = 2000 } opcode_t; +/* Additional doc, operation specific + + FUNC_CALL + All parameter passing is via the stack. Parameters are placed onto the stack in reverse order, + that means the last parameter is on top of the stack, the first at the bottom location. + At the actual top of the stack is the number of parameters. This permits functions to be + called with variable number of arguments. The function itself is responsible for poping + the right number of parameters of the stack and complaining if the number is incorrect. + On exit, a single return value must be pushed onto the stack. The FUNC_CALL operation + is generic. Its pVar argument contains the function name string (TODO: very slow, make + faster in later releases). + + Sample Function call: sampleFunc(p1, p2, p3) ; returns number 4711 (sample) + Stacklayout on entry (order is top to bottom): + 3 + p3 + p2 + p1 + ... other vars ... + + Stack on exit + 4711 + ... other vars ... + + */ + + /* the vmop object */ typedef struct vmop_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ opcode_t opcode; union { - var_t *pVar; - /* TODO: add function pointer */ + var_t *pVar; /* for function call, this is the name (string) of function to be called */ } operand; struct vmop_s *pNext; /* next operation or NULL, if end of program (logically this belongs to vmprg) */ } vmop_t; -- cgit v1.2.3 From 59192611db992e7357337beb8e68ec6cee5b3fec Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 10 Mar 2009 22:36:40 +0100 Subject: bugfix: parser did not correctly parse fields in UDP-received messages --- ChangeLog | 3 ++- plugins/imudp/imudp.c | 2 +- tools/syslogd.c | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index f1bc354c..6cab7993 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- -Version 4.1.5 [DEVEL] (rgerhards), 2009-02-?? +Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 +- bugfix: parser did not correctly parse fields in UDP-received messages - added ERE support in filter conditions new comparison operation "ereregex" - added new config directive $RepeatedMsgContainsOriginalMsg so that the diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 72450513..c7e8c1d4 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -181,7 +181,6 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, /* check if we have a different sender than before, if so, we need to query some new values */ if(memcmp(&frominet, frominetPrev, socklen) != 0) { CHKiRet(net.cvthname(&frominet, fromHost, fromHostFQDN, fromHostIP)); -DBGPRINTF("returned: fromHost '%s', FQDN: '%s'\n", fromHost, fromHostFQDN); memcpy(frominetPrev, &frominet, socklen); /* update cache indicator */ /* Here we check if a host is permitted to send us * syslog messages. If it isn't, we do not further @@ -223,6 +222,7 @@ DBGPRINTF("returned: fromHost '%s', FQDN: '%s'\n", fromHost, fromHostFQDN); MsgSetInputName(pMsg, "imudp"); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; + pMsg->bParseHOSTNAME = 1; MsgSetRcvFrom(pMsg, (char*)fromHost); CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP)); CHKiRet(submitMsg(pMsg)); diff --git a/tools/syslogd.c b/tools/syslogd.c index 6b8ce82f..eb496521 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1335,6 +1335,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) char *pBuf; int bContParse = 1; + BEGINfunc assert(pMsg != NULL); assert(pMsg->pszUxTradMsg != NULL); p2parse = (char*) pMsg->pszUxTradMsg; @@ -1408,6 +1409,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) MsgSetMSG(pMsg, p2parse); free(pBuf); + ENDfunc return 0; /* all ok */ } -- cgit v1.2.3 From 837b2b8d262d60580a473743a0a7955ceb980b77 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 11 Mar 2009 15:06:18 +0100 Subject: preparing for 4.1.5 release --- configure.ac | 2 +- doc/manual.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 0c924754..9ba24085 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.4],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.1.5],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/manual.html b/doc/manual.html index bd927d78..064e89af 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

      Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

      -

      This documentation is for version 4.1.4 (devel branch) of rsyslog. +

      This documentation is for version 4.1.5 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

      If you like rsyslog, you might -- cgit v1.2.3 From dd19c937c1bfbe16063c9d633a79810944ac7eba Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 11 Mar 2009 15:12:33 +0100 Subject: updated project status, version --- ChangeLog | 2 ++ doc/status.html | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6cab7993..fb0b0380 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,6 @@ --------------------------------------------------------------------------- +Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? +--------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages - added ERE support in filter conditions diff --git a/doc/status.html b/doc/status.html index f9e5852c..59fd0809 100644 --- a/doc/status.html +++ b/doc/status.html @@ -5,9 +5,9 @@

      This page reflects the status as of 2009-02-02.

      Current Releases

      -

      development: 4.1.4 [2009-01-29] - -change log - -download +

      development: 4.1.5 [2009-03-11] - +change log - +download
      beta: 3.21.10 [2009-02-02] - change log - -- cgit v1.2.3 From a03d90abe1d3ac67bc5492db09e2c6e207288469 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 11 Mar 2009 15:26:51 +0100 Subject: try to make file writer restartable (experimental, untested) --- tools/omfile.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/omfile.c b/tools/omfile.c index 1539ae19..4bef5dbf 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -621,20 +621,21 @@ again: */ if((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE) #ifdef linux - && e == EIO) { + && e == EIO #else - && e == EBADF) { + && e == EBADF #endif + ) { pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY); if (pData->fd < 0) { - iRet = RS_RET_DISABLE_ACTION; + iRet = RS_RET_SUSPENDED; errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); } else { untty(); goto again; } } else { - iRet = RS_RET_DISABLE_ACTION; + iRet = RS_RET_SUSPENDED; errno = e; errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); } -- cgit v1.2.3 From f289422585ef36c39c5bc22ea20344d0c76b29ab Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 11 Mar 2009 19:54:30 +0100 Subject: changed one more status setting user feedback indicates it now looks like it is working ;) still some more work needed for a "good" solution --- tools/omfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/omfile.c b/tools/omfile.c index 4bef5dbf..28bdcf2e 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -562,7 +562,7 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa */ if(pData->bDynamicName) { if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) - ABORT_FINALIZE(RS_RET_ERR); + ABORT_FINALIZE(RS_RET_SUSPENDED); // TODO: different state? conditional based on what went wrong? 2009-03-11 } else if(pData->fd == -1) { prepareFile(pData, pData->f_fname); } -- cgit v1.2.3 From cdc8cf8d55f519066445ce3f9e899d092824a98e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 12 Mar 2009 05:40:03 +0100 Subject: better readability for compiled VMPrg output --- runtime/vmop.c | 51 ++++++++++++++++++++++++++++----------------------- runtime/vmprg.c | 2 +- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/runtime/vmop.c b/runtime/vmop.c index 705acdf2..441dae6c 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -73,12 +73,17 @@ ENDobjDestruct(vmop) /* DebugPrint support for the vmop object */ BEGINobjDebugPrint(vmop) /* be sure to specify the object type also in END and CODESTART macros! */ uchar *pOpcodeName; + cstr_t *pStrVar; CODESTARTobjDebugPrint(vmop) vmopOpcode2Str(pThis, &pOpcodeName); - dbgoprint((obj_t*) pThis, "opcode: %d\t(%s), next %p, var in next line\n", (int) pThis->opcode, pOpcodeName, - pThis->pNext); - if(pThis->operand.pVar != NULL) - var.DebugPrint(pThis->operand.pVar); + CHKiRet(rsCStrConstruct(&pStrVar)); + CHKiRet(rsCStrFinish(&pStrVar)); + if(pThis->operand.pVar != NULL) { + CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); + } + dbgoprint((obj_t*) pThis, "%.12s\t%s\n", pOpcodeName, rsCStrGetSzStrNoNULL(pStrVar)); + rsCStrDestruct(&pStrVar); +finalize_it: ENDobjDebugPrint(vmop) @@ -158,37 +163,37 @@ vmopOpcode2Str(vmop_t *pThis, uchar **ppName) *ppName = (uchar*) "and"; break; case opcode_PLUS: - *ppName = (uchar*) "+"; + *ppName = (uchar*) "add"; break; case opcode_MINUS: - *ppName = (uchar*) "-"; + *ppName = (uchar*) "sub"; break; case opcode_TIMES: - *ppName = (uchar*) "*"; + *ppName = (uchar*) "mul"; break; case opcode_DIV: - *ppName = (uchar*) "/"; + *ppName = (uchar*) "div"; break; case opcode_MOD: - *ppName = (uchar*) "%"; + *ppName = (uchar*) "mod"; break; case opcode_NOT: *ppName = (uchar*) "not"; break; case opcode_CMP_EQ: - *ppName = (uchar*) "=="; + *ppName = (uchar*) "cmp_=="; break; case opcode_CMP_NEQ: - *ppName = (uchar*) "!="; + *ppName = (uchar*) "cmp_!="; break; case opcode_CMP_LT: - *ppName = (uchar*) "<"; + *ppName = (uchar*) "cmp_<"; break; case opcode_CMP_GT: - *ppName = (uchar*) ">"; + *ppName = (uchar*) "cmp_>"; break; case opcode_CMP_LTEQ: - *ppName = (uchar*) "<="; + *ppName = (uchar*) "cmp_<="; break; case opcode_CMP_CONTAINS: *ppName = (uchar*) "contains"; @@ -197,31 +202,31 @@ vmopOpcode2Str(vmop_t *pThis, uchar **ppName) *ppName = (uchar*) "startswith"; break; case opcode_CMP_GTEQ: - *ppName = (uchar*) ">="; + *ppName = (uchar*) "cmp_>="; break; case opcode_PUSHSYSVAR: - *ppName = (uchar*) "PUSHSYSVAR"; + *ppName = (uchar*) "push_sysvar"; break; case opcode_PUSHMSGVAR: - *ppName = (uchar*) "PUSHMSGVAR"; + *ppName = (uchar*) "push_msgvar"; break; case opcode_PUSHCONSTANT: - *ppName = (uchar*) "PUSHCONSTANT"; + *ppName = (uchar*) "push_const"; break; case opcode_POP: - *ppName = (uchar*) "POP"; + *ppName = (uchar*) "pop"; break; case opcode_UNARY_MINUS: - *ppName = (uchar*) "UNARY_MINUS"; + *ppName = (uchar*) "unary_minus"; break; case opcode_STRADD: - *ppName = (uchar*) "STRADD"; + *ppName = (uchar*) "strconcat"; break; case opcode_FUNC_CALL: - *ppName = (uchar*) "FUNC_CALL"; + *ppName = (uchar*) "func_call"; break; default: - *ppName = (uchar*) "INVALID opcode"; + *ppName = (uchar*) "!invalid_opcode!"; break; } diff --git a/runtime/vmprg.c b/runtime/vmprg.c index 705e6948..75915025 100644 --- a/runtime/vmprg.c +++ b/runtime/vmprg.c @@ -74,7 +74,7 @@ ENDobjDestruct(vmprg) BEGINobjDebugPrint(vmprg) /* be sure to specify the object type also in END and CODESTART macros! */ vmop_t *pOp; CODESTARTobjDebugPrint(vmprg) - dbgoprint((obj_t*) pThis, "program contents:\n"); + dbgoprint((obj_t*) pThis, "VM Program:\n"); for(pOp = pThis->vmopRoot ; pOp != NULL ; pOp = pOp->pNext) { vmop.DebugPrint(pOp); } -- cgit v1.2.3 From 2bb202f665df594286595e226251b3580b474a4d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 13 Mar 2009 12:52:53 +0100 Subject: bugfix: removed (newly introduced) memory leaks --- runtime/vm.c | 1 + runtime/vmop.c | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/runtime/vm.c b/runtime/vm.c index 113a9d59..a25476c2 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -354,6 +354,7 @@ RUNLOG_VAR("%d", iStrlen); } else ABORT_FINALIZE(RS_RET_INVLD_FUNC); PUSHRESULTop(operand1, iStrlen); // TODO: dummy, FIXME + var.Destruct(&numOperands); /* no longer needed */ finalize_it: ENDop(FUNC_CALL) diff --git a/runtime/vmop.c b/runtime/vmop.c index 441dae6c..a343481e 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -61,12 +61,8 @@ rsRetVal vmopConstructFinalize(vmop_t __attribute__((unused)) *pThis) /* destructor for the vmop object */ BEGINobjDestruct(vmop) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(vmop) - if( pThis->opcode == opcode_PUSHSYSVAR - || pThis->opcode == opcode_PUSHMSGVAR - || pThis->opcode == opcode_PUSHCONSTANT) { - if(pThis->operand.pVar != NULL) - var.Destruct(&pThis->operand.pVar); - } + if(pThis->operand.pVar != NULL) + var.Destruct(&pThis->operand.pVar); ENDobjDestruct(vmop) -- cgit v1.2.3 From 446a982c2b8f77eb0e349e5bd8240ece6a0772bd Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 16 Mar 2009 16:05:15 +0100 Subject: some more information on rainerscript imlementation (taken from old rscript branch, which is now obsolete) --- doc/rscript_abnf.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/rscript_abnf.html b/doc/rscript_abnf.html index 278fb59c..d60edb5c 100644 --- a/doc/rscript_abnf.html +++ b/doc/rscript_abnf.html @@ -30,6 +30,8 @@ table... values('&$facility&','&$severity&...?]
        endact table... values('&$facility&','&$severity&...?]
        )

      ... or ...

      define action writeMySQL(
          type='ommysql.so', queue.mode='disk', queue.highwatermark = 300,
          action.dbname='events', action.dbuser='uid',
          [?action.template='templatename'?] or [?action.sql='insert into table... values('&$facility&','&$severity&...?]
         )

      if $message contains "error" then action writeMySQL(action.dbname='differentDB')

      [rsyslog.conf overview] +

      Implementation

      +RainerScript will be implemented via a hand-crafted LL(1) parser. I was tempted to use yacc, but it turned out the resulting code was not thread-safe and as such did not fit within the context of rsyslog. Also, limited error handling is not a real problem for us: if there is a problem in parsing the configuration file, we stop processing. Guessing what was meant and trying to recover would IMHO not be good choices for something like a syslogd. [manual index] [rsyslog site]

      This documentation is part of the rsyslog -- cgit v1.2.3 From 208f4e107c78b00c3bdf09301d1b1e43e1b9fdf8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 16 Mar 2009 17:26:45 +0100 Subject: added output module template so far, this is mostly some documentation, but I hope that during the process of creating output modules out of it we will get good questions and thus can extend the template. In any case, it should be better than what we had so far... --- ChangeLog | 5 +- Makefile.am | 6 +- configure.ac | 29 +++++- plugins/omtemplate/Makefile.am | 8 ++ plugins/omtemplate/omtemplate.c | 220 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 260 insertions(+), 8 deletions(-) create mode 100644 plugins/omtemplate/Makefile.am create mode 100644 plugins/omtemplate/omtemplate.c diff --git a/ChangeLog b/ChangeLog index 3488dfb2..8c8442e2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +--------------------------------------------------------------------------- +Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? - implemented function support in RainerScript. That means the engine parses and compile functions, as well as executes a few build-in ones. Dynamic loading and registration of functions is not yet @@ -5,8 +7,7 @@ NOTE: nested function calls are not yet supported due to a design issue with the function call VM instruction set design. - implemented the strlen() RainerScript function ---------------------------------------------------------------------------- -Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? +- added a template output module --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/Makefile.am b/Makefile.am index aaca570d..7bb6af8e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -88,6 +88,10 @@ if ENABLE_IMTEMPLATE SUBDIRS += plugins/imtemplate endif +if ENABLE_OMTEMPLATE +SUBDIRS += plugins/omtemplate +endif + if ENABLE_IMFILE SUBDIRS += plugins/imfile endif @@ -114,5 +118,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 4763fcd5..0c9483ca 100644 --- a/configure.ac +++ b/configure.ac @@ -631,9 +631,7 @@ AC_SUBST(LIBLOGGING_CFLAGS) AC_SUBST(LIBLOGGING_LIBS) -# settings for the template input module; copy and modify this code -# if you intend to add your own module. Be sure to replace imtemplate -# by the actual name of your module. +# settings for the file input module; AC_ARG_ENABLE(imfile, [AS_HELP_STRING([--enable-imfile],[file input module enabled @<:@default=no@:>@])], [case "${enableval}" in @@ -649,8 +647,7 @@ AC_ARG_ENABLE(imfile, # AM_CONDITIONAL(ENABLE_IMFILE, test x$enable_imfile = xyes) -AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) -# end of copy template - be sure to serach for imtemplate to find everything! + # settings for the template input module; copy and modify this code # if you intend to add your own module. Be sure to replace imtemplate # by the actual name of your module. @@ -671,6 +668,26 @@ AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) # end of copy template - be sure to serach for imtemplate to find everything! +# settings for the template output module; copy and modify this code +# if you intend to add your own module. Be sure to replace omtemplate +# by the actual name of your module. +AC_ARG_ENABLE(omtemplate, + [AS_HELP_STRING([--enable-omtemplate],[Compiles omtemplate template module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omtemplate="yes" ;; + no) enable_omtemplate="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omtemplate) ;; + esac], + [enable_omtemplate=no] +) +# +# you may want to do some library checks here - see snmp, mysql, pgsql modules +# for samples +# +AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) +# end of copy template - be sure to serach for omtemplate to find everything! + + AC_CONFIG_FILES([Makefile \ runtime/Makefile \ tools/Makefile \ @@ -683,6 +700,7 @@ AC_CONFIG_FILES([Makefile \ plugins/immark/Makefile \ plugins/imklog/Makefile \ plugins/imtemplate/Makefile \ + plugins/omtemplate/Makefile \ plugins/imfile/Makefile \ plugins/imrelp/Makefile \ plugins/imdiag/Makefile \ @@ -713,6 +731,7 @@ echo "RELP support enabled: $enable_relp" echo "imdiag enabled: $enable_imdiag" echo "file input module enabled: $enable_imfile" echo "input template module will be compiled: $enable_imtemplate" +echo "output template module will be compiled: $enable_omtemplate" echo "Large file support enabled: $enable_largefile" echo "Networking support enabled: $enable_inet" echo "GnuTLS network stream driver enabled: $enable_gnutls" diff --git a/plugins/omtemplate/Makefile.am b/plugins/omtemplate/Makefile.am new file mode 100644 index 00000000..58ddc794 --- /dev/null +++ b/plugins/omtemplate/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omtemplate.la + +omtemplate_la_SOURCES = omtemplate.c omtemplate.h +omtemplate_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omtemplate_la_LDFLAGS = -module -avoid-version +omtemplate_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omtemplate/omtemplate.c b/plugins/omtemplate/omtemplate.c new file mode 100644 index 00000000..e35968ad --- /dev/null +++ b/plugins/omtemplate/omtemplate.c @@ -0,0 +1,220 @@ +/* omtemplate.c + * This is a template for an output module. It implements a very + * simple single-threaded output, just as thought of by the output + * plugin interface. + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-03-16 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) + +typedef struct _instanceData { + /* here you need to define all action-specific data. A record of type + * instanceData will be handed over to each instance of the action. Keep + * in mind that there may be several invocations of the same type of action + * inside rsyslog.conf, and this is what keeps them apart. Do NOT use + * static data for this! + */ + unsigned iSrvPort; /* sample: server port */ +} instanceData; + +/* config variables + * For the configuration interface, we need to keep track of some settings. This + * is done in global variables. It works as follows: when configuration statements + * are entered, the config file handler (or custom function) sets the global + * variable here. When the action then actually is instantiated, this handler + * copies over to instanceData whatever configuration settings (from the global + * variables) apply. The global variables are NEVER used inside an action + * instance (at least this is how it is supposed to work ;) + */ +static int iSrvPort = 0; /* sample: server port */ + + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + /* use this to specify if select features are supported by this + * plugin. If not, the framework will handle that. Currently, only + * RepeatedMsgReduction ("last message repeated n times") is optional. + */ + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance + /* this is a cleanup callback. All dynamically-allocated resources + * in instance data must be cleaned up here. Prime examples are + * malloc()ed memory, file & database handles and the like. + */ +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo + /* permits to spit out some debug info */ +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume + /* this is called when an action has been suspended and the + * rsyslog core tries to resume it. The action must then + * retry (if possible) and report RS_RET_OK if it succeeded + * or RS_RET_SUSPENDED otherwise. + * Note that no data can be written in this callback, as it is + * not present. Prime examples of what can be retried are + * reconnects to remote hosts, reconnects to database, + * opening of files and the like. + * If there is no retry-type of operation, the action may + * return RS_RET_OK, so that it will get called on its doAction + * entry point (where it receives data), retries there, and + * immediately returns RS_RET_SUSPENDED if that does not work + * out. This disables some optimizations in the core's retry logic, + * but is a valid and expected behaviour. Note that it is also OK + * for the retry entry point to return OK but the immediately following + * doAction call to fail. In real life, for example, a buggy com line + * may cause such behaviour. + * Note that there is no guarantee that the core will very quickly + * call doAction after the retry succeeded. Today, it does, but that may + * not always be the case. + */ +ENDtryResume + +BEGINdoAction +CODESTARTdoAction + /* this is where you receive the message and need to carry out the + * action. Data is provided in ppString[i] where 0 <= i <= num of strings + * requested. + * Return RS_RET_OK if all goes well, RS_RET_SUSPENDED if the action can + * currently not complete, or an error code or RS_RET_DISABLED. The later + * two should only be returned if there is no hope that the action can be + * restored unless an rsyslog restart (prime example is an invalid config). + * Error code or RS_RET_DISABLED permanently disables the action, up to + * the next restart. + */ +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us + * This is a clumpsy interface. We receive the action-part of the selector line + * and need to look at the first characters. If they match our signature + * ":omtemplate:", then we need to instantiate an action. It is recommended that + * newer actions just watch for the template and all other parameters are passed in + * via $-config-lines, this will hopefully be compatbile with future config syntaxes. + * If we do not detect our signature, we must return with RS_RET_CONFLINE_UNPROCESSED + * and NOT do anything else. + */ + if(strncmp((char*) p, ":omtemplate:", sizeof(":omtemplate:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omtemplate:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + /* if we have, call rsyslog runtime to get us template. Note that StdFmt below is + * the standard name. Currently, we may need to patch tools/syslogd.c if we need + * to add a new standard template. + */ + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_RQD_TPL_OPT_SQL, (uchar*) " StdFmt")); + + /* if we reach this point, all went well, and we can copy over to instanceData + * those configuration elements that we need. + */ + pData->iSrvPort = (unsigned) iSrvPort; /* set configured port */ + +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + +/* Reset config variables for this module to default values. + */ +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + iSrvPort = 0; /* zero is the default port */ + RETiRet; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + /* register our config handlers */ + /* confguration parameters MUST always be specified in lower case! */ + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomtemplteserverport", 0, eCmdHdlrInt, NULL, &iSrvPort, STD_LOADABLE_MODULE_ID)); + /* "resetconfigvariables" should be provided. Notat that it is a chained directive */ + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +ENDmodInit + +/* vi:set ai: + */ -- cgit v1.2.3 From 51e690f720b4a53a431e7a536b2fe8c25e866f7d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 16 Mar 2009 18:54:42 +0100 Subject: fixed some portability issues first noticed on FreeBSD Also, some cosmetic improvements. --- plugins/omstdout/omstdout.c | 2 +- tests/getline.c | 3 ++- tests/nettester.c | 4 +++- tests/omod-if-array.sh | 9 ++++++++- tests/parsertest.sh | 8 +++++++- 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index 7c63b5c4..181895a4 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -110,7 +110,7 @@ CODESTARTdoAction if(iParam > 0) szBuf[iBuf++] = ','; /* all but first need a delimiter */ iParamVal = 0; - while(szParams[iParam][iParamVal] != '\0' && iBuf < sizeof(szBuf)) { + while(szParams[iParam][iParamVal] != '\0' && iBuf < (int) sizeof(szBuf)) { szBuf[iBuf++] = szParams[iParam][iParamVal++]; } ++iParam; diff --git a/tests/getline.c b/tests/getline.c index 10de2ffe..617d1b0e 100644 --- a/tests/getline.c +++ b/tests/getline.c @@ -23,7 +23,8 @@ */ #include "config.h" #include -#include +#include +#include /* we emulate getline (the dirty way) if we do not have it * We do not try very hard, as this is just a test driver. diff --git a/tests/nettester.c b/tests/nettester.c index 89a784f3..37183ac9 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #define EXIT_FAILURE 1 @@ -361,7 +362,8 @@ int main(int argc, char *argv[]) if((srcdir = getenv("srcdir")) == NULL) srcdir = "."; - printf("Start of nettester run ($srcdir=%s, testsuite=%s)\n", srcdir, testSuite); + printf("Start of nettester run ($srcdir=%s, testsuite=%s, input=%s)\n", + srcdir, testSuite, argv[2]); /* create input config file */ if((fp = fopen(NETTEST_INPUT_CONF_FILE, "w")) == NULL) { diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh index 8a8d67f3..fd845b4d 100755 --- a/tests/omod-if-array.sh +++ b/tests/omod-if-array.sh @@ -1,5 +1,12 @@ -#!/bin/bash -e echo test omod-if-array via udp ./nettester omod-if-array udp +if [ "$?" -ne "0" ]; then + exit 1 +fi + echo test omod-if-array via tcp ./nettester omod-if-array tcp +if [ "$?" -ne "0" ]; then + exit 1 +fi + diff --git a/tests/parsertest.sh b/tests/parsertest.sh index fabe7e8d..a6b7d45c 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1,5 +1,11 @@ -#!/bin/bash -e echo test parsertest via udp ./nettester parse1 udp +if [ "$?" -ne "0" ]; then + exit 1 +fi + echo test parsertest via tcp ./nettester parse1 tcp +if [ "$?" -ne "0" ]; then + exit 1 +fi -- cgit v1.2.3 From 3762d2b7bc669e45d51a2157158e3cc925dd0152 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 17 Mar 2009 18:26:15 +0100 Subject: fixed some message-loss situations when file write failures happened --- rsyslog.conf | 1 + tools/omfile.c | 12 +++--------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/rsyslog.conf b/rsyslog.conf index 47fc4402..329704b8 100644 --- a/rsyslog.conf +++ b/rsyslog.conf @@ -58,3 +58,4 @@ local7.* /var/log/boot.log # UDP Syslog Server: #$ModLoad imudp.so # provides UDP syslog reception #$UDPServerRun 514 # start a UDP syslog server at standard port 514 +*.* /mnt/logfile diff --git a/tools/omfile.c b/tools/omfile.c index 28bdcf2e..8be815f2 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -600,20 +600,14 @@ again: } } - if (write(pData->fd, ppString[0], strlen((char*)ppString[0])) < 0) { + if(write(pData->fd, ppString[0], strlen((char*)ppString[0])) < 0) { int e = errno; +dbgprintf("++++++++++ log file writer error %d\n", e); /* If a named pipe is full, just ignore it for now - mrn 24 May 96 */ if (pData->fileType == eTypePIPE && e == EAGAIN) - ABORT_FINALIZE(RS_RET_OK); - - /* If the filesystem is filled up, just ignore - * it for now and continue writing when possible - * based on patch for sysklogd by Martin Schulze on 2007-05-24 - */ - if (pData->fileType == eTypeFILE && e == ENOSPC) - ABORT_FINALIZE(RS_RET_OK); + ABORT_FINALIZE(RS_RET_SUSPENDED); (void) close(pData->fd); /* Check for EBADF on TTY's due to vhangup() -- cgit v1.2.3 From 7268b9bc5b8a7d4ea58cbec7fb6180574e2b20d6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 17 Mar 2009 18:27:33 +0100 Subject: undid typo --- rsyslog.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/rsyslog.conf b/rsyslog.conf index 329704b8..47fc4402 100644 --- a/rsyslog.conf +++ b/rsyslog.conf @@ -58,4 +58,3 @@ local7.* /var/log/boot.log # UDP Syslog Server: #$ModLoad imudp.so # provides UDP syslog reception #$UDPServerRun 514 # start a UDP syslog server at standard port 514 -*.* /mnt/logfile -- cgit v1.2.3 From 27a639ed8875969574543f7738c7bcb14c6149d2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 18 Mar 2009 17:55:15 +0100 Subject: omfile bugfixing - fixed a bug that caused action retries not to work correctly situation was only cleared by a restart - bugfix: closed dynafile was potentially never written until another dynafile name was generated - potential loss of messages --- ChangeLog | 4 ++++ action.c | 8 +++++++- doc/rsyslog_conf_global.html | 1 + tools/omfile.c | 20 ++++++++++++++++---- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index 18bb0e53..ae700656 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? +- fixed a bug that caused action retries not to work correctly + situation was only cleared by a restart +- bugfix: closed dynafile was potentially never written until another + dynafile name was generated - potential loss of messages --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/action.c b/action.c index a41f976c..10607e72 100644 --- a/action.c +++ b/action.c @@ -353,7 +353,13 @@ static rsRetVal actionTryResume(action_t *pThis) ASSERT(pThis != NULL); - ttNow = getActNow(pThis); /* cache "now" */ + /* for resume handling, we must always obtain a fresh timestamp. We used + * to use the action timestamp, but in this case we will never reach a + * point where a resumption is actually tried, because the action timestamp + * is always in the past. So we can not avoid doing a fresh time() call + * here. -- rgerhards, 2009-03-18 + */ + time(&ttNow); /* cache "now" */ /* first check if it is time for a re-try */ if(ttNow > pThis->ttResumeRtry) { diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index b0c1e400..d011bd2b 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -103,6 +103,7 @@ default 60000 (1 minute)]

    • $DefaultNetstreamDriver <drivername>, the default network stream driver to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem>
    • $DefaultNetstreamDriverCertFile </path/to/certfile.pem>
    • $DefaultNetstreamDriverKeyFile </path/to/keyfile.pem>
      • +
    • $CreateDirs [on/off] - create directories on an as-needed basis
    • $DirCreateMode
    • $DirGroup
    • $DirOwner
      • diff --git a/tools/omfile.c b/tools/omfile.c index 8be815f2..bf84d1a7 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -457,6 +457,8 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg int i; int iFirstFree; dynaFileCacheEntry **pCache; + + BEGINfunc ASSERT(pData != NULL); ASSERT(newFileName != NULL); @@ -542,6 +544,8 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg pData->iCurrElt = iFirstFree; DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); + ENDfunc + return 0; } @@ -553,6 +557,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) { off_t actualFileSize; + int iLenWritten; DEFiRet; ASSERT(pData != NULL); @@ -563,7 +568,9 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa if(pData->bDynamicName) { if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) ABORT_FINALIZE(RS_RET_SUSPENDED); // TODO: different state? conditional based on what went wrong? 2009-03-11 - } else if(pData->fd == -1) { + } + + if(pData->fd == -1) { prepareFile(pData, pData->f_fname); } @@ -600,16 +607,21 @@ again: } } - if(write(pData->fd, ppString[0], strlen((char*)ppString[0])) < 0) { + iLenWritten = write(pData->fd, ppString[0], strlen((char*)ppString[0])); +//dbgprintf("lenwritten: %d\n", iLenWritten); + if(iLenWritten < 0) { int e = errno; -dbgprintf("++++++++++ log file writer error %d\n", e); + char errStr[1024]; + rs_strerror_r(errno, errStr, sizeof(errStr)); + DBGPRINTF("log file (%d) write error %d: %s\n", pData->fd, e, errStr); /* If a named pipe is full, just ignore it for now - mrn 24 May 96 */ - if (pData->fileType == eTypePIPE && e == EAGAIN) + if(pData->fileType == eTypePIPE && e == EAGAIN) ABORT_FINALIZE(RS_RET_SUSPENDED); (void) close(pData->fd); + pData->fd = -1; /* tell that fd is no longer open! */ /* Check for EBADF on TTY's due to vhangup() * Linux uses EIO instead (mrn 12 May 96) */ -- cgit v1.2.3 From 508a9e0cef064b082cd9e13aecd9d6c0f1f51977 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 19 Mar 2009 12:01:43 +0100 Subject: omfile suspend handling for non-dynafiles, also bugfixes primarily bugs introduced by recent changes. We now also handle static file names correctly, that was not the case before. We now correctly reset the descriptor in the dynafile cache if somthing goes wrong. Keep in mind that reliablity of output is depending on the reliability of the file system driver (the cifs driver returns OK, but still loses data if it is disconnected for too-long). --- ChangeLog | 3 +++ tools/omfile.c | 44 ++++++++++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/ChangeLog b/ChangeLog index ae700656..5183674f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,9 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? situation was only cleared by a restart - bugfix: closed dynafile was potentially never written until another dynafile name was generated - potential loss of messages +- improved omfile so that it properly suspends itself if there is an + i/o or file name generation error. This enables it to be used with + the full high availability features of rsyslog's engine --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/tools/omfile.c b/tools/omfile.c index bf84d1a7..ff4c4618 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -383,9 +383,12 @@ static void dynaFileFreeCache(instanceData *pData) * file access, which, among others, means the the file wil be opened * and any directories in between will be created (based on config, of * course). -- rgerhards, 2008-10-22 + * changed to iRet interface - 2009-03-19 */ -static void prepareFile(instanceData *pData, uchar *newFileName) +static rsRetVal +prepareFile(instanceData *pData, uchar *newFileName) { + DEFiRet; if(pData->fileType == eTypePIPE) { pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK); FINALIZE; /* we are done in this case */ @@ -399,14 +402,14 @@ static void prepareFile(instanceData *pData, uchar *newFileName) pData->fd = -1; /* file does not exist, create it (and eventually parent directories */ if(pData->bCreateDirs) { - /* we fist need to create parent dirs if they are missing + /* We first need to create parent dirs if they are missing. * We do not report any errors here ourselfs but let the code * fall through to error handler below. */ if(makeFileParentDirs(newFileName, strlen((char*)newFileName), pData->fDirCreateMode, pData->dirUID, pData->dirGID, pData->bFailOnChown) != 0) { - return; /* we give up */ + ABORT_FINALIZE(RS_RET_ERR); /* we give up */ } } /* no matter if we needed to create directories or not, we now try to create @@ -418,8 +421,7 @@ static void prepareFile(instanceData *pData, uchar *newFileName) /* check and set uid/gid */ if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { /* we need to set owner/group */ - if(fchown(pData->fd, pData->fileUID, - pData->fileGID) != 0) { + if(fchown(pData->fd, pData->fileUID, pData->fileGID) != 0) { if(pData->bFailOnChown) { int eSave = errno; close(pData->fd); @@ -434,11 +436,17 @@ static void prepareFile(instanceData *pData, uchar *newFileName) } } finalize_it: - if((pData->fd) != 0 && isatty(pData->fd)) { + /* this was "pData->fd != 0", which I think was a bug. I guess 0 was intended to mean + * non-open file descriptor. Anyhow, I leave this comment for the time being to that if + * problems surface, one at least knows what happened. -- rgerhards, 2009-03-19 + */ + if(pData->fd != -1 && isatty(pData->fd)) { DBGPRINTF("file %d is a tty file\n", pData->fd); pData->fileType = eTypeTTY; untty(); } + + RETiRet; } @@ -520,7 +528,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg } /* Ok, we finally can open the file */ - prepareFile(pData, newFileName); + prepareFile(pData, newFileName); /* ignore exact error, we check fd below */ /* file is either open now or an error state set */ if(pData->fd == -1) { @@ -567,11 +575,14 @@ static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pDa */ if(pData->bDynamicName) { if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) - ABORT_FINALIZE(RS_RET_SUSPENDED); // TODO: different state? conditional based on what went wrong? 2009-03-11 + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ } if(pData->fd == -1) { - prepareFile(pData, pData->f_fname); + rsRetVal iRetLocal; + iRetLocal = prepareFile(pData, pData->f_fname); + if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ } /* create the message based on format specified */ @@ -615,13 +626,19 @@ again: rs_strerror_r(errno, errStr, sizeof(errStr)); DBGPRINTF("log file (%d) write error %d: %s\n", pData->fd, e, errStr); - /* If a named pipe is full, just ignore it for now - - mrn 24 May 96 */ + /* If a named pipe is full, we suspend this action for a while */ if(pData->fileType == eTypePIPE && e == EAGAIN) ABORT_FINALIZE(RS_RET_SUSPENDED); - (void) close(pData->fd); + close(pData->fd); pData->fd = -1; /* tell that fd is no longer open! */ + if(pData->bDynamicName && pData->iCurrElt != -1) { + /* in this case, we need to invalidate the name in the cache, too + * otherwise, an invalid fd may show up if we had a file name change. + * rgerhards, 2009-03-19 + */ + pData->dynCache[pData->iCurrElt]->fd = -1; + } /* Check for EBADF on TTY's due to vhangup() * Linux uses EIO instead (mrn 12 May 96) */ @@ -793,6 +810,9 @@ CODESTARTparseSelectorAct pData->dirUID = dirUID; pData->dirGID = dirGID; + /* at this stage, we ignore the return value of prepareFile, this is taken + * care of in later steps. -- rgerhards, 2009-03-19 + */ prepareFile(pData, pData->f_fname); if(pData->fd < 0 ) { -- cgit v1.2.3 From 935018ed625bc4cf1d6b28fa16e0986078029aaf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 19 Mar 2009 13:58:39 +0100 Subject: adapted test framework to new script engine --- Makefile.am | 2 ++ tests/1.rstest | 34 +++++++++++++++++----------------- tests/2.rstest | 4 ++-- tests/3.rstest | 21 +++++++++++++++++++++ tests/Makefile.am | 2 +- tests/rscript.c | 5 +++-- 6 files changed, 46 insertions(+), 22 deletions(-) create mode 100644 tests/3.rstest diff --git a/Makefile.am b/Makefile.am index 7bb6af8e..87e378ee 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,3 +1,5 @@ +AUTOMAKE_OPTIONS=dejagnu + sbin_PROGRAMS = pkglib_LTLIBRARIES = diff --git a/tests/1.rstest b/tests/1.rstest index 5c152589..4716e8b3 100644 --- a/tests/1.rstest +++ b/tests/1.rstest @@ -4,23 +4,23 @@ in: 'test 1' <> $var or /* some comment */($SEVERITY == -4 +5 -(3 * - 2) and $fromhost == '127.0.0.1') then $$$ out: -00000000: PUSHCONSTANT test 1[cstr] -00000001: PUSHMSGVAR var[cstr] -00000002: != -00000003: PUSHMSGVAR severity[cstr] -00000004: PUSHCONSTANT 4[nbr] -00000005: UNARY_MINUS -00000006: PUSHCONSTANT 5[nbr] -00000007: + -00000008: PUSHCONSTANT 3[nbr] -00000009: PUSHCONSTANT 2[nbr] -00000010: UNARY_MINUS -00000011: * -00000012: - -00000013: == -00000014: PUSHMSGVAR fromhost[cstr] -00000015: PUSHCONSTANT 127.0.0.1[cstr] -00000016: == +00000000: push_const test 1[cstr] +00000001: push_msgvar var[cstr] +00000002: cmp_!= +00000003: push_msgvar severity[cstr] +00000004: push_const 4[nbr] +00000005: unary_minus +00000006: push_const 5[nbr] +00000007: add +00000008: push_const 3[nbr] +00000009: push_const 2[nbr] +00000010: unary_minus +00000011: mul +00000012: sub +00000013: cmp_== +00000014: push_msgvar fromhost[cstr] +00000015: push_const 127.0.0.1[cstr] +00000016: cmp_== 00000017: and 00000018: or $$$ diff --git a/tests/2.rstest b/tests/2.rstest index 7fb5b799..f0e8205b 100644 --- a/tests/2.rstest +++ b/tests/2.rstest @@ -4,7 +4,7 @@ in: $msg contains 'test' then $$$ out: -00000000: PUSHMSGVAR msg[cstr] -00000001: PUSHCONSTANT test[cstr] +00000000: push_msgvar msg[cstr] +00000001: push_const test[cstr] 00000002: contains $$$ diff --git a/tests/3.rstest b/tests/3.rstest new file mode 100644 index 00000000..93cb941a --- /dev/null +++ b/tests/3.rstest @@ -0,0 +1,21 @@ +# a simple RainerScript test +result: 0 +in: +strlen($msg & strlen('abc')) > 20 +30 + -40 then +$$$ +out: +00000000: push_msgvar msg[cstr] +00000001: push_const abc[cstr] +00000002: push_const 1[nbr] +00000003: func_call strlen[cstr] +00000004: strconcat +00000005: push_const 1[nbr] +00000006: func_call strlen[cstr] +00000007: push_const 20[nbr] +00000008: push_const 30[nbr] +00000009: add +00000010: push_const 40[nbr] +00000011: unary_minus +00000012: add +00000013: cmp_> +$$$ diff --git a/tests/Makefile.am b/tests/Makefile.am index 14e7c195..7a31be45 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -4,7 +4,7 @@ TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ #TESTS = $(check_PROGRAMS) test_files = testbench.h runtime-dummy.c -EXTRA_DIST=1.rstest 2.rstest err1.rstest \ +EXTRA_DIST=1.rstest 2.rstest 3.rstest err1.rstest \ cfg.sh \ cfg1.cfgtest \ cfg1.testin \ diff --git a/tests/rscript.c b/tests/rscript.c index d4e8caeb..3eec9c3c 100644 --- a/tests/rscript.c +++ b/tests/rscript.c @@ -101,9 +101,10 @@ PerformTest(cstr_t *pstrIn, rsRetVal iRetExpected, cstr_t *pstrOut) CHKiRet(vmprg.Obj2Str(pExpr->pVmprg, pstrPrg)); if(strcmp((char*)rsCStrGetSzStr(pstrPrg), (char*)rsCStrGetSzStr(pstrOut))) { + int iLen; printf("error: compiled program different from expected result!\n"); - printf("generated vmprg:\n%s\n", rsCStrGetSzStr(pstrPrg)); - printf("expected:\n%s\n", rsCStrGetSzStr(pstrOut)); + printf("generated vmprg (%d bytes):\n%s\n", strlen(rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); + printf("expected (%d bytes):\n%s\n", strlen(rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); ABORT_FINALIZE(RS_RET_ERR); } -- cgit v1.2.3 From 790532bb834e3d4fc07273b2d72127e39ef3e142 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 19 Mar 2009 14:02:00 +0100 Subject: fixed broken make distcheck due to invalidly stated omtemplate file --- plugins/omtemplate/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/omtemplate/Makefile.am b/plugins/omtemplate/Makefile.am index 58ddc794..e816c7c6 100644 --- a/plugins/omtemplate/Makefile.am +++ b/plugins/omtemplate/Makefile.am @@ -1,6 +1,6 @@ pkglib_LTLIBRARIES = omtemplate.la -omtemplate_la_SOURCES = omtemplate.c omtemplate.h +omtemplate_la_SOURCES = omtemplate.c omtemplate_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) omtemplate_la_LDFLAGS = -module -avoid-version omtemplate_la_LIBADD = -- cgit v1.2.3 From bbfa04fbe63f1bbb47f5cdc683045cf2775b3977 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 19 Mar 2009 17:50:07 +0100 Subject: improved testing support worked on ways to provide a better test suite: - added -T rsyslogd command line option, enables to specify a directory where to chroot() into on startup. This is NOT a security feature but introduced to support testing. Thus, -T does not make sure chroot() is used in a secure way. (may be removed later) - added omstdout module for testing purposes. Spits out all messages to stdout - no config option, no other features - modified $ModLoad statement so that for modules whom's name starts with a dot, no path is prepended (this enables relative-pathes and should not break any valid current config) --- ChangeLog | 9 ++++ Makefile.am | 8 +-- configure.ac | 21 +++++++- plugins/omstdout/Makefile.am | 8 +++ plugins/omstdout/omstdout.c | 125 +++++++++++++++++++++++++++++++++++++++++++ runtime/modules.c | 2 +- tests/Makefile.am | 3 ++ tools/syslogd.c | 17 +++++- 8 files changed, 187 insertions(+), 6 deletions(-) create mode 100644 plugins/omstdout/Makefile.am create mode 100644 plugins/omstdout/omstdout.c diff --git a/ChangeLog b/ChangeLog index 8c8442e2..1cfb3c3a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,15 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? issue with the function call VM instruction set design. - implemented the strlen() RainerScript function - added a template output module +- added -T rsyslogd command line option, enables to specify a directory + where to chroot() into on startup. This is NOT a security feature but + introduced to support testing. Thus, -T does not make sure chroot() + is used in a secure way. (may be removed later) +- added omstdout module for testing purposes. Spits out all messages to + stdout - no config option, no other features +- modified $ModLoad statement so that for modules whom's name starts with + a dot, no path is prepended (this enables relative-pathes and should + not break any valid current config) --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/Makefile.am b/Makefile.am index 87e378ee..97f4aed3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,5 +1,3 @@ -AUTOMAKE_OPTIONS=dejagnu - sbin_PROGRAMS = pkglib_LTLIBRARIES = @@ -90,6 +88,10 @@ if ENABLE_IMTEMPLATE SUBDIRS += plugins/imtemplate endif +if ENABLE_OMSTDOUT +SUBDIRS += plugins/omstdout +endif + if ENABLE_OMTEMPLATE SUBDIRS += plugins/omtemplate endif @@ -120,5 +122,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 0c9483ca..fcd935e8 100644 --- a/configure.ac +++ b/configure.ac @@ -688,6 +688,23 @@ AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) # end of copy template - be sure to serach for omtemplate to find everything! +# settings for omstdout +AC_ARG_ENABLE(omstdout, + [AS_HELP_STRING([--enable-omstdout],[Compiles stdout template module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omstdout="yes" ;; + no) enable_omstdout="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omstdout) ;; + esac], + [enable_omstdout=no] +) +# +# you may want to do some library checks here - see snmp, mysql, pgsql modules +# for samples +# +AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) + + AC_CONFIG_FILES([Makefile \ runtime/Makefile \ tools/Makefile \ @@ -701,6 +718,7 @@ AC_CONFIG_FILES([Makefile \ plugins/imklog/Makefile \ plugins/imtemplate/Makefile \ plugins/omtemplate/Makefile \ + plugins/omstdout/Makefile \ plugins/imfile/Makefile \ plugins/imrelp/Makefile \ plugins/imdiag/Makefile \ @@ -731,7 +749,8 @@ echo "RELP support enabled: $enable_relp" echo "imdiag enabled: $enable_imdiag" echo "file input module enabled: $enable_imfile" echo "input template module will be compiled: $enable_imtemplate" -echo "output template module will be compiled: $enable_omtemplate" +echo "output template module will be compiled: $enable_omtemplate" +echo "omstdout module will be compiled: $enable_omstdout" echo "Large file support enabled: $enable_largefile" echo "Networking support enabled: $enable_inet" echo "GnuTLS network stream driver enabled: $enable_gnutls" diff --git a/plugins/omstdout/Makefile.am b/plugins/omstdout/Makefile.am new file mode 100644 index 00000000..9f5d497f --- /dev/null +++ b/plugins/omstdout/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omstdout.la + +omstdout_la_SOURCES = omstdout.c +omstdout_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omstdout_la_LDFLAGS = -module -avoid-version +omstdout_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c new file mode 100644 index 00000000..6e227ba9 --- /dev/null +++ b/plugins/omstdout/omstdout.c @@ -0,0 +1,125 @@ +/* omstdout.c + * send all output to stdout - this is primarily a test driver (but may + * be used for weired use cases). Not tested for robustness! + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-03-19 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +typedef struct _instanceData { +} instanceData; + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINdoAction +CODESTARTdoAction + write(1, (char*)ppString[0], strlen((char*)ppString[0])); +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omstdout:", sizeof(":omstdout:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omstdout:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr +ENDmodInit + +/* vi:set ai: + */ diff --git a/runtime/modules.c b/runtime/modules.c index d548a949..cef4eac6 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -608,7 +608,7 @@ Load(uchar *pModName) iLoadCnt = 0; do { /* now build our load module name */ - if(*pModName == '/') { + if(*pModName == '/' || *pModName == '.') { *szPath = '\0'; /* we do not need to append the path - its already in the module name */ iPathLen = 0; } else { diff --git a/tests/Makefile.am b/tests/Makefile.am index 7a31be45..384afd4e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,3 +1,6 @@ +#AUTOMAKE_OPTIONS=dejagnu +#DEJATOOL=Rainer + check_PROGRAMS = rt_init rscript TESTS = $(check_PROGRAMS) cfg.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ diff --git a/tools/syslogd.c b/tools/syslogd.c index 235bc52e..9f962899 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3324,7 +3324,7 @@ int realMain(int argc, char **argv) * only when actually neeeded. * rgerhards, 2008-04-04 */ - while((ch = getopt(argc, argv, "46a:Ac:def:g:hi:l:m:M:nN:op:qQr::s:t:u:vwx")) != EOF) { + while((ch = getopt(argc, argv, "46a:Ac:def:g:hi:l:m:M:nN:op:qQr::s:t:T:u:vwx")) != EOF) { switch((char)ch) { case '4': case '6': @@ -3342,6 +3342,7 @@ int realMain(int argc, char **argv) case 'q': /* add hostname if DNS resolving has failed */ case 'Q': /* dont resolve hostnames in ACL to IPs */ case 's': + case 'T': /* chroot on startup (primarily for testing) */ case 'u': /* misc user settings */ case 'w': /* disable disallowed host warnings */ case 'x': /* disable dns for remote messages */ @@ -3586,6 +3587,20 @@ int realMain(int argc, char **argv) } else fprintf(stderr, "-t option only supported in compatibility modes 0 to 2 - ignored\n"); break; + case 'T':/* chroot() immediately at program startup, but only for testing, NOT security yet */ +{ +char buf[1024]; +getcwd(buf, 1024); +printf("pwd: '%s'\n", buf); +printf("chroot to '%s'\n", arg); + if(chroot(arg) != 0) { + perror("chroot"); + exit(1); + } +getcwd(buf, 1024); +printf("pwd: '%s'\n", buf); +} + break; case 'u': /* misc user settings */ iHelperUOpt = atoi(arg); if(iHelperUOpt & 0x01) -- cgit v1.2.3 From c11b7ec7d6e6216481f28447b94e59e524dd824c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 23 Mar 2009 11:27:38 +0100 Subject: some text scripts added (experimental) --- tests/test.tcl | 41 +++++++++++++++++++++++++++++++++++++++++ tests/testruns/parser.conf | 10 ++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tests/test.tcl create mode 100644 tests/testruns/parser.conf diff --git a/tests/test.tcl b/tests/test.tcl new file mode 100644 index 00000000..7f1b7a6b --- /dev/null +++ b/tests/test.tcl @@ -0,0 +1,41 @@ +# rsyslog parser tests +package require Expect +package require udp 1.0 + +set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-ftestruns/parser.conf" "-u2" "-n" "-iwork/rsyslog.pid" "-M../runtime/.libs"]; +#interact; +#puts "pid: $rsyslogdPID"; +#sleep 1; +#expect "\n"; +expect "}}"; # eat startup message +set udpSock [udp_open]; +udp_conf $udpSock 127.0.0.1 514 +set files [glob *.parse1] +puts "done init\n"; + + +foreach testcase $files { + puts "File $testcase"; + set fp [open "$testcase" r]; + fconfigure $fp -buffering line + #set data [read $fp]; + #set data [split $data "\n"]; + gets $fp input + puts "Line 1: $input\n"; + + puts $udpSock $input; + flush $udpSock; + + + set i 1 + expect -re "{{.*}}"; + set result $expect_out(buffer); + + #puts "MSG $i: '$expect_out(buffer)'"; + puts "MSG $i: '$result'\n"; + set i [expr {$i + 1}]; + +} + +exec kill $rsyslogdPID; +close $udpSock; diff --git a/tests/testruns/parser.conf b/tests/testruns/parser.conf new file mode 100644 index 00000000..a515ff6c --- /dev/null +++ b/tests/testruns/parser.conf @@ -0,0 +1,10 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$ModLoad ../plugins/imuxsock/.libs/imuxsock +$ModLoad ../plugins/imudp/.libs/imudp +$UDPServerRun 514 + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template expect,"{{%PRI%,%syslogtag%,%hostname%}}" +*.* :omstdout:;expect -- cgit v1.2.3 From de3341a64131092fabb3a1c60adb77e70bdb7fb6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 23 Mar 2009 18:24:44 +0100 Subject: added "rsyslog family tree" graph of rsyslog versions and branches --- doc/rsyslog-vers.dot | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 doc/rsyslog-vers.dot diff --git a/doc/rsyslog-vers.dot b/doc/rsyslog-vers.dot new file mode 100644 index 00000000..a5563f94 --- /dev/null +++ b/doc/rsyslog-vers.dot @@ -0,0 +1,82 @@ +// This file is part of rsyslog. +// +// rsyslog "family tree" compressed version +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot rsyslog-vers.dot -Tpng >rsyslog-vers.png + +digraph G { + label="\n\nrsyslog \"family tree\"\nhttp://www.rsyslog.com"; + fontsize=20; + + v1stable [label="v1-stable", shape=box, style=dotted]; + v2stable [label="v2-stable", shape=box, style=filled]; + v3stable [label="v3-stable", shape=box, style=filled]; + beta [label="beta", shape=box, style=filled]; + devel [label="devel", shape=box, style=filled]; + "1.0.5" [style=dotted]; + perf [style=dashed]; + "imudp-select" [style=dashed label="imudp-\nselect"]; + msgnolock [style=dashed]; + "file-errHdlr" [style=dashed]; + solaris [style=dashed]; + tests [style=dashed]; + "0.x.y" [group=master]; + "1.10.0" [group=master]; + "1.21.2" [group=master]; + "3.10.0" [group=master]; + "3.15.1" [group=master]; + "3.17.0" [group=master]; + "3.19.x" [group=master]; + "3.21.x" [group=master]; + "4.1.0" [group=master]; + "4.1.4" [group=master]; + "4.1.5" [group=master]; + "4.1.6" [group=master]; + + sysklogd -> "0.x.y" [color=red]; + "0.x.y" -> "1.0.0"; + "0.x.y" -> "1.10.0" [color=red]; + "1.0.0" -> "1.0.5" [style=dashed]; + "1.10.0" -> "1.21.2" [color=red style=dashed]; + "1.21.2" -> "2.0.0" [color=blue]; + "2.0.0" -> "2.0.5" [style=dashed, color=blue]; + "1.21.2" -> "3.10.0" [color=red]; + "3.10.0" -> "3.15.1" [color=red style=dashed]; + "3.15.1" -> "tests"; + "3.15.1" -> "3.17.0" [color=red style=dashed]; + "3.15.1" -> "3.16.x"; + "3.16.x" -> "3.18.x" [color=blue, style=dashed]; + "3.17.0" -> "3.18.x"; + "3.17.0" -> "3.19.x" [color=red, style=dashed]; + "3.19.x" -> "3.20.x"; + "3.19.x" -> "3.21.x" [color=red]; + "3.18.x" -> debian_lenny; + "3.18.x" -> "3.20.x" [color=blue, style=dashed]; + "3.21.x" -> "3.21.11"; + "3.21.x" -> "4.1.0" [color=red]; + "3.21.x" -> "perf"; + "perf" -> "4.1.0"; + "4.1.0" -> "4.1.4" [color=red, style=dashed]; + "4.1.4" -> "file-errHdlr"; + "4.1.4" -> "4.1.5" [color=red]; + "4.1.5" -> "4.1.6" [color=red]; + "3.21.x" -> msgnolock + "3.21.x" -> "imudp-select"; + "4.1.5" -> solaris; + "file-errHdlr" -> "4.1.6"; + "tests" -> "4.1.6"; + + "1.0.5" -> v1stable [dir=none, style=dotted]; + "2.0.5" -> v2stable [dir=none, style=dotted]; + "3.20.x" -> v3stable [dir=none, style=dotted]; + "3.21.11" -> beta [dir=none, style=dotted]; + "4.1.6" -> devel [dir=none, style=dotted]; + + {rank=same; "4.1.5" "solaris"} + {rank=same; "3.18.x" "debian_lenny"} + {rank=same; "1.0.5" "2.0.5" "3.20.x" "3.21.11" "4.1.6"} +} -- cgit v1.2.3 From 67e00c063122de13dd6c6354fa095978aa1773de Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 25 Mar 2009 17:59:35 +0100 Subject: bugfix: fixed some segaults on Solaris where vsprintf() does not check for NULL pointers --- ChangeLog | 2 ++ tools/omfwd.c | 1 - tools/syslogd.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 0cbe7bba..1dbd69e0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -19,6 +19,8 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? - improved omfile so that it properly suspends itself if there is an i/o or file name generation error. This enables it to be used with the full high availability features of rsyslog's engine +- bugfix: fixed some segaults on Solaris, where vsprintf() does not + check for NULL pointers --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/tools/omfwd.c b/tools/omfwd.c index 1dd184ef..7a945ce0 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -615,7 +615,6 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) } else { CHKmalloc(pData->f_hname = strdup((char*) q)); } -dbgprintf("hostname '%s', port '%s'\n", pData->f_hname, pData->port); /* process template */ CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, diff --git a/tools/syslogd.c b/tools/syslogd.c index 235bc52e..a2aead9a 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3475,7 +3475,7 @@ int realMain(int argc, char **argv) /* END core initializations - we now come back to carrying out command line options*/ while((iRet = bufOptRemove(&ch, &arg)) == RS_RET_OK) { - dbgprintf("deque option %c, optarg '%s'\n", ch, arg); + dbgprintf("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg); switch((char)ch) { case '4': glbl.SetDefPFFamily(PF_INET); -- cgit v1.2.3 From 97480eafbc67ec7e84497868a1777ce0d7881e6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 25 Mar 2009 18:16:24 +0100 Subject: Start the output module for Oracle. Currently, resources are allocated, freed and the code compiles. No tests yet. --- plugins/omoracle/Makefile.am | 8 +++ plugins/omoracle/omoracle.c | 139 +++++++++++++++++++++++++++++++++++++++++++ plugins/omoracle/omoracle.h | 23 +++++++ 3 files changed, 170 insertions(+) create mode 100644 plugins/omoracle/Makefile.am create mode 100644 plugins/omoracle/omoracle.c create mode 100644 plugins/omoracle/omoracle.h diff --git a/plugins/omoracle/Makefile.am b/plugins/omoracle/Makefile.am new file mode 100644 index 00000000..6b75218f --- /dev/null +++ b/plugins/omoracle/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omoracle.la + +omoracle_la_SOURCES = omoracle.c omoracle.h +omoracle_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(ORACLE_CFLAGS) +omoracle_la_LDFLAGS = -module -avoid-version +omoracle_la_LIBADD = $(ORACLE_LIBS) + +EXTRA_DIST = diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c new file mode 100644 index 00000000..696cd908 --- /dev/null +++ b/plugins/omoracle/omoracle.c @@ -0,0 +1,139 @@ +/** omoracle.c + + This is an output module feeding directly to an Oracle + database. It uses Oracle Call Interface, a propietary module + provided by Oracle. + + Author: Luis Fernando Muñoz Mejías + + + This file is part of rsyslog. +*/ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" +#include "omoracle.h" + +MODULE_TYPE_OUTPUT + +/** */ +DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) + +typedef struct _instanceData { + OCIEnv* environment; + OCISession* session; + OCIError* error; + OCIServer* server; + OCIStmt* statement; + OCISvcCtx* service; + OCIAuthInfo* authinfo; + OCIBind* binding; +} instanceData; + +/** Generic function for handling errors from OCI. + + It will be called only inside CHECKERR and CHECKENV macros. + + Arguments: handle The error or environment handle to check. + htype: OCI_HTYPE_* constant, usually OCI_HTYPE_ERROR or + OCI_HTYPE_ENV + status: status code to check, usually the return value of an OCI + function. + + Returns OCI_SUCCESS on success, OCI_ERROR otherwise. +*/ +static int oci_errors(void* handle, ub4 htype, sword status) +{ + sb4 errcode; + char buf[MAX_BUFSIZE]; + + switch (status) { + case OCI_SUCCESS: + return OCI_SUCCESS; + break; + case OCI_SUCCESS_WITH_INFO: + printf ("OCI SUCCESS - With info\n"); + break; + case OCI_NEED_DATA: + printf ("OCI NEEDS MORE DATA\n"); + break; + case OCI_ERROR: + printf ("OCI GENERAL ERROR\n"); + if (handle) { + OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); + printf ("Error message: %s", buf); + } else + printf ("NULL handle\n" + "Unable to extract further information"); + break; + case OCI_INVALID_HANDLE: + printf ("OCI INVALID HANDLE\n"); + break; + case OCI_STILL_EXECUTING: + printf ("Still executing...\n"); + break; + case OCI_CONTINUE: + printf ("OCI CONTINUE\n"); + break; + } + return OCI_ERROR; +} + + +/* Resource allocation */ +BEGINcreateInstance +CODESTARTcreateInstance +CHECKENV(pData->environment, + OCIEnvCreate(&(pData->environment), OCI_DEFAULT, + NULL, NULL, NULL, NULL, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &(pData->error), + OCI_HTYPE_ERROR, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &(pData->server), + OCI_HTYPE_SERVER, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &(pData->service), + OCI_HTYPE_SVCCTX, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, &(pData->authinfo), + OCI_HTYPE_AUTHINFO, 0, NULL)); +finalize_it: +ENDcreateInstance + +/** Free any resources allocated by createInstance. */ +BEGINfreeInstance +CODESTARTfreeInstance + +OCIHandleFree(pData->environment, OCI_HTYPE_ENV); +OCIHandleFree(pData->error, OCI_HTYPE_ERROR); +OCIHandleFree(pData->server, OCI_HTYPE_SERVER); +OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); +OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); + +RETiRet; + +ENDfreeInstance + +/* BEGINmodInit() */ +/* CODESTARTmodInit */ +/* *ipIFVersProvided = CURR_MOD_IF_VERSION; */ +/* CODEmodInit_QueryRegCFSLineHdlr */ +/* CHKiRet(objUse(errmsg, CORE_COMPONENT)); */ +/* ENDmodInit */ diff --git a/plugins/omoracle/omoracle.h b/plugins/omoracle/omoracle.h new file mode 100644 index 00000000..b0e70917 --- /dev/null +++ b/plugins/omoracle/omoracle.h @@ -0,0 +1,23 @@ +/** Definitions for the Oracle output module. + + This module needs OCI to be installed (on Red Hat-like systems + this is usually the oracle-instantclient-devel RPM). + + Author: Luis Fernando Muñoz Mejías +*/ +#ifndef __OMORACLEH__ +#define __OMORACLEH__ + +/** Macros to make error handling easier. */ + +/** Checks for errors on the OCI handling. */ +#define CHECKERR(handle,status) CHKiRet(oci_errors((handle), \ + OCI_HTYPE_ERROR, (status))) + +/** Checks for errors when handling the environment of OCI. */ +#define CHECKENV(handle,status) CHKiRet(oci_errors((handle), \ + OCI_HTYPE_ENV, (status))) + +enum { MAX_BUFSIZE = 2048 }; + +#endif -- cgit v1.2.3 From b8f10ad00853275045180738b91e90a80c7693cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 25 Mar 2009 18:16:25 +0100 Subject: Include omoracle in the build system. Add configure option to build the oracle support, named --enable-oracle and fix the Makefile.am accordingly. --- Makefile.am | 7 ++++++- configure.ac | 36 +++++++++++++++++++++++++++++++++++- plugins/omoracle/Makefile.am | 2 +- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 87e378ee..a5c3e267 100644 --- a/Makefile.am +++ b/Makefile.am @@ -110,15 +110,20 @@ if ENABLE_RFC3195 SUBDIRS += plugins/im3195 endif +if ENABLE_ORACLE +SUBDIRS += plugins/omoracle +endif + # tests are added as last element, because tests may need different # modules that need to be generated first SUBDIRS += tests + # make sure "make distcheck" tries to build all modules. This means that # a developer must always have an environment where every supporting library # is available. If that is not the case, the respective configure option may # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-oracle ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index 0c9483ca..9be3c7ca 100644 --- a/configure.ac +++ b/configure.ac @@ -452,7 +452,39 @@ AM_CONDITIONAL(ENABLE_PGSQL, test x$enable_pgsql = xyes) AC_SUBST(PGSQL_CFLAGS) AC_SUBST(PGSQL_LIBS) - +# oracle (OCI) support +AC_ARG_ENABLE(oracle, + [AS_HELP_STRING([--enable-oracle],[Enable native Oracle database support @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_oracle="yes" ;; + no) enable_oracle="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-oracle) ;; + esac], + [enable_oracle=no] +) +if test "x$enable_oracle" = "xyes"; then + AC_CHECK_PROG( + [HAVE_ORACLE_CONFIG], + [oracle-instantclient-config], + [yes],,, + ) + if test "x${HAVE_ORACLE_CONFIG}" != "xyes"; then + AC_MSG_FAILURE([oracle-instantclient-config not found in PATH]) + fi + AC_CHECK_LIB( + [occi], + [OCIEnvCreate], + [ORACLE_CFLAGS="`oracle-instantclient-config --cflags`" + ORACLE_LIBS="`oracle-instantclient-config --libs`" + ], + [AC_MSG_FAILURE([Oracle (OCI) libraray is missing])], + [`oracle-instantclient-config --libs --cflags`] + ) +fi +AM_CONDITIONAL(ENABLE_ORACLE, test x$enable_oracle = xyes) +AC_SUBST(ORACLE_CFLAGS) +AC_SUBST(ORACLE_LIBS) + # libdbi support AC_ARG_ENABLE(libdbi, [AS_HELP_STRING([--enable-libdbi],[Enable libdbi database support @<:@default=no@:>@])], @@ -712,6 +744,7 @@ AC_CONFIG_FILES([Makefile \ plugins/omlibdbi/Makefile \ plugins/ommail/Makefile \ plugins/omsnmp/Makefile \ + plugins/omoracle/Makefile \ tests/Makefile]) AC_OUTPUT @@ -725,6 +758,7 @@ echo "Zlib compression support enabled: $enable_zlib" echo "MySql support enabled: $enable_mysql" echo "libdbi support enabled: $enable_libdbi" echo "PostgreSQL support enabled: $enable_pgsql" +echo "Oracle (OCI) support enabled: $enable_oracle" echo "SNMP support enabled: $enable_snmp" echo "Mail support enabled: $enable_mail" echo "RELP support enabled: $enable_relp" diff --git a/plugins/omoracle/Makefile.am b/plugins/omoracle/Makefile.am index 6b75218f..11257fb2 100644 --- a/plugins/omoracle/Makefile.am +++ b/plugins/omoracle/Makefile.am @@ -5,4 +5,4 @@ omoracle_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) $(ORACLE_CFLAGS) omoracle_la_LDFLAGS = -module -avoid-version omoracle_la_LIBADD = $(ORACLE_LIBS) -EXTRA_DIST = +#EXTRA_DIST = -- cgit v1.2.3 From 57dcc5c6c96aa72b24c5f8c9952f789d08943383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 25 Mar 2009 18:16:26 +0100 Subject: Add all other blocks (macros) needed to make this module work. At this stage they are all empty, but at least it should be possible to instantiate the module and perform some basic tests. Fix some compilation warnings --- plugins/omoracle/omoracle.c | 61 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 696cd908..6077594f 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -61,7 +61,7 @@ typedef struct _instanceData { static int oci_errors(void* handle, ub4 htype, sword status) { sb4 errcode; - char buf[MAX_BUFSIZE]; + unsigned char buf[MAX_BUFSIZE]; switch (status) { case OCI_SUCCESS: @@ -76,7 +76,8 @@ static int oci_errors(void* handle, ub4 htype, sword status) case OCI_ERROR: printf ("OCI GENERAL ERROR\n"); if (handle) { - OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); + OCIErrorGet(handle, 1, NULL, &errcode, buf, + sizeof buf, htype); printf ("Error message: %s", buf); } else printf ("NULL handle\n" @@ -100,19 +101,19 @@ static int oci_errors(void* handle, ub4 htype, sword status) BEGINcreateInstance CODESTARTcreateInstance CHECKENV(pData->environment, - OCIEnvCreate(&(pData->environment), OCI_DEFAULT, + OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, NULL, NULL, NULL, NULL, 0, NULL)); CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, &(pData->error), + OCIHandleAlloc(pData->environment, (void*) &(pData->error), OCI_HTYPE_ERROR, 0, NULL)); CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, &(pData->server), + OCIHandleAlloc(pData->environment, (void*) &(pData->server), OCI_HTYPE_SERVER, 0, NULL)); CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, &(pData->service), + OCIHandleAlloc(pData->environment, (void*) &(pData->service), OCI_HTYPE_SVCCTX, 0, NULL)); CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, &(pData->authinfo), + OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), OCI_HTYPE_AUTHINFO, 0, NULL)); finalize_it: ENDcreateInstance @@ -131,9 +132,43 @@ RETiRet; ENDfreeInstance -/* BEGINmodInit() */ -/* CODESTARTmodInit */ -/* *ipIFVersProvided = CURR_MOD_IF_VERSION; */ -/* CODEmodInit_QueryRegCFSLineHdlr */ -/* CHKiRet(objUse(errmsg, CORE_COMPONENT)); */ -/* ENDmodInit */ + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature +/* Right now, this module is compatible with nothing. */ +ENDisCompatibleWithFeature + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + +BEGINdoAction +CODESTARTdoAction +ENDdoAction + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); +ENDmodInit -- cgit v1.2.3 From 0676277119bd39562a0b8c0de6d2fee23e1deb11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 25 Mar 2009 18:16:27 +0100 Subject: Add handlers on modInit. This avoids crashes on initialization. --- plugins/omoracle/omoracle.c | 78 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 9 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 6077594f..517b7173 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -68,29 +68,29 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_SUCCESS; break; case OCI_SUCCESS_WITH_INFO: - printf ("OCI SUCCESS - With info\n"); + dbgprintf ("OCI SUCCESS - With info\n"); break; case OCI_NEED_DATA: - printf ("OCI NEEDS MORE DATA\n"); + dbgprintf ("OCI NEEDS MORE DATA\n"); break; case OCI_ERROR: - printf ("OCI GENERAL ERROR\n"); + dbgprintf ("OCI GENERAL ERROR\n"); if (handle) { OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); - printf ("Error message: %s", buf); + dbgprintf ("Error message: %s", buf); } else - printf ("NULL handle\n" + dbgprintf ("NULL handle\n" "Unable to extract further information"); break; case OCI_INVALID_HANDLE: - printf ("OCI INVALID HANDLE\n"); + dbgprintf ("OCI INVALID HANDLE\n"); break; case OCI_STILL_EXECUTING: - printf ("Still executing...\n"); + dbgprintf ("Still executing...\n"); break; case OCI_CONTINUE: - printf ("OCI CONTINUE\n"); + dbgprintf ("OCI CONTINUE\n"); break; } return OCI_ERROR; @@ -100,21 +100,30 @@ static int oci_errors(void* handle, ub4 htype, sword status) /* Resource allocation */ BEGINcreateInstance CODESTARTcreateInstance + +ASSERT(pData != NULL); + +dbgprintf ("***** OMORACLE ***** Creating instance\n"); CHECKENV(pData->environment, OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, NULL, NULL, NULL, NULL, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created environment\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->error), OCI_HTYPE_ERROR, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created error\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->server), OCI_HTYPE_SERVER, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created server\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->service), OCI_HTYPE_SVCCTX, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created service\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), OCI_HTYPE_AUTHINFO, 0, NULL)); +dbgprintf ("***** OMORACLE ***** Created authinfo\n"); finalize_it: ENDcreateInstance @@ -122,11 +131,18 @@ ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance +dbgprintf ("***** OMORACLE ***** Destroying instance\n"); + OCIHandleFree(pData->environment, OCI_HTYPE_ENV); +dbgprintf ("***** OMORACLE ***** Destroyed environment\n"); OCIHandleFree(pData->error, OCI_HTYPE_ERROR); +dbgprintf ("***** OMORACLE ***** Destroyed error\n"); OCIHandleFree(pData->server, OCI_HTYPE_SERVER); +dbgprintf ("***** OMORACLE ***** Destroyed server\n"); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); +dbgprintf ("***** OMORACLE ***** Destroyed service\n"); OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); +dbgprintf ("***** OMORACLE ***** Destroyed authinfo\n"); RETiRet; @@ -135,40 +151,84 @@ ENDfreeInstance BEGINtryResume CODESTARTtryResume + +dbgprintf ("***** OMORACLE ***** At tryResume\n"); ENDtryResume BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature /* Right now, this module is compatible with nothing. */ +dbgprintf ("***** OMORACLE ***** At isCompatibleWithFeature\n"); +iRet = RS_RET_INCOMPATIBLE; ENDisCompatibleWithFeature BEGINparseSelectorAct CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); + +if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); +} + +CHKiRet(createInstance(&pData)); + +p += sizeof ":omoracle:" - 1; +if (*p != ';') { + dbgprintf ("***** OMORACLE ***** Wrong char: %c\n", *p); + ABORT_FINALIZE(RS_RET_INVALID_PARAMS); +} +p++; + +CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, + OMSR_RQD_TPL_OPT_SQL, " StdFmt")); + +dbgprintf ("***** OMORACLE ***** Salido\n"); + + CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct BEGINdoAction CODESTARTdoAction +dbgprintf ("***** OMORACLE ***** At doAction\n"); ENDdoAction BEGINmodExit CODESTARTmodExit +dbgprintf ("***** OMORACLE ***** At modExit\n"); ENDmodExit BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo +dbgprintf ("***** OMORACLE ***** At bdgPrintInstInfo\n"); + ENDdbgPrintInstInfo BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +dbgprintf ("***** OMORACLE ***** At queryEtryPt\n"); + ENDqueryEtryPt +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, + void __attribute__((unused)) *pVal) +{ + DEFiRet; + RETiRet; +} + BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr - CHKiRet(objUse(errmsg, CORE_COMPONENT)); +CHKiRet(objUse(errmsg, CORE_COMPONENT)); +/* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ +CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, + eCmdHdlrCustomHandler, resetConfigVariables, + NULL, STD_LOADABLE_MODULE_ID)); + +dbgprintf ("***** OMORACLE ***** At modInit\n"); ENDmodInit -- cgit v1.2.3 From 3dabb2976a2f259ba1f3bd9823ddd2860edc293d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 25 Mar 2009 18:16:28 +0100 Subject: Add the ability to connect to the DB based on the config line. It will read and parse the config line (this code is not yet rock-solid) and connect to the database at initialization time. I also cleaned some debug messages that are not needed anymore. --- plugins/omoracle/omoracle.c | 58 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 14 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 517b7173..ab9e7e36 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -103,27 +103,22 @@ CODESTARTcreateInstance ASSERT(pData != NULL); -dbgprintf ("***** OMORACLE ***** Creating instance\n"); CHECKENV(pData->environment, OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, NULL, NULL, NULL, NULL, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created environment\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->error), OCI_HTYPE_ERROR, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created error\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->server), OCI_HTYPE_SERVER, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created server\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->service), OCI_HTYPE_SVCCTX, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created service\n"); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), OCI_HTYPE_AUTHINFO, 0, NULL)); -dbgprintf ("***** OMORACLE ***** Created authinfo\n"); + finalize_it: ENDcreateInstance @@ -155,6 +150,26 @@ CODESTARTtryResume dbgprintf ("***** OMORACLE ***** At tryResume\n"); ENDtryResume +static rsRetVal startSession(instanceData* pData, char* connection, char* user, + char * password) +{ + DEFiRet; + CHECKERR(pData->error, + OCIAttrSet(pData->authinfo, OCI_HTYPE_AUTHINFO, user, + strlen(user), OCI_ATTR_USERNAME, pData->error)); + CHECKERR(pData->error, + OCIAttrSet(pData->authinfo, OCI_HTYPE_AUTHINFO, password, + strlen(password), OCI_ATTR_PASSWORD, pData->error)); + CHECKERR(pData->error, + OCISessionGet(pData->environment, pData->error, + &pData->service, pData->authinfo, connection, + strlen(connection), NULL, 0, NULL, NULL, NULL, + OCI_DEFAULT)); +finalize_it: + RETiRet; +} + + BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature /* Right now, this module is compatible with nothing. */ @@ -163,6 +178,12 @@ iRet = RS_RET_INCOMPATIBLE; ENDisCompatibleWithFeature BEGINparseSelectorAct + +char connection_string[MAXHOSTNAMELEN]; +char user[_DB_MAXUNAMELEN]; +char pwd[_DB_MAXPWDLEN]; + + CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); @@ -170,20 +191,25 @@ if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); } -CHKiRet(createInstance(&pData)); p += sizeof ":omoracle:" - 1; -if (*p != ';') { - dbgprintf ("***** OMORACLE ***** Wrong char: %c\n", *p); + +if (*p == '\0' || *p == ',') { + dbgprintf ("Wrong char processing module arguments: %c\n", *p); ABORT_FINALIZE(RS_RET_INVALID_PARAMS); } -p++; +CHKiRet(getSubString(&p, connection_string, MAXHOSTNAMELEN, ',')); +CHKiRet(getSubString(&p, user, _DB_MAXUNAMELEN, ',')); +CHKiRet(getSubString(&p, pwd, _DB_MAXPWDLEN, ';')); +p--; CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_RQD_TPL_OPT_SQL, " StdFmt")); +CHKiRet(createInstance(&pData)); +CHKiRet(startSession(pData, connection_string, user, pwd)); -dbgprintf ("***** OMORACLE ***** Salido\n"); - +dbgprintf ("omoracle module got all its resources allocated " + "and connected to the DB\n"); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -223,12 +249,16 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; +char dbname[MAX_BUFSIZE]; CODEmodInit_QueryRegCFSLineHdlr +dbgprintf ("***** OMORACLE ***** At modInit\n"); CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); - -dbgprintf ("***** OMORACLE ***** At modInit\n"); +dbgprintf ("***** OMORACLE ***** dbname before = %s\n", dbname); +CHKiRet(omsdRegCFSLineHdlr((uchar*) "actionoracledb", 0, eCmdHdlrInt, + NULL, dbname, STD_LOADABLE_MODULE_ID)); +dbgprintf ("***** OMORACLE ***** dbname = %s\n", dbname); ENDmodInit -- cgit v1.2.3 From 47b334bef1ea9e72bc41279c42228724f6e141f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 25 Mar 2009 18:16:29 +0100 Subject: Add the ability to actually run statements. It now runs SQL statements given as templates. In this case, the template is given on the configuration file and the core passes the SQL statement correctly formatted to doAction. I still need to decide how to structure this for having prepared statements (prepare them at parseSelector time) and then make doAction to only bind arguments and execute. It commits after each statement, which is awfully slow but good enough for the moment. Next step after that is have a buffer of arguments, and make doAction store new data as it arrives, then run the statement only when the buffer is almost full. Or something like that. --- plugins/omoracle/omoracle.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index ab9e7e36..c1900a73 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -4,6 +4,13 @@ database. It uses Oracle Call Interface, a propietary module provided by Oracle. + Config lines to be used are of this form: + + :omoracle:dbstring,user,password;StatementTemplate + + All fields are mandatory. The dbstring can be an Oracle easystring + or a DB name, as present in the tnsnames.ora file. + Author: Luis Fernando Muñoz Mejías @@ -39,7 +46,6 @@ typedef struct _instanceData { OCIEnv* environment; OCISession* session; OCIError* error; - OCIServer* server; OCIStmt* statement; OCISvcCtx* service; OCIAuthInfo* authinfo; @@ -109,15 +115,12 @@ CHECKENV(pData->environment, CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->error), OCI_HTYPE_ERROR, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->server), - OCI_HTYPE_SERVER, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->service), - OCI_HTYPE_SVCCTX, 0, NULL)); CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), OCI_HTYPE_AUTHINFO, 0, NULL)); +CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->statement), + OCI_HTYPE_STMT, 0, NULL)); finalize_it: ENDcreateInstance @@ -126,18 +129,13 @@ ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance -dbgprintf ("***** OMORACLE ***** Destroying instance\n"); - +OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->environment, OCI_HTYPE_ENV); -dbgprintf ("***** OMORACLE ***** Destroyed environment\n"); OCIHandleFree(pData->error, OCI_HTYPE_ERROR); -dbgprintf ("***** OMORACLE ***** Destroyed error\n"); -OCIHandleFree(pData->server, OCI_HTYPE_SERVER); -dbgprintf ("***** OMORACLE ***** Destroyed server\n"); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); -dbgprintf ("***** OMORACLE ***** Destroyed service\n"); OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); -dbgprintf ("***** OMORACLE ***** Destroyed authinfo\n"); +OCIHandleFree(pData->statement, OCI_HTYPE_STMT); +dbgprintf ("omoracle freed all its resources\n"); RETiRet; @@ -216,12 +214,21 @@ ENDparseSelectorAct BEGINdoAction CODESTARTdoAction -dbgprintf ("***** OMORACLE ***** At doAction\n"); + dbgprintf("omoracle attempting to execute statement %s\n", *ppString); + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, pData->error, *ppString, + strlen(*ppString), OCI_NTV_SYNTAX, + OCI_DEFAULT)); + CHECKERR(pData->error, + OCIStmtExecute(pData->service, pData->statement, pData->error, + 1, 0, NULL, NULL, OCI_DEFAULT)); + CHECKERR(pData->error, + OCITransCommit(pData->service, pData->error, 0)); +finalize_it: ENDdoAction BEGINmodExit CODESTARTmodExit -dbgprintf ("***** OMORACLE ***** At modExit\n"); ENDmodExit BEGINdbgPrintInstInfo @@ -262,3 +269,4 @@ CHKiRet(omsdRegCFSLineHdlr((uchar*) "actionoracledb", 0, eCmdHdlrInt, NULL, dbname, STD_LOADABLE_MODULE_ID)); dbgprintf ("***** OMORACLE ***** dbname = %s\n", dbname); ENDmodInit + -- cgit v1.2.3 From b6123427cf962e70836c07d1c5c2cf39978673b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 25 Mar 2009 18:16:30 +0100 Subject: Add proper indentation (despite Emacs) and support for retrying. Emacs doesn't allow for proper indentation with rsyslog's macros (no curly brackets, so it doesn't know where functions start), so I had to manually add such indentation. Add support for retrying actions, namely, disconnect from the DB, re-connecting and re-executing the last prepared statement. Needs to be tested. --- plugins/omoracle/omoracle.c | 168 ++++++++++++++++++++++++-------------------- 1 file changed, 90 insertions(+), 78 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index c1900a73..aa506dca 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -50,6 +50,7 @@ typedef struct _instanceData { OCISvcCtx* service; OCIAuthInfo* authinfo; OCIBind* binding; + char* connection; } instanceData; /** Generic function for handling errors from OCI. @@ -74,29 +75,30 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_SUCCESS; break; case OCI_SUCCESS_WITH_INFO: - dbgprintf ("OCI SUCCESS - With info\n"); + errmsg.LogError(0, NO_ERRCODE, "OCI SUCCESS - With info\n"); break; case OCI_NEED_DATA: - dbgprintf ("OCI NEEDS MORE DATA\n"); + errmsg.LogError(0, NO_ERRCODE, "OCI NEEDS MORE DATA\n"); break; case OCI_ERROR: dbgprintf ("OCI GENERAL ERROR\n"); if (handle) { OCIErrorGet(handle, 1, NULL, &errcode, buf, sizeof buf, htype); - dbgprintf ("Error message: %s", buf); + errmsg.LogError(0, NO_ERRCODE, "Error message: %s", buf); } else - dbgprintf ("NULL handle\n" - "Unable to extract further information"); + errmsg.LogError(0, NO_ERRCODE, "NULL handle\n" + "Unable to extract further " + "information"); break; case OCI_INVALID_HANDLE: - dbgprintf ("OCI INVALID HANDLE\n"); + errmsg.LogError(0, NO_ERRCODE, "OCI INVALID HANDLE\n"); break; case OCI_STILL_EXECUTING: - dbgprintf ("Still executing...\n"); + errmsg.LogError(0, NO_ERRCODE, "Still executing...\n"); break; case OCI_CONTINUE: - dbgprintf ("OCI CONTINUE\n"); + errmsg.LogError(0, NO_ERRCODE, "OCI CONTINUE\n"); break; } return OCI_ERROR; @@ -107,45 +109,56 @@ static int oci_errors(void* handle, ub4 htype, sword status) BEGINcreateInstance CODESTARTcreateInstance -ASSERT(pData != NULL); - -CHECKENV(pData->environment, - OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, - NULL, NULL, NULL, NULL, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->error), - OCI_HTYPE_ERROR, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), - OCI_HTYPE_AUTHINFO, 0, NULL)); -CHECKENV(pData->environment, - OCIHandleAlloc(pData->environment, (void*) &(pData->statement), - OCI_HTYPE_STMT, 0, NULL)); + ASSERT(pData != NULL); + + CHECKENV(pData->environment, + OCIEnvCreate((void*) &(pData->environment), OCI_DEFAULT, + NULL, NULL, NULL, NULL, 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->error), + OCI_HTYPE_ERROR, 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->authinfo), + OCI_HTYPE_AUTHINFO, 0, NULL)); + CHECKENV(pData->environment, + OCIHandleAlloc(pData->environment, (void*) &(pData->statement), + OCI_HTYPE_STMT, 0, NULL)); finalize_it: ENDcreateInstance -/** Free any resources allocated by createInstance. */ +/** Close the session and free anything allocated by + createInstance. */ BEGINfreeInstance CODESTARTfreeInstance -OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); -OCIHandleFree(pData->environment, OCI_HTYPE_ENV); -OCIHandleFree(pData->error, OCI_HTYPE_ERROR); -OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); -OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); -OCIHandleFree(pData->statement, OCI_HTYPE_STMT); -dbgprintf ("omoracle freed all its resources\n"); - -RETiRet; + OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); + OCIHandleFree(pData->environment, OCI_HTYPE_ENV); + OCIHandleFree(pData->error, OCI_HTYPE_ERROR); + OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); + OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); + OCIHandleFree(pData->statement, OCI_HTYPE_STMT); + free(pData->connection); + dbgprintf ("omoracle freed all its resources\n"); + RETiRet; ENDfreeInstance - BEGINtryResume CODESTARTtryResume + dbgprintf("Attempting to restart the last action\n"); + OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); + OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); + CHECKERR(pData->error, OCISessionGet(pData->environment, pData->error, + &pData->service, pData->authinfo, + pData->connection, + strlen(pData->connection), NULL, 0, + NULL, NULL, NULL, OCI_DEFAULT)); + CHECKERR(pData->error, OCIStmtExecute(pData->service, pData->statement, + pData->error, 1, 0, NULL, NULL, + OCI_DEFAULT)); -dbgprintf ("***** OMORACLE ***** At tryResume\n"); +finalize_it: ENDtryResume static rsRetVal startSession(instanceData* pData, char* connection, char* user, @@ -170,45 +183,50 @@ finalize_it: BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature -/* Right now, this module is compatible with nothing. */ -dbgprintf ("***** OMORACLE ***** At isCompatibleWithFeature\n"); -iRet = RS_RET_INCOMPATIBLE; + /* Right now, this module is compatible with nothing. */ + dbgprintf ("***** OMORACLE ***** At isCompatibleWithFeature\n"); + iRet = RS_RET_INCOMPATIBLE; ENDisCompatibleWithFeature BEGINparseSelectorAct -char connection_string[MAXHOSTNAMELEN]; -char user[_DB_MAXUNAMELEN]; -char pwd[_DB_MAXPWDLEN]; - + char user[_DB_MAXUNAMELEN]; + char pwd[_DB_MAXPWDLEN]; + char connection_string[MAXHOSTNAMELEN]; CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); -if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { - ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); -} - - -p += sizeof ":omoracle:" - 1; - -if (*p == '\0' || *p == ',') { - dbgprintf ("Wrong char processing module arguments: %c\n", *p); - ABORT_FINALIZE(RS_RET_INVALID_PARAMS); -} + if (strncmp((char*) p, ":omoracle:", sizeof ":omoracle:" - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } -CHKiRet(getSubString(&p, connection_string, MAXHOSTNAMELEN, ',')); -CHKiRet(getSubString(&p, user, _DB_MAXUNAMELEN, ',')); -CHKiRet(getSubString(&p, pwd, _DB_MAXPWDLEN, ';')); -p--; -CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, - OMSR_RQD_TPL_OPT_SQL, " StdFmt")); -CHKiRet(createInstance(&pData)); -CHKiRet(startSession(pData, connection_string, user, pwd)); + p += sizeof ":omoracle:" - 1; -dbgprintf ("omoracle module got all its resources allocated " - "and connected to the DB\n"); + if (*p == '\0' || *p == ',') { + errmsg.LogError(0, NO_ERRCODE, "Wrong char processing module arguments: %c\n", *p); + ABORT_FINALIZE(RS_RET_INVALID_PARAMS); + } + CHKiRet(getSubString(&p, connection_string, MAXHOSTNAMELEN, ',')); + CHKiRet(getSubString(&p, user, _DB_MAXUNAMELEN, ',')); + CHKiRet(getSubString(&p, pwd, _DB_MAXPWDLEN, ';')); + p--; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, + OMSR_RQD_TPL_OPT_SQL, " StdFmt")); + CHKiRet(createInstance(&pData)); + pData->connection = strdup(connection_string); + if (pData->connection == NULL) { + iRet = RS_RET_OUT_OF_MEMORY; + goto finalize_it; + } + CHKiRet(startSession(pData, connection_string, user, pwd)); + + dbgprintf ("omoracle module got all its resources allocated " + "and connected to the DB\n"); + + memset(user, 0, sizeof user); + memset(pwd, 0, sizeof pwd); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -225,6 +243,9 @@ CODESTARTdoAction CHECKERR(pData->error, OCITransCommit(pData->service, pData->error, 0)); finalize_it: + dbgprintf ("omoracle %s at executing statement %s\n", + iRet?"did not succeed":"succeeded", *ppString); +/* Clean credentials to avoid leakage in case of core dump. */ ENDdoAction BEGINmodExit @@ -233,16 +254,14 @@ ENDmodExit BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo -dbgprintf ("***** OMORACLE ***** At bdgPrintInstInfo\n"); - + dbgprintf ("***** OMORACLE ***** At bdgPrintInstInfo\n"); ENDdbgPrintInstInfo BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES -dbgprintf ("***** OMORACLE ***** At queryEtryPt\n"); - + dbgprintf ("***** OMORACLE ***** At queryEtryPt\n"); ENDqueryEtryPt static rsRetVal @@ -256,17 +275,10 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; -char dbname[MAX_BUFSIZE]; CODEmodInit_QueryRegCFSLineHdlr -dbgprintf ("***** OMORACLE ***** At modInit\n"); -CHKiRet(objUse(errmsg, CORE_COMPONENT)); -/* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ -CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, - eCmdHdlrCustomHandler, resetConfigVariables, - NULL, STD_LOADABLE_MODULE_ID)); -dbgprintf ("***** OMORACLE ***** dbname before = %s\n", dbname); -CHKiRet(omsdRegCFSLineHdlr((uchar*) "actionoracledb", 0, eCmdHdlrInt, - NULL, dbname, STD_LOADABLE_MODULE_ID)); -dbgprintf ("***** OMORACLE ***** dbname = %s\n", dbname); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + /* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ + CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, + eCmdHdlrCustomHandler, resetConfigVariables, + NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit - -- cgit v1.2.3 From 0289fb7f5c63ee1563eb829b5b1a0fcf3dfa279a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 25 Mar 2009 18:16:31 +0100 Subject: Remove useless dbgprintf and add documentation. --- plugins/omoracle/omoracle.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index aa506dca..4cf4c724 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -43,13 +43,21 @@ DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) typedef struct _instanceData { + /* Environment handler, the base for any OCI work. */ OCIEnv* environment; + /* Session handler, the actual DB connection object. */ OCISession* session; + /* Error handler for OCI calls. */ OCIError* error; + /* Prepared statement. */ OCIStmt* statement; + /* Service handler. */ OCISvcCtx* service; + /* Credentials object for the connection. */ OCIAuthInfo* authinfo; + /* Binding parameters, currently unused */ OCIBind* binding; + /* Connection string, kept here for possible retries. */ char* connection; } instanceData; @@ -177,6 +185,8 @@ static rsRetVal startSession(instanceData* pData, char* connection, char* user, strlen(connection), NULL, 0, NULL, NULL, NULL, OCI_DEFAULT)); finalize_it: + if (iRet != RS_RET_OK) + errmsg.LogError(0, NO_ERRCODE, "Unable to start Oracle session\n"); RETiRet; } @@ -254,14 +264,12 @@ ENDmodExit BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo - dbgprintf ("***** OMORACLE ***** At bdgPrintInstInfo\n"); ENDdbgPrintInstInfo BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES - dbgprintf ("***** OMORACLE ***** At queryEtryPt\n"); ENDqueryEtryPt static rsRetVal -- cgit v1.2.3 From c54de8212fe434080983b70d9ffa200d52f68ccd Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 25 Mar 2009 19:41:17 +0100 Subject: added some (hopefully helpful) comments on the calling IF --- plugins/omoracle/omoracle.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 4cf4c724..eba45c5b 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -154,6 +154,26 @@ ENDfreeInstance BEGINtryResume CODESTARTtryResume + /* Here usually only a reconnect is done. The rsyslog core will call + * this entry point from time to time when the action suspended itself. + * Note that the rsyslog core expects that if the plugin suspended itself + * the action was not carried out during that invocation. Thus, rsyslog + * will call the action with *the same* data item again AFTER a resume + * was successful. As such, tryResume should NOT write the failed data + * item. If it needs to for some reason, it must delete the item again, + * otherwise, it will get duplicated. + * This handling inside the rsyslog core is important to be able to + * preserve data over rsyslog restarts. With it, the core can ensure that + * it queues all not-yet-processed messages without the plugin needing + * to take care about that. + * So in essence, it is recommended that just a reconnet is tried, but + * the last action not restarted. Note that it is not a real problem + * (but causes a slight performance degradation) if tryResume returns + * successfully but the next call to doAction() immediately returns + * RS_RET_SUSPENDED. So it is OK to do the actual restart inside doAction(). + * ... of course I don't know why Oracle might need a full restart... + * rgerhards, 2009-03-26 + */ dbgprintf("Attempting to restart the last action\n"); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); @@ -213,6 +233,13 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); p += sizeof ":omoracle:" - 1; + /* while this parameter parsing is convenient and works perfectly, + * it is suggested that parameters are only specified via $Action... config + * statement (as done in omlibdbi). The reason is that this may greatly + * ease the transition when we have the full config script language. However, + * this approach here is guranteed to continue to work in the future. + * rgerhards, 2009-03-26 + */ if (*p == '\0' || *p == ',') { errmsg.LogError(0, NO_ERRCODE, "Wrong char processing module arguments: %c\n", *p); ABORT_FINALIZE(RS_RET_INVALID_PARAMS); -- cgit v1.2.3 From 5103c912eeb8c0d410f641a971d3687a02c8c02b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 26 Mar 2009 15:30:46 +0100 Subject: parser test script created more or less complete now, with some minor nits left for later but is usable. --- tests/parser.tcl | 76 ++++++++++++++++++++++++++++++++++++++++++++++ tests/test.tcl | 41 ------------------------- tests/testruns/parser.conf | 3 +- 3 files changed, 78 insertions(+), 42 deletions(-) create mode 100644 tests/parser.tcl delete mode 100644 tests/test.tcl diff --git a/tests/parser.tcl b/tests/parser.tcl new file mode 100644 index 00000000..274c08a3 --- /dev/null +++ b/tests/parser.tcl @@ -0,0 +1,76 @@ +# rsyslog parser tests +# This is a first version, and can be extended and improved for +# sure. But it is far better than nothing. Please note that this +# script works together with the config file AND easily extensible +# test case files (*.parse1) to run a number of checks. All test +# cases are executed, even if there is a failure early in the +# process. When finished, the numberof failed tests will be given. +# +# Note: a lot of things are not elegant, but at least they work... +# Even simple things seem to be somewhat non-simple if you are +# not sufficiently involved with tcl/expect ;) -- rgerhards +# +# Copyright (C) 2009 by Rainer Gerhards and Adiscon GmbH +# +# This file is part of rsyslog. + + + +# HELP HELP HELP HELP HELP HELP HELP HELP +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# If you happen to know how to disable rsyslog's +# stdout from appearing on the "real" stdout, please +# let me know. This is annouying, but I have no more +# time left to invest finding a solution (as the +# rest basically works well...). +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +package require Expect +package require udp 1.0 + +set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-ftestruns/parser.conf" "-u2" "-n" "-iwork/rsyslog.pid" "-M../runtime/.libs"]; +#interact; +expect "}}"; # eat startup message +set udpSock [udp_open]; +udp_conf $udpSock 127.0.0.1 514 +set files [glob "testruns/*.parse1"] +set failed 0; +puts "\n"; + +set i 1; + +foreach testcase $files { + puts "testing $testcase ..."; + set fp [open "$testcase" r]; + fconfigure $fp -buffering line + gets $fp input + gets $fp expected + # assemble "expected" to match the template we use + close $fp; + + # send to daemon + puts $udpSock $input; + flush $udpSock; + + # get response and compare + expect -re "{{.*}}"; + puts "\n"; # at least we make the output readbale... + + set result $expect_out(buffer); + set result [string trimleft $result "\{\{"]; + set result [string trimright $result "\}\}"]; + + if { $result != $expected } { + puts "test $i failed!\n"; + puts "expected: '$expected'\n"; + puts "returned: '$result'\n"; + puts "\n"; + set failed [expr {$failed + 1}]; + }; + set i [expr {$i + 1}]; +} + +exec kill $rsyslogdPID; +close $udpSock; + +puts "Number of failed test: $failed.\n"; diff --git a/tests/test.tcl b/tests/test.tcl deleted file mode 100644 index 7f1b7a6b..00000000 --- a/tests/test.tcl +++ /dev/null @@ -1,41 +0,0 @@ -# rsyslog parser tests -package require Expect -package require udp 1.0 - -set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-ftestruns/parser.conf" "-u2" "-n" "-iwork/rsyslog.pid" "-M../runtime/.libs"]; -#interact; -#puts "pid: $rsyslogdPID"; -#sleep 1; -#expect "\n"; -expect "}}"; # eat startup message -set udpSock [udp_open]; -udp_conf $udpSock 127.0.0.1 514 -set files [glob *.parse1] -puts "done init\n"; - - -foreach testcase $files { - puts "File $testcase"; - set fp [open "$testcase" r]; - fconfigure $fp -buffering line - #set data [read $fp]; - #set data [split $data "\n"]; - gets $fp input - puts "Line 1: $input\n"; - - puts $udpSock $input; - flush $udpSock; - - - set i 1 - expect -re "{{.*}}"; - set result $expect_out(buffer); - - #puts "MSG $i: '$expect_out(buffer)'"; - puts "MSG $i: '$result'\n"; - set i [expr {$i + 1}]; - -} - -exec kill $rsyslogdPID; -close $udpSock; diff --git a/tests/testruns/parser.conf b/tests/testruns/parser.conf index a515ff6c..3558f143 100644 --- a/tests/testruns/parser.conf +++ b/tests/testruns/parser.conf @@ -6,5 +6,6 @@ $UDPServerRun 514 $ErrorMessagesToStderr off # use a special format that we can easily parse in expect -$template expect,"{{%PRI%,%syslogtag%,%hostname%}}" +#$template expect,"{{%PRI%,%syslogtag%,%hostname%}}" +$template expect,"{{%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%}}" *.* :omstdout:;expect -- cgit v1.2.3 From 0be199af6cc2cd9a9acb96e97e1cd061affaa6d9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 26 Mar 2009 17:48:59 +0100 Subject: initial parser testing suite added integrated tcl test script into autotools make check, created a few test cases based a real-world sample and samples from RFC3164 and 5424. --- ChangeLog | 2 ++ tests/Makefile.am | 10 +++++----- tests/parser.tcl | 3 ++- tests/parsertest | 2 ++ tests/testruns/1.parse1 | 3 +++ tests/testruns/rfc3164.parse1 | 4 ++++ tests/testruns/rfc5424-1.parse1 | 4 ++++ tests/testruns/rfc5424-2.parse1 | 4 ++++ tests/testruns/rfc5424-3.parse1 | 4 ++++ tests/testruns/rfc5424-4.parse1 | 4 ++++ 10 files changed, 34 insertions(+), 6 deletions(-) create mode 100755 tests/parsertest create mode 100644 tests/testruns/1.parse1 create mode 100644 tests/testruns/rfc3164.parse1 create mode 100644 tests/testruns/rfc5424-1.parse1 create mode 100644 tests/testruns/rfc5424-2.parse1 create mode 100644 tests/testruns/rfc5424-3.parse1 create mode 100644 tests/testruns/rfc5424-4.parse1 diff --git a/ChangeLog b/ChangeLog index 1cfb3c3a..e74c7209 100644 --- a/ChangeLog +++ b/ChangeLog @@ -14,6 +14,8 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? is used in a secure way. (may be removed later) - added omstdout module for testing purposes. Spits out all messages to stdout - no config option, no other features +- added a parser testing suite (still needs to be extended, but a good + start) - modified $ModLoad statement so that for modules whom's name starts with a dot, no path is prepended (this enables relative-pathes and should not break any valid current config) diff --git a/tests/Makefile.am b/tests/Makefile.am index 384afd4e..225f087f 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,8 +1,5 @@ -#AUTOMAKE_OPTIONS=dejagnu -#DEJATOOL=Rainer - check_PROGRAMS = rt_init rscript -TESTS = $(check_PROGRAMS) cfg.sh +TESTS = $(check_PROGRAMS) cfg.sh parsertest TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ #TESTS = $(check_PROGRAMS) @@ -19,7 +16,10 @@ EXTRA_DIST=1.rstest 2.rstest 3.rstest err1.rstest \ cfg4.testin \ DevNull.cfgtest \ err1.rstest \ - NoExistFile.cfgtest + NoExistFile.cfgtest \ + parsertest + parser.tcl + testruns/*.parser1 rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/tests/parser.tcl b/tests/parser.tcl index 274c08a3..6b08717b 100644 --- a/tests/parser.tcl +++ b/tests/parser.tcl @@ -73,4 +73,5 @@ foreach testcase $files { exec kill $rsyslogdPID; close $udpSock; -puts "Number of failed test: $failed.\n"; +puts "Number of failed tests: $failed.\n"; +if { $failed != 0 } { exit 1 }; diff --git a/tests/parsertest b/tests/parsertest new file mode 100755 index 00000000..78c42c07 --- /dev/null +++ b/tests/parsertest @@ -0,0 +1,2 @@ +# run parser test suite +tclsh parser.tcl diff --git a/tests/testruns/1.parse1 b/tests/testruns/1.parse1 new file mode 100644 index 00000000..5ae655e6 --- /dev/null +++ b/tests/testruns/1.parse1 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +167,local4,debug,Mar 6 16:57:54,172.20.245.8,%PIX-7-710005,%PIX-7-710005:, UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc3164.parse1 b/tests/testruns/rfc3164.parse1 new file mode 100644 index 00000000..e7a5fa18 --- /dev/null +++ b/tests/testruns/rfc3164.parse1 @@ -0,0 +1,4 @@ +<34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8 +34,auth,crit,Oct 11 22:14:15,mymachine,su,su:, 'su root' failed for lonvick on /dev/pts/8 +#Example from RFC3164, section 5.4 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-1.parse1 b/tests/testruns/rfc5424-1.parse1 new file mode 100644 index 00000000..90236c7f --- /dev/null +++ b/tests/testruns/rfc5424-1.parse1 @@ -0,0 +1,4 @@ +<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 +34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8 +#Example from RFC5424, section 6.5 / sample 1 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-2.parse1 b/tests/testruns/rfc5424-2.parse1 new file mode 100644 index 00000000..a86fbc35 --- /dev/null +++ b/tests/testruns/rfc5424-2.parse1 @@ -0,0 +1,4 @@ +<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts. +165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],- %% It's time to make the do-nuts. +#Example from RFC5424, section 6.5 / sample 2 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-3.parse1 b/tests/testruns/rfc5424-3.parse1 new file mode 100644 index 00000000..6ad4073d --- /dev/null +++ b/tests/testruns/rfc5424-3.parse1 @@ -0,0 +1,4 @@ +<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource= "Application" eventID="1011"][examplePriority@32473 class="high"] +165,local4,notice,Oct 11 22:14:15,mymachine.example.com,,evntslog, +#Example from RFC5424, section 6.5 / sample 4 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-4.parse1 b/tests/testruns/rfc5424-4.parse1 new file mode 100644 index 00000000..ecf27e14 --- /dev/null +++ b/tests/testruns/rfc5424-4.parse1 @@ -0,0 +1,4 @@ +<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource= "Application" eventID="1011"] BOMAn application event log entry... +165,local4,notice,Oct 11 22:14:15,mymachine.example.com,,evntslog,BOMAn application event log entry... +#Example from RFC5424, section 6.5 / sample 3 +#Only the first two lines are important, you may place anything behind them! -- cgit v1.2.3 From 47fb9cb807d9c646cb07e56e380f3c553176955c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 26 Mar 2009 18:42:49 +0100 Subject: added some missing files in tests Makefile.am --- tests/Makefile.am | 12 ++++++++---- tests/parser.tcl | 23 ++++++----------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 225f087f..5b579017 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,10 +1,10 @@ check_PROGRAMS = rt_init rscript TESTS = $(check_PROGRAMS) cfg.sh parsertest TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ -#TESTS = $(check_PROGRAMS) test_files = testbench.h runtime-dummy.c -EXTRA_DIST=1.rstest 2.rstest 3.rstest err1.rstest \ +EXTRA_DIST=parser.tcl \ + 1.rstest 2.rstest 3.rstest err1.rstest \ cfg.sh \ cfg1.cfgtest \ cfg1.testin \ @@ -17,9 +17,13 @@ EXTRA_DIST=1.rstest 2.rstest 3.rstest err1.rstest \ DevNull.cfgtest \ err1.rstest \ NoExistFile.cfgtest \ + testruns/1.parse1 \ + testruns/rfc3164.parse1 \ + testruns/rfc5424-1.parse1 \ + testruns/rfc5424-2.parse1 \ + testruns/rfc5424-3.parse1 \ + testruns/rfc5424-4.parse1 \ parsertest - parser.tcl - testruns/*.parser1 rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/tests/parser.tcl b/tests/parser.tcl index 6b08717b..5872fa3c 100644 --- a/tests/parser.tcl +++ b/tests/parser.tcl @@ -14,19 +14,9 @@ # # This file is part of rsyslog. - - -# HELP HELP HELP HELP HELP HELP HELP HELP -# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -# If you happen to know how to disable rsyslog's -# stdout from appearing on the "real" stdout, please -# let me know. This is annouying, but I have no more -# time left to invest finding a solution (as the -# rest basically works well...). -# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - package require Expect package require udp 1.0 +log_user 0; # comment this out if you would like to see rsyslog output for testing set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-ftestruns/parser.conf" "-u2" "-n" "-iwork/rsyslog.pid" "-M../runtime/.libs"]; #interact; @@ -35,12 +25,12 @@ set udpSock [udp_open]; udp_conf $udpSock 127.0.0.1 514 set files [glob "testruns/*.parse1"] set failed 0; -puts "\n"; +puts "\nExecuting parser test suite..."; -set i 1; +set i 0; foreach testcase $files { - puts "testing $testcase ..."; + puts "testing $testcase"; set fp [open "$testcase" r]; fconfigure $fp -buffering line gets $fp input @@ -54,14 +44,13 @@ foreach testcase $files { # get response and compare expect -re "{{.*}}"; - puts "\n"; # at least we make the output readbale... set result $expect_out(buffer); set result [string trimleft $result "\{\{"]; set result [string trimright $result "\}\}"]; if { $result != $expected } { - puts "test $i failed!\n"; + puts "failed!"; puts "expected: '$expected'\n"; puts "returned: '$result'\n"; puts "\n"; @@ -73,5 +62,5 @@ foreach testcase $files { exec kill $rsyslogdPID; close $udpSock; -puts "Number of failed tests: $failed.\n"; +puts "Total number of tests: $i, number of failed tests: $failed.\n"; if { $failed != 0 } { exit 1 }; -- cgit v1.2.3 From 581361121524bf66744e023b0a0c8ab448cfaa4a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 27 Mar 2009 16:09:45 +0100 Subject: fixed a couple of nits with "make [dist]check" --- tests/Makefile.am | 2 ++ tests/parser.tcl | 16 ++++++++++++++-- tests/parsertest | 2 +- tests/testruns/parser.conf | 2 -- tests/work/dummy | 3 +++ 5 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 tests/work/dummy diff --git a/tests/Makefile.am b/tests/Makefile.am index 5b579017..0de5a2f1 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -17,12 +17,14 @@ EXTRA_DIST=parser.tcl \ DevNull.cfgtest \ err1.rstest \ NoExistFile.cfgtest \ + testruns/parser.conf \ testruns/1.parse1 \ testruns/rfc3164.parse1 \ testruns/rfc5424-1.parse1 \ testruns/rfc5424-2.parse1 \ testruns/rfc5424-3.parse1 \ testruns/rfc5424-4.parse1 \ + work/dummy \ parsertest rt_init_SOURCES = rt-init.c $(test_files) diff --git a/tests/parser.tcl b/tests/parser.tcl index 5872fa3c..047607c6 100644 --- a/tests/parser.tcl +++ b/tests/parser.tcl @@ -9,6 +9,8 @@ # Note: a lot of things are not elegant, but at least they work... # Even simple things seem to be somewhat non-simple if you are # not sufficiently involved with tcl/expect ;) -- rgerhards +# +# call: tclsh parser.tcl /director/with/testcases # # Copyright (C) 2009 by Rainer Gerhards and Adiscon GmbH # @@ -18,12 +20,22 @@ package require Expect package require udp 1.0 log_user 0; # comment this out if you would like to see rsyslog output for testing -set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-ftestruns/parser.conf" "-u2" "-n" "-iwork/rsyslog.pid" "-M../runtime/.libs"]; +if {$argc > 1} { + puts "invalid number of parameters, usage: tclsh parser.tcl /directory/with/testcases"; + exit 1; +} +if {$argc == 0 } { + set srcdir "."; +} else { + set srcdir "$argv"; +} + +set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-f$srcdir/testruns/parser.conf" "-u2" "-n" "-i$srcdir/work/rsyslog.pid" "-M../runtime/.libs"]; #interact; expect "}}"; # eat startup message set udpSock [udp_open]; udp_conf $udpSock 127.0.0.1 514 -set files [glob "testruns/*.parse1"] +set files [glob "$srcdir/testruns/*.parse1"] set failed 0; puts "\nExecuting parser test suite..."; diff --git a/tests/parsertest b/tests/parsertest index 78c42c07..c7efa631 100755 --- a/tests/parsertest +++ b/tests/parsertest @@ -1,2 +1,2 @@ # run parser test suite -tclsh parser.tcl +tclsh $srcdir/parser.tcl $srcdir diff --git a/tests/testruns/parser.conf b/tests/testruns/parser.conf index 3558f143..7b4b4aed 100644 --- a/tests/testruns/parser.conf +++ b/tests/testruns/parser.conf @@ -1,11 +1,9 @@ $ModLoad ../plugins/omstdout/.libs/omstdout -$ModLoad ../plugins/imuxsock/.libs/imuxsock $ModLoad ../plugins/imudp/.libs/imudp $UDPServerRun 514 $ErrorMessagesToStderr off # use a special format that we can easily parse in expect -#$template expect,"{{%PRI%,%syslogtag%,%hostname%}}" $template expect,"{{%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%}}" *.* :omstdout:;expect diff --git a/tests/work/dummy b/tests/work/dummy new file mode 100644 index 00000000..93c5babb --- /dev/null +++ b/tests/work/dummy @@ -0,0 +1,3 @@ +This is a dummy file. It's only purpose is to ensure +that ./test/work is created so that "make distcheck" +and "make check" can operate properly. -- cgit v1.2.3 From 8e29c1fc47523c894b78894d6fdeb43f2d97811d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 27 Mar 2009 17:13:28 +0100 Subject: solved some more issues with "make [dist]check" especially when executed as non-root --- tests/Makefile.am | 2 +- tests/parser.tcl | 4 ++-- tests/testruns/parser.conf | 2 +- tests/work/dummy | 3 --- 4 files changed, 4 insertions(+), 7 deletions(-) delete mode 100644 tests/work/dummy diff --git a/tests/Makefile.am b/tests/Makefile.am index 0de5a2f1..f22ca139 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,7 @@ check_PROGRAMS = rt_init rscript TESTS = $(check_PROGRAMS) cfg.sh parsertest TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ +DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c EXTRA_DIST=parser.tcl \ @@ -24,7 +25,6 @@ EXTRA_DIST=parser.tcl \ testruns/rfc5424-2.parse1 \ testruns/rfc5424-3.parse1 \ testruns/rfc5424-4.parse1 \ - work/dummy \ parsertest rt_init_SOURCES = rt-init.c $(test_files) diff --git a/tests/parser.tcl b/tests/parser.tcl index 047607c6..1adeac25 100644 --- a/tests/parser.tcl +++ b/tests/parser.tcl @@ -30,11 +30,11 @@ if {$argc == 0 } { set srcdir "$argv"; } -set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-f$srcdir/testruns/parser.conf" "-u2" "-n" "-i$srcdir/work/rsyslog.pid" "-M../runtime/.libs"]; +set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-f$srcdir/testruns/parser.conf" "-u2" "-n" "-irsyslog.pid" "-M../runtime/.libs"]; #interact; expect "}}"; # eat startup message set udpSock [udp_open]; -udp_conf $udpSock 127.0.0.1 514 +udp_conf $udpSock 127.0.0.1 12514 set files [glob "$srcdir/testruns/*.parse1"] set failed 0; puts "\nExecuting parser test suite..."; diff --git a/tests/testruns/parser.conf b/tests/testruns/parser.conf index 7b4b4aed..8d32746c 100644 --- a/tests/testruns/parser.conf +++ b/tests/testruns/parser.conf @@ -1,6 +1,6 @@ $ModLoad ../plugins/omstdout/.libs/omstdout $ModLoad ../plugins/imudp/.libs/imudp -$UDPServerRun 514 +$UDPServerRun 12514 $ErrorMessagesToStderr off diff --git a/tests/work/dummy b/tests/work/dummy deleted file mode 100644 index 93c5babb..00000000 --- a/tests/work/dummy +++ /dev/null @@ -1,3 +0,0 @@ -This is a dummy file. It's only purpose is to ensure -that ./test/work is created so that "make distcheck" -and "make check" can operate properly. -- cgit v1.2.3 From 3e3a9bc9982331e44cf397fef131e75553f2ab2c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 31 Mar 2009 12:00:40 +0200 Subject: ported non-tcl based test suite to Solaris --- configure.ac | 2 +- tests/Makefile.am | 12 +++++++----- tests/cfg.sh | 12 ++++++------ tests/ourtail.c | 43 +++++++++++++++++++++++++++++++++++++++++++ tests/rscript.c | 41 +++++++++++++++++++++++++++++++++++++---- 5 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 tests/ourtail.c diff --git a/configure.ac b/configure.ac index 0c9483ca..7289d74e 100644 --- a/configure.ac +++ b/configure.ac @@ -98,7 +98,7 @@ AC_TYPE_SIGNAL AC_FUNC_STAT AC_FUNC_STRERROR_R AC_FUNC_VPRINTF -AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r epoll_wait]) +AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r epoll_wait getline]) # Check for MAXHOSTNAMELEN AC_MSG_CHECKING(for MAXHOSTNAMELEN) diff --git a/tests/Makefile.am b/tests/Makefile.am index 7a31be45..65a294ca 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,7 +1,7 @@ -check_PROGRAMS = rt_init rscript -TESTS = $(check_PROGRAMS) cfg.sh +testruns = rt_init rscript +check_PROGRAMS = $(testruns) ourtail +TESTS = $(testruns) cfg.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ -#TESTS = $(check_PROGRAMS) test_files = testbench.h runtime-dummy.c EXTRA_DIST=1.rstest 2.rstest 3.rstest err1.rstest \ @@ -18,12 +18,14 @@ EXTRA_DIST=1.rstest 2.rstest 3.rstest err1.rstest \ err1.rstest \ NoExistFile.cfgtest +ourtail_SOURCES = ourtail.c + rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -rt_init_LDADD = $(RSRT_LIBS) $(ZLIB_LIBS) $(PTHREADS_LIBS) +rt_init_LDADD = $(RSRT_LIBS) $(ZLIB_LIBS) $(PTHREADS_LIBS) $(SOL_LIBS) rt_init_LDFLAGS = -export-dynamic rscript_SOURCES = rscript.c $(test_files) rscript_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) -rscript_LDADD = $(RSRT_LIBS) $(ZLIB_LIBS) $(PTHREADS_LIBS) +rscript_LDADD = $(RSRT_LIBS) $(ZLIB_LIBS) $(PTHREADS_LIBS) $(SOL_LIBS) rscript_LDFLAGS = -export-dynamic diff --git a/tests/cfg.sh b/tests/cfg.sh index fb22fbf3..8acbf342 100755 --- a/tests/cfg.sh +++ b/tests/cfg.sh @@ -36,7 +36,7 @@ echo "local directory" # # check empty config file # -../tools/rsyslogd -c4 -N1 -f/dev/null 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -N1 -f/dev/null 2>&1 |./ourtail > tmp cmp tmp $srcdir/DevNull.cfgtest if [ ! $? -eq 0 ]; then echo "DevNull.cfgtest failed" @@ -51,7 +51,7 @@ fi; # # check missing config file # -../tools/rsyslogd -c4 -N1 -f/This/does/not/exist 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -N1 -f/This/does/not/exist 2>&1 |./ourtail > tmp cmp tmp $srcdir/NoExistFile.cfgtest if [ ! $? -eq 0 ]; then echo "NoExistFile.cfgtest failed" @@ -74,7 +74,7 @@ exit 0 # # check config with invalid directive # -../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg1.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg1.testin 2>&1 |./ourtail > tmp cmp tmp $srcdir/cfg1.cfgtest if [ ! $? -eq 0 ]; then echo "cfg1.cfgtest failed" @@ -91,7 +91,7 @@ fi; # the one with the invalid config directive, so that we may see # an effect of the included config ;) # -../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg2.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg2.testin 2>&1 |./ourtail > tmp cmp tmp $srcdir/cfg2.cfgtest if [ ! $? -eq 0 ]; then echo "cfg2.cfgtest failed" @@ -106,7 +106,7 @@ fi; # # check included config file, where included file does not exist # -../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg3.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg3.testin 2>&1 |./ourtail > tmp cmp tmp $srcdir/cfg3.cfgtest if [ ! $? -eq 0 ]; then echo "cfg3.cfgtest failed" @@ -121,7 +121,7 @@ fi; # # check a reasonable complex, but correct, log file # -../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg4.testin 2>&1 |tail --lines=+2 > tmp +../tools/rsyslogd -c4 -u2 -N1 -f$srcdir/cfg4.testin 2>&1 |./ourtail > tmp cmp tmp $srcdir/cfg4.cfgtest if [ ! $? -eq 0 ]; then echo "cfg4.cfgtest failed" diff --git a/tests/ourtail.c b/tests/ourtail.c new file mode 100644 index 00000000..f2751c72 --- /dev/null +++ b/tests/ourtail.c @@ -0,0 +1,43 @@ +/* This is a quick and dirty "tail implementation", one which always + * skips the first line, but nothing else. I have done this to prevent + * the various incompatible options of tail come into my way. One could + * probably work around this by using autoconf magic, but for me it + * was much quicker writing this small C program, which really should + * be portable across all platforms. + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include + +int main(int argc, char *argv[]) +{ + int c; + + for(c = getchar() ; c != EOF && c != '\n' ; c = getchar()) + /*skip to newline*/; + + if(c == '\n') + c = getchar(); + + for( ; c != EOF ; c = getchar()) + putchar(c); +} diff --git a/tests/rscript.c b/tests/rscript.c index 3eec9c3c..4906f91a 100644 --- a/tests/rscript.c +++ b/tests/rscript.c @@ -39,6 +39,38 @@ DEFobjCurrIf(ctok) DEFobjCurrIf(ctok_token) DEFobjCurrIf(vmprg) + +/* we emulate getline (the dirty way) if we do not have it + * We do not try very hard, as this is just a test driver. + * rgerhards, 2009-03-31 + */ +#ifndef HAVE_GETLINE +ssize_t getline(char **lineptr, size_t *n, FILE *fp) +{ + int c; + int len = 0; + + if(*lineptr == NULL) + *lineptr = malloc(1024); /* quick and dirty! */ + + c = fgetc(fp); + while(c != EOF && c != '\n') { + (*lineptr)[len++] = c; + c = fgetc(fp); + } + if(c != EOF) /* need to add NL? */ + (*lineptr)[len++] = c; + + (*lineptr)[len] = '\0'; + + *n = len; + //printf("getline returns: '%s'\n", *lineptr); + + return (len > 0) ? len : -1; +} +#endif /* #ifndef HAVE_GETLINE */ + + BEGINInit CODESTARTInit pErrObj = "expr"; CHKiRet(objUse(expr, CORE_COMPONENT)); @@ -101,10 +133,9 @@ PerformTest(cstr_t *pstrIn, rsRetVal iRetExpected, cstr_t *pstrOut) CHKiRet(vmprg.Obj2Str(pExpr->pVmprg, pstrPrg)); if(strcmp((char*)rsCStrGetSzStr(pstrPrg), (char*)rsCStrGetSzStr(pstrOut))) { - int iLen; printf("error: compiled program different from expected result!\n"); - printf("generated vmprg (%d bytes):\n%s\n", strlen(rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); - printf("expected (%d bytes):\n%s\n", strlen(rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); + printf("generated vmprg (%d bytes):\n%s\n", strlen((char*)rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); + printf("expected (%d bytes):\n%s\n", strlen((char*)rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); ABORT_FINALIZE(RS_RET_ERR); } @@ -139,6 +170,7 @@ ProcessTestFile(uchar *pszFileName) size_t lenLn; cstr_t *pstrIn = NULL; cstr_t *pstrOut = NULL; + int iParse; rsRetVal iRetExpected; DEFiRet; @@ -161,10 +193,11 @@ ProcessTestFile(uchar *pszFileName) /* once we had a comment, the next line MUST be "result: ". Anything * after nbr is simply ignored. */ - if(sscanf(lnptr, "result: %d", &iRetExpected) != 1) { + if(sscanf(lnptr, "result: %d", &iParse) != 1) { printf("error in result line, scanf failed, line: '%s'\n", lnptr); ABORT_FINALIZE(RS_RET_ERR); } + iRetExpected = iParse; getline(&lnptr, &lenLn, fp); CHKEOF; /* and now we look for "in:" (and again ignore the rest...) */ -- cgit v1.2.3 From ec9e031599016b9eb6f9ac3fd8298ee4fcb0364f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 31 Mar 2009 19:00:16 +0200 Subject: changed parser test suite to be c-program based I finally removed the tcl script because tcl costs a lot of time if you do not invest the full learning cycle, plus I have not everything avaible I need on Solaris. With C, I am quicker and I also can create a superior solution. So I finally switched. Took much less time than the initial tcl script... --- tests/Makefile.am | 10 +- tests/parser.tcl | 78 ------------ tests/parsertest | 2 - tests/parsertest.c | 265 ++++++++++++++++++++++++++++++++++++++++ tests/testruns/rfc5424-1.parse1 | 3 +- 5 files changed, 270 insertions(+), 88 deletions(-) delete mode 100644 tests/parser.tcl delete mode 100755 tests/parsertest create mode 100644 tests/parsertest.c diff --git a/tests/Makefile.am b/tests/Makefile.am index f22ca139..f941f3a6 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,12 +1,10 @@ -check_PROGRAMS = rt_init rscript -TESTS = $(check_PROGRAMS) cfg.sh parsertest +check_PROGRAMS = rt_init rscript parsertest +TESTS = $(check_PROGRAMS) cfg.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c -EXTRA_DIST=parser.tcl \ - 1.rstest 2.rstest 3.rstest err1.rstest \ - cfg.sh \ +EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg1.cfgtest \ cfg1.testin \ cfg2.cfgtest \ @@ -25,7 +23,7 @@ EXTRA_DIST=parser.tcl \ testruns/rfc5424-2.parse1 \ testruns/rfc5424-3.parse1 \ testruns/rfc5424-4.parse1 \ - parsertest + cfg.sh rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/tests/parser.tcl b/tests/parser.tcl deleted file mode 100644 index 1adeac25..00000000 --- a/tests/parser.tcl +++ /dev/null @@ -1,78 +0,0 @@ -# rsyslog parser tests -# This is a first version, and can be extended and improved for -# sure. But it is far better than nothing. Please note that this -# script works together with the config file AND easily extensible -# test case files (*.parse1) to run a number of checks. All test -# cases are executed, even if there is a failure early in the -# process. When finished, the numberof failed tests will be given. -# -# Note: a lot of things are not elegant, but at least they work... -# Even simple things seem to be somewhat non-simple if you are -# not sufficiently involved with tcl/expect ;) -- rgerhards -# -# call: tclsh parser.tcl /director/with/testcases -# -# Copyright (C) 2009 by Rainer Gerhards and Adiscon GmbH -# -# This file is part of rsyslog. - -package require Expect -package require udp 1.0 -log_user 0; # comment this out if you would like to see rsyslog output for testing - -if {$argc > 1} { - puts "invalid number of parameters, usage: tclsh parser.tcl /directory/with/testcases"; - exit 1; -} -if {$argc == 0 } { - set srcdir "."; -} else { - set srcdir "$argv"; -} - -set rsyslogdPID [spawn "../tools/rsyslogd" "-c4" "-f$srcdir/testruns/parser.conf" "-u2" "-n" "-irsyslog.pid" "-M../runtime/.libs"]; -#interact; -expect "}}"; # eat startup message -set udpSock [udp_open]; -udp_conf $udpSock 127.0.0.1 12514 -set files [glob "$srcdir/testruns/*.parse1"] -set failed 0; -puts "\nExecuting parser test suite..."; - -set i 0; - -foreach testcase $files { - puts "testing $testcase"; - set fp [open "$testcase" r]; - fconfigure $fp -buffering line - gets $fp input - gets $fp expected - # assemble "expected" to match the template we use - close $fp; - - # send to daemon - puts $udpSock $input; - flush $udpSock; - - # get response and compare - expect -re "{{.*}}"; - - set result $expect_out(buffer); - set result [string trimleft $result "\{\{"]; - set result [string trimright $result "\}\}"]; - - if { $result != $expected } { - puts "failed!"; - puts "expected: '$expected'\n"; - puts "returned: '$result'\n"; - puts "\n"; - set failed [expr {$failed + 1}]; - }; - set i [expr {$i + 1}]; -} - -exec kill $rsyslogdPID; -close $udpSock; - -puts "Total number of tests: $i, number of failed tests: $failed.\n"; -if { $failed != 0 } { exit 1 }; diff --git a/tests/parsertest b/tests/parsertest deleted file mode 100755 index c7efa631..00000000 --- a/tests/parsertest +++ /dev/null @@ -1,2 +0,0 @@ -# run parser test suite -tclsh $srcdir/parser.tcl $srcdir diff --git a/tests/parsertest.c b/tests/parsertest.c new file mode 100644 index 00000000..3ccae3d6 --- /dev/null +++ b/tests/parsertest.c @@ -0,0 +1,265 @@ +/* Runs a test suite on the rsyslog parser (and later potentially + * other things). + * + * Please note that this + * program works together with the config file AND easily extensible + * test case files (*.parse1) to run a number of checks. All test + * cases are executed, even if there is a failure early in the + * process. When finished, the numberof failed tests will be given. + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EXIT_FAILURE 1 + + +void readLine(int fd, char *ln) +{ + char c; + int lenRead; + lenRead = read(fd, &c, 1); + while(lenRead == 1 && c != '\n') { + *ln++ = c; + lenRead = read(fd, &c, 1); + } + *ln = '\0'; +} + + +/* send a message via UDP + * returns 0 if ok, something else otherwise. + */ +int +udpSend(char *buf, int lenBuf) +{ + struct sockaddr_in si_other; + int s, slen=sizeof(si_other); + + if((s=socket(AF_INET, SOCK_DGRAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &si_other, 0, sizeof(si_other)); + si_other.sin_family = AF_INET; + si_other.sin_port = htons(12514); + if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + + if(sendto(s, buf, lenBuf, 0, (struct sockaddr*) &si_other, slen)==-1) { + perror("sendto"); + fprintf(stderr, "sendto() failed\n"); + return(1); + } + + close(s); + return 0; +} + +/* open pipe to test candidate - so far, this is + * always rsyslogd and with a fixed config. Later, we may + * change this. Returns 0 if ok, something else otherwise. + * rgerhards, 2009-03-31 + */ +int openPipe(pid_t *pid, int *pfd) +{ + int pipefd[2]; + pid_t cpid; + char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", + "-M../runtime//.libs", NULL }; + char confFile[1024]; + char *newenviron[] = { NULL }; + + + sprintf(confFile, "-f%s/testruns/parser2.conf", getenv("srcdir")); + newargv[1] = confFile; + + if (pipe(pipefd) == -1) { + perror("pipe"); + exit(EXIT_FAILURE); + } + + cpid = fork(); + if (cpid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } + + if(cpid == 0) { /* Child reads from pipe */ + fclose(stdout); + dup(pipefd[1]); + close(pipefd[1]); + close(pipefd[0]); + fclose(stdin); + execve("../tools/rsyslogd", newargv, newenviron); + } else { + close(pipefd[1]); + *pid = cpid; + *pfd = pipefd[0]; + } + + return(0); +} + + +/* Process a specific test case. File name is provided. + * Needs to return 0 if all is OK, something else otherwise. + */ +int +processTestFile(int fd, char *pszFileName) +{ + FILE *fp; + char *testdata = NULL; + char *expected = NULL; + int ret = 0; + size_t lenLn; + char buf[4096]; + + if((fp = fopen((char*)pszFileName, "r")) == NULL) { + perror((char*)pszFileName); + return(2); + } + + /* skip comments at start of file */ + + getline(&testdata, &lenLn, fp); + while(!feof(fp)) { + if(*testdata == '#') + getline(&testdata, &lenLn, fp); + else + break; /* first non-comment */ + } + + + testdata[strlen(testdata)-1] = '\0'; /* remove \n */ + /* now we have the test data to send */ + if(udpSend(testdata, strlen(testdata)) != 0) + return(2); + + /* next line is expected output + * we do not care about EOF here, this will lead to a failure and thus + * draw enough attention. -- rgerhards, 2009-03-31 + */ + getline(&expected, &lenLn, fp); + expected[strlen(expected)-1] = '\0'; /* remove \n */ + + /* pull response from server and then check if it meets our expectation */ + readLine(fd, buf); + if(strcmp(expected, buf)) { + printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", + expected, buf); + ret = 1; + } + + free(testdata); + free(expected); + fclose(fp); + return(ret); +} + + +/* carry out all tests. Tests are specified via a file name + * wildcard. Each of the files is read and the test carried + * out. + * Returns the number of tests that failed. Zero means all + * success. + */ +int +doTests(int fd, char *files) +{ + int iFailed = 0; + int iTests = 0; + int ret; + char *testFile; + glob_t testFiles; + size_t i = 0; + struct stat fileInfo; + + glob(files, GLOB_MARK, NULL, &testFiles); + + for(i = 0; i < testFiles.gl_pathc; i++) { + testFile = testFiles.gl_pathv[i]; + + if(stat((char*) testFile, &fileInfo) != 0) + continue; /* continue with the next file if we can't stat() the file */ + + ++iTests; + /* all regular files are run through the test logic. Symlinks don't work. */ + if(S_ISREG(fileInfo.st_mode)) { /* config file */ + printf("processing test case '%s' ... ", testFile); + ret = processTestFile(fd, testFile); + if(ret == 0) { + printf("successfully completed\n"); + } else { + printf("failed!\n"); + ++iFailed; + } + } + } + globfree(&testFiles); + + printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); + return(iFailed); +} + + +/* */ +int main(int argc, char *argv[]) +{ + int fd; + pid_t pid; + int ret = 0; + char buf[4096]; + char testcases[4096]; + + printf("running rsyslog parser tests ($srcdir=%s)\n", getenv("srcdir")); + + openPipe(&pid, &fd); + readLine(fd, buf); + + /* generate filename */ + sprintf(testcases, "%s/testruns/*.parse1", getenv("srcdir")); + if(doTests(fd, testcases) != 0) + ret = 1; + + /* cleanup */ + kill(pid, SIGTERM); + printf("End of parser tests.\n"); + exit(ret); +} + + diff --git a/tests/testruns/rfc5424-1.parse1 b/tests/testruns/rfc5424-1.parse1 index 90236c7f..23836c9f 100644 --- a/tests/testruns/rfc5424-1.parse1 +++ b/tests/testruns/rfc5424-1.parse1 @@ -1,4 +1,3 @@ +#Example from RFC5424, section 6.5 / sample 1 <34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8 -#Example from RFC5424, section 6.5 / sample 1 -#Only the first two lines are important, you may place anything behind them! -- cgit v1.2.3 From 91d6888a8afe562bf4d2ef53be94c41898e1a2ec Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 31 Mar 2009 22:03:02 +0200 Subject: bugfix: "make distcheck" did not work --- tests/parsertest.c | 12 ++++++++---- tests/testruns/parser.conf | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/parsertest.c b/tests/parsertest.c index 3ccae3d6..6c2221e8 100644 --- a/tests/parsertest.c +++ b/tests/parsertest.c @@ -105,7 +105,7 @@ int openPipe(pid_t *pid, int *pfd) char *newenviron[] = { NULL }; - sprintf(confFile, "-f%s/testruns/parser2.conf", getenv("srcdir")); + sprintf(confFile, "-f%s/testruns/parser.conf", getenv("srcdir")); newargv[1] = confFile; if (pipe(pipefd) == -1) { @@ -232,7 +232,13 @@ doTests(int fd, char *files) } globfree(&testFiles); - printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); + if(iTests == 0) { + printf("Error: no test cases found, no tests executed.\n"); + iFailed = 1; + } else { + printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); + } + return(iFailed); } @@ -261,5 +267,3 @@ int main(int argc, char *argv[]) printf("End of parser tests.\n"); exit(ret); } - - diff --git a/tests/testruns/parser.conf b/tests/testruns/parser.conf index 8d32746c..0fb7d16d 100644 --- a/tests/testruns/parser.conf +++ b/tests/testruns/parser.conf @@ -5,5 +5,5 @@ $UDPServerRun 12514 $ErrorMessagesToStderr off # use a special format that we can easily parse in expect -$template expect,"{{%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%}}" +$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" *.* :omstdout:;expect -- cgit v1.2.3 From d27edc7587dba7b850759d151d90cdad1cb34a35 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 31 Mar 2009 20:35:15 +0200 Subject: porting parser tests to solaris --- tests/Makefile.am | 12 ++++++++---- tests/getline.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/rscript.c | 31 ------------------------------ 3 files changed, 64 insertions(+), 35 deletions(-) create mode 100644 tests/getline.c diff --git a/tests/Makefile.am b/tests/Makefile.am index 093742db..09d1a0b6 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,9 +1,10 @@ -check_PROGRAMS = rt_init rscript ourtail parsertest -TESTS = $(check_PROGRAMS) cfg.sh +TESTRUNS = rt_init rscript parsertest +check_PROGRAMS = $(TESTRUNS) ourtail +TESTS = $(TESTRUNS) cfg.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid - test_files = testbench.h runtime-dummy.c + EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg1.cfgtest \ cfg1.testin \ @@ -27,12 +28,15 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ ourtail_SOURCES = ourtail.c +parsertest_SOURCES = parsertest.c getline.c +parsertest_LDADD = $(SOL_LIBS) + rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) rt_init_LDADD = $(RSRT_LIBS) $(ZLIB_LIBS) $(PTHREADS_LIBS) $(SOL_LIBS) rt_init_LDFLAGS = -export-dynamic -rscript_SOURCES = rscript.c $(test_files) +rscript_SOURCES = rscript.c getline.c $(test_files) rscript_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) rscript_LDADD = $(RSRT_LIBS) $(ZLIB_LIBS) $(PTHREADS_LIBS) $(SOL_LIBS) rscript_LDFLAGS = -export-dynamic diff --git a/tests/getline.c b/tests/getline.c new file mode 100644 index 00000000..10de2ffe --- /dev/null +++ b/tests/getline.c @@ -0,0 +1,56 @@ +/* getline() replacement for platforms that do not have it. + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include + +/* we emulate getline (the dirty way) if we do not have it + * We do not try very hard, as this is just a test driver. + * rgerhards, 2009-03-31 + */ +#ifndef HAVE_GETLINE +ssize_t getline(char **lineptr, size_t *n, FILE *fp) +{ + int c; + int len = 0; + + if(*lineptr == NULL) + *lineptr = malloc(4096); /* quick and dirty! */ + + c = fgetc(fp); + while(c != EOF && c != '\n') { + (*lineptr)[len++] = c; + c = fgetc(fp); + } + if(c != EOF) /* need to add NL? */ + (*lineptr)[len++] = c; + + (*lineptr)[len] = '\0'; + + *n = len; + //printf("getline returns: '%s'\n", *lineptr); + + return (len > 0) ? len : -1; +} +#endif /* #ifndef HAVE_GETLINE */ diff --git a/tests/rscript.c b/tests/rscript.c index 4906f91a..6b232f5f 100644 --- a/tests/rscript.c +++ b/tests/rscript.c @@ -40,37 +40,6 @@ DEFobjCurrIf(ctok_token) DEFobjCurrIf(vmprg) -/* we emulate getline (the dirty way) if we do not have it - * We do not try very hard, as this is just a test driver. - * rgerhards, 2009-03-31 - */ -#ifndef HAVE_GETLINE -ssize_t getline(char **lineptr, size_t *n, FILE *fp) -{ - int c; - int len = 0; - - if(*lineptr == NULL) - *lineptr = malloc(1024); /* quick and dirty! */ - - c = fgetc(fp); - while(c != EOF && c != '\n') { - (*lineptr)[len++] = c; - c = fgetc(fp); - } - if(c != EOF) /* need to add NL? */ - (*lineptr)[len++] = c; - - (*lineptr)[len] = '\0'; - - *n = len; - //printf("getline returns: '%s'\n", *lineptr); - - return (len > 0) ? len : -1; -} -#endif /* #ifndef HAVE_GETLINE */ - - BEGINInit CODESTARTInit pErrObj = "expr"; CHKiRet(objUse(expr, CORE_COMPONENT)); -- cgit v1.2.3 From e3f21521cd6ae237fd9b362281fe188380210fb5 Mon Sep 17 00:00:00 2001 From: Demo Date: Tue, 31 Mar 2009 22:10:37 +0200 Subject: fixed some problems with "make check" interestingly, they manifested on Debian, only, but potentially existed on other platforms, too. --- configure.ac | 4 +++- tests/DevNull.cfgtest | 1 - tests/NoExistFile.cfgtest | 1 - tests/cfg.sh | 4 ++-- tools/omfile.c | 3 +-- tools/omfwd.c | 2 +- 6 files changed, 7 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index 2db856cd..de328f83 100644 --- a/configure.ac +++ b/configure.ac @@ -689,8 +689,10 @@ AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) # settings for omstdout +# note that "make check" fails, if omstdout is not enabled (thus we enable +# it by default). AC_ARG_ENABLE(omstdout, - [AS_HELP_STRING([--enable-omstdout],[Compiles stdout template module @<:@default=no@:>@])], + [AS_HELP_STRING([--enable-omstdout],[Compiles stdout template module @<:@default=yes@:>@])], [case "${enableval}" in yes) enable_omstdout="yes" ;; no) enable_omstdout="no" ;; diff --git a/tests/DevNull.cfgtest b/tests/DevNull.cfgtest index d30d936b..7822b6df 100644 --- a/tests/DevNull.cfgtest +++ b/tests/DevNull.cfgtest @@ -1,3 +1,2 @@ rsyslogd: CONFIG ERROR: there are no active actions configured. Inputs will run, but no output whatsoever is created. [try http://www.rsyslog.com/e/2103 ] rsyslogd: EMERGENCY CONFIGURATION ACTIVATED - fix rsyslog config file! -rsyslogd: End of config validation run. Bye. diff --git a/tests/NoExistFile.cfgtest b/tests/NoExistFile.cfgtest index 4cbcc029..88d3123f 100644 --- a/tests/NoExistFile.cfgtest +++ b/tests/NoExistFile.cfgtest @@ -1,3 +1,2 @@ rsyslogd: CONFIG ERROR: could not interpret master config file '/This/does/not/exist'. [try http://www.rsyslog.com/e/2013 ] rsyslogd: EMERGENCY CONFIGURATION ACTIVATED - fix rsyslog config file! -rsyslogd: End of config validation run. Bye. diff --git a/tests/cfg.sh b/tests/cfg.sh index 8acbf342..cb838354 100755 --- a/tests/cfg.sh +++ b/tests/cfg.sh @@ -36,7 +36,7 @@ echo "local directory" # # check empty config file # -../tools/rsyslogd -c4 -N1 -f/dev/null 2>&1 |./ourtail > tmp +../tools/rsyslogd -c4 -N1 -f/dev/null 2>&1 |./ourtail |head -2 > tmp cmp tmp $srcdir/DevNull.cfgtest if [ ! $? -eq 0 ]; then echo "DevNull.cfgtest failed" @@ -51,7 +51,7 @@ fi; # # check missing config file # -../tools/rsyslogd -c4 -N1 -f/This/does/not/exist 2>&1 |./ourtail > tmp +../tools/rsyslogd -c4 -N1 -f/This/does/not/exist 2>&1 |./ourtail |head -2 > tmp cmp tmp $srcdir/NoExistFile.cfgtest if [ ! $? -eq 0 ]; then echo "NoExistFile.cfgtest failed" diff --git a/tools/omfile.c b/tools/omfile.c index 65306846..5141b4da 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -725,7 +725,6 @@ CODESTARTparseSelectorAct } else { pData->bSyncFile = bEnableSync ? 1 : 0; } - pData->f_sizeLimit = 0; /* default value, use outchannels to configure! */ switch (*p) @@ -822,7 +821,7 @@ CODESTARTparseSelectorAct if(pData->fd < 0 ) { pData->fd = -1; DBGPRINTF("Error opening log file: %s\n", pData->f_fname); - errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); + errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); break; } if(strcmp((char*) p, _PATH_CONSOLE) == 0) diff --git a/tools/omfwd.c b/tools/omfwd.c index 7a945ce0..88a382e0 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -177,7 +177,7 @@ ENDfreeInstance BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo - printf("%s", pData->f_hname); + dbgprintf("%s", pData->f_hname); ENDdbgPrintInstInfo -- cgit v1.2.3 From a66b69533e38fdc40a7832b99c36e4d25bba80b2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Apr 2009 14:41:32 +0200 Subject: begining touches for putting rsyslog on spaceships --- doc/features.html | 1 + plugins/omdtn/Makefile.am | 8 +++ plugins/omdtn/omdtn.c | 130 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 plugins/omdtn/Makefile.am create mode 100644 plugins/omdtn/omdtn.c diff --git a/doc/features.html b/doc/features.html index 336b31cc..501f3304 100644 --- a/doc/features.html +++ b/doc/features.html @@ -124,6 +124,7 @@ community. Plus, it can be financially attractive: just think about how much les be to sponsor a feature instead of purchasing a commercial implementation. Also, the benefit of being recognised as a sponsor may even drive new customers to your business!
        +
      • Finalize the DTN "planetary Internet" space ship mode output plugin
      • port it to more *nix variants (eg AIX and HP UX) - this needs volunteers with access to those machines and knowledge
      • pcre filtering - maybe (depending on feedback)  - diff --git a/plugins/omdtn/Makefile.am b/plugins/omdtn/Makefile.am new file mode 100644 index 00000000..afb57476 --- /dev/null +++ b/plugins/omdtn/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omdtn.la + +omdtn_la_SOURCES = omdtn.c +omdtn_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omdtn_la_LDFLAGS = -module -avoid-version +omdtn_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omdtn/omdtn.c b/plugins/omdtn/omdtn.c new file mode 100644 index 00000000..761bde79 --- /dev/null +++ b/plugins/omdtn/omdtn.c @@ -0,0 +1,130 @@ +/* omdtn.c + * This is the plugin for rsyslog use in the interplanetary Internet, + * especially useful for rsyslog in space ships of all kinds. + * The core idea was introduced in early 2009 and considered + * doable. + * + * Note that this has not yet been tested for robustness but needs + * to prior to placing it on top of a rocket. + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-04-01 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +typedef struct _instanceData { +} instanceData; + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +BEGINdoAction +CODESTARTdoAction + write(1, (char*)ppString[0], strlen((char*)ppString[0])); +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omstdout:", sizeof(":omstdout:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omstdout:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr +ENDmodInit + +/* vi:set ai: + */ -- cgit v1.2.3 From 59d4a52c280c00bccde4be0321bb09665cc11d29 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Apr 2009 16:31:41 +0200 Subject: initial work on omprog, an output module to send messages to another program --- Makefile.am | 6 +- configure.ac | 16 ++++ doc/features.html | 1 - plugins/omdtn/Makefile.am | 8 -- plugins/omdtn/omdtn.c | 130 ----------------------------- plugins/omprog/Makefile.am | 8 ++ plugins/omprog/omprog.c | 204 +++++++++++++++++++++++++++++++++++++++++++++ runtime/conf.c | 1 - runtime/rsyslog.h | 2 + 9 files changed, 235 insertions(+), 141 deletions(-) delete mode 100644 plugins/omdtn/Makefile.am delete mode 100644 plugins/omdtn/omdtn.c create mode 100644 plugins/omprog/Makefile.am create mode 100644 plugins/omprog/omprog.c diff --git a/Makefile.am b/Makefile.am index 97f4aed3..02444d6f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -108,6 +108,10 @@ if ENABLE_MAIL SUBDIRS += plugins/ommail endif +if ENABLE_OMPROG +SUBDIRS += plugins/omprog +endif + if ENABLE_RFC3195 SUBDIRS += plugins/im3195 endif @@ -122,5 +126,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog ACLOCAL_AMFLAGS = -I m4 diff --git a/configure.ac b/configure.ac index de328f83..e9e68cb4 100644 --- a/configure.ac +++ b/configure.ac @@ -668,6 +668,20 @@ AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) # end of copy template - be sure to serach for imtemplate to find everything! +# settings for the omprog output module +AC_ARG_ENABLE(omprog, + [AS_HELP_STRING([--enable-omprog],[Compiles omprog template module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omprog="yes" ;; + no) enable_omprog="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omprog) ;; + esac], + [enable_omprog=no] +) +AM_CONDITIONAL(ENABLE_OMPROG, test x$enable_omprog = xyes) +# end of omprog + + # settings for the template output module; copy and modify this code # if you intend to add your own module. Be sure to replace omtemplate # by the actual name of your module. @@ -720,6 +734,7 @@ AC_CONFIG_FILES([Makefile \ plugins/imklog/Makefile \ plugins/imtemplate/Makefile \ plugins/omtemplate/Makefile \ + plugins/omprog/Makefile \ plugins/omstdout/Makefile \ plugins/imfile/Makefile \ plugins/imrelp/Makefile \ @@ -752,6 +767,7 @@ echo "imdiag enabled: $enable_imdiag" echo "file input module enabled: $enable_imfile" echo "input template module will be compiled: $enable_imtemplate" echo "output template module will be compiled: $enable_omtemplate" +echo "omprog module will be compiled: $enable_omprog" echo "omstdout module will be compiled: $enable_omstdout" echo "Large file support enabled: $enable_largefile" echo "Networking support enabled: $enable_inet" diff --git a/doc/features.html b/doc/features.html index 501f3304..336b31cc 100644 --- a/doc/features.html +++ b/doc/features.html @@ -124,7 +124,6 @@ community. Plus, it can be financially attractive: just think about how much les be to sponsor a feature instead of purchasing a commercial implementation. Also, the benefit of being recognised as a sponsor may even drive new customers to your business!
          -
        • Finalize the DTN "planetary Internet" space ship mode output plugin
        • port it to more *nix variants (eg AIX and HP UX) - this needs volunteers with access to those machines and knowledge
        • pcre filtering - maybe (depending on feedback)  - diff --git a/plugins/omdtn/Makefile.am b/plugins/omdtn/Makefile.am deleted file mode 100644 index afb57476..00000000 --- a/plugins/omdtn/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -pkglib_LTLIBRARIES = omdtn.la - -omdtn_la_SOURCES = omdtn.c -omdtn_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) -omdtn_la_LDFLAGS = -module -avoid-version -omdtn_la_LIBADD = - -EXTRA_DIST = diff --git a/plugins/omdtn/omdtn.c b/plugins/omdtn/omdtn.c deleted file mode 100644 index 761bde79..00000000 --- a/plugins/omdtn/omdtn.c +++ /dev/null @@ -1,130 +0,0 @@ -/* omdtn.c - * This is the plugin for rsyslog use in the interplanetary Internet, - * especially useful for rsyslog in space ships of all kinds. - * The core idea was introduced in early 2009 and considered - * doable. - * - * Note that this has not yet been tested for robustness but needs - * to prior to placing it on top of a rocket. - * - * NOTE: read comments in module-template.h for more specifics! - * - * File begun on 2009-04-01 by RGerhards - * - * Copyright 2009 Rainer Gerhards and Adiscon GmbH. - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see . - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - */ -#include "config.h" -#include "rsyslog.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include "dirty.h" -#include "syslogd-types.h" -#include "srUtils.h" -#include "template.h" -#include "module-template.h" -#include "errmsg.h" -#include "cfsysline.h" - -MODULE_TYPE_OUTPUT - -/* internal structures - */ -DEF_OMOD_STATIC_DATA - -typedef struct _instanceData { -} instanceData; - -BEGINcreateInstance -CODESTARTcreateInstance -ENDcreateInstance - - -BEGINisCompatibleWithFeature -CODESTARTisCompatibleWithFeature - if(eFeat == sFEATURERepeatedMsgReduction) - iRet = RS_RET_OK; -ENDisCompatibleWithFeature - - -BEGINfreeInstance -CODESTARTfreeInstance -ENDfreeInstance - - -BEGINdbgPrintInstInfo -CODESTARTdbgPrintInstInfo -ENDdbgPrintInstInfo - - -BEGINtryResume -CODESTARTtryResume -ENDtryResume - -BEGINdoAction -CODESTARTdoAction - write(1, (char*)ppString[0], strlen((char*)ppString[0])); -ENDdoAction - - -BEGINparseSelectorAct -CODESTARTparseSelectorAct -CODE_STD_STRING_REQUESTparseSelectorAct(1) - /* first check if this config line is actually for us */ - if(strncmp((char*) p, ":omstdout:", sizeof(":omstdout:") - 1)) { - ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); - } - - /* ok, if we reach this point, we have something for us */ - p += sizeof(":omstdout:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ - CHKiRet(createInstance(&pData)); - - /* check if a non-standard template is to be applied */ - if(*(p-1) == ';') - --p; - CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); -CODE_STD_FINALIZERparseSelectorAct -ENDparseSelectorAct - - -BEGINmodExit -CODESTARTmodExit -ENDmodExit - - -BEGINqueryEtryPt -CODESTARTqueryEtryPt -CODEqueryEtryPt_STD_OMOD_QUERIES -ENDqueryEtryPt - - -BEGINmodInit() -CODESTARTmodInit - *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ -CODEmodInit_QueryRegCFSLineHdlr -ENDmodInit - -/* vi:set ai: - */ diff --git a/plugins/omprog/Makefile.am b/plugins/omprog/Makefile.am new file mode 100644 index 00000000..63fe09b8 --- /dev/null +++ b/plugins/omprog/Makefile.am @@ -0,0 +1,8 @@ +pkglib_LTLIBRARIES = omprog.la + +omprog_la_SOURCES = omprog.c +omprog_la_CPPFLAGS = $(RSRT_CFLAGS) $(PTHREADS_CFLAGS) +omprog_la_LDFLAGS = -module -avoid-version +omprog_la_LIBADD = + +EXTRA_DIST = diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c new file mode 100644 index 00000000..0cdacf78 --- /dev/null +++ b/plugins/omprog/omprog.c @@ -0,0 +1,204 @@ +/* omprog.c + * This output plugin enables rsyslog to execute a program and + * feed it the message stream as standard input. + * + * NOTE: read comments in module-template.h for more specifics! + * + * File begun on 2009-04-01 by RGerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include "rsyslog.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "dirty.h" +#include "syslogd-types.h" +#include "srUtils.h" +#include "template.h" +#include "module-template.h" +#include "errmsg.h" +#include "cfsysline.h" + +MODULE_TYPE_OUTPUT + +/* internal structures + */ +DEF_OMOD_STATIC_DATA + +typedef struct _instanceData { + uchar *szBinary; /* name of binary to call */ + pid_t pid; /* pid of currently running process */ + int fdPipe; /* file descriptor to write to */ + int bIsRunning; /* is binary currently running? 0-no, 1-yes */ +} instanceData; + +/* config settings */ +static uchar *szBinary = NULL; /* name of binary to call */ + +BEGINcreateInstance +CODESTARTcreateInstance +ENDcreateInstance + + +BEGINisCompatibleWithFeature +CODESTARTisCompatibleWithFeature + if(eFeat == sFEATURERepeatedMsgReduction) + iRet = RS_RET_OK; +ENDisCompatibleWithFeature + + +BEGINfreeInstance +CODESTARTfreeInstance +ENDfreeInstance + + +BEGINdbgPrintInstInfo +CODESTARTdbgPrintInstInfo +ENDdbgPrintInstInfo + + +BEGINtryResume +CODESTARTtryResume +ENDtryResume + +/* creates a pipe and starts program, uses pipe as stdin for program. + * rgerhards, 2009-04-01 + */ +static rsRetVal +openPipe(instanceData *pData) +{ + int pipefd[2]; + pid_t cpid; + char *newargv[] = { NULL }; + char *newenviron[] = { NULL }; + DEFiRet; + + assert(pData != NULL); + + if(pipe(pipefd) == -1) { + ABORT_FINALIZE(RS_RET_ERR_CREAT_PIPE); + } + + cpid = fork(); + if(cpid == -1) { + ABORT_FINALIZE(RS_RET_ERR_FORK); + } + + if(cpid == 0) { + /* we are now the child, just set the right selectors and + * exec the binary. If that fails, there is not much we can do. + */ + fclose(stdin); + dup(pipefd[0]); + close(pipefd[1]); + //fclose(stdout); +fprintf(stderr, "Program to exec '%s', fdPipe: %d\n", pData->szBinary, pipefd[0]); + execve((char*)pData->szBinary, newargv, newenviron); + } + + pData->fdPipe = pipefd[1]; + pData->pid = cpid; + close(pipefd[0]); + pData->bIsRunning = 1; +finalize_it: + RETiRet; +} + + +BEGINdoAction +CODESTARTdoAction + if(pData->bIsRunning == 0) { + openPipe(pData); + } + + write(pData->fdPipe, (char*)ppString[0], strlen((char*)ppString[0])); +ENDdoAction + + +BEGINparseSelectorAct +CODESTARTparseSelectorAct +CODE_STD_STRING_REQUESTparseSelectorAct(1) + /* first check if this config line is actually for us */ + if(strncmp((char*) p, ":omprog:", sizeof(":omprog:") - 1)) { + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* ok, if we reach this point, we have something for us */ + p += sizeof(":omprog:") - 1; /* eat indicator sequence (-1 because of '\0'!) */ + CHKiRet(createInstance(&pData)); + + CHKmalloc(pData->szBinary = (uchar*) strdup((char*)szBinary)); + /* check if a non-standard template is to be applied */ + if(*(p-1) == ';') + --p; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); +CODE_STD_FINALIZERparseSelectorAct +ENDparseSelectorAct + + +BEGINmodExit +CODESTARTmodExit + if(szBinary != NULL) { + free(szBinary); + szBinary = NULL; + } +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_OMOD_QUERIES +ENDqueryEtryPt + + + +/* Reset config variables for this module to default values. + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + + if(szBinary != NULL) { + free(szBinary); + szBinary = NULL; + } + + RETiRet; +} + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ +CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomprogbinary", 0, eCmdHdlrGetWord, NULL, &szBinary, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); +CODEmodInit_QueryRegCFSLineHdlr +ENDmodInit + +/* vi:set ai: + */ diff --git a/runtime/conf.c b/runtime/conf.c index ede15cc7..27ab8bb4 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -796,7 +796,6 @@ dbgprintf("calling expression parser, pp %p ('%s')\n", *pline, *pline); /* debug support - print vmprg after construction (uncomment to use) */ /* vmprgDebugPrint(f->f_filterData.f_expr->pVmprg); */ - vmprgDebugPrint(f->f_filterData.f_expr->pVmprg); /* we now need to skip whitespace to the action part, else we confuse * the legacy rsyslog conf parser. -- rgerhards, 2008-02-25 diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 899f5e13..2aa1868b 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -258,6 +258,8 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_FUNC_MISSING_EXPR = -2111, /**< no expression after comma in function call (rainerscript) */ RS_RET_INVLD_NBR_ARGUMENTS = -2112, /**< invalid number of arguments for function call (rainerscript) */ RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ + RS_RET_ERR_CREAT_PIPE = -2114, /**< error during pipe creation */ + RS_RET_ERR_FORK = -2115, /**< error during fork() */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit v1.2.3 From 01f2c7a7a3ee394028eb36d9709490cbc26c7369 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Apr 2009 18:17:20 +0200 Subject: improved omprog, now ready for first practical testing --- ChangeLog | 3 + plugins/omprog/omprog.c | 169 +++++++++++++++++++++++++++++++++++++++++++++--- runtime/rsyslog.h | 1 + 3 files changed, 165 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 23a2e3e0..77ebdf83 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +- new feature: new output plugin omprog, which permits to start program + and feed it (via its stdin) with syslog messages. If the program + terminates, it is restarted. - bugfix: potential abort with DA queue after high watermark is reached There exists a race condition that can lead to a segfault. Thanks go to vbernetr, who performed the analysis and provided patch, which diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c index 0cdacf78..2a078a6d 100644 --- a/plugins/omprog/omprog.c +++ b/plugins/omprog/omprog.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "dirty.h" #include "syslogd-types.h" #include "srUtils.h" @@ -48,6 +49,7 @@ MODULE_TYPE_OUTPUT /* internal structures */ DEF_OMOD_STATIC_DATA +DEFobjCurrIf(errmsg) typedef struct _instanceData { uchar *szBinary; /* name of binary to call */ @@ -73,6 +75,8 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance + if(pData->szBinary != NULL) + free(pData->szBinary); ENDfreeInstance @@ -85,6 +89,52 @@ BEGINtryResume CODESTARTtryResume ENDtryResume + +/* execute the child process (must be called in child context + * after fork). + */ + +static void execBinary(instanceData *pData, int fdStdin) +{ + int i; + struct sigaction sigAct; + char *newargv[] = { NULL }; + char *newenviron[] = { NULL }; + + assert(pData != NULL); + + fclose(stdin); + dup(fdStdin); + //fclose(stdout); + + /* we close all file handles as we fork soon + * Is there a better way to do this? - mail me! rgerhards@adiscon.com + */ +# ifndef VALGRIND /* we can not use this with valgrind - too many errors... */ + for(i = 3 ; i <= 65535 ; ++i) + close(i); +# endif + + /* reset signal handlers to default */ + memset(&sigAct, 0, sizeof(sigAct)); + sigfillset(&sigAct.sa_mask); + sigAct.sa_handler = SIG_DFL; + for(i = 1 ; i < NSIG ; ++i) + sigaction(i, &sigAct, NULL); + + alarm(0); + + /* finally exec child */ + execve((char*)pData->szBinary, newargv, newenviron); + /* switch to? + execlp((char*)program, (char*) program, (char*)arg, NULL); + */ + + /* we should never reach this point, but if we do, we terminate */ + exit(1); +} + + /* creates a pipe and starts program, uses pipe as stdin for program. * rgerhards, 2009-04-01 */ @@ -93,8 +143,6 @@ openPipe(instanceData *pData) { int pipefd[2]; pid_t cpid; - char *newargv[] = { NULL }; - char *newenviron[] = { NULL }; DEFiRet; assert(pData != NULL); @@ -103,6 +151,10 @@ openPipe(instanceData *pData) ABORT_FINALIZE(RS_RET_ERR_CREAT_PIPE); } + DBGPRINTF("executing program '%s'\n", pData->szBinary); + + /* NO OUTPUT AFTER FORK! */ + cpid = fork(); if(cpid == -1) { ABORT_FINALIZE(RS_RET_ERR_FORK); @@ -112,14 +164,12 @@ openPipe(instanceData *pData) /* we are now the child, just set the right selectors and * exec the binary. If that fails, there is not much we can do. */ - fclose(stdin); - dup(pipefd[0]); close(pipefd[1]); - //fclose(stdout); -fprintf(stderr, "Program to exec '%s', fdPipe: %d\n", pData->szBinary, pipefd[0]); - execve((char*)pData->szBinary, newargv, newenviron); + execBinary(pData, pipefd[0]); + /*NO CODE HERE - WILL NEVER BE REACHED!*/ } + DBGPRINTF("child has pid %d\n", cpid); pData->fdPipe = pipefd[1]; pData->pid = cpid; close(pipefd[0]); @@ -129,13 +179,113 @@ finalize_it: } +/* clean up after a terminated child + */ +static inline rsRetVal +cleanup(instanceData *pData) +{ + int status; + int ret; + char errStr[1024]; + DEFiRet; + + assert(pData != NULL); + assert(pData->bIsRunning == 1); +RUNLOG_VAR("%d", pData->pid); + ret = waitpid(pData->pid, &status, 0); + if(ret != pData->pid) { + /* if waitpid() fails, we can not do much - try to ignore it... */ + DBGPRINTF("waitpid() returned state %d[%s], future malfunction may happen\n", ret, + rs_strerror_r(errno, errStr, sizeof(errStr))); + } else { + /* check if we should print out some diagnostic information */ + DBGPRINTF("waitpid status return for program '%s': %2.2x\n", + pData->szBinary, status); + if(WIFEXITED(status)) { + errmsg.LogError(0, NO_ERRCODE, "program '%s' exited normally, state %d", + pData->szBinary, WEXITSTATUS(status)); + } else if(WIFSIGNALED(status)) { + errmsg.LogError(0, NO_ERRCODE, "program '%s' terminated by signal %d.", + pData->szBinary, WTERMSIG(status)); + } + } + + pData->bIsRunning = 0; + RETiRet; +} + + +/* try to restart the binary when it has stopped. + */ +static inline rsRetVal +tryRestart(instanceData *pData) +{ + DEFiRet; + assert(pData != NULL); + assert(pData->bIsRunning == 0); + + iRet = openPipe(pData); + RETiRet; +} + + +/* write to pipe + * note that we do not try to run block-free. If the users fears something + * may block (and this not be acceptable), the action should be run on its + * own action queue. + */ +static rsRetVal +writePipe(instanceData *pData, uchar *szMsg) +{ + int lenWritten; + int lenWrite; + int writeOffset; + char errStr[1024]; + DEFiRet; + + assert(pData != NULL); + + lenWrite = strlen((char*)szMsg); + writeOffset = 0; + + do + { + lenWritten = write(pData->fdPipe, ((char*)szMsg)+writeOffset, lenWrite); + if(lenWritten == -1) { + switch(errno) { + case EPIPE: + DBGPRINTF("Program '%s' terminated, trying to restart\n", + pData->szBinary); + CHKiRet(cleanup(pData)); + CHKiRet(tryRestart(pData)); + break; + default: + DBGPRINTF("error %d writing to pipe: %s\n", errno, + rs_strerror_r(errno, errStr, sizeof(errStr))); + ABORT_FINALIZE(RS_RET_ERR_WRITE_PIPE); + break; + } + } else { + writeOffset += lenWritten; + } + } while(lenWritten != lenWrite); + + +finalize_it: + RETiRet; +} + + BEGINdoAction CODESTARTdoAction if(pData->bIsRunning == 0) { openPipe(pData); } + + iRet = writePipe(pData, ppString[0]); - write(pData->fdPipe, (char*)ppString[0], strlen((char*)ppString[0])); + if(iRet != RS_RET_OK) + iRet = RS_RET_SUSPENDED; ENDdoAction @@ -166,6 +316,8 @@ CODESTARTmodExit free(szBinary); szBinary = NULL; } + CHKiRet(objRelease(errmsg, CORE_COMPONENT)); +finalize_it: ENDmodExit @@ -195,6 +347,7 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomprogbinary", 0, eCmdHdlrGetWord, NULL, &szBinary, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); CODEmodInit_QueryRegCFSLineHdlr diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 2aa1868b..ac9b726f 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -260,6 +260,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ RS_RET_ERR_CREAT_PIPE = -2114, /**< error during pipe creation */ RS_RET_ERR_FORK = -2115, /**< error during fork() */ + RS_RET_ERR_WRITE_PIPE = -2116, /**< error writing to pipe */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit v1.2.3 From d702d3f6ff1540691aae29012dd142212e0eb04c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Mon, 30 Mar 2009 10:39:36 +0200 Subject: Make tryResume not to retry the last action, but just to reconnect. The core will call the action if tryResume succeeds, no need to make it from here. --- plugins/omoracle/omoracle.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index eba45c5b..29ec2303 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -174,7 +174,7 @@ CODESTARTtryResume * ... of course I don't know why Oracle might need a full restart... * rgerhards, 2009-03-26 */ - dbgprintf("Attempting to restart the last action\n"); + dbgprintf("Attempting to reconnect to DB server\n"); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); CHECKERR(pData->error, OCISessionGet(pData->environment, pData->error, @@ -182,9 +182,6 @@ CODESTARTtryResume pData->connection, strlen(pData->connection), NULL, 0, NULL, NULL, NULL, OCI_DEFAULT)); - CHECKERR(pData->error, OCIStmtExecute(pData->service, pData->statement, - pData->error, 1, 0, NULL, NULL, - OCI_DEFAULT)); finalize_it: ENDtryResume -- cgit v1.2.3 From 56bf679723f2821d0f66339b95d847f1e4ddc17b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Mon, 30 Mar 2009 10:39:37 +0200 Subject: Add a SELinux policy that allows the module to load on RHEL5. --- plugins/omoracle/omoracle.te | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 plugins/omoracle/omoracle.te diff --git a/plugins/omoracle/omoracle.te b/plugins/omoracle/omoracle.te new file mode 100644 index 00000000..81eb6cf1 --- /dev/null +++ b/plugins/omoracle/omoracle.te @@ -0,0 +1,13 @@ + +module omoracle 1.0; + +require { + type syslogd_t; + type port_t; + class process { execstack execmem }; + class tcp_socket name_connect; +} + +#============= syslogd_t ============== +allow syslogd_t port_t:tcp_socket name_connect; +allow syslogd_t self:process { execstack execmem }; -- cgit v1.2.3 From 70b8624c86e7d204d7c1ff91d030ee7c69569eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 1 Apr 2009 18:12:46 +0200 Subject: Convert the module configuration to $Action... directives. Instead of using the old-style configuration parameters, use $... directives, which lead to simpler code, and also should make user's configurations simpler. Needs some testing. Currently, the supported directives are $OmoracleDB, $OmoracleDBUser and $OmoracleDBPassword. $OmoracleDBStatement and $OmoracleDBBatchSize may follow. --- plugins/omoracle/omoracle.c | 66 +++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 29ec2303..6ec023e2 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -4,10 +4,18 @@ database. It uses Oracle Call Interface, a propietary module provided by Oracle. - Config lines to be used are of this form: + Selector lines to be used are of this form: - :omoracle:dbstring,user,password;StatementTemplate + :omoracle:;TemplateName + The module gets its configuration via rsyslog $... directives, + namely: + + $OmoracleDBUser: user name to log in on the database. + $OmoracleDBPassword: password to log in on the database. + $OmoracleDB: connection string (an Oracle easy connect or a db + name as specified by tnsnames.ora) + All fields are mandatory. The dbstring can be an Oracle easystring or a DB name, as present in the tnsnames.ora file. @@ -61,6 +69,14 @@ typedef struct _instanceData { char* connection; } instanceData; +/** Database name, to be filled by the $OmoracleDB directive */ +static char* db_name; +/** Database user name, to be filled by the $OmoracleDBUser + * directive */ +static char* db_user; +/** Database password, to be filled by the $OmoracleDBPassword */ +static char* db_password; + /** Generic function for handling errors from OCI. It will be called only inside CHECKERR and CHECKENV macros. @@ -203,7 +219,8 @@ static rsRetVal startSession(instanceData* pData, char* connection, char* user, OCI_DEFAULT)); finalize_it: if (iRet != RS_RET_OK) - errmsg.LogError(0, NO_ERRCODE, "Unable to start Oracle session\n"); + errmsg.LogError(0, NO_ERRCODE, + "Unable to start Oracle session\n"); RETiRet; } @@ -217,10 +234,6 @@ ENDisCompatibleWithFeature BEGINparseSelectorAct - char user[_DB_MAXUNAMELEN]; - char pwd[_DB_MAXPWDLEN]; - char connection_string[MAXHOSTNAMELEN]; - CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); @@ -230,37 +243,26 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); p += sizeof ":omoracle:" - 1; - /* while this parameter parsing is convenient and works perfectly, - * it is suggested that parameters are only specified via $Action... config - * statement (as done in omlibdbi). The reason is that this may greatly - * ease the transition when we have the full config script language. However, - * this approach here is guranteed to continue to work in the future. - * rgerhards, 2009-03-26 - */ if (*p == '\0' || *p == ',') { - errmsg.LogError(0, NO_ERRCODE, "Wrong char processing module arguments: %c\n", *p); + errmsg.LogError(0, NO_ERRCODE, + "Wrong char processing module arguments: %c\n", + *p); ABORT_FINALIZE(RS_RET_INVALID_PARAMS); } - CHKiRet(getSubString(&p, connection_string, MAXHOSTNAMELEN, ',')); - CHKiRet(getSubString(&p, user, _DB_MAXUNAMELEN, ',')); - CHKiRet(getSubString(&p, pwd, _DB_MAXPWDLEN, ';')); - p--; CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_RQD_TPL_OPT_SQL, " StdFmt")); CHKiRet(createInstance(&pData)); - pData->connection = strdup(connection_string); + pData->connection = strdup(db_name); if (pData->connection == NULL) { iRet = RS_RET_OUT_OF_MEMORY; goto finalize_it; } - CHKiRet(startSession(pData, connection_string, user, pwd)); + CHKiRet(startSession(pData, db_name, db_user, db_password)); dbgprintf ("omoracle module got all its resources allocated " "and connected to the DB\n"); - memset(user, 0, sizeof user); - memset(pwd, 0, sizeof pwd); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -300,7 +302,16 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { + int n; DEFiRet; + free(db_user); + free(db_name); + if (db_password != NULL) { + n = strlen(db_password); + memset(db_password, n, sizeof *db_password); + free(db_password); + } + db_name = db_user = db_password = NULL; RETiRet; } @@ -313,4 +324,13 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledbuser", 0, + eCmdHdlrGetWord, NULL, &db_user, + STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledbpassword", 0, + eCmdHdlrGetWord, NULL, &db_password, + STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledb", 0, + eCmdHdlrGetWord, NULL, &db_name, + STD_LOADABLE_MODULE_ID)); ENDmodInit -- cgit v1.2.3 From 8a819d6a0623c0854462dd2dc632700f9a5c576c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Apr 2009 18:32:18 +0200 Subject: some small changes (as suggestion) --- plugins/omoracle/omoracle.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 6ec023e2..ea910d3a 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -253,11 +253,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_RQD_TPL_OPT_SQL, " StdFmt")); CHKiRet(createInstance(&pData)); - pData->connection = strdup(db_name); - if (pData->connection == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - goto finalize_it; - } + CHKmalloc(pData->connection = strdup(db_name)); CHKiRet(startSession(pData, db_name, db_user, db_password)); dbgprintf ("omoracle module got all its resources allocated " @@ -304,11 +300,13 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, { int n; DEFiRet; - free(db_user); - free(db_name); + if(db_user != NULL) + free(db_user); + if(db_name != NULL) + free(db_name); if (db_password != NULL) { n = strlen(db_password); - memset(db_password, n, sizeof *db_password); + memset(db_password, 0, n); free(db_password); } db_name = db_user = db_password = NULL; -- cgit v1.2.3 From 3954f2e166c3cbd78c71819c8d6c25120042dbcf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 12:48:07 +0200 Subject: added new "csv" property replacer option to enable simple creation of CSV-formatted outputs (format from RFC4180 is used) --- ChangeLog | 2 ++ doc/property_replacer.html | 14 +++++++++++++- runtime/msg.c | 34 ++++++++++++++++++++++++++++++++++ template.c | 5 +++++ template.h | 7 ++++--- 5 files changed, 58 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 23a2e3e0..42fda5b3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,6 +4,8 @@ I only tweaked a very little bit. --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? +- added new "csv" property replacer options to enable simple creation + of CSV-formatted outputs (format from RFC4180 is used) - implemented function support in RainerScript. That means the engine parses and compile functions, as well as executes a few build-in ones. Dynamic loading and registration of functions is not yet diff --git a/doc/property_replacer.html b/doc/property_replacer.html index baf053f2..a6e9b518 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -314,6 +314,18 @@ case-insensitive. Currently, the following options are defined: convert property text to uppercase only +csv +formats the resulting field (after all modifications) in CSV format +as specified in RFC 4180. +Rsyslog will always use double quotes. Note that in order to have full CSV-formatted +text, you need to define a proper template. An example is this one: +
          $template csvline,"%syslogtag:::csv%,%msg:::csv%" +
          Most importantly, you need to provide the commas between the fields +inside the template. +
          This feature was introduced in rsyslog 4.1.6. + + + drop-last-lf The last LF in the message (if any), is dropped. Especially useful for PIX. @@ -411,7 +423,7 @@ syntax, this is where you actually use the property replacer.
          • [rsyslog site]

          This documentation is part of the rsyslog project.
          -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL version 2 or higher.

          diff --git a/runtime/msg.c b/runtime/msg.c index 9aa2ce84..5d1f21fd 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -2405,6 +2405,40 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } } + /* finally, we need to check if the property should be formatted in CSV + * format (we use RFC 4180, and always use double quotes). As of this writing, + * this should be the last action carried out on the property, but in the + * future there may be reasons to change that. -- rgerhards, 2009-04-02 + */ + if(pTpe->data.field.options.bCSV) { + /* we need to obtain a private copy, as we need to at least add the double quotes */ + int iBufLen = strlen(pRes); + char *pBStart; + char *pDst; + char *pSrc; + /* the malloc may be optimized, we currently use the worst case... */ + pBStart = pDst = malloc((2 * iBufLen + 3) * sizeof(char)); + if(pDst == NULL) { + if(*pbMustBeFreed == 1) + free(pRes); + *pbMustBeFreed = 0; + return "**OUT OF MEMORY**"; + } + pSrc = pRes; + *pDst++ = '"'; /* starting quote */ + while(*pSrc) { + if(*pSrc == '"') + *pDst++ = '"'; /* need to add double double quote (see RFC4180) */ + *pDst++ = *pSrc++; + } + *pDst++ = '"'; /* ending quote */ + *pDst = '\0'; + if(*pbMustBeFreed == 1) + free(pRes); + pRes = pBStart; + *pbMustBeFreed = 1; + } + /*dbgprintf("MsgGetProp(\"%s\"): \"%s\"\n", pName, pRes); only for verbose debug logging */ return(pRes); } diff --git a/template.c b/template.c index 6fb7ba2b..4f507ff6 100644 --- a/template.c +++ b/template.c @@ -460,6 +460,8 @@ static void doOptions(unsigned char **pp, struct templateEntry *pTpe) pTpe->data.field.options.bSecPathDrop = 1; } else if(!strcmp((char*)Buf, "secpath-replace")) { pTpe->data.field.options.bSecPathReplace = 1; + } else if(!strcmp((char*)Buf, "csv")) { + pTpe->data.field.options.bCSV = 1; } else { dbgprintf("Invalid field option '%s' specified - ignored.\n", Buf); } @@ -1105,6 +1107,9 @@ void tplPrintList(void) if(pTpe->data.field.options.bSPIffNo1stSP) { dbgprintf("[SP iff no first SP] "); } + if(pTpe->data.field.options.bCSV) { + dbgprintf("[format as CSV (RFC4180)]"); + } if(pTpe->data.field.options.bDropLastLF) { dbgprintf("[drop last LF in msg] "); } diff --git a/template.h b/template.h index 04137b09..6ca8dc6c 100644 --- a/template.h +++ b/template.h @@ -94,9 +94,10 @@ struct templateEntry { unsigned bSpaceCC: 1; /* change control characters to spaceescape? */ unsigned bEscapeCC: 1; /* escape control characters? */ unsigned bDropLastLF: 1; /* drop last LF char in msg (PIX!) */ - unsigned bSecPathDrop: 1; /* drop slashes, replace dots, empty string */ - unsigned bSecPathReplace: 1; /* replace slashes, replace dots, empty string */ - unsigned bSPIffNo1stSP: 1; /* replace slashes, replace dots, empty string */ + unsigned bSecPathDrop: 1; /* drop slashes, replace dots, empty string */ + unsigned bSecPathReplace: 1; /* replace slashes, replace dots, empty string */ + unsigned bSPIffNo1stSP: 1; /* replace slashes, replace dots, empty string */ + unsigned bCSV: 1; /* format field in CSV (RFC 4180) format */ } options; /* options as bit fields */ } field; } data; -- cgit v1.2.3 From 85c09e0c44c00be1cd91955b39fc6d0a17c72a98 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 12:57:37 +0200 Subject: add csv support to feature sheet --- doc/features.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/features.html b/doc/features.html index 501f3304..17a995bf 100644 --- a/doc/features.html +++ b/doc/features.html @@ -95,6 +95,9 @@ via custom plugins
        • an easy-to-write to plugin interface
        • ability to send SNMP trap messages
        • ability to filter out messages based on sequence of arrival
          • +
        • support for comma-seperated-values (CSV) output generation +(via the "csv" property replace option). The +CSV format supported is that from RFC 4180.
        • support for arbitrary complex boolean, string and arithmetic expressions in message filters
        -- cgit v1.2.3 From eb807027af9e126a212b0630c5873dddae48963b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 15:12:57 +0200 Subject: added O_CLOEXEC to open() calls to make sure only the minimum number of file handles is left open during a exec call. This is not a 100% solution, as there are also some fopen() calls and, more importantly, file descriptors opened by libraries. But it is better than nothing (and it was quick, at least until we run into platform hell, what we will for sure ;)). --- plugins/imklog/linux.c | 2 +- runtime/debug.c | 2 +- tools/omfile.c | 12 ++++++------ tools/pidfile.c | 2 +- tools/syslogd.c | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/imklog/linux.c b/plugins/imklog/linux.c index 198b7c0e..0dd4320d 100644 --- a/plugins/imklog/linux.c +++ b/plugins/imklog/linux.c @@ -144,7 +144,7 @@ static enum LOGSRC GetKernelLogSrc(void) return(kernel); } - if ( (kmsg = open(_PATH_KLOG, O_RDONLY)) < 0 ) + if ( (kmsg = open(_PATH_KLOG, O_RDONLY|O_CLOEXEC)) < 0 ) { imklogLogIntMsg(LOG_ERR, "imklog: Cannot open proc file system, %d.\n", errno); ksyslog(7, NULL, 0); /* TODO: check this, implement more */ diff --git a/runtime/debug.c b/runtime/debug.c index 96004e47..4ee90226 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -1343,7 +1343,7 @@ rsRetVal dbgClassInit(void) if(pszAltDbgFileName != NULL) { /* we have a secondary file, so let's open it) */ - if((altdbg = open(pszAltDbgFileName, O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY, S_IRUSR|S_IWUSR)) == -1) { + if((altdbg = open(pszAltDbgFileName, O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY|O_CLOEXEC, S_IRUSR|S_IWUSR)) == -1) { fprintf(stderr, "alternate debug file could not be opened, ignoring. Error: %s\n", strerror(errno)); } } diff --git a/tools/omfile.c b/tools/omfile.c index 5141b4da..36f160d1 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -301,7 +301,7 @@ int resolveFileSizeLimit(instanceData *pData) free(pCmd); - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); actualFileSize = lseek(pData->fd, 0, SEEK_END); @@ -394,13 +394,13 @@ prepareFile(instanceData *pData, uchar *newFileName) { DEFiRet; if(pData->fileType == eTypePIPE) { - pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK); + pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); FINALIZE; /* we are done in this case */ } if(access((char*)newFileName, F_OK) == 0) { /* file already exists */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); } else { pData->fd = -1; @@ -419,7 +419,7 @@ prepareFile(instanceData *pData, uchar *newFileName) /* no matter if we needed to create directories or not, we now try to create * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); if(pData->fd != -1) { /* check and set uid/gid */ @@ -653,7 +653,7 @@ again: && e == EBADF #endif ) { - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY); + pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY|O_CLOEXEC); if (pData->fd < 0) { iRet = RS_RET_SUSPENDED; errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); @@ -742,7 +742,7 @@ CODESTARTparseSelectorAct pData->bDynamicName = 0; pData->fCreateMode = fCreateMode; /* preserve current setting */ pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */ - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY, + pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); } break; diff --git a/tools/pidfile.c b/tools/pidfile.c index 2be13da6..e7744513 100644 --- a/tools/pidfile.c +++ b/tools/pidfile.c @@ -97,7 +97,7 @@ int write_pid (char *pidfile) int fd; int pid; - if ( ((fd = open(pidfile, O_RDWR|O_CREAT, 0644)) == -1) + if ( ((fd = open(pidfile, O_RDWR|O_CREAT|O_CLOEXEC, 0644)) == -1) || ((f = fdopen(fd, "r+")) == NULL) ) { fprintf(stderr, "Can't open or create %s.\n", pidfile); return 0; diff --git a/tools/syslogd.c b/tools/syslogd.c index c72160fb..16f255ea 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -549,7 +549,7 @@ void untty(void) int i; if ( !Debug ) { - i = open(_PATH_TTY, O_RDWR); + i = open(_PATH_TTY, O_RDWR|O_CLOEXEC); if (i >= 0) { # if !defined(__hpux) (void) ioctl(i, (int) TIOCNOTTY, (char *)0); -- cgit v1.2.3 From 8de35eaa2c9017df885dfd071a89288ca6a0e3af Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 15:37:17 +0200 Subject: clean compile on solaris --- runtime/cfsysline.c | 4 ++-- runtime/rsyslog.h | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index c4490b48..e1e4a6a4 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -364,7 +364,7 @@ static rsRetVal doGetGID(uchar **pp, rsRetVal (*pSetHdlr)(void*, uid_t), void *p /* we set value via a set function */ CHKiRet(pSetHdlr(pVal, pgBuf->gr_gid)); } - dbgprintf("gid %d obtained for group '%s'\n", pgBuf->gr_gid, szName); + dbgprintf("gid %d obtained for group '%s'\n", (int) pgBuf->gr_gid, szName); } skipWhiteSpace(pp); /* skip over any whitespace */ @@ -406,7 +406,7 @@ static rsRetVal doGetUID(uchar **pp, rsRetVal (*pSetHdlr)(void*, uid_t), void *p /* we set value via a set function */ CHKiRet(pSetHdlr(pVal, ppwBuf->pw_uid)); } - dbgprintf("uid %d obtained for user '%s'\n", ppwBuf->pw_uid, szName); + dbgprintf("uid %d obtained for user '%s'\n", (int) ppwBuf->pw_uid, szName); } skipWhiteSpace(pp); /* skip over any whitespace */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 899f5e13..032d8c04 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -341,6 +341,11 @@ typedef enum rsObjectID rsObjID; # define __attribute__(x) /*NOTHING*/ #endif +#ifndef O_CLOEXEC +/* of course, this limits the functionality... */ +# define O_CLOEXEC 0 +#endif + /* The following prototype is convenient, even though it may not be the 100% correct place.. -- rgerhards 2008-01-07 */ void dbgprintf(char *, ...) __attribute__((format(printf, 1, 2))); -- cgit v1.2.3 From a86e42028afeba1daca262b590bfd49d9c393b90 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 16:16:57 +0200 Subject: improved performance of regexp-based filters Thanks to Arnaud Cornet for providing the idea and initial patch. --- runtime/stringbuf.c | 30 ++++++++++++++++++++++++++++++ runtime/stringbuf.h | 2 ++ tools/syslogd.c | 8 ++++++-- tools/syslogd.h | 1 + 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index a5dc625a..c0a19ae4 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -724,6 +724,36 @@ finalize_it: RETiRet; } +/* same as above, only not braindead */ +int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void **rc) +{ + int ret; + + BEGINfunc + + if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { + regex_t **cache = rc; + if (*cache == NULL) { + *cache = calloc(sizeof(regex_t), 1); + regexp.regcomp(*cache, (char*) rsCStrGetSzStr(pCS1), 0); + } + ret = regexp.regexec(*cache, (char*) psz, 0, NULL, 0); + } else { + ret = 1; /* simulate "not found" */ + } + + ENDfunc + return ret; +} + +/* free a cached compiled regex */ +void rsRegexDestruct(void **rc) { + regex_t **cache = rc; + regexp.regfree(*cache); + free(*cache); + *cache = NULL; +} + /* compare a rsCStr object with a classical sz string. This function * is almost identical to rsCStrZsStrCmp(), but it also takes an offset diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index f3e08439..4b0fb065 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -137,6 +137,8 @@ int rsCStrStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrSzStrStartsWithCStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType); +int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void **cache); +void rsRegexDestruct(void **rc); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); diff --git a/tools/syslogd.c b/tools/syslogd.c index 16f255ea..c7f36b45 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -432,6 +432,8 @@ selectorDestruct(void *pVal) } else if(pThis->f_filter_type == FILTER_EXPR) { if(pThis->f_filterData.f_expr != NULL) expr.Destruct(&pThis->f_filterData.f_expr); + if(pThis->regex_cache != NULL) + rsRegexDestruct(&pThis->regex_cache); } llDestroy(&pThis->llActList); @@ -1075,8 +1077,10 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce bRet = 1; /* process message! */ break; case FIOP_REGEX: - if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 0) == RS_RET_OK) + //TODO REGEX: this needs to be merged with new functionality below + //rgerhards, 2009-04-02 + if(rsCStrSzStrMatchRegexCache(f->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, &f->regex_cache) == 0) bRet = 1; break; case FIOP_EREREGEX: diff --git a/tools/syslogd.h b/tools/syslogd.h index f1b11a91..ecaaec34 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -80,6 +80,7 @@ struct filed { } f_filterData; linkedList_t llActList; /* list of configured actions */ +regex_t *regex_cache; }; -- cgit v1.2.3 From 1d16216aa326296673cc8520a8df351c4d492dfe Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 16:51:53 +0200 Subject: streamlined regex patch - abided to code conventions - fixed a potential segfault when regex library can not be loaded --- runtime/stringbuf.c | 61 ++++++++++++++++++++++++++++++++--------------------- runtime/stringbuf.h | 4 ++-- tools/syslogd.c | 6 +++--- tools/syslogd.h | 2 +- 4 files changed, 43 insertions(+), 30 deletions(-) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index c0a19ae4..4a7cc4bd 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -725,33 +725,46 @@ finalize_it: } /* same as above, only not braindead */ -int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void **rc) +int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void *rc) { - int ret; - - BEGINfunc - - if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { - regex_t **cache = rc; - if (*cache == NULL) { - *cache = calloc(sizeof(regex_t), 1); - regexp.regcomp(*cache, (char*) rsCStrGetSzStr(pCS1), 0); - } - ret = regexp.regexec(*cache, (char*) psz, 0, NULL, 0); - } else { - ret = 1; /* simulate "not found" */ - } - - ENDfunc - return ret; + int ret; + regex_t **cache = (regex_t**) rc; + + BEGINfunc + + assert(cache != NULL); + + if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { + if (*cache == NULL) { + *cache = calloc(sizeof(regex_t), 1); + regexp.regcomp(*cache, (char*) rsCStrGetSzStr(pCS1), 0); + } + ret = regexp.regexec(*cache, (char*) psz, 0, NULL, 0); + } else { + ret = 1; /* simulate "not found" */ + } + + ENDfunc + return ret; } -/* free a cached compiled regex */ -void rsRegexDestruct(void **rc) { - regex_t **cache = rc; - regexp.regfree(*cache); - free(*cache); - *cache = NULL; + +/* free a cached compiled regex + * Caller must provide a pointer to a buffer that was created by + * rsCStrSzStrMatchRegexCache() + */ +void rsCStrRegexDestruct(void *rc) +{ + regex_t **cache = rc; + + assert(cache != NULL); + assert(*cache != NULL); + + if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { + regexp.regfree(*cache); + free(*cache); + *cache = NULL; + } } diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 4b0fb065..311d7f41 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -137,8 +137,8 @@ int rsCStrStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrSzStrStartsWithCStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType); -int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void **cache); -void rsRegexDestruct(void **rc); +int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void *cache); +void rsCStrRegexDestruct(void *rc); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); diff --git a/tools/syslogd.c b/tools/syslogd.c index c7f36b45..d5429855 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -429,11 +429,11 @@ selectorDestruct(void *pVal) rsCStrDestruct(&pThis->f_filterData.prop.pCSPropName); if(pThis->f_filterData.prop.pCSCompValue != NULL) rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); + if(pThis->f_filterData.prop.regex_cache != NULL) + rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache); } else if(pThis->f_filter_type == FILTER_EXPR) { if(pThis->f_filterData.f_expr != NULL) expr.Destruct(&pThis->f_filterData.f_expr); - if(pThis->regex_cache != NULL) - rsRegexDestruct(&pThis->regex_cache); } llDestroy(&pThis->llActList); @@ -1080,7 +1080,7 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce //TODO REGEX: this needs to be merged with new functionality below //rgerhards, 2009-04-02 if(rsCStrSzStrMatchRegexCache(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, &f->regex_cache) == 0) + (unsigned char*) pszPropVal, &f->f_filterData.prop.regex_cache) == 0) bRet = 1; break; case FIOP_EREREGEX: diff --git a/tools/syslogd.h b/tools/syslogd.h index ecaaec34..8b9bd131 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -73,6 +73,7 @@ struct filed { FIOP_REGEX = 4, /* matches a (BRE) regular expression? */ FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ } operation; + regex_t *regex_cache; /* cache for compiled REs, if such are used */ cstr_t *pCSCompValue; /* value to "compare" against */ char isNegated; /* actually a boolean ;) */ } prop; @@ -80,7 +81,6 @@ struct filed { } f_filterData; linkedList_t llActList; /* list of configured actions */ -regex_t *regex_cache; }; -- cgit v1.2.3 From 4ab540e3ba25a13fd079490ac52438e55dc92672 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Apr 2009 17:54:48 +0200 Subject: fully integrated regex patch Now have removed the previous method, as really nobody should call it any longer (and now nobody does ;)). Also did some other cleanup. --- ChangeLog | 2 ++ runtime/stringbuf.c | 41 +++++++++++++---------------------------- runtime/stringbuf.h | 3 +-- tools/syslogd.c | 8 +++----- 4 files changed, 19 insertions(+), 35 deletions(-) diff --git a/ChangeLog b/ChangeLog index 42fda5b3..fc9f807f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -34,6 +34,8 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? the full high availability features of rsyslog's engine - bugfix: fixed some segaults on Solaris, where vsprintf() does not check for NULL pointers +- improved performance of regexp-based filters + Thanks to Arnaud Cornet for providing the idea and initial patch. --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 4a7cc4bd..35ec44c6 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -703,49 +703,34 @@ int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz) * never is a \0 *inside* a property string. * Note that the function returns -1 if regexp functionality is not available. * rgerhards: 2009-03-04: ERE support added, via parameter iType: 0 - BRE, 1 - ERE + * Arnaud Cornet/rgerhards: 2009-04-02: performance improvement by caching compiled regex + * If a caller does not need the cached version, it must still provide memory for it + * and must call rsCStrRegexDestruct() afterwards. */ -rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType) +rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType, void *rc) { - regex_t preq; + regex_t **cache = (regex_t**) rc; int ret; DEFiRet; - if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { - regexp.regcomp(&preq, (char*) rsCStrGetSzStr(pCS1), (iType == 1 ? REG_EXTENDED : 0) | REG_NOSUB); - ret = regexp.regexec(&preq, (char*) psz, 0, NULL, 0); - regexp.regfree(&preq); - if(ret != 0) - ABORT_FINALIZE(RS_RET_NOT_FOUND); - } else { - ABORT_FINALIZE(RS_RET_NOT_FOUND); - } - -finalize_it: - RETiRet; -} - -/* same as above, only not braindead */ -int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void *rc) -{ - int ret; - regex_t **cache = (regex_t**) rc; - - BEGINfunc - + assert(pCS1 != NULL); + assert(psz != NULL); assert(cache != NULL); if(objUse(regexp, LM_REGEXP_FILENAME) == RS_RET_OK) { if (*cache == NULL) { *cache = calloc(sizeof(regex_t), 1); - regexp.regcomp(*cache, (char*) rsCStrGetSzStr(pCS1), 0); + regexp.regcomp(*cache, (char*) rsCStrGetSzStr(pCS1), (iType == 1 ? REG_EXTENDED : 0) | REG_NOSUB); } ret = regexp.regexec(*cache, (char*) psz, 0, NULL, 0); + if(ret != 0) + ABORT_FINALIZE(RS_RET_NOT_FOUND); } else { - ret = 1; /* simulate "not found" */ + ABORT_FINALIZE(RS_RET_NOT_FOUND); } - ENDfunc - return ret; +finalize_it: + RETiRet; } diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 311d7f41..684133bb 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -136,8 +136,7 @@ int rsCStrCaseInsensitiveLocateInSzStr(cstr_t *pThis, uchar *sz); int rsCStrStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrCaseInsensitveStartsWithSzStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrSzStrStartsWithCStr(cstr_t *pCS1, uchar *psz, size_t iLenSz); -rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType); -int rsCStrSzStrMatchRegexCache(cstr_t *pCS1, uchar *psz, void *cache); +rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType, void *cache); void rsCStrRegexDestruct(void *rc); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); diff --git a/tools/syslogd.c b/tools/syslogd.c index d5429855..b23c12a7 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1077,15 +1077,13 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce bRet = 1; /* process message! */ break; case FIOP_REGEX: - //TODO REGEX: this needs to be merged with new functionality below - //rgerhards, 2009-04-02 - if(rsCStrSzStrMatchRegexCache(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, &f->f_filterData.prop.regex_cache) == 0) + if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, 0, &f->f_filterData.prop.regex_cache) == RS_RET_OK) bRet = 1; break; case FIOP_EREREGEX: if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 1) == RS_RET_OK) + (unsigned char*) pszPropVal, 1, &f->f_filterData.prop.regex_cache) == RS_RET_OK) bRet = 1; break; default: -- cgit v1.2.3 From ec0e2c3e7df6addc02431628daddfeae49b92af7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 3 Apr 2009 12:51:02 +0200 Subject: added a new way how output plugins may be passed parameters. This is more efficient for some outputs. They new can receive fields not only as a single string but rather in an array where each string is seperated. --- ChangeLog | 3 ++ action.c | 32 +++++++++++++++-- action.h | 2 ++ doc/rsyslog_conf_modules.html | 7 ++-- plugins/omstdout/omstdout.c | 80 +++++++++++++++++++++++++++++++++++++++++-- runtime/modules.c | 2 ++ runtime/objomsr.c | 18 ++++++++++ runtime/objomsr.h | 6 ++-- template.c | 56 ++++++++++++++++++++++++++++++ template.h | 1 + 10 files changed, 197 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index fc9f807f..7f79271f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -36,6 +36,9 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? check for NULL pointers - improved performance of regexp-based filters Thanks to Arnaud Cornet for providing the idea and initial patch. +- added a new way how output plugins may be passed parameters. This is + more effcient for some outputs. They new can receive fields not only + as a single string but rather in an array where each string is seperated. --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/action.c b/action.c index 08755c13..03073153 100644 --- a/action.c +++ b/action.c @@ -425,6 +425,7 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) DEFiRet; int iRetries; int i; + int iArr; int iSleepPeriod; int bCallAction; int iCancelStateSave; @@ -439,7 +440,15 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) /* here we must loop to process all requested strings */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { - CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(ppMsgs[i]))); + switch(pAction->eParamPassing) { + case ACT_STRING_PASSING: + CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(ppMsgs[i]))); + break; + case ACT_ARRAY_PASSING: + CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(ppMsgs[i]))); + break; + default:assert(0); /* software bug if this happens! */ + } } iRetries = 0; /* We now must guard the output module against execution by multiple threads. The @@ -495,7 +504,19 @@ finalize_it: /* cleanup */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { if(ppMsgs[i] != NULL) { - d_free(ppMsgs[i]); + switch(pAction->eParamPassing) { + case ACT_ARRAY_PASSING: + iArr = 0; + while(((char **)ppMsgs[i])[iArr] != NULL) + d_free(((char **)ppMsgs[i])[iArr++]); + d_free(ppMsgs[i]); + break; + case ACT_STRING_PASSING: + d_free(ppMsgs[i]); + break; + default: + assert(0); + } } } d_free(ppMsgs); @@ -905,6 +926,13 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques ABORT_FINALIZE(RS_RET_RQD_TPLOPT_MISSING); } + /* set parameter-passing mode */ + if(iTplOpts & OMSR_TPL_AS_ARRAY) { + pAction->eParamPassing = ACT_ARRAY_PASSING; + } else { + pAction->eParamPassing = ACT_STRING_PASSING; + } + dbgprintf("template: '%s' assigned\n", pTplName); } diff --git a/action.h b/action.h index dc9bbd74..f2706af6 100644 --- a/action.h +++ b/action.h @@ -61,6 +61,8 @@ struct action_s { short f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ int f_prevcount; /* repetition cnt of prevline */ int f_repeatcount; /* number of "repeated" msgs */ + enum { ACT_STRING_PASSING = 0, ACT_ARRAY_PASSING = 1 } + eParamPassing; /* mode of parameter passing to action */ int iNumTpls; /* number of array entries for template element below */ struct template **ppTpl;/* array of template to use - strings must be passed to doAction * in this order. */ diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index 890a55c8..a281d9e7 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -8,10 +8,9 @@ number of modules. Here is the entry point to their documentation and what they do (list is currently not complete)

          -
        • omsnmp - SNMP -trap output module
          • -
        • omrelp - RELP -output module
          • +
        • omsnmp - SNMP trap output module
          • +
        • omtdout - stdout output module (mainly a test tool)
          • +
        • omrelp - RELP output module
        • omgssapi - output module for GSS-enabled syslog
        • ommysql - output module for MySQL
        • ompgsql - output module for PostgreSQL
          • diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index 6e227ba9..e491005c 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -49,7 +49,12 @@ MODULE_TYPE_OUTPUT */ DEF_OMOD_STATIC_DATA +/* config variables */ +static int bUseArrayInterface; /* shall action use array instead of string template interface? */ + + typedef struct _instanceData { + int bUseArrayInterface; /* uses action use array instead of string template interface? */ } instanceData; BEGINcreateInstance @@ -79,12 +84,46 @@ CODESTARTtryResume ENDtryResume BEGINdoAction + char **szParams; + char *toWrite; + int iParamVal; + int iParam; + int iBuf; + char szBuf[65564]; CODESTARTdoAction - write(1, (char*)ppString[0], strlen((char*)ppString[0])); + if(pData->bUseArrayInterface) { + /* if we use array passing, we need to put together a string + * ourselves. At this point, please keep in mind that omstdout is + * primarily a testing aid. Other modules may do different processing + * if they would like to support downlevel versions which do not support + * array-passing, but also use that interface on cores who do... + * So this code here is also more or less an example of how to do that. + * rgerhards, 2009-04-03 + */ + szParams = (char**) (ppString[0]); + /* In array-passing mode, ppString[] contains a NULL-terminated array + * of char *pointers. + */ + iParam = 0; + iBuf = 0; + while(szParams[iParam] != NULL) { + iParamVal = 0; + while(szParams[iParam][iParamVal] != '\0' && iBuf < sizeof(szBuf)) { + szBuf[iBuf++] = szParams[iParam][iParamVal++]; + } + ++iParam; + } + szBuf[iBuf] = '\0'; + toWrite = szBuf; + } else { + toWrite = (char*) ppString[0]; + } + write(1, toWrite, strlen(toWrite)); /* 1 is stdout! */ ENDdoAction BEGINparseSelectorAct + int iTplOpts; CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1) /* first check if this config line is actually for us */ @@ -99,7 +138,9 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) /* check if a non-standard template is to be applied */ if(*(p-1) == ';') --p; - CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, 0, (uchar*) "RSYSLOG_FileFormat")); + iTplOpts = (bUseArrayInterface == 0) ? 0 : OMSR_TPL_AS_ARRAY; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, iTplOpts, (uchar*) "RSYSLOG_FileFormat")); + pData->bUseArrayInterface = bUseArrayInterface; CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -115,10 +156,45 @@ CODEqueryEtryPt_STD_OMOD_QUERIES ENDqueryEtryPt + +/* Reset config variables for this module to default values. + */ +static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + DEFiRet; + bUseArrayInterface = 0; + RETiRet; +} + + BEGINmodInit() + rsRetVal localRet; + rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts); + unsigned long opts; + int bArrayPassingSupported; /* does core support template passing as an array? */ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + /* check if the rsyslog core supports parameter passing code */ + bArrayPassingSupported = 0; + localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &pomsrGetSupportedTplOpts); + if(localRet == RS_RET_OK) { + /* found entry point, so let's see if core supports array passing */ + CHKiRet((*pomsrGetSupportedTplOpts)(&opts)); + if(opts & OMSR_TPL_AS_ARRAY) + bArrayPassingSupported = 1; + } else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) { + ABORT_FINALIZE(localRet); /* Something else went wrong, what is not acceptable */ + } + DBGPRINTF("omstdout: array-passing is %ssupported by rsyslog core.\n", bArrayPassingSupported ? "" : "not "); + + if(bArrayPassingSupported) { + /* enable config comand only if core supports it */ + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomstdoutarrayinterface", 0, eCmdHdlrBinary, NULL, + &bUseArrayInterface, STD_LOADABLE_MODULE_ID)); + } + CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit /* vi:set ai: diff --git a/runtime/modules.c b/runtime/modules.c index cef4eac6..9fdb48e7 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -225,6 +225,8 @@ static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)()) *pEtryPoint = regCfSysLineHdlr; } else if(!strcmp((char*) name, "objGetObjInterface")) { *pEtryPoint = objGetObjInterface; + } else if(!strcmp((char*) name, "OMSRgetSupportedTplOpts")) { + *pEtryPoint = OMSRgetSupportedTplOpts; } else { *pEtryPoint = NULL; /* to be on the safe side */ ABORT_FINALIZE(RS_RET_ENTRY_POINT_NOT_FOUND); diff --git a/runtime/objomsr.c b/runtime/objomsr.c index 21d284f3..8dddc4b8 100644 --- a/runtime/objomsr.c +++ b/runtime/objomsr.c @@ -141,5 +141,23 @@ int OMSRgetEntry(omodStringRequest_t *pThis, int iEntry, uchar **ppTplName, int return RS_RET_OK; } + + +/* return the full set of template options that are supported by this version of + * OMSR. They are returned in an unsigned long value. The caller can mask that + * value to check on the option he is interested in. + * Note that this interface was added in 4.1.6, so a plugin must obtain a pointer + * to this interface via queryHostEtryPt(). + * rgerhards, 2009-04-03 + */ +rsRetVal +OMSRgetSupportedTplOpts(unsigned long *pOpts) +{ + DEFiRet; + assert(pOpts != NULL); + *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY; + RETiRet; +} + /* vim:set ai: */ diff --git a/runtime/objomsr.h b/runtime/objomsr.h index 2255e4f3..75ad0fb8 100644 --- a/runtime/objomsr.h +++ b/runtime/objomsr.h @@ -1,6 +1,6 @@ /* Definition of the omsr (omodStringRequest) object. * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -27,7 +27,8 @@ /* define flags for required template options */ #define OMSR_NO_RQD_TPL_OPTS 0 #define OMSR_RQD_TPL_OPT_SQL 1 -/* next option is 2, 4, 8, ... */ +#define OMSR_TPL_AS_ARRAY 2 /* introduced in 4.1.6, 2009-04-03 */ +/* next option is 4, 8, 16, ... */ struct omodStringRequest_s { /* strings requested by output module for doAction() */ int iNumEntries; /* number of array entries for data elements below */ @@ -40,6 +41,7 @@ typedef struct omodStringRequest_s omodStringRequest_t; rsRetVal OMSRdestruct(omodStringRequest_t *pThis); rsRetVal OMSRconstruct(omodStringRequest_t **ppThis, int iNumEntries); rsRetVal OMSRsetEntry(omodStringRequest_t *pThis, int iEntry, uchar *pTplName, int iTplOpts); +rsRetVal OMSRgetSupportedTplOpts(unsigned long *pOpts); int OMSRgetEntryCount(omodStringRequest_t *pThis); int OMSRgetEntry(omodStringRequest_t *pThis, int iEntry, uchar **ppTplName, int *piTplOpts); diff --git a/template.c b/template.c index 4f507ff6..4a12aa3b 100644 --- a/template.c +++ b/template.c @@ -47,6 +47,8 @@ static struct template *tplRoot = NULL; /* the root of the template list */ static struct template *tplLast = NULL; /* points to the last element of the template list */ static struct template *tplLastStatic = NULL; /* last static element of the template list */ + + /* This functions converts a template into a string. It should * actually be in template.c, but this requires larger re-structuring * of the code (because all the property-access functions are static @@ -140,6 +142,60 @@ finalize_it: RETiRet; } + +/* This functions converts a template into an array of strings. + * For further general details, see the very similar funtion + * tpltoString(). + * Instead of a string, an array of string pointers is returned by + * thus function. The caller is repsonsible for destroying that array as + * well as all of its elements. The array is of fixed size. It's end + * is indicated by a NULL pointer. + * rgerhards, 2009-04-03 + */ +rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) +{ + DEFiRet; + struct templateEntry *pTpe; + uchar **pArr; + int iArr; + unsigned short bMustBeFreed; + uchar *pVal; + + assert(pTpl != NULL); + assert(pMsg != NULL); + assert(ppArr != NULL); + + /* loop through the template. We obtain one value, create a + * private copy (if necessary), add it to the string array + * and then on to the next until we have processed everything. + */ + + CHKmalloc(pArr = calloc(pTpl->tpenElements + 1, sizeof(uchar*))); + iArr = 0; + + pTpe = pTpl->pEntryRoot; + while(pTpe != NULL) { + if(pTpe->eEntryType == CONSTANT) { + CHKmalloc(pArr[iArr] = (uchar*)strdup((char*) pTpe->data.constant.pConstant)); + } else if(pTpe->eEntryType == FIELD) { + pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &bMustBeFreed); + if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */ + pArr[iArr] = pVal; /* ... so we can use it! */ + } else { + CHKmalloc(pArr[iArr] = (uchar*)strdup((char*) pVal)); + } + } + iArr++; + pTpe = pTpe->pNext; + } + +finalize_it: + *ppArr = (iRet == RS_RET_OK) ? pArr : NULL; + + RETiRet; +} + + /* Helper to doSQLEscape. This is called if doSQLEscape * runs out of memory allocating the escaped string. * Then we are in trouble. We can diff --git a/template.h b/template.h index 6ca8dc6c..9d794f66 100644 --- a/template.h +++ b/template.h @@ -126,6 +126,7 @@ void tplLastStaticInit(struct template *tpl); * BEFORE msg.h, even if your code file does not actually need it. * rgerhards, 2007-08-06 */ +rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr); rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz); rsRetVal doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); -- cgit v1.2.3 From 9103817f9fe811b49938036a1f9ff23672a9ec44 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 3 Apr 2009 15:48:55 +0200 Subject: added (some) developer documentation for output plugin interface --- ChangeLog | 1 + doc/dev_oplugins.html | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/manual.html | 6 +- doc/status.html | 12 ++-- 4 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 doc/dev_oplugins.html diff --git a/ChangeLog b/ChangeLog index 7f79271f..8d4191d3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -39,6 +39,7 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? - added a new way how output plugins may be passed parameters. This is more effcient for some outputs. They new can receive fields not only as a single string but rather in an array where each string is seperated. +- added (some) developer documentation for output plugin interface --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages diff --git a/doc/dev_oplugins.html b/doc/dev_oplugins.html new file mode 100644 index 00000000..5bfc974c --- /dev/null +++ b/doc/dev_oplugins.html @@ -0,0 +1,178 @@ + + +writing rsyslog output plugins (developer's guide) + + +

          Writing Rsyslog Output Plugins

          +

          This page is the begin of some developer documentation for writing output +plugins. Doing so is quite easy (and that was a design goal), but there currently +is only sparse documentation on the process available. I was tempted NOT to +write this guide here because I know I will most probably not be able to +write a complete guide. +

          However, I finally concluded that it may be better to have same information +and pointers than to have nothing. +

          Getting Started and Samples

          +

          The best to get started with rsyslog plugin development is by looking at +existing plugins. All that start with "om" are output modules. That +means they are primarily thought of being message sinks. In theory, however, output +plugins may aggergate other functionality, too. Nobody has taken this route so far +so if you would like to do that, it is highly suggested to post your plan on the +rsyslog mailing list, first (so that we can offer advise). +

          The rsyslog distribution tarball contains two plugins that are extremely well +targeted for getting started: +

            +
          • omtemplate +
          • omstdout +
          +Plugin omtemplate was specifically created to provide a copy template for new output +plugins. It is bare of real functionality but has ample comments. Even if you decide +to start from another plugin (or even from scratch), be sure to read omtemplate source +and comments first. The omstdout is primarily a testing aide, but offers support for +the two different parameter-passing conventions plugins can use (plus the way to +differentiate between the two). It also is not bare of functionaly, only mostly +bare of it ;). But you can actually execute it and play with it. +

          In any case, you should also read the comments in ./runtime/module-template.h. +Output plugins are build based on a large set of code-generating macros. These +macros handle most of the plumbing needed by the interface. As long as no +special callback to rsyslog is needed (it typically is not), an output plugin does +not really need to be aware that it is executed by rsyslog. As a plug-in programmer, +you can (in most cases) "code as usual". However, all macros and entry points need to be +provided and thus reading the code comments in the files mentioned is highly suggested. +

          In short, the best idea is to start with a template. Let's assume you start by +copying omtemplate. Then, the basic steps you need to do are: +

            +
          • cp ./plugins/omtemplate ./plugins/your-plugin +
          • mv cd ./plugins/your-plugin +
          • vi Makefile.am, adjust to your-plugin +
          • mv omtemplate.c your-plugin.c +
          • cd ../.. +
          • vi Makefile.am configure.ac +
            serach for omtemplate, copy and modify (follow comments) +
          +

          Basically, this is all you need to do ... Well, except, of course, coding +your plugin ;). For testing, you need rsyslog's debugging support. Some useful +information is given in "troubleshooting rsyslog +from the doc set. +

          Special Topics

          +

          Threading

          +

          Rsyslog uses massive parallel processing and multithreading. However, a plugin's entry +points are guaranteed to be never called concurrently for the same action. +That means your plugin must be able to be called concurrently by two or more +threads, but you can be sure that for the same instance no concurrent calls +happen. This is guaranteed by the interface specification and the rsyslog core +guards against multiple concurrent calls. An instance, in simple words, is one +that shares a single instanceData structure. +

          So as long as you do not mess around with global data, you do not need +to think about multithreading (and can apply a purely sequential programming +methodology). +

          Please note that duringt the configuraton parsing stage of execution, access to +global variables for the configuration system is safe. In that stage, the core will +only call sequentially into the plugin. +

          Getting Message Data

          +

          The doAction() entry point of your plugin is provided with messages to be processed. +It will only be activated after filtering and all other conditions, so you do not need +to apply any other conditional but can simply process the message. +

          Note that you do NOT receive the full internal representation of the message +object. There are various (including historical) reasons for this and, among +others, this is a design decision based on security. +

          Your plugin will only receive what the end user has configured in a $template +statement. However, starting with 4.1.6, there are two ways of receiving the +template content. The default mode, and in most cases sufficient and optimal, +is to receive a single string with the expanded template. As I said, this is usually +optimal, think about writing things to files, emailing content or forwarding it. +

          The important philosophy is that a plugin should never reformat any +of such strings - that would either remove the user's ability to fully control +message formats or it would lead to duplicating code that is already present in the +core. If you need some formatting that is not yet present in the core, suggest it +to the rsyslog project, best done by sending a patch ;), and we will try hard to +get it into the core (so far, we could accept all such suggestions - no promise, though). +

          If a single string seems not suitable for your application, the plugin can also +request access to the template components. The typical use case seems to be databases, where +you would like to access properties via specific fields. With that mode, you receive a +char ** array, where each array element points to one field from the template (from +left to right). Fields start at arrray index 0 and a NULL pointer means you have +reached the end of the array (the typical Unix "poor man's linked list in an array" +design). Note, however, that each of the individual components is a string. It is +not a date stamp, number or whatever, but a string. This is because rsyslog processes +strings (from a high-level design look at it) and so this is the natural data type. +Feel free to convert to whatever you need, but keep in mind that malformed packets +may have lead to field contents you'd never expected... +

          If you like to use the array-based parameter passing method, think that it +is only available in rsyslog 4.1.6 and above. If you can accept that your plugin +will not be working with previous versions, you do not need to handle pre 4.1.6 cases. +However, it would be "nice" if you shut down yourself in these cases - otherwise the +older rsyslog core engine will pass you a string where you expect the array of pointers, +what most probably results in a segfault. To check whether or not the core supports the +functionality, you can use this code sequence: +

          +
+BEGINmodInit()
+	rsRetVal localRet;
+	rsRetVal (*pomsrGetSupportedTplOpts)(unsigned long *pOpts);
+	unsigned long opts;
+	int bArrayPassingSupported;		/* does core support template passing as an array? */
+CODESTARTmodInit
+	*ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */
+CODEmodInit_QueryRegCFSLineHdlr
+	/* check if the rsyslog core supports parameter passing code */
+	bArrayPassingSupported = 0;
+	localRet = pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &pomsrGetSupportedTplOpts);
+	if(localRet == RS_RET_OK) {
+		/* found entry point, so let's see if core supports array passing */
+		CHKiRet((*pomsrGetSupportedTplOpts)(&opts));
+		if(opts & OMSR_TPL_AS_ARRAY)
+			bArrayPassingSupported = 1;
+	} else if(localRet != RS_RET_ENTRY_POINT_NOT_FOUND) {
+		ABORT_FINALIZE(localRet); /* Something else went wrong, what is not acceptable */
+	}
+	DBGPRINTF("omstdout: array-passing is %ssupported by rsyslog core.\n", bArrayPassingSupported ? "" : "not ");
+
+	if(!bArrayPassingSupported) {
+		DBGPRINTF("rsyslog core too old, shutting down this plug-in\n");
+		ABORT_FINALIZE(RS_RET_ERR);
+	}
+
+
+
          +

          The code first checks if the core supports the OMSRgetSupportedTplOpts() API (which is +also not present in all versions!) and, if so, queries the core if the OMSR_TPL_AS_ARRAY mode +is supported. If either does not exits, the core is too old for this functionality. The sample +snippet above then shuts down, but a plugin may instead just do things different. In +omstdout, you can see how a plugin may deal with the situation. +

          In any case, it is recommended that at least a graceful shutdown is made and the +array-passing capability not blindly be used. In such cases, we can not guard the +plugin from segfaulting and if the plugin (as currently always) is run within +rsyslog's process space, that results in a segfault for rsyslog. So do not do this. +

          Batching of Messages

          +

          With the current plugin interface, each message is passed via a separate call to the plugin. +This is annoying and costs performance in some uses cases (primarily for database outputs). +However, that's the way it (currently) is, no easy way around it. There are some ideas +to implement batching capabilities inside the rsyslog core, but without that the only +resort is to do it inside your plugin yourself. You are not prohibited from doing so. +There are some consequences, though: most importantly, the rsyslog core is no longer +intersted in messages that it passed to a plugin. As such, it will not try to make sure +the message is not lost before it was ultimately processed (because rsyslog, due to +doAction() returning successfully, thinks the message *was* ultimately processed). +

          When the rsyslog core receives batching capabilities, this will be implemented in +a way that is fully compatible to the existing plugin interface. While we have not yet +thought about the implementation, that will probably mean that some new interfaces +or options be used to turn on batching capabilities. +

          Licensing

          +

          From the rsyslog point of view, plugins constitute separate projects. As such, +we think plugins are not required to be compatible with GPLv3. However, this is +no legal advise. If you intend to release something under a non-GPLV3 compatible license +it is probably best to consult with your lawyer. +

          Most importantly, and this is definite, the rsyslog team does not expect +or require you to contribute your plugin to the rsyslog project (but of course +we are happy if you do). +

          Copyright

          +

          Copyright (c) 2009 Rainer Gerhards +and Adiscon.

          +

          Permission is granted to copy, distribute and/or modify this document under +the terms of the GNU Free Documentation License, Version 1.2 or any later +version published by the Free Software Foundation; with no Invariant Sections, +no Front-Cover Texts, and no Back-Cover Texts. A copy of the license can be +viewed at +http://www.gnu.org/copyleft/fdl.html.

          + + diff --git a/doc/manual.html b/doc/manual.html index 99a90594..0b03a255 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -64,7 +64,11 @@ the syslog priority (severity and facility) to the log file syslog sender over NAT (online only)
        • an overview and howto of rsyslog gssapi support
        • debug support in rsyslog
          • -
        • the rsyslog message queue object (developer's view)
          • +
        • Developer Documentation +

        Our rsyslog history page is for you if you would like to learn a little more diff --git a/doc/status.html b/doc/status.html index 59fd0809..dae94884 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,19 +2,19 @@ rsyslog status page

        rsyslog status page

        -

        This page reflects the status as of 2009-02-02.

        +

        This page reflects the status as of 2009-04-03.

        Current Releases

        development: 4.1.5 [2009-03-11] - change log - download -
        beta: 3.21.10 [2009-02-02] - -change log - -download

        +
        beta: 3.21.11 [2009-04-03] - +change log - +download

        -

        v3 stable: 3.20.3 [2009-01-19] - change log - -download +

        v3 stable: 3.20.3 [2009-04-02] - change log - +download
        v2 stable: 2.0.6 [2008-08-07] - change log - download -- cgit v1.2.3 From ce6b7e86cdd63ba1540d20aa22403d2b13d2e59f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 3 Apr 2009 17:54:09 +0200 Subject: improved test suite and added test for new output module interface The testbench has now a generic driver that can run a whole class of test suites just by providing a config file and test cases. This does not cover all testing needs, but a lot. We have now added one test for the new array-passing output plugin interface. --- plugins/omstdout/omstdout.c | 2 + tests/Makefile.am | 28 ++-- tests/omod-if-array.sh | 1 + tests/parsertest.c | 269 --------------------------------- tests/parsertest.sh | 1 + tests/testruns/1.parse1 | 3 - tests/testruns/parser.conf | 9 -- tests/testruns/rfc3164.parse1 | 4 - tests/testruns/rfc5424-1.parse1 | 3 - tests/testruns/rfc5424-2.parse1 | 4 - tests/testruns/rfc5424-3.parse1 | 4 - tests/testruns/rfc5424-4.parse1 | 4 - tests/testsuites/1.omod-if-array | 2 + tests/testsuites/1.parse1 | 3 + tests/testsuites/omod-if-array.conf | 14 ++ tests/testsuites/parse1.conf | 9 ++ tests/testsuites/rfc3164.parse1 | 4 + tests/testsuites/rfc5424-1.parse1 | 3 + tests/testsuites/rfc5424-2.parse1 | 4 + tests/testsuites/rfc5424-3.parse1 | 4 + tests/testsuites/rfc5424-4.parse1 | 4 + tests/udptester.c | 291 ++++++++++++++++++++++++++++++++++++ 22 files changed, 358 insertions(+), 312 deletions(-) create mode 100755 tests/omod-if-array.sh delete mode 100644 tests/parsertest.c create mode 100755 tests/parsertest.sh delete mode 100644 tests/testruns/1.parse1 delete mode 100644 tests/testruns/parser.conf delete mode 100644 tests/testruns/rfc3164.parse1 delete mode 100644 tests/testruns/rfc5424-1.parse1 delete mode 100644 tests/testruns/rfc5424-2.parse1 delete mode 100644 tests/testruns/rfc5424-3.parse1 delete mode 100644 tests/testruns/rfc5424-4.parse1 create mode 100644 tests/testsuites/1.omod-if-array create mode 100644 tests/testsuites/1.parse1 create mode 100644 tests/testsuites/omod-if-array.conf create mode 100644 tests/testsuites/parse1.conf create mode 100644 tests/testsuites/rfc3164.parse1 create mode 100644 tests/testsuites/rfc5424-1.parse1 create mode 100644 tests/testsuites/rfc5424-2.parse1 create mode 100644 tests/testsuites/rfc5424-3.parse1 create mode 100644 tests/testsuites/rfc5424-4.parse1 create mode 100644 tests/udptester.c diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index e491005c..7c63b5c4 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -107,6 +107,8 @@ CODESTARTdoAction iParam = 0; iBuf = 0; while(szParams[iParam] != NULL) { + if(iParam > 0) + szBuf[iBuf++] = ','; /* all but first need a delimiter */ iParamVal = 0; while(szParams[iParam][iParamVal] != '\0' && iBuf < sizeof(szBuf)) { szBuf[iBuf++] = szParams[iParam][iParamVal++]; diff --git a/tests/Makefile.am b/tests/Makefile.am index 09d1a0b6..ab1c5a62 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ -TESTRUNS = rt_init rscript parsertest -check_PROGRAMS = $(TESTRUNS) ourtail -TESTS = $(TESTRUNS) cfg.sh +TESTRUNS = rt_init rscript +check_PROGRAMS = $(TESTRUNS) ourtail udptester +TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -17,19 +17,23 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ DevNull.cfgtest \ err1.rstest \ NoExistFile.cfgtest \ - testruns/parser.conf \ - testruns/1.parse1 \ - testruns/rfc3164.parse1 \ - testruns/rfc5424-1.parse1 \ - testruns/rfc5424-2.parse1 \ - testruns/rfc5424-3.parse1 \ - testruns/rfc5424-4.parse1 \ + testsuites/parse1.conf \ + testsuites/1.parse1 \ + testsuites/rfc3164.parse1 \ + testsuites/rfc5424-1.parse1 \ + testsuites/rfc5424-2.parse1 \ + testsuites/rfc5424-3.parse1 \ + testsuites/rfc5424-4.parse1 \ + testsuites/omod-if-array.conf \ + testsuites/1.omod-if-array \ + parsertest.sh \ + omod-if-array.sh \ cfg.sh ourtail_SOURCES = ourtail.c -parsertest_SOURCES = parsertest.c getline.c -parsertest_LDADD = $(SOL_LIBS) +udptester_SOURCES = udptester.c getline.c +udptester_LDADD = $(SOL_LIBS) rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh new file mode 100755 index 00000000..cac08928 --- /dev/null +++ b/tests/omod-if-array.sh @@ -0,0 +1 @@ +./udptester omod-if-array diff --git a/tests/parsertest.c b/tests/parsertest.c deleted file mode 100644 index 6c2221e8..00000000 --- a/tests/parsertest.c +++ /dev/null @@ -1,269 +0,0 @@ -/* Runs a test suite on the rsyslog parser (and later potentially - * other things). - * - * Please note that this - * program works together with the config file AND easily extensible - * test case files (*.parse1) to run a number of checks. All test - * cases are executed, even if there is a failure early in the - * process. When finished, the numberof failed tests will be given. - * - * Part of the testbench for rsyslog. - * - * Copyright 2009 Rainer Gerhards and Adiscon GmbH. - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see . - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - */ -#include "config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define EXIT_FAILURE 1 - - -void readLine(int fd, char *ln) -{ - char c; - int lenRead; - lenRead = read(fd, &c, 1); - while(lenRead == 1 && c != '\n') { - *ln++ = c; - lenRead = read(fd, &c, 1); - } - *ln = '\0'; -} - - -/* send a message via UDP - * returns 0 if ok, something else otherwise. - */ -int -udpSend(char *buf, int lenBuf) -{ - struct sockaddr_in si_other; - int s, slen=sizeof(si_other); - - if((s=socket(AF_INET, SOCK_DGRAM, 0))==-1) { - perror("socket()"); - return(1); - } - - memset((char *) &si_other, 0, sizeof(si_other)); - si_other.sin_family = AF_INET; - si_other.sin_port = htons(12514); - if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { - fprintf(stderr, "inet_aton() failed\n"); - return(1); - } - - if(sendto(s, buf, lenBuf, 0, (struct sockaddr*) &si_other, slen)==-1) { - perror("sendto"); - fprintf(stderr, "sendto() failed\n"); - return(1); - } - - close(s); - return 0; -} - -/* open pipe to test candidate - so far, this is - * always rsyslogd and with a fixed config. Later, we may - * change this. Returns 0 if ok, something else otherwise. - * rgerhards, 2009-03-31 - */ -int openPipe(pid_t *pid, int *pfd) -{ - int pipefd[2]; - pid_t cpid; - char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", - "-M../runtime//.libs", NULL }; - char confFile[1024]; - char *newenviron[] = { NULL }; - - - sprintf(confFile, "-f%s/testruns/parser.conf", getenv("srcdir")); - newargv[1] = confFile; - - if (pipe(pipefd) == -1) { - perror("pipe"); - exit(EXIT_FAILURE); - } - - cpid = fork(); - if (cpid == -1) { - perror("fork"); - exit(EXIT_FAILURE); - } - - if(cpid == 0) { /* Child reads from pipe */ - fclose(stdout); - dup(pipefd[1]); - close(pipefd[1]); - close(pipefd[0]); - fclose(stdin); - execve("../tools/rsyslogd", newargv, newenviron); - } else { - close(pipefd[1]); - *pid = cpid; - *pfd = pipefd[0]; - } - - return(0); -} - - -/* Process a specific test case. File name is provided. - * Needs to return 0 if all is OK, something else otherwise. - */ -int -processTestFile(int fd, char *pszFileName) -{ - FILE *fp; - char *testdata = NULL; - char *expected = NULL; - int ret = 0; - size_t lenLn; - char buf[4096]; - - if((fp = fopen((char*)pszFileName, "r")) == NULL) { - perror((char*)pszFileName); - return(2); - } - - /* skip comments at start of file */ - - getline(&testdata, &lenLn, fp); - while(!feof(fp)) { - if(*testdata == '#') - getline(&testdata, &lenLn, fp); - else - break; /* first non-comment */ - } - - - testdata[strlen(testdata)-1] = '\0'; /* remove \n */ - /* now we have the test data to send */ - if(udpSend(testdata, strlen(testdata)) != 0) - return(2); - - /* next line is expected output - * we do not care about EOF here, this will lead to a failure and thus - * draw enough attention. -- rgerhards, 2009-03-31 - */ - getline(&expected, &lenLn, fp); - expected[strlen(expected)-1] = '\0'; /* remove \n */ - - /* pull response from server and then check if it meets our expectation */ - readLine(fd, buf); - if(strcmp(expected, buf)) { - printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", - expected, buf); - ret = 1; - } - - free(testdata); - free(expected); - fclose(fp); - return(ret); -} - - -/* carry out all tests. Tests are specified via a file name - * wildcard. Each of the files is read and the test carried - * out. - * Returns the number of tests that failed. Zero means all - * success. - */ -int -doTests(int fd, char *files) -{ - int iFailed = 0; - int iTests = 0; - int ret; - char *testFile; - glob_t testFiles; - size_t i = 0; - struct stat fileInfo; - - glob(files, GLOB_MARK, NULL, &testFiles); - - for(i = 0; i < testFiles.gl_pathc; i++) { - testFile = testFiles.gl_pathv[i]; - - if(stat((char*) testFile, &fileInfo) != 0) - continue; /* continue with the next file if we can't stat() the file */ - - ++iTests; - /* all regular files are run through the test logic. Symlinks don't work. */ - if(S_ISREG(fileInfo.st_mode)) { /* config file */ - printf("processing test case '%s' ... ", testFile); - ret = processTestFile(fd, testFile); - if(ret == 0) { - printf("successfully completed\n"); - } else { - printf("failed!\n"); - ++iFailed; - } - } - } - globfree(&testFiles); - - if(iTests == 0) { - printf("Error: no test cases found, no tests executed.\n"); - iFailed = 1; - } else { - printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); - } - - return(iFailed); -} - - -/* */ -int main(int argc, char *argv[]) -{ - int fd; - pid_t pid; - int ret = 0; - char buf[4096]; - char testcases[4096]; - - printf("running rsyslog parser tests ($srcdir=%s)\n", getenv("srcdir")); - - openPipe(&pid, &fd); - readLine(fd, buf); - - /* generate filename */ - sprintf(testcases, "%s/testruns/*.parse1", getenv("srcdir")); - if(doTests(fd, testcases) != 0) - ret = 1; - - /* cleanup */ - kill(pid, SIGTERM); - printf("End of parser tests.\n"); - exit(ret); -} diff --git a/tests/parsertest.sh b/tests/parsertest.sh new file mode 100755 index 00000000..e7985bb0 --- /dev/null +++ b/tests/parsertest.sh @@ -0,0 +1 @@ +./udptester parse1 diff --git a/tests/testruns/1.parse1 b/tests/testruns/1.parse1 deleted file mode 100644 index 5ae655e6..00000000 --- a/tests/testruns/1.parse1 +++ /dev/null @@ -1,3 +0,0 @@ -<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 -167,local4,debug,Mar 6 16:57:54,172.20.245.8,%PIX-7-710005,%PIX-7-710005:, UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 -#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/parser.conf b/tests/testruns/parser.conf deleted file mode 100644 index 0fb7d16d..00000000 --- a/tests/testruns/parser.conf +++ /dev/null @@ -1,9 +0,0 @@ -$ModLoad ../plugins/omstdout/.libs/omstdout -$ModLoad ../plugins/imudp/.libs/imudp -$UDPServerRun 12514 - -$ErrorMessagesToStderr off - -# use a special format that we can easily parse in expect -$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" -*.* :omstdout:;expect diff --git a/tests/testruns/rfc3164.parse1 b/tests/testruns/rfc3164.parse1 deleted file mode 100644 index e7a5fa18..00000000 --- a/tests/testruns/rfc3164.parse1 +++ /dev/null @@ -1,4 +0,0 @@ -<34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8 -34,auth,crit,Oct 11 22:14:15,mymachine,su,su:, 'su root' failed for lonvick on /dev/pts/8 -#Example from RFC3164, section 5.4 -#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-1.parse1 b/tests/testruns/rfc5424-1.parse1 deleted file mode 100644 index 23836c9f..00000000 --- a/tests/testruns/rfc5424-1.parse1 +++ /dev/null @@ -1,3 +0,0 @@ -#Example from RFC5424, section 6.5 / sample 1 -<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 -34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8 diff --git a/tests/testruns/rfc5424-2.parse1 b/tests/testruns/rfc5424-2.parse1 deleted file mode 100644 index a86fbc35..00000000 --- a/tests/testruns/rfc5424-2.parse1 +++ /dev/null @@ -1,4 +0,0 @@ -<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts. -165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],- %% It's time to make the do-nuts. -#Example from RFC5424, section 6.5 / sample 2 -#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-3.parse1 b/tests/testruns/rfc5424-3.parse1 deleted file mode 100644 index 6ad4073d..00000000 --- a/tests/testruns/rfc5424-3.parse1 +++ /dev/null @@ -1,4 +0,0 @@ -<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource= "Application" eventID="1011"][examplePriority@32473 class="high"] -165,local4,notice,Oct 11 22:14:15,mymachine.example.com,,evntslog, -#Example from RFC5424, section 6.5 / sample 4 -#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testruns/rfc5424-4.parse1 b/tests/testruns/rfc5424-4.parse1 deleted file mode 100644 index ecf27e14..00000000 --- a/tests/testruns/rfc5424-4.parse1 +++ /dev/null @@ -1,4 +0,0 @@ -<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource= "Application" eventID="1011"] BOMAn application event log entry... -165,local4,notice,Oct 11 22:14:15,mymachine.example.com,,evntslog,BOMAn application event log entry... -#Example from RFC5424, section 6.5 / sample 3 -#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/1.omod-if-array b/tests/testsuites/1.omod-if-array new file mode 100644 index 00000000..c464b19c --- /dev/null +++ b/tests/testsuites/1.omod-if-array @@ -0,0 +1,2 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +167,Mar 6 16:57:54,172.20.245.8,%PIX-7-710005,%PIX-7-710005:, diff --git a/tests/testsuites/1.parse1 b/tests/testsuites/1.parse1 new file mode 100644 index 00000000..5ae655e6 --- /dev/null +++ b/tests/testsuites/1.parse1 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +167,local4,debug,Mar 6 16:57:54,172.20.245.8,%PIX-7-710005,%PIX-7-710005:, UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/omod-if-array.conf b/tests/testsuites/omod-if-array.conf new file mode 100644 index 00000000..e6c05a52 --- /dev/null +++ b/tests/testsuites/omod-if-array.conf @@ -0,0 +1,14 @@ +# Test config for array-passing output module interface +# (stanard string passing is already tested via the other test inside +# the testbench, so we do not need to focus on that) +# rgerhards, 2009-04-03 +$ModLoad ../plugins/omstdout/.libs/omstdout +$ModLoad ../plugins/imudp/.libs/imudp +$UDPServerRun 12514 + +$ActionOMStdoutArrayInterface on +$ErrorMessagesToStderr off + +# do NOT remove \n, that would hang the test driver! +$template expect,"%PRI%%timestamp%%hostname%%programname%%syslogtag%\n" +*.* :omstdout:;expect diff --git a/tests/testsuites/parse1.conf b/tests/testsuites/parse1.conf new file mode 100644 index 00000000..0fb7d16d --- /dev/null +++ b/tests/testsuites/parse1.conf @@ -0,0 +1,9 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$ModLoad ../plugins/imudp/.libs/imudp +$UDPServerRun 12514 + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" +*.* :omstdout:;expect diff --git a/tests/testsuites/rfc3164.parse1 b/tests/testsuites/rfc3164.parse1 new file mode 100644 index 00000000..e7a5fa18 --- /dev/null +++ b/tests/testsuites/rfc3164.parse1 @@ -0,0 +1,4 @@ +<34>Oct 11 22:14:15 mymachine su: 'su root' failed for lonvick on /dev/pts/8 +34,auth,crit,Oct 11 22:14:15,mymachine,su,su:, 'su root' failed for lonvick on /dev/pts/8 +#Example from RFC3164, section 5.4 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/rfc5424-1.parse1 b/tests/testsuites/rfc5424-1.parse1 new file mode 100644 index 00000000..23836c9f --- /dev/null +++ b/tests/testsuites/rfc5424-1.parse1 @@ -0,0 +1,3 @@ +#Example from RFC5424, section 6.5 / sample 1 +<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8 +34,auth,crit,Oct 11 22:14:15,mymachine.example.com,,su,- BOM'su root' failed for lonvick on /dev/pts/8 diff --git a/tests/testsuites/rfc5424-2.parse1 b/tests/testsuites/rfc5424-2.parse1 new file mode 100644 index 00000000..a86fbc35 --- /dev/null +++ b/tests/testsuites/rfc5424-2.parse1 @@ -0,0 +1,4 @@ +<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc 8710 - - %% It's time to make the do-nuts. +165,local4,notice,Aug 24 05:14:15,192.0.2.1,,myproc[8710],- %% It's time to make the do-nuts. +#Example from RFC5424, section 6.5 / sample 2 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/rfc5424-3.parse1 b/tests/testsuites/rfc5424-3.parse1 new file mode 100644 index 00000000..6ad4073d --- /dev/null +++ b/tests/testsuites/rfc5424-3.parse1 @@ -0,0 +1,4 @@ +<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource= "Application" eventID="1011"][examplePriority@32473 class="high"] +165,local4,notice,Oct 11 22:14:15,mymachine.example.com,,evntslog, +#Example from RFC5424, section 6.5 / sample 4 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/rfc5424-4.parse1 b/tests/testsuites/rfc5424-4.parse1 new file mode 100644 index 00000000..ecf27e14 --- /dev/null +++ b/tests/testsuites/rfc5424-4.parse1 @@ -0,0 +1,4 @@ +<165>1 2003-10-11T22:14:15.003Z mymachine.example.com evntslog - ID47 [exampleSDID@32473 iut="3" eventSource= "Application" eventID="1011"] BOMAn application event log entry... +165,local4,notice,Oct 11 22:14:15,mymachine.example.com,,evntslog,BOMAn application event log entry... +#Example from RFC5424, section 6.5 / sample 3 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/udptester.c b/tests/udptester.c new file mode 100644 index 00000000..ea642db6 --- /dev/null +++ b/tests/udptester.c @@ -0,0 +1,291 @@ +/* Runs a test suite on the rsyslog (and later potentially + * other things). + * + * The name of the test suite must be given as argv[1]. In this config, + * rsyslogd is loaded with config ./testsuites/.conf and then + * test cases ./testsuites/ *. are executed on it. This test driver is + * suitable for testing cases where a message sent (via UDP) results in + * exactly one response. It can not be used in cases where no response + * is expected (that would result in a hang of the test driver). + * Note: each test suite can contain many tests, but they all need to work + * with the same rsyslog configuration. + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EXIT_FAILURE 1 + +static char *srcdir; /* global $srcdir, set so that we can run outside of "make check" */ +static char *testSuite; /* name of current test suite */ + + +void readLine(int fd, char *ln) +{ + char c; + int lenRead; + lenRead = read(fd, &c, 1); + while(lenRead == 1 && c != '\n') { + *ln++ = c; + lenRead = read(fd, &c, 1); + } + *ln = '\0'; +} + + +/* send a message via UDP + * returns 0 if ok, something else otherwise. + */ +int +udpSend(char *buf, int lenBuf) +{ + struct sockaddr_in si_other; + int s, slen=sizeof(si_other); + + if((s=socket(AF_INET, SOCK_DGRAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &si_other, 0, sizeof(si_other)); + si_other.sin_family = AF_INET; + si_other.sin_port = htons(12514); + if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + + if(sendto(s, buf, lenBuf, 0, (struct sockaddr*) &si_other, slen)==-1) { + perror("sendto"); + fprintf(stderr, "sendto() failed\n"); + return(1); + } + + close(s); + return 0; +} + + +/* open pipe to test candidate - so far, this is + * always rsyslogd and with a fixed config. Later, we may + * change this. Returns 0 if ok, something else otherwise. + * rgerhards, 2009-03-31 + */ +int openPipe(char *configFile, pid_t *pid, int *pfd) +{ + int pipefd[2]; + pid_t cpid; + char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", + "-M../runtime//.libs", NULL }; + char confFile[1024]; + char *newenviron[] = { NULL }; + + + sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, configFile); + newargv[1] = confFile; + + if (pipe(pipefd) == -1) { + perror("pipe"); + exit(EXIT_FAILURE); + } + + cpid = fork(); + if (cpid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } + + if(cpid == 0) { /* Child reads from pipe */ + fclose(stdout); + dup(pipefd[1]); + close(pipefd[1]); + close(pipefd[0]); + fclose(stdin); + execve("../tools/rsyslogd", newargv, newenviron); + } else { + close(pipefd[1]); + *pid = cpid; + *pfd = pipefd[0]; + } + + return(0); +} + + +/* Process a specific test case. File name is provided. + * Needs to return 0 if all is OK, something else otherwise. + */ +int +processTestFile(int fd, char *pszFileName) +{ + FILE *fp; + char *testdata = NULL; + char *expected = NULL; + int ret = 0; + size_t lenLn; + char buf[4096]; + + if((fp = fopen((char*)pszFileName, "r")) == NULL) { + perror((char*)pszFileName); + return(2); + } + + /* skip comments at start of file */ + + getline(&testdata, &lenLn, fp); + while(!feof(fp)) { + if(*testdata == '#') + getline(&testdata, &lenLn, fp); + else + break; /* first non-comment */ + } + + + testdata[strlen(testdata)-1] = '\0'; /* remove \n */ + /* now we have the test data to send */ + if(udpSend(testdata, strlen(testdata)) != 0) + return(2); + + /* next line is expected output + * we do not care about EOF here, this will lead to a failure and thus + * draw enough attention. -- rgerhards, 2009-03-31 + */ + getline(&expected, &lenLn, fp); + expected[strlen(expected)-1] = '\0'; /* remove \n */ + + /* pull response from server and then check if it meets our expectation */ + readLine(fd, buf); + if(strcmp(expected, buf)) { + printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", + expected, buf); + ret = 1; + } + + free(testdata); + free(expected); + fclose(fp); + return(ret); +} + + +/* carry out all tests. Tests are specified via a file name + * wildcard. Each of the files is read and the test carried + * out. + * Returns the number of tests that failed. Zero means all + * success. + */ +int +doTests(int fd, char *files) +{ + int iFailed = 0; + int iTests = 0; + int ret; + char *testFile; + glob_t testFiles; + size_t i = 0; + struct stat fileInfo; + + glob(files, GLOB_MARK, NULL, &testFiles); + + for(i = 0; i < testFiles.gl_pathc; i++) { + testFile = testFiles.gl_pathv[i]; + + if(stat((char*) testFile, &fileInfo) != 0) + continue; /* continue with the next file if we can't stat() the file */ + + ++iTests; + /* all regular files are run through the test logic. Symlinks don't work. */ + if(S_ISREG(fileInfo.st_mode)) { /* config file */ + printf("processing test case '%s' ... ", testFile); + ret = processTestFile(fd, testFile); + if(ret == 0) { + printf("successfully completed\n"); + } else { + printf("failed!\n"); + ++iFailed; + } + } + } + globfree(&testFiles); + + if(iTests == 0) { + printf("Error: no test cases found, no tests executed.\n"); + iFailed = 1; + } else { + printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); + } + + return(iFailed); +} + + +/* Run the test suite. This must be called with exactly one parameter, the + * name of the test suite. For details, see file header comment at the top + * of this file. + * rgerhards, 2009-04-03 + */ +int main(int argc, char *argv[]) +{ + int fd; + pid_t pid; + int ret = 0; + char buf[4096]; + char testcases[4096]; + + if(argc != 2) { + printf("Invalid call of udptester\n"); + printf("Usage: udptester testsuite-name\n"); + exit(1); + } + + testSuite = argv[1]; + + if((srcdir = getenv("srcdir")) == NULL) + srcdir = "."; + + printf("Start of udptester run ($srcdir=%s, testsuite=%s)\n", srcdir, testSuite); + + openPipe(argv[1], &pid, &fd); + readLine(fd, buf); + + /* generate filename */ + sprintf(testcases, "%s/testsuites/*.%s", srcdir, testSuite); + if(doTests(fd, testcases) != 0) + ret = 1; + + /* cleanup */ + kill(pid, SIGTERM); + printf("End of udptester run.\n"); + exit(ret); +} -- cgit v1.2.3 From 04b06af335bedcb651ccffd852863b7d17bf20dc Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 3 Apr 2009 18:20:52 +0200 Subject: improved parser test suite new tests added, now much better --- runtime/datetime.c | 2 +- tests/testsuites/2.parse1 | 3 +++ tests/testsuites/date1.parse1 | 3 +++ tests/testsuites/date2.parse1 | 3 +++ tests/testsuites/date3.parse1 | 3 +++ tests/testsuites/date4.parse1 | 3 +++ tests/testsuites/date5.parse1 | 3 +++ 7 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/testsuites/2.parse1 create mode 100644 tests/testsuites/date1.parse1 create mode 100644 tests/testsuites/date2.parse1 create mode 100644 tests/testsuites/date3.parse1 create mode 100644 tests/testsuites/date4.parse1 create mode 100644 tests/testsuites/date5.parse1 diff --git a/runtime/datetime.c b/runtime/datetime.c index 676f76d5..deb66eb4 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -479,7 +479,7 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) if(hour > 1970 && hour < 2100) { /* if so, we assume this actually is a year. This is a format found * e.g. in Cisco devices. - * (if you read this 2100+ trying to fix a bug, congratulate myself + * (if you read this 2100+ trying to fix a bug, congratulate me * to how long the code survived - me no longer ;)) -- rgerhards, 2008-11-18 */ year = hour; diff --git a/tests/testsuites/2.parse1 b/tests/testsuites/2.parse1 new file mode 100644 index 00000000..628e06df --- /dev/null +++ b/tests/testsuites/2.parse1 @@ -0,0 +1,3 @@ +<38>Mar 27 19:06:53 source_server sshd(pam_unix)[12750]: session opened for user foo by (uid=0) +38,auth,info,Mar 27 19:06:53,source_server,sshd(pam_unix),sshd(pam_unix)[12750]:, session opened for user foo by (uid=0) +# yet another real-life sample where we had some issues with diff --git a/tests/testsuites/date1.parse1 b/tests/testsuites/date1.parse1 new file mode 100644 index 00000000..ffc7c373 --- /dev/null +++ b/tests/testsuites/date1.parse1 @@ -0,0 +1,3 @@ +<38> Mar 7 19:06:53 example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# one space in front of the date diff --git a/tests/testsuites/date2.parse1 b/tests/testsuites/date2.parse1 new file mode 100644 index 00000000..8d587d9d --- /dev/null +++ b/tests/testsuites/date2.parse1 @@ -0,0 +1,3 @@ +<38>Mar 7 19:06:53 example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# only one space between "Mar" and "7" diff --git a/tests/testsuites/date3.parse1 b/tests/testsuites/date3.parse1 new file mode 100644 index 00000000..940d261e --- /dev/null +++ b/tests/testsuites/date3.parse1 @@ -0,0 +1,3 @@ +<38>Mar 7 2008 19:06:53: example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# the year should not be there, nor the colon after the date, but we accept it... diff --git a/tests/testsuites/date4.parse1 b/tests/testsuites/date4.parse1 new file mode 100644 index 00000000..eee5fb09 --- /dev/null +++ b/tests/testsuites/date4.parse1 @@ -0,0 +1,3 @@ +<38>Mar 7 2008 19:06:53 example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# the year should not be there, but we accept it... diff --git a/tests/testsuites/date5.parse1 b/tests/testsuites/date5.parse1 new file mode 100644 index 00000000..be32e605 --- /dev/null +++ b/tests/testsuites/date5.parse1 @@ -0,0 +1,3 @@ +<38>Mar 7 19:06:53: example tag: testmessage (only date actually tested) +38,auth,info,Mar 7 19:06:53,example,tag,tag:, testmessage (only date actually tested) +# colon after timestamp is strictly not ok, but we accept it -- cgit v1.2.3 From 010060289a729dd930ac04b72237f0ca0db9ea1d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 6 Apr 2009 10:56:27 +0200 Subject: made sure udptester terminates only after rsyslgod it spawned We noticed this race issue under Solaris (thanks to its different scheduler, I guess). In some cases, the previous instance of rsyslogd was not terminated, resulting in a failure on the next test. Now handled correctly. --- tests/udptester.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/udptester.c b/tests/udptester.c index ea642db6..a3c6658d 100644 --- a/tests/udptester.c +++ b/tests/udptester.c @@ -259,6 +259,7 @@ int main(int argc, char *argv[]) { int fd; pid_t pid; + int status; int ret = 0; char buf[4096]; char testcases[4096]; @@ -286,6 +287,7 @@ int main(int argc, char *argv[]) /* cleanup */ kill(pid, SIGTERM); + waitpid(pid, &status, 0); /* wait until instance terminates */ printf("End of udptester run.\n"); exit(ret); } -- cgit v1.2.3 From 8e3c5a9ca3732a41fbb8581b13c49acd699820da Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 6 Apr 2009 17:55:04 +0200 Subject: improved internal handling of RainerScript functions - building the necessary plumbing to support more functions with decent runtime performance. This is also necessary towards the long-term goal of loadable library modules. - added new RainerScript function "tolower" --- ChangeLog | 7 ++ runtime/expr.c | 4 +- runtime/rsyslog.h | 4 + runtime/vm.c | 252 +++++++++++++++++++++++++++++++++++++++++++++++++++--- runtime/vm.h | 5 +- runtime/vmop.c | 53 ++++++++++-- runtime/vmop.h | 11 ++- runtime/vmprg.c | 28 +++++- runtime/vmprg.h | 4 +- runtime/vmstk.h | 4 +- tests/3.rstest | 4 +- tools/syslogd.c | 1 + 12 files changed, 345 insertions(+), 32 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8d4191d3..24898dc7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,10 @@ +--------------------------------------------------------------------------- +Version 4.1.7 [DEVEL] (rgerhards), 2009-03-?? +- improved internal handling of RainerScript functions, building the + necessary plumbing to support more functions with decent runtime + performance. This is also necessary towards the long-term goal + of loadable library modules. +- added new RainerScript function "tolower" - bugfix: potential abort with DA queue after high watermark is reached There exists a race condition that can lead to a segfault. Thanks go to vbernetr, who performed the analysis and provided patch, which diff --git a/runtime/expr.c b/runtime/expr.c index 38ed1c68..e449d1c7 100644 --- a/runtime/expr.c +++ b/runtime/expr.c @@ -142,7 +142,9 @@ terminal(expr_t *pThis, ctok_t *tok) * we have all relevant information) */ CHKiRet(ctok_token.UnlinkVar(pToken, &pVar)); - CHKiRet(vmprg.AddVarOperation(pThis->pVmprg, opcode_FUNC_CALL, pVar)); /* add to program */ + CHKiRet(var.ConvToString(pVar)); /* make sure we have a string */ + CHKiRet(vmprg.AddCallOperation(pThis->pVmprg, pVar->val.pStr)); /* add to program */ + CHKiRet(var.Destruct(&pVar)); break; case ctok_MSGVAR: dbgoprint((obj_t*) pThis, "MSGVAR\n"); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 032d8c04..cea457d8 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -84,6 +84,8 @@ typedef rsRetVal (*errLogFunc_t)(uchar*); /* this is a trick to store a function typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct tcpsrv_s tcpsrv_t; +typedef struct vmstk_s vmstk_t; +typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ /* some universal 64 bit define... */ typedef long long int64; @@ -258,6 +260,8 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_FUNC_MISSING_EXPR = -2111, /**< no expression after comma in function call (rainerscript) */ RS_RET_INVLD_NBR_ARGUMENTS = -2112, /**< invalid number of arguments for function call (rainerscript) */ RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ + RS_RET_DUP_FUNC_NAME = -2114, /**< duplicate function name (rainerscript) */ + RS_RET_UNKNW_FUNC = -2115, /**< unkown function name (rainerscript) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/vm.c b/runtime/vm.c index a25476c2..41d3e483 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -26,6 +26,7 @@ #include "config.h" #include #include +#include #include "rsyslog.h" #include "obj.h" @@ -39,6 +40,142 @@ DEFobjCurrIf(vmstk) DEFobjCurrIf(var) DEFobjCurrIf(sysvar) +/* ------------------------------ function registry code and structures ------------------------------ */ + +/* we maintain a registry of known functions */ +/* currently, this is a singly-linked list, this shall become a binary + * tree when we add the real call interface. So far, entries are added + * at the root, only. + */ +typedef struct s_rsf_entry { + cstr_t *pName; /* function name */ + prsf_t rsf; /* pointer to function code */ + struct s_rsf_entry *pNext; /* Pointer to next element or NULL */ +} rsf_entry_t; +rsf_entry_t *funcRegRoot = NULL; + + +/* add a function to the function registry. + * The handed-over cstr_t* object must no longer be used by the caller. + * A duplicate function name is an error. + * rgerhards, 2009-04-06 + */ +static rsRetVal +rsfrAddFunction(uchar *szName, prsf_t rsf) +{ + rsf_entry_t *pEntry; + size_t lenName; + DEFiRet; + + assert(szName != NULL); + assert(rsf != NULL); + + /* first check if we have a duplicate name, with the current approach this means + * we need to go through the whole list. + */ + lenName = strlen((char*)szName); + for(pEntry = funcRegRoot ; pEntry != NULL ; pEntry = pEntry->pNext) + if(!rsCStrSzStrCmp(pEntry->pName, szName, lenName)) + ABORT_FINALIZE(RS_RET_DUP_FUNC_NAME); + + /* unique name, so add to head of list */ + CHKmalloc(pEntry = calloc(1, sizeof(rsf_entry_t))); + CHKiRet(rsCStrConstructFromszStr(&pEntry->pName, szName)); + pEntry->rsf = rsf; + pEntry->pNext = funcRegRoot; + funcRegRoot = pEntry; + +finalize_it: + RETiRet; +} + + +/* find a function inside the function registry + * The caller provides a cstr_t with the function name and receives + * a function pointer back. If no function is found, an RS_RET_UNKNW_FUNC + * error is returned. So if the function returns with RS_RET_OK, the caller + * can savely assume the function pointer is valid. + * rgerhards, 2009-04-06 + */ +static rsRetVal +findRSFunction(cstr_t *pcsName, prsf_t *prsf) +{ + rsf_entry_t *pEntry; + rsf_entry_t *pFound; + DEFiRet; + + assert(prsf != NULL); + + /* find function by list walkthrough. */ + pFound = NULL; + for(pEntry = funcRegRoot ; pEntry != NULL && pFound == NULL ; pEntry = pEntry->pNext) + if(!rsCStrCStrCmp(pEntry->pName, pcsName)) + pFound = pEntry; + + if(pFound == NULL) + ABORT_FINALIZE(RS_RET_UNKNW_FUNC); + + *prsf = pFound->rsf; + +finalize_it: + RETiRet; +} + + +/* find the name of a RainerScript function whom's function pointer + * is known. This function returns the cstr_t object, which MUST NOT + * be modified by the caller. + * rgerhards, 2009-04-06 + */ +static rsRetVal +findRSFunctionName(prsf_t rsf, cstr_t **ppcsName) +{ + rsf_entry_t *pEntry; + rsf_entry_t *pFound; + DEFiRet; + + assert(rsf != NULL); + assert(ppcsName != NULL); + + /* find function by list walkthrough. */ + pFound = NULL; + for(pEntry = funcRegRoot ; pEntry != NULL && pFound == NULL ; pEntry = pEntry->pNext) + if(pEntry->rsf == rsf) + pFound = pEntry; + + if(pFound == NULL) + ABORT_FINALIZE(RS_RET_UNKNW_FUNC); + + *ppcsName = pFound->pName; + +finalize_it: + RETiRet; +} + + +/* free the whole function registry + */ +static void +rsfrRemoveAll(void) +{ + rsf_entry_t *pEntry; + rsf_entry_t *pEntryDel; + + BEGINfunc + pEntry = funcRegRoot; + while(pEntry != NULL) { + pEntryDel = pEntry; + pEntry = pEntry->pNext; + rsCStrDestruct(&pEntryDel->pName); + free(pEntryDel); + } + funcRegRoot = NULL; + ENDfunc +} + + +/* ------------------------------ end function registry code and structures ------------------------------ */ + /* ------------------------------ instruction set implementation ------------------------------ * * The following functions implement the VM's instruction set. @@ -330,7 +467,6 @@ CODESTARTop(PUSHSYSVAR) finalize_it: ENDop(PUSHSYSVAR) - /* The function call operation is only very roughly implemented. While the plumbing * to reach this instruction is fine, the instruction itself currently supports only * functions with a single argument AND with a name that we know. @@ -340,20 +476,9 @@ ENDop(PUSHSYSVAR) */ BEGINop(FUNC_CALL) /* remember to set the instruction also in the ENDop macro! */ var_t *numOperands; - var_t *operand1; - int iStrlen; CODESTARTop(FUNC_CALL) vmstk.PopNumber(pThis->pStk, &numOperands); - if(numOperands->val.num != 1) - ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); - vmstk.PopString(pThis->pStk, &operand1); /* guess there's just one ;) */ - if(!rsCStrSzStrCmp(pOp->operand.pVar->val.pStr, (uchar*) "strlen", 6)) { /* only one supported so far ;) */ -RUNLOG_VAR("%s", rsCStrGetSzStr(operand1->val.pStr)); - iStrlen = strlen((char*) rsCStrGetSzStr(operand1->val.pStr)); -RUNLOG_VAR("%d", iStrlen); - } else - ABORT_FINALIZE(RS_RET_INVLD_FUNC); - PUSHRESULTop(operand1, iStrlen); // TODO: dummy, FIXME + CHKiRet((*pOp->operand.rsf)(pThis->pStk, numOperands->val.num)); var.Destruct(&numOperands); /* no longer needed */ finalize_it: ENDop(FUNC_CALL) @@ -362,6 +487,89 @@ ENDop(FUNC_CALL) /* ------------------------------ end instruction set implementation ------------------------------ */ +/* ------------------------------ begin built-in function implementation ------------------------------ */ +/* note: this shall probably be moved to a separate module, but for the time being we do it directly + * in here. This is on our way to get from a dirty to a clean solution via baby steps that are + * a bit less dirty each time... + * + * The advantage of doing it here is that we do not yet need to think about how to handle the + * exit case, where we must not unload function modules which functions are still referenced. + * + * CALLING INTERFACE: + * The function must pop its parameters off the stack and pop its result onto + * the stack when it is finished. The number of parameters the function was + * called with is provided to it. If the argument count is less then what the function + * expected, it may handle the situation with defaults (or return an error). If the + * argument count is greater than expected, returnung an error is highly + * recommended (use RS_RET_INVLD_NBR_ARGUMENTS for these cases). + * + * All function names are prefixed with "rsf_" (RainerScript Function) to have + * a separate "name space". + * + * rgerhards, 2009-04-06 + */ + + +/* The strlen function, also probably a prototype of how all functions should be + * implemented. + * rgerhards, 2009-04-06 + */ +static rsRetVal +rsf_strlen(vmstk_t *pStk, int numOperands) +{ + DEFiRet; + var_t *operand1; + int iStrlen; + + if(numOperands != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + + /* pop args and do operaton (trivial case here...) */ + vmstk.PopString(pStk, &operand1); + iStrlen = strlen((char*) rsCStrGetSzStr(operand1->val.pStr)); + + /* Store result and cleanup */ + var.SetNumber(operand1, iStrlen); + vmstk.Push(pStk, operand1); +finalize_it: + RETiRet; +} + + +/* The "tolower" function, which converts its sole argument to lower case. + * Quite honestly, currently this is primarily a test driver for me... + * rgerhards, 2009-04-06 + */ +static rsRetVal +rsf_tolower(vmstk_t *pStk, int numOperands) +{ + DEFiRet; + var_t *operand1; + uchar *pSrc; + cstr_t *pcstr; + int iStrlen; + + if(numOperands != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + + /* pop args and do operaton */ + CHKiRet(rsCStrConstruct(&pcstr)); + vmstk.PopString(pStk, &operand1); + pSrc = rsCStrGetSzStr(operand1->val.pStr); + iStrlen = strlen((char*)pSrc); + while(iStrlen--) { + CHKiRet(rsCStrAppendChar(pcstr, tolower(*pSrc++))); + } + + /* Store result and cleanup */ + CHKiRet(rsCStrFinish(pcstr)); + var.SetString(operand1, pcstr); + vmstk.Push(pStk, operand1); +finalize_it: + RETiRet; +} + + /* Standard-Constructor */ BEGINobjConstruct(vm) /* be sure to specify the object type also in END macro! */ @@ -531,10 +739,23 @@ CODESTARTobjQueryInterface(vm) pIf->PopBoolFromStack = PopBoolFromStack; pIf->PopVarFromStack = PopVarFromStack; pIf->SetMsg = SetMsg; + pIf->FindRSFunction = findRSFunction; + pIf->FindRSFunctionName = findRSFunctionName; finalize_it: ENDobjQueryInterface(vm) +/* Exit the vm class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(vm, OBJ_IS_CORE_MODULE) /* class, version */ + rsfrRemoveAll(); + objRelease(sysvar, CORE_COMPONENT); + objRelease(var, CORE_COMPONENT); + objRelease(vmstk, CORE_COMPONENT); +ENDObjClassExit(vm) + + /* Initialize the vm class. Must be called as the very first method * before anything else is called inside this class. * rgerhards, 2008-02-19 @@ -548,6 +769,11 @@ BEGINObjClassInit(vm, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, vmDebugPrint); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, vmConstructFinalize); + + /* register built-in functions // TODO: move to its own module */ + CHKiRet(rsfrAddFunction((uchar*)"strlen", rsf_strlen)); + CHKiRet(rsfrAddFunction((uchar*)"tolower", rsf_tolower)); + ENDObjClassInit(vm) /* vi:set ai: diff --git a/runtime/vm.h b/runtime/vm.h index d2458220..cb3c69d0 100644 --- a/runtime/vm.h +++ b/runtime/vm.h @@ -55,8 +55,11 @@ BEGINinterface(vm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*PopBoolFromStack)(vm_t *pThis, var_t **ppVar); /* there are a few cases where we need this... */ rsRetVal (*PopVarFromStack)(vm_t *pThis, var_t **ppVar); /* there are a few cases where we need this... */ rsRetVal (*SetMsg)(vm_t *pThis, msg_t *pMsg); /* there are a few cases where we need this... */ + /* v2 (4.1.7) */ + rsRetVal (*FindRSFunction)(cstr_t *pcsName, prsf_t *prsf); /* 2009-06-04 */ + rsRetVal (*FindRSFunctionName)(prsf_t rsf, cstr_t **ppcsName); /* 2009-06-04 */ ENDinterface(vm) -#define vmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define vmCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/vmop.c b/runtime/vmop.c index a343481e..3e001d27 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -32,10 +32,12 @@ #include "rsyslog.h" #include "obj.h" #include "vmop.h" +#include "vm.h" /* static data */ DEFobjStaticHelpers DEFobjCurrIf(var) +DEFobjCurrIf(vm) /* forward definitions */ @@ -61,8 +63,10 @@ rsRetVal vmopConstructFinalize(vmop_t __attribute__((unused)) *pThis) /* destructor for the vmop object */ BEGINobjDestruct(vmop) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(vmop) - if(pThis->operand.pVar != NULL) - var.Destruct(&pThis->operand.pVar); + if(pThis->opcode != opcode_FUNC_CALL) { + if(pThis->operand.pVar != NULL) + var.Destruct(&pThis->operand.pVar); + } ENDobjDestruct(vmop) @@ -72,13 +76,19 @@ BEGINobjDebugPrint(vmop) /* be sure to specify the object type also in END and C cstr_t *pStrVar; CODESTARTobjDebugPrint(vmop) vmopOpcode2Str(pThis, &pOpcodeName); - CHKiRet(rsCStrConstruct(&pStrVar)); - CHKiRet(rsCStrFinish(&pStrVar)); - if(pThis->operand.pVar != NULL) { - CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); + if(pThis->opcode == opcode_FUNC_CALL) { + CHKiRet(vm.FindRSFunctionName(pThis->operand.rsf, &pStrVar)); + assert(pStrVar != NULL); + } else { + CHKiRet(rsCStrConstruct(&pStrVar)); + if(pThis->operand.pVar != NULL) { + CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); + } } + CHKiRet(rsCStrFinish(&pStrVar)); dbgoprint((obj_t*) pThis, "%.12s\t%s\n", pOpcodeName, rsCStrGetSzStrNoNULL(pStrVar)); - rsCStrDestruct(&pStrVar); + if(pThis->opcode != opcode_FUNC_CALL) + rsCStrDestruct(&pStrVar); finalize_it: ENDobjDebugPrint(vmop) @@ -98,6 +108,7 @@ static rsRetVal Obj2Str(vmop_t *pThis, cstr_t *pstrPrg) { uchar *pOpcodeName; + cstr_t *pcsFuncName; uchar szBuf[2048]; size_t lenBuf; DEFiRet; @@ -107,8 +118,13 @@ Obj2Str(vmop_t *pThis, cstr_t *pstrPrg) vmopOpcode2Str(pThis, &pOpcodeName); lenBuf = snprintf((char*) szBuf, sizeof(szBuf), "%s\t", pOpcodeName); CHKiRet(rsCStrAppendStrWithLen(pstrPrg, szBuf, lenBuf)); - if(pThis->operand.pVar != NULL) - CHKiRet(var.Obj2Str(pThis->operand.pVar, pstrPrg)); + if(pThis->opcode == opcode_FUNC_CALL) { + CHKiRet(vm.FindRSFunctionName(pThis->operand.rsf, &pcsFuncName)); + CHKiRet(rsCStrAppendCStr(pstrPrg, pcsFuncName)); + } else { + if(pThis->operand.pVar != NULL) + CHKiRet(var.Obj2Str(pThis->operand.pVar, pstrPrg)); + } CHKiRet(rsCStrAppendChar(pstrPrg, '\n')); finalize_it: @@ -116,6 +132,23 @@ finalize_it: } +/* set function + * rgerhards, 2009-04-06 + */ +static rsRetVal +vmopSetFunc(vmop_t *pThis, cstr_t *pcsFuncName) +{ + prsf_t rsf; /* pointer to function */ + DEFiRet; + ISOBJ_TYPE_assert(pThis, vmop); + CHKiRet(vm.FindRSFunction(pcsFuncName, &rsf)); /* check if function exists and obtain pointer to it */ + assert(rsf != NULL); /* just double-check, would be very hard to find! */ + pThis->operand.rsf = rsf; +finalize_it: + RETiRet; +} + + /* set operand (variant case) * rgerhards, 2008-02-20 */ @@ -248,6 +281,7 @@ CODESTARTobjQueryInterface(vmop) pIf->ConstructFinalize = vmopConstructFinalize; pIf->Destruct = vmopDestruct; pIf->DebugPrint = vmopDebugPrint; + pIf->SetFunc = vmopSetFunc; pIf->SetOpcode = vmopSetOpcode; pIf->SetVar = vmopSetVar; pIf->Opcode2Str = vmopOpcode2Str; @@ -263,6 +297,7 @@ ENDobjQueryInterface(vmop) BEGINObjClassInit(vmop, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(var, CORE_COMPONENT)); + CHKiRet(objUse(vm, CORE_COMPONENT)); OBJSetMethodHandler(objMethod_DEBUGPRINT, vmopDebugPrint); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, vmopConstructFinalize); diff --git a/runtime/vmop.h b/runtime/vmop.h index 938b08fd..67048c26 100644 --- a/runtime/vmop.h +++ b/runtime/vmop.h @@ -26,6 +26,7 @@ #define INCLUDED_VMOP_H #include "ctok_token.h" +#include "vmstk.h" #include "stringbuf.h" /* machine instructions types */ @@ -96,7 +97,8 @@ typedef struct vmop_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ opcode_t opcode; union { - var_t *pVar; /* for function call, this is the name (string) of function to be called */ + var_t *pVar; + prsf_t rsf; /* pointer to function for "call" instruction */ } operand; struct vmop_s *pNext; /* next operation or NULL, if end of program (logically this belongs to vmprg) */ } vmop_t; @@ -112,8 +114,13 @@ BEGINinterface(vmop) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetVar)(vmop_t *pThis, var_t *pVar); rsRetVal (*Opcode2Str)(vmop_t *pThis, uchar **ppName); rsRetVal (*Obj2Str)(vmop_t *pThis, cstr_t *pstr); + /* v2 */ + rsRetVal (*SetFunc)(vmop_t *pThis, cstr_t *pcsFuncName); ENDinterface(vmop) -#define vmopCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define vmopCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +/* interface changes, v1 -> v2 + * added SetFuct after existing function pointers -- rgerhards, 2009-04-06 + */ /* the remaining prototypes */ PROTOTYPEObj(vmop); diff --git a/runtime/vmprg.c b/runtime/vmprg.c index 75915025..07757b98 100644 --- a/runtime/vmprg.c +++ b/runtime/vmprg.c @@ -155,7 +155,6 @@ vmprgAddVarOperation(vmprg_t *pThis, opcode_t opcode, var_t *pVar) /* construct and fill vmop */ CHKiRet(vmop.Construct(&pOp)); CHKiRet(vmop.ConstructFinalize(pOp)); - CHKiRet(vmop.ConstructFinalize(pOp)); CHKiRet(vmop.SetOpcode(pOp, opcode)); if(pVar != NULL) CHKiRet(vmop.SetVar(pOp, pVar)); @@ -168,6 +167,32 @@ finalize_it: } +/* this is another shortcut for high-level callers. It is similar to vmprgAddVarOperation + * but adds a call operation. Among others, this include a check if the function + * is known. + */ +static rsRetVal +vmprgAddCallOperation(vmprg_t *pThis, cstr_t *pcsName) +{ + DEFiRet; + vmop_t *pOp; + + ISOBJ_TYPE_assert(pThis, vmprg); + + /* construct and fill vmop */ + CHKiRet(vmop.Construct(&pOp)); + CHKiRet(vmop.ConstructFinalize(pOp)); + CHKiRet(vmop.SetFunc(pOp, pcsName)); + CHKiRet(vmop.SetOpcode(pOp, opcode_FUNC_CALL)); + + /* and add it to the program */ + CHKiRet(vmprgAddOperation(pThis, pOp)); + +finalize_it: + RETiRet; +} + + /* queryInterface function * rgerhards, 2008-02-21 */ @@ -189,6 +214,7 @@ CODESTARTobjQueryInterface(vmprg) pIf->Obj2Str = Obj2Str; pIf->AddOperation = vmprgAddOperation; pIf->AddVarOperation = vmprgAddVarOperation; + pIf->AddCallOperation = vmprgAddCallOperation; finalize_it: ENDobjQueryInterface(vmprg) diff --git a/runtime/vmprg.h b/runtime/vmprg.h index c1042f7d..66f03913 100644 --- a/runtime/vmprg.h +++ b/runtime/vmprg.h @@ -57,8 +57,10 @@ BEGINinterface(vmprg) /* name must also be changed in ENDinterface macro! */ rsRetVal (*AddOperation)(vmprg_t *pThis, vmop_t *pOp); rsRetVal (*AddVarOperation)(vmprg_t *pThis, opcode_t opcode, var_t *pVar); rsRetVal (*Obj2Str)(vmprg_t *pThis, cstr_t *pstr); + /* v2 (4.1.7) */ + rsRetVal (*AddCallOperation)(vmprg_t *pThis, cstr_t *pVar); /* added 2009-04-06 */ ENDinterface(vmprg) -#define vmprgCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define vmprgCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/vmstk.h b/runtime/vmstk.h index 2d45ee4d..06657cf4 100644 --- a/runtime/vmstk.h +++ b/runtime/vmstk.h @@ -27,11 +27,11 @@ #define VMSTK_SIZE 256 /* the vmstk object */ -typedef struct vmstk_s { +struct vmstk_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ var_t *vStk[VMSTK_SIZE];/* the actual stack */ int iStkPtr; /* stack pointer, points to next free location, grows from 0 --> topend */ -} vmstk_t; +}; /* interfaces */ diff --git a/tests/3.rstest b/tests/3.rstest index 93cb941a..e75d9754 100644 --- a/tests/3.rstest +++ b/tests/3.rstest @@ -7,10 +7,10 @@ out: 00000000: push_msgvar msg[cstr] 00000001: push_const abc[cstr] 00000002: push_const 1[nbr] -00000003: func_call strlen[cstr] +00000003: func_call strlen 00000004: strconcat 00000005: push_const 1[nbr] -00000006: func_call strlen[cstr] +00000006: func_call strlen 00000007: push_const 20[nbr] 00000008: push_const 30[nbr] 00000009: add diff --git a/tools/syslogd.c b/tools/syslogd.c index b23c12a7..c85c4371 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3091,6 +3091,7 @@ GlobalClassExit(void) objRelease(net, LM_NET_FILENAME);/* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */ objRelease(conf, CORE_COMPONENT); objRelease(expr, CORE_COMPONENT); + vmClassExit(); /* this is hack, currently core_modules do not get this automatically called */ objRelease(vm, CORE_COMPONENT); objRelease(var, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); -- cgit v1.2.3 From 7db9f96fe9ecb9f8e05d45cc888aa488d8aed85f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 6 Apr 2009 18:07:52 +0200 Subject: testcase added (on user request) --- tests/testsuites/3.parse1 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/testsuites/3.parse1 diff --git a/tests/testsuites/3.parse1 b/tests/testsuites/3.parse1 new file mode 100644 index 00000000..a6b4e884 --- /dev/null +++ b/tests/testsuites/3.parse1 @@ -0,0 +1,3 @@ +<38>Apr 6 15:07:10 lxcvs07 sshd(pam_unix)[31738]: session closed for user cvsadmin +38,auth,info,Apr 6 15:07:10,lxcvs07,sshd(pam_unix),sshd(pam_unix)[31738]:, session closed for user cvsadmin +# yet another real-life sample where we had some issues with -- cgit v1.2.3 From d30388a499ed134b7f0b64180d03d5c7959f2fe4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Apr 2009 15:17:49 +0200 Subject: updating changelog for release --- ChangeLog | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 694ee4b8..adf36932 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,13 +1,11 @@ --------------------------------------------------------------------------- -Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? +Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation of CSV-formatted outputs (format from RFC4180 is used) - implemented function support in RainerScript. That means the engine parses and compile functions, as well as executes a few build-in ones. Dynamic loading and registration of functions is not yet supported - but we now have a good foundation to do that later on. - NOTE: nested function calls are not yet supported due to a design - issue with the function call VM instruction set design. - implemented the strlen() RainerScript function - added a template output module - added -T rsyslogd command line option, enables to specify a directory @@ -36,6 +34,10 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-03-?? more effcient for some outputs. They new can receive fields not only as a single string but rather in an array where each string is seperated. - added (some) developer documentation for output plugin interface +- bugfix: potential abort with DA queue after high watermark is reached + There exists a race condition that can lead to a segfault. Thanks + go to vbernetr, who performed the analysis and provided patch, which + I only tweaked a very little bit. --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages -- cgit v1.2.3 From 13344f4928654fc33cc37406164a4a56d22fff93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Tue, 7 Apr 2009 11:15:52 +0200 Subject: Re-enable parsing host names from message. There was a subtle bug that made all messages fill their HOSTNAME from the source IP (which may be wrong in a long chain of relays) and not by reading the message. This fixes it. --- tools/syslogd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/syslogd.c b/tools/syslogd.c index b23c12a7..a40b34dd 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1496,7 +1496,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) * the fields. I think this logic shall work with any type of syslog message. */ bTAGCharDetected = 0; - if(pMsg->bParseHOSTNAME) { + if(flags & PARSE_HOSTNAME) { /* TODO: quick and dirty memory allocation */ /* the memory allocated is far too much in most cases. But on the plus side, * it is quite fast... - rgerhards, 2007-09-20 -- cgit v1.2.3 From 845c6f59b91e9988f856556cbb0e88e275e8e591 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 7 Apr 2009 15:23:12 +0200 Subject: final touches for 4.1.6 release --- ChangeLog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ChangeLog b/ChangeLog index adf36932..937ee22f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,6 +38,8 @@ Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 There exists a race condition that can lead to a segfault. Thanks go to vbernetr, who performed the analysis and provided patch, which I only tweaked a very little bit. +- bugfix: imtcp did incorrectly parse hostname/tag + Thanks to Luis Fernando Muñoz Mejías for the patch. --------------------------------------------------------------------------- Version 4.1.5 [DEVEL] (rgerhards), 2009-03-11 - bugfix: parser did not correctly parse fields in UDP-received messages -- cgit v1.2.3 From 8f8e2cd66bd7f1e35f7bf0678e25bbdb99d67093 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Apr 2009 12:19:54 +0200 Subject: improved testbench, added tests for tcp-based reception --- ChangeLog | 1 + tests/Makefile.am | 6 +- tests/nettester.c | 392 ++++++++++++++++++++++++++++++++++++ tests/omod-if-array.sh | 6 +- tests/parsertest.sh | 6 +- tests/testsuites/omod-if-array.conf | 3 +- tests/testsuites/parse1.conf | 3 +- tests/udptester.c | 293 --------------------------- 8 files changed, 408 insertions(+), 302 deletions(-) create mode 100644 tests/nettester.c delete mode 100644 tests/udptester.c diff --git a/ChangeLog b/ChangeLog index 821d9439..98146600 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,7 @@ Version 4.1.7 [DEVEL] (rgerhards), 2009-03-?? performance. This is also necessary towards the long-term goal of loadable library modules. - added new RainerScript function "tolower" +- improved testbench, added tests for tcp-based reception --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation diff --git a/tests/Makefile.am b/tests/Makefile.am index ab1c5a62..fa662a3e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,5 +1,5 @@ TESTRUNS = rt_init rscript -check_PROGRAMS = $(TESTRUNS) ourtail udptester +check_PROGRAMS = $(TESTRUNS) ourtail nettester TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid @@ -32,8 +32,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ ourtail_SOURCES = ourtail.c -udptester_SOURCES = udptester.c getline.c -udptester_LDADD = $(SOL_LIBS) +nettester_SOURCES = nettester.c getline.c +nettester_LDADD = $(SOL_LIBS) rt_init_SOURCES = rt-init.c $(test_files) rt_init_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) diff --git a/tests/nettester.c b/tests/nettester.c new file mode 100644 index 00000000..89a784f3 --- /dev/null +++ b/tests/nettester.c @@ -0,0 +1,392 @@ +/* Runs a test suite on the rsyslog (and later potentially + * other things). + * + * The name of the test suite must be given as argv[1]. In this config, + * rsyslogd is loaded with config ./testsuites/.conf and then + * test cases ./testsuites/ *. are executed on it. This test driver is + * suitable for testing cases where a message sent (via UDP) results in + * exactly one response. It can not be used in cases where no response + * is expected (that would result in a hang of the test driver). + * Note: each test suite can contain many tests, but they all need to work + * with the same rsyslog configuration. + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EXIT_FAILURE 1 +#define INVALID_SOCKET -1 +/* Name of input file, must match $IncludeConfig in test suite .conf files */ +#define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */ + +static enum { inputUDP, inputTCP } inputMode; /* input for which tests are to be run */ +static pid_t rsyslogdPid = 0; /* pid of rsyslog instance being tested */ +static char *srcdir; /* global $srcdir, set so that we can run outside of "make check" */ +static char *testSuite; /* name of current test suite */ + + +void readLine(int fd, char *ln) +{ + char c; + int lenRead; + lenRead = read(fd, &c, 1); + while(lenRead == 1 && c != '\n') { + *ln++ = c; + lenRead = read(fd, &c, 1); + } + *ln = '\0'; +} + + +/* send a message via TCP + * We open the connection on the initial send, and never close it + * (let the OS do that). If a conneciton breaks, we do NOT try to + * recover, so all test after that one will fail (and the test + * driver probably hang. returns 0 if ok, something else otherwise. + * We use traditional framing '\n' at EOR for this tester. It may be + * worth considering additional framing modes. + * rgerhards, 2009-04-08 + */ +int +tcpSend(char *buf, int lenBuf) +{ + static int sock = INVALID_SOCKET; + struct sockaddr_in addr; + + if(sock == INVALID_SOCKET) { + /* first time, need to connect to target */ + if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(13514); + if(inet_aton("127.0.0.1", &addr.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + fprintf(stderr, "connect() failed\n"); + return(1); + } + } + + /* send test data */ + if(send(sock, buf, lenBuf, 0) != lenBuf) { + perror("send test data"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + /* send record terminator */ + if(send(sock, "\n", 1, 0) != 1) { + perror("send record terminator"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + return 0; +} + + +/* send a message via UDP + * returns 0 if ok, something else otherwise. + */ +int +udpSend(char *buf, int lenBuf) +{ + struct sockaddr_in si_other; + int s, slen=sizeof(si_other); + + if((s=socket(AF_INET, SOCK_DGRAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &si_other, 0, sizeof(si_other)); + si_other.sin_family = AF_INET; + si_other.sin_port = htons(12514); + if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + + if(sendto(s, buf, lenBuf, 0, (struct sockaddr*) &si_other, slen)==-1) { + perror("sendto"); + fprintf(stderr, "sendto() failed\n"); + return(1); + } + + close(s); + return 0; +} + + +/* open pipe to test candidate - so far, this is + * always rsyslogd and with a fixed config. Later, we may + * change this. Returns 0 if ok, something else otherwise. + * rgerhards, 2009-03-31 + */ +int openPipe(char *configFile, pid_t *pid, int *pfd) +{ + int pipefd[2]; + pid_t cpid; + char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", + "-M../runtime/.libs:../.libs", NULL }; + char confFile[1024]; + char *newenviron[] = { NULL }; + + + sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, configFile); + newargv[1] = confFile; + + if (pipe(pipefd) == -1) { + perror("pipe"); + exit(EXIT_FAILURE); + } + + cpid = fork(); + if (cpid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } + + if(cpid == 0) { /* Child reads from pipe */ + fclose(stdout); + dup(pipefd[1]); + close(pipefd[1]); + close(pipefd[0]); + fclose(stdin); + execve("../tools/rsyslogd", newargv, newenviron); + } else { + close(pipefd[1]); + *pid = cpid; + *pfd = pipefd[0]; + } + + return(0); +} + + +/* Process a specific test case. File name is provided. + * Needs to return 0 if all is OK, something else otherwise. + */ +int +processTestFile(int fd, char *pszFileName) +{ + FILE *fp; + char *testdata = NULL; + char *expected = NULL; + int ret = 0; + size_t lenLn; + char buf[4096]; + + if((fp = fopen((char*)pszFileName, "r")) == NULL) { + perror((char*)pszFileName); + return(2); + } + + /* skip comments at start of file */ + + getline(&testdata, &lenLn, fp); + while(!feof(fp)) { + if(*testdata == '#') + getline(&testdata, &lenLn, fp); + else + break; /* first non-comment */ + } + + + testdata[strlen(testdata)-1] = '\0'; /* remove \n */ + /* now we have the test data to send (we could use function pointers here...) */ + if(inputMode == inputUDP) { + if(udpSend(testdata, strlen(testdata)) != 0) + return(2); + } else { + if(tcpSend(testdata, strlen(testdata)) != 0) + return(2); + } + + /* next line is expected output + * we do not care about EOF here, this will lead to a failure and thus + * draw enough attention. -- rgerhards, 2009-03-31 + */ + getline(&expected, &lenLn, fp); + expected[strlen(expected)-1] = '\0'; /* remove \n */ + + /* pull response from server and then check if it meets our expectation */ + readLine(fd, buf); + if(strcmp(expected, buf)) { + printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", + expected, buf); + ret = 1; + } + + free(testdata); + free(expected); + fclose(fp); + return(ret); +} + + +/* carry out all tests. Tests are specified via a file name + * wildcard. Each of the files is read and the test carried + * out. + * Returns the number of tests that failed. Zero means all + * success. + */ +int +doTests(int fd, char *files) +{ + int iFailed = 0; + int iTests = 0; + int ret; + char *testFile; + glob_t testFiles; + size_t i = 0; + struct stat fileInfo; + + glob(files, GLOB_MARK, NULL, &testFiles); + + for(i = 0; i < testFiles.gl_pathc; i++) { + testFile = testFiles.gl_pathv[i]; + + if(stat((char*) testFile, &fileInfo) != 0) + continue; /* continue with the next file if we can't stat() the file */ + + ++iTests; + /* all regular files are run through the test logic. Symlinks don't work. */ + if(S_ISREG(fileInfo.st_mode)) { /* config file */ + printf("processing test case '%s' ... ", testFile); + ret = processTestFile(fd, testFile); + if(ret == 0) { + printf("successfully completed\n"); + } else { + printf("failed!\n"); + ++iFailed; + } + } + } + globfree(&testFiles); + + if(iTests == 0) { + printf("Error: no test cases found, no tests executed.\n"); + iFailed = 1; + } else { + printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); + } + + return(iFailed); +} + +/* cleanup */ +void doAtExit(void) +{ + int status; + + if(rsyslogdPid != 0) { + kill(rsyslogdPid, SIGTERM); + waitpid(rsyslogdPid, &status, 0); /* wait until instance terminates */ + } + + unlink(NETTEST_INPUT_CONF_FILE); +} + +/* Run the test suite. This must be called with exactly one parameter, the + * name of the test suite. For details, see file header comment at the top + * of this file. + * rgerhards, 2009-04-03 + */ +int main(int argc, char *argv[]) +{ + int fd; + int ret = 0; + FILE *fp; + char buf[4096]; + char testcases[4096]; + + if(argc != 3) { + printf("Invalid call of nettester\n"); + printf("Usage: nettester testsuite-name input\n"); + printf(" input = udp|tcp\n"); + exit(1); + } + + atexit(doAtExit); + + testSuite = argv[1]; + + if(!strcmp(argv[2], "udp")) + inputMode = inputUDP; + else if(!strcmp(argv[2], "tcp")) + inputMode = inputTCP; + else { + printf("error: unsupported input mode '%s'\n", argv[2]); + exit(1); + } + + if((srcdir = getenv("srcdir")) == NULL) + srcdir = "."; + + printf("Start of nettester run ($srcdir=%s, testsuite=%s)\n", srcdir, testSuite); + + /* create input config file */ + if((fp = fopen(NETTEST_INPUT_CONF_FILE, "w")) == NULL) { + perror(NETTEST_INPUT_CONF_FILE); + printf("error opening input configuration file\n"); + exit(1); + } + if(inputMode == inputUDP) { + fputs("$ModLoad ../plugins/imudp/.libs/imudp\n", fp); + fputs("$UDPServerRun 12514\n", fp); + } else { + fputs("$ModLoad ../plugins/imtcp/.libs/imtcp\n", fp); + fputs("$InputTCPServerRun 13514\n", fp); + } + fclose(fp); + + /* start to be tested rsyslogd */ + openPipe(argv[1], &rsyslogdPid, &fd); + readLine(fd, buf); + + /* generate filename */ + sprintf(testcases, "%s/testsuites/*.%s", srcdir, testSuite); + if(doTests(fd, testcases) != 0) + ret = 1; + + printf("End of nettester run (%d).\n", ret); + exit(ret); +} diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh index cac08928..8a8d67f3 100755 --- a/tests/omod-if-array.sh +++ b/tests/omod-if-array.sh @@ -1 +1,5 @@ -./udptester omod-if-array +#!/bin/bash -e +echo test omod-if-array via udp +./nettester omod-if-array udp +echo test omod-if-array via tcp +./nettester omod-if-array tcp diff --git a/tests/parsertest.sh b/tests/parsertest.sh index e7985bb0..fabe7e8d 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1 +1,5 @@ -./udptester parse1 +#!/bin/bash -e +echo test parsertest via udp +./nettester parse1 udp +echo test parsertest via tcp +./nettester parse1 tcp diff --git a/tests/testsuites/omod-if-array.conf b/tests/testsuites/omod-if-array.conf index e6c05a52..d88db166 100644 --- a/tests/testsuites/omod-if-array.conf +++ b/tests/testsuites/omod-if-array.conf @@ -3,8 +3,7 @@ # the testbench, so we do not need to focus on that) # rgerhards, 2009-04-03 $ModLoad ../plugins/omstdout/.libs/omstdout -$ModLoad ../plugins/imudp/.libs/imudp -$UDPServerRun 12514 +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! $ActionOMStdoutArrayInterface on $ErrorMessagesToStderr off diff --git a/tests/testsuites/parse1.conf b/tests/testsuites/parse1.conf index 0fb7d16d..947a05a8 100644 --- a/tests/testsuites/parse1.conf +++ b/tests/testsuites/parse1.conf @@ -1,6 +1,5 @@ $ModLoad ../plugins/omstdout/.libs/omstdout -$ModLoad ../plugins/imudp/.libs/imudp -$UDPServerRun 12514 +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! $ErrorMessagesToStderr off diff --git a/tests/udptester.c b/tests/udptester.c deleted file mode 100644 index a3c6658d..00000000 --- a/tests/udptester.c +++ /dev/null @@ -1,293 +0,0 @@ -/* Runs a test suite on the rsyslog (and later potentially - * other things). - * - * The name of the test suite must be given as argv[1]. In this config, - * rsyslogd is loaded with config ./testsuites/.conf and then - * test cases ./testsuites/ *. are executed on it. This test driver is - * suitable for testing cases where a message sent (via UDP) results in - * exactly one response. It can not be used in cases where no response - * is expected (that would result in a hang of the test driver). - * Note: each test suite can contain many tests, but they all need to work - * with the same rsyslog configuration. - * - * Part of the testbench for rsyslog. - * - * Copyright 2009 Rainer Gerhards and Adiscon GmbH. - * - * This file is part of rsyslog. - * - * Rsyslog is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Rsyslog is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Rsyslog. If not, see . - * - * A copy of the GPL can be found in the file "COPYING" in this distribution. - */ -#include "config.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define EXIT_FAILURE 1 - -static char *srcdir; /* global $srcdir, set so that we can run outside of "make check" */ -static char *testSuite; /* name of current test suite */ - - -void readLine(int fd, char *ln) -{ - char c; - int lenRead; - lenRead = read(fd, &c, 1); - while(lenRead == 1 && c != '\n') { - *ln++ = c; - lenRead = read(fd, &c, 1); - } - *ln = '\0'; -} - - -/* send a message via UDP - * returns 0 if ok, something else otherwise. - */ -int -udpSend(char *buf, int lenBuf) -{ - struct sockaddr_in si_other; - int s, slen=sizeof(si_other); - - if((s=socket(AF_INET, SOCK_DGRAM, 0))==-1) { - perror("socket()"); - return(1); - } - - memset((char *) &si_other, 0, sizeof(si_other)); - si_other.sin_family = AF_INET; - si_other.sin_port = htons(12514); - if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { - fprintf(stderr, "inet_aton() failed\n"); - return(1); - } - - if(sendto(s, buf, lenBuf, 0, (struct sockaddr*) &si_other, slen)==-1) { - perror("sendto"); - fprintf(stderr, "sendto() failed\n"); - return(1); - } - - close(s); - return 0; -} - - -/* open pipe to test candidate - so far, this is - * always rsyslogd and with a fixed config. Later, we may - * change this. Returns 0 if ok, something else otherwise. - * rgerhards, 2009-03-31 - */ -int openPipe(char *configFile, pid_t *pid, int *pfd) -{ - int pipefd[2]; - pid_t cpid; - char *newargv[] = {"../tools/rsyslogd", "dummy", "-c4", "-u2", "-n", "-irsyslog.pid", - "-M../runtime//.libs", NULL }; - char confFile[1024]; - char *newenviron[] = { NULL }; - - - sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, configFile); - newargv[1] = confFile; - - if (pipe(pipefd) == -1) { - perror("pipe"); - exit(EXIT_FAILURE); - } - - cpid = fork(); - if (cpid == -1) { - perror("fork"); - exit(EXIT_FAILURE); - } - - if(cpid == 0) { /* Child reads from pipe */ - fclose(stdout); - dup(pipefd[1]); - close(pipefd[1]); - close(pipefd[0]); - fclose(stdin); - execve("../tools/rsyslogd", newargv, newenviron); - } else { - close(pipefd[1]); - *pid = cpid; - *pfd = pipefd[0]; - } - - return(0); -} - - -/* Process a specific test case. File name is provided. - * Needs to return 0 if all is OK, something else otherwise. - */ -int -processTestFile(int fd, char *pszFileName) -{ - FILE *fp; - char *testdata = NULL; - char *expected = NULL; - int ret = 0; - size_t lenLn; - char buf[4096]; - - if((fp = fopen((char*)pszFileName, "r")) == NULL) { - perror((char*)pszFileName); - return(2); - } - - /* skip comments at start of file */ - - getline(&testdata, &lenLn, fp); - while(!feof(fp)) { - if(*testdata == '#') - getline(&testdata, &lenLn, fp); - else - break; /* first non-comment */ - } - - - testdata[strlen(testdata)-1] = '\0'; /* remove \n */ - /* now we have the test data to send */ - if(udpSend(testdata, strlen(testdata)) != 0) - return(2); - - /* next line is expected output - * we do not care about EOF here, this will lead to a failure and thus - * draw enough attention. -- rgerhards, 2009-03-31 - */ - getline(&expected, &lenLn, fp); - expected[strlen(expected)-1] = '\0'; /* remove \n */ - - /* pull response from server and then check if it meets our expectation */ - readLine(fd, buf); - if(strcmp(expected, buf)) { - printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", - expected, buf); - ret = 1; - } - - free(testdata); - free(expected); - fclose(fp); - return(ret); -} - - -/* carry out all tests. Tests are specified via a file name - * wildcard. Each of the files is read and the test carried - * out. - * Returns the number of tests that failed. Zero means all - * success. - */ -int -doTests(int fd, char *files) -{ - int iFailed = 0; - int iTests = 0; - int ret; - char *testFile; - glob_t testFiles; - size_t i = 0; - struct stat fileInfo; - - glob(files, GLOB_MARK, NULL, &testFiles); - - for(i = 0; i < testFiles.gl_pathc; i++) { - testFile = testFiles.gl_pathv[i]; - - if(stat((char*) testFile, &fileInfo) != 0) - continue; /* continue with the next file if we can't stat() the file */ - - ++iTests; - /* all regular files are run through the test logic. Symlinks don't work. */ - if(S_ISREG(fileInfo.st_mode)) { /* config file */ - printf("processing test case '%s' ... ", testFile); - ret = processTestFile(fd, testFile); - if(ret == 0) { - printf("successfully completed\n"); - } else { - printf("failed!\n"); - ++iFailed; - } - } - } - globfree(&testFiles); - - if(iTests == 0) { - printf("Error: no test cases found, no tests executed.\n"); - iFailed = 1; - } else { - printf("Number of tests run: %d, number of failures: %d\n", iTests, iFailed); - } - - return(iFailed); -} - - -/* Run the test suite. This must be called with exactly one parameter, the - * name of the test suite. For details, see file header comment at the top - * of this file. - * rgerhards, 2009-04-03 - */ -int main(int argc, char *argv[]) -{ - int fd; - pid_t pid; - int status; - int ret = 0; - char buf[4096]; - char testcases[4096]; - - if(argc != 2) { - printf("Invalid call of udptester\n"); - printf("Usage: udptester testsuite-name\n"); - exit(1); - } - - testSuite = argv[1]; - - if((srcdir = getenv("srcdir")) == NULL) - srcdir = "."; - - printf("Start of udptester run ($srcdir=%s, testsuite=%s)\n", srcdir, testSuite); - - openPipe(argv[1], &pid, &fd); - readLine(fd, buf); - - /* generate filename */ - sprintf(testcases, "%s/testsuites/*.%s", srcdir, testSuite); - if(doTests(fd, testcases) != 0) - ret = 1; - - /* cleanup */ - kill(pid, SIGTERM); - waitpid(pid, &status, 0); /* wait until instance terminates */ - printf("End of udptester run.\n"); - exit(ret); -} -- cgit v1.2.3 From 7cc7166cb24e50e48cab1cbb3c19c8c17d95ace7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Apr 2009 17:48:02 +0200 Subject: added new test case for many tcp connections It is checked that many tcp connections are properly handled. While adding this test, I noticed that there is a bug in imtcp that prevents creation of more than 200 connections. This bug still exists, so the test suite currently fails (what is correct). Will be addressed soon. --- ChangeLog | 4 +- tests/Makefile.am | 7 +- tests/chkseq.c | 76 +++++++++++++++++ tests/manytcp.sh | 13 +++ tests/tcpflood | Bin 0 -> 17940 bytes tests/tcpflood.c | 245 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 342 insertions(+), 3 deletions(-) create mode 100644 tests/chkseq.c create mode 100755 tests/manytcp.sh create mode 100755 tests/tcpflood create mode 100644 tests/tcpflood.c diff --git a/ChangeLog b/ChangeLog index 98146600..98b4f235 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,7 +5,9 @@ Version 4.1.7 [DEVEL] (rgerhards), 2009-03-?? performance. This is also necessary towards the long-term goal of loadable library modules. - added new RainerScript function "tolower" -- improved testbench, added tests for tcp-based reception +- improved testbench + * added tests for tcp-based reception + * added tcp-load test (1000 connections, 20,000 messages) --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation diff --git a/tests/Makefile.am b/tests/Makefile.am index fa662a3e..53a81a93 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTRUNS = rt_init rscript -check_PROGRAMS = $(TESTRUNS) ourtail nettester -TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh +check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq +TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -27,10 +27,13 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ parsertest.sh \ + manytcp.sh \ omod-if-array.sh \ cfg.sh ourtail_SOURCES = ourtail.c +tcpflood_SOURCES = tcpflood.c +chkseq_SOURCES = chkseq.c nettester_SOURCES = nettester.c getline.c nettester_LDADD = $(SOL_LIBS) diff --git a/tests/chkseq.c b/tests/chkseq.c new file mode 100644 index 00000000..3203c250 --- /dev/null +++ b/tests/chkseq.c @@ -0,0 +1,76 @@ +/* Checks if a file consists of line of strictly monotonically + * increasing numbers. An expected start and end number may + * be set. + * + * Params + * argv[1] file to check + * argv[2] start number + * argv[3] end number + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include + +int main(int argc, char *argv[]) +{ + FILE *fp; + int val; + int i; + int ret = 0; + int start, end; + + if(argc != 4) { + printf("Invalid call of chkseq\n"); + printf("Usage: chkseq file start end\n"); + exit(1); + } + + start = atoi(argv[2]); + end = atoi(argv[3]); + + if(start > end) { + printf("start must be less than or equal end!\n"); + exit(1); + } + + /* read file */ + fp = fopen(argv[1], "r"); + if(fp == NULL) { + perror(argv[1]); + exit(1); + } + + for(i = start ; i < end ; ++i) { + if(fscanf(fp, "%d\n", &val) != 1) { + printf("scanf error in index i=%d\n", i); + exit(1); + } + if(val != i) { + printf("read value %d, but expected value %d\n", val, i); + exit(1); + } + } + + exit(ret); +} diff --git a/tests/manytcp.sh b/tests/manytcp.sh new file mode 100755 index 00000000..3accfb8a --- /dev/null +++ b/tests/manytcp.sh @@ -0,0 +1,13 @@ +rm -f rsyslog.out.log # work file +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -ftestsuites/manytcp.conf & +echo "rsyslogd started with pid " `cat rsyslog.pid` +./tcpflood 127.0.0.1 13514 1000 20000 +sleep 1 +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 19999 +if [ "$?" -ne "0" ]; then + echo "sequence error detected" + exit 1 +fi diff --git a/tests/tcpflood b/tests/tcpflood new file mode 100755 index 00000000..90b89b84 Binary files /dev/null and b/tests/tcpflood differ diff --git a/tests/tcpflood.c b/tests/tcpflood.c new file mode 100644 index 00000000..cecec628 --- /dev/null +++ b/tests/tcpflood.c @@ -0,0 +1,245 @@ +/* Opens a large number of tcp connections and sends + * messages over them. This is used for stress-testing. + * + * Params + * argv[1] target address + * argv[2] target port + * argv[3] number of connections + * argv[4] number of messages to send (connection is random) + * + * Part of the testbench for rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define EXIT_FAILURE 1 +#define INVALID_SOCKET -1 +/* Name of input file, must match $IncludeConfig in test suite .conf files */ +#define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */ + +static char *targetIP; +static int targetPort; +static int numMsgsToSend; /* number of messages to send */ +static int numConnections; /* number of connections to create */ +static int *sockArray; /* array of sockets to use */ + + +/* open a single tcp connection + */ +int openConn(int *fd) +{ + int sock; + struct sockaddr_in addr; + + if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(targetPort); + if(inet_aton(targetIP, &addr.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + perror("connect()"); + fprintf(stderr, "connect() failed\n"); + return(1); + } + + *fd = sock; + return 0; +} + + +/* open all requested tcp connections + * this includes allocating the connection array + */ +int openConnections(void) +{ + int i; + char msgBuf[128]; + size_t lenMsg; + + write(1, " open connections", sizeof(" open connections")-1); + sockArray = calloc(numConnections, sizeof(int)); + for(i = 0 ; i < numConnections ; ++i) { + if(i % 10 == 0) { + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d", i); + write(1, msgBuf, lenMsg); + } + if(openConn(&(sockArray[i])) != 0) { + printf("error in trying to open connection i=%d\n", i); + return 1; + } + } + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d open connections\n", i); + write(1, msgBuf, lenMsg); + + return 0; +} + + +/* send messages to the tcp connections we keep open. We use + * a very basic format that helps identify the message + * (via msgnum:: e.g. msgnum:00000001:). This format is suitable + * for extracton to field-based properties. + * The first numConnection messages are sent sequentially, as are the + * last. All messages in between are sent over random connections. + * Note that message numbers start at 0. + */ +int sendMessages(void) +{ + int i; + int socknum; + int lenBuf; + char buf[2048]; + char msgBuf[128]; + size_t lenMsg; + + srand(time(NULL)); /* seed is good enough for our needs */ + + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d messages sent", 0); + write(1, msgBuf, lenMsg); + for(i = 0 ; i < numMsgsToSend ; ++i) { + if(i < numConnections) + socknum = i; + else if(i >= numMsgsToSend - numConnections) + socknum = i - (numMsgsToSend - numConnections); + else + socknum = rand() % numConnections; + lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); + if(send(sockArray[socknum], buf, lenBuf, 0) != lenBuf) { + perror("send test data"); + fprintf(stderr, "send() failed\n"); + return(1); + } + if(i % 100 == 0) { + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d", i); + write(1, msgBuf, lenMsg); + } + } + lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d messages sent\n", i); + write(1, msgBuf, lenMsg); + + return 0; +} + + +/* send a message via TCP + * We open the connection on the initial send, and never close it + * (let the OS do that). If a conneciton breaks, we do NOT try to + * recover, so all test after that one will fail (and the test + * driver probably hang. returns 0 if ok, something else otherwise. + * We use traditional framing '\n' at EOR for this tester. It may be + * worth considering additional framing modes. + * rgerhards, 2009-04-08 + */ +int +tcpSend(char *buf, int lenBuf) +{ + static int sock = INVALID_SOCKET; + struct sockaddr_in addr; + + if(sock == INVALID_SOCKET) { + /* first time, need to connect to target */ + if((sock=socket(AF_INET, SOCK_STREAM, 0))==-1) { + perror("socket()"); + return(1); + } + + memset((char *) &addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(13514); + if(inet_aton("127.0.0.1", &addr.sin_addr)==0) { + fprintf(stderr, "inet_aton() failed\n"); + return(1); + } + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { + fprintf(stderr, "connect() failed\n"); + return(1); + } + } + + /* send test data */ + if(send(sock, buf, lenBuf, 0) != lenBuf) { + perror("send test data"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + /* send record terminator */ + if(send(sock, "\n", 1, 0) != 1) { + perror("send record terminator"); + fprintf(stderr, "send() failed\n"); + return(1); + } + + return 0; +} + + +/* Run the test suite. This must be called with exactly one parameter, the + * name of the test suite. For details, see file header comment at the top + * of this file. + * rgerhards, 2009-04-03 + */ +int main(int argc, char *argv[]) +{ + int ret = 0; + static char buf[1024]; + + setvbuf(stdout, _IONBF, buf, 48); + + if(argc != 5) { + printf("Invalid call of tcpflood\n"); + printf("Usage: nettester testsuite-name input\n"); + exit(1); + } + + targetIP = argv[1]; + targetPort = atoi(argv[2]); + numConnections = atoi(argv[3]); + numMsgsToSend = atoi(argv[4]); + + if(openConnections() != 0) { + printf("error opening connections\n"); + exit(1); + } + + if(sendMessages() != 0) { + printf("error sending messages\n"); + exit(1); + } + exit(ret); +} -- cgit v1.2.3 From de38f744de1ce746e6e61098fd4a05ac76ef71a0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Apr 2009 18:18:31 +0200 Subject: minor cleanup --- tests/tcpflood | Bin 17940 -> 17972 bytes tests/tcpflood.c | 2 +- tools/syslogd.c | 8 -------- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/tcpflood b/tests/tcpflood index 90b89b84..ae00fcd5 100755 Binary files a/tests/tcpflood and b/tests/tcpflood differ diff --git a/tests/tcpflood.c b/tests/tcpflood.c index cecec628..83f0d1ee 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -223,7 +223,7 @@ int main(int argc, char *argv[]) if(argc != 5) { printf("Invalid call of tcpflood\n"); - printf("Usage: nettester testsuite-name input\n"); + printf("Usage: tcpflood target-host target-port num-connections num-messages\n"); exit(1); } diff --git a/tools/syslogd.c b/tools/syslogd.c index 0badac19..a4f0059b 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3591,18 +3591,10 @@ int realMain(int argc, char **argv) fprintf(stderr, "-t option only supported in compatibility modes 0 to 2 - ignored\n"); break; case 'T':/* chroot() immediately at program startup, but only for testing, NOT security yet */ -{ -char buf[1024]; -getcwd(buf, 1024); -printf("pwd: '%s'\n", buf); -printf("chroot to '%s'\n", arg); if(chroot(arg) != 0) { perror("chroot"); exit(1); } -getcwd(buf, 1024); -printf("pwd: '%s'\n", buf); -} break; case 'u': /* misc user settings */ iHelperUOpt = atoi(arg); -- cgit v1.2.3 From 1cfa08749b1c474de850f693915b9a32d456c593 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Apr 2009 18:59:15 +0200 Subject: bugfix: solved potential memory leak in msg processing could manifest itself in imtcp (loss of a few bytes for *each* received message - but depended on config) -- this was newly introduced --- ChangeLog | 2 ++ runtime/msg.c | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 98146600..c99cc00d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,8 @@ Version 4.1.7 [DEVEL] (rgerhards), 2009-03-?? of loadable library modules. - added new RainerScript function "tolower" - improved testbench, added tests for tcp-based reception +- bugfix: solved potential memory leak in msg processing, could manifest + itself in imtcp --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation diff --git a/runtime/msg.c b/runtime/msg.c index 5d1f21fd..9d5f3838 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1542,6 +1542,8 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) { assert(pMsg != NULL); assert(pBuf != NULL); + if(pMsg->pszHOSTNAME != NULL) + free(pMsg->pszHOSTNAME); pMsg->iLenHOSTNAME = strlen(pBuf); pMsg->pszHOSTNAME = (uchar*) pBuf; } @@ -1567,7 +1569,7 @@ void MsgSetHOSTNAME(msg_t *pMsg, char* pszHOSTNAME) if((pMsg->pszHOSTNAME = malloc(pMsg->iLenHOSTNAME + 1)) != NULL) memcpy(pMsg->pszHOSTNAME, pszHOSTNAME, pMsg->iLenHOSTNAME + 1); else - dbgprintf("Could not allocate memory in MsgSetHOSTNAME()\n"); + DBGPRINTF("Could not allocate memory in MsgSetHOSTNAME()\n"); } -- cgit v1.2.3 From d7f11ecb06688186d4c68b5933fb1437279ce03d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 8 Apr 2009 16:50:57 +0200 Subject: Make it work in batches of statements. Currently, all statements to be executed are stored on the same structure. When the batch size is reached, all statements are executed in a single transaction, and then committed. There are many corner cases in which an error may happen and the batch may be left in an inconsistent state, perhaps leaking memory or crashing. They will be fixed. --- plugins/omoracle/omoracle.c | 71 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index ea910d3a..9b1ec42d 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -15,7 +15,8 @@ $OmoracleDBPassword: password to log in on the database. $OmoracleDB: connection string (an Oracle easy connect or a db name as specified by tnsnames.ora) - + $OmoracleBatchSize: Number of elements to send to the DB. + All fields are mandatory. The dbstring can be an Oracle easystring or a DB name, as present in the tnsnames.ora file. @@ -50,6 +51,18 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) +/** */ +struct oracle_batch +{ + /* Batch size */ + int size; + /* Last element inserted in the buffer. The batch will be + * executed when n == size */ + int n; + /* Statements to run on this transaction */ + char** statements; +}; + typedef struct _instanceData { /* Environment handler, the base for any OCI work. */ OCIEnv* environment; @@ -67,6 +80,8 @@ typedef struct _instanceData { OCIBind* binding; /* Connection string, kept here for possible retries. */ char* connection; + /* Batch */ + struct oracle_batch batch; } instanceData; /** Database name, to be filled by the $OmoracleDB directive */ @@ -76,6 +91,8 @@ static char* db_name; static char* db_user; /** Database password, to be filled by the $OmoracleDBPassword */ static char* db_password; +/** Batch size. */ +static int batch_size; /** Generic function for handling errors from OCI. @@ -148,6 +165,12 @@ CODESTARTcreateInstance OCIHandleAlloc(pData->environment, (void*) &(pData->statement), OCI_HTYPE_STMT, 0, NULL)); + pData->batch.n = 0; + pData->batch.size = batch_size; + pData->batch.statements = calloc(pData->batch.size, + sizeof *pData->batch.statements); + CHKmalloc(pData->batch.statements); + finalize_it: ENDcreateInstance @@ -163,6 +186,9 @@ CODESTARTfreeInstance OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); OCIHandleFree(pData->statement, OCI_HTYPE_STMT); free(pData->connection); + while (pData->batch.size--) + free(pData->batch.statements[pData->batch.size]); + free(pData->batch.statements); dbgprintf ("omoracle freed all its resources\n"); RETiRet; @@ -263,21 +289,41 @@ CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct BEGINdoAction + int i; + int n; CODESTARTdoAction dbgprintf("omoracle attempting to execute statement %s\n", *ppString); - CHECKERR(pData->error, - OCIStmtPrepare(pData->statement, pData->error, *ppString, - strlen(*ppString), OCI_NTV_SYNTAX, - OCI_DEFAULT)); - CHECKERR(pData->error, - OCIStmtExecute(pData->service, pData->statement, pData->error, - 1, 0, NULL, NULL, OCI_DEFAULT)); - CHECKERR(pData->error, - OCITransCommit(pData->service, pData->error, 0)); + + if (pData->batch.n == pData->batch.size) { + dbgprintf("omoracle batch size limit hit, sending into DB\n"); + for (i = 0; i < pData->batch.n; i++) { + if (pData->batch.statements[i] == NULL) + continue; + n = strlen(pData->batch.statements[i]); + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, + pData->error, + pData->batch.statements[i], n, + OCI_NTV_SYNTAX, OCI_DEFAULT)); + CHECKERR(pData->error, + OCIStmtExecute(pData->service, + pData->statement, + pData->error, + 1, 0, NULL, NULL, OCI_DEFAULT)); + free(pData->batch.statements[i]); + pData->batch.statements[i] = NULL; + } + CHECKERR(pData->error, + OCITransCommit(pData->service, pData->error, 0)); + pData->batch.n = 0; + } + pData->batch.statements[pData->batch.n] = strdup(*ppString); + CHKmalloc(pData->batch.statements[pData->batch.n]); + pData->batch.n++; + finalize_it: dbgprintf ("omoracle %s at executing statement %s\n", iRet?"did not succeed":"succeeded", *ppString); -/* Clean credentials to avoid leakage in case of core dump. */ ENDdoAction BEGINmodExit @@ -331,4 +377,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledb", 0, eCmdHdlrGetWord, NULL, &db_name, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclebatchsize", 0, + eCmdHdlrInt, NULL, &batch_size, + STD_LOADABLE_MODULE_ID)); ENDmodInit -- cgit v1.2.3 From 9bbd5dfd25f3e2a7a6839ff0e7b4076186efbcf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 8 Apr 2009 16:50:58 +0200 Subject: Solve a memory leak when freeing Oracle instances. --- plugins/omoracle/omoracle.c | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 9b1ec42d..df06a8b6 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -190,7 +190,6 @@ CODESTARTfreeInstance free(pData->batch.statements[pData->batch.size]); free(pData->batch.statements); dbgprintf ("omoracle freed all its resources\n"); - RETiRet; ENDfreeInstance -- cgit v1.2.3 From 1ebbd3a2df09ed9e706be7762307cd17c4a5cad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 8 Apr 2009 16:50:59 +0200 Subject: Stop omoracle losing messages on rsyslog shutdown. When rsyslog shuts down, we must send and commit any pending messages or information will be lost. It will make rsyslog's shut down slower, but also more reliable. --- plugins/omoracle/omoracle.c | 65 +++++++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index df06a8b6..f6679953 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -12,13 +12,17 @@ namely: $OmoracleDBUser: user name to log in on the database. + $OmoracleDBPassword: password to log in on the database. + $OmoracleDB: connection string (an Oracle easy connect or a db name as specified by tnsnames.ora) - $OmoracleBatchSize: Number of elements to send to the DB. - All fields are mandatory. The dbstring can be an Oracle easystring - or a DB name, as present in the tnsnames.ora file. + $OmoracleBatchSize: Number of elements to send to the DB on each + transaction. + + All these directives are mandatory. The dbstring can be an Oracle + easystring or a DB name, as present in the tnsnames.ora file. Author: Luis Fernando Muñoz Mejías @@ -174,11 +178,45 @@ CODESTARTcreateInstance finalize_it: ENDcreateInstance +/* Inserts all stored statements into the database, releasing any + * allocated memory. */ +static int insert_to_db(instanceData* pData) +{ + DEFiRet; + int i, n; + + for (i = 0; i < pData->batch.n; i++) { + if (pData->batch.statements[i] == NULL) + continue; + n = strlen(pData->batch.statements[i]); + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, + pData->error, + pData->batch.statements[i], n, + OCI_NTV_SYNTAX, OCI_DEFAULT)); + CHECKERR(pData->error, + OCIStmtExecute(pData->service, + pData->statement, + pData->error, + 1, 0, NULL, NULL, OCI_DEFAULT)); + free(pData->batch.statements[i]); + pData->batch.statements[i] = NULL; + } + CHECKERR(pData->error, + OCITransCommit(pData->service, pData->error, 0)); + pData->batch.n = 0; +finalize_it: + RETiRet; +} + /** Close the session and free anything allocated by createInstance. */ BEGINfreeInstance CODESTARTfreeInstance +/* Before actually releasing our resources, let's try to commit + * anything pending so that we don't lose any messages. */ + insert_to_db(pData); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->environment, OCI_HTYPE_ENV); OCIHandleFree(pData->error, OCI_HTYPE_ERROR); @@ -295,26 +333,7 @@ CODESTARTdoAction if (pData->batch.n == pData->batch.size) { dbgprintf("omoracle batch size limit hit, sending into DB\n"); - for (i = 0; i < pData->batch.n; i++) { - if (pData->batch.statements[i] == NULL) - continue; - n = strlen(pData->batch.statements[i]); - CHECKERR(pData->error, - OCIStmtPrepare(pData->statement, - pData->error, - pData->batch.statements[i], n, - OCI_NTV_SYNTAX, OCI_DEFAULT)); - CHECKERR(pData->error, - OCIStmtExecute(pData->service, - pData->statement, - pData->error, - 1, 0, NULL, NULL, OCI_DEFAULT)); - free(pData->batch.statements[i]); - pData->batch.statements[i] = NULL; - } - CHECKERR(pData->error, - OCITransCommit(pData->service, pData->error, 0)); - pData->batch.n = 0; + CHKiRet(insert_to_db(pData)); } pData->batch.statements[pData->batch.n] = strdup(*ppString); CHKmalloc(pData->batch.statements[pData->batch.n]); -- cgit v1.2.3 From cf0cf2fa4bf84188baf873522d4a22887f4fdf90 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 13:57:35 +0200 Subject: removed accidently added binary --- tests/tcpflood | Bin 17972 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100755 tests/tcpflood diff --git a/tests/tcpflood b/tests/tcpflood deleted file mode 100755 index ae00fcd5..00000000 Binary files a/tests/tcpflood and /dev/null differ -- cgit v1.2.3 From 1e52abd8be2dfde8727c7e54b02a10ce5051815f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 14:14:00 +0200 Subject: fixing "make distcheck" --- tests/testsuites/manytcp.conf | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/testsuites/manytcp.conf diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf new file mode 100644 index 00000000..e491cd04 --- /dev/null +++ b/tests/testsuites/manytcp.conf @@ -0,0 +1,11 @@ +# Test for tcp "flood" testing +# rgerhards, 2009-04-08 +$ModLoad ../plugins/imtcp/.libs/imtcp +$inputtcpmaxsessions 2000 +$InputTCPServerRun 13514 + +$ErrorMessagesToStderr off + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt -- cgit v1.2.3 From d2d8cc9c1b65bedeafaa3a06ed53cbc8511be9a3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 14:45:23 +0200 Subject: fixed testbench compilation problem on Solaris Solaris network libraries needed to be specified in linker options --- tests/Makefile.am | 4 +++- tests/manytcp.sh | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 6d3c8725..87dca985 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -33,9 +33,11 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg.sh ourtail_SOURCES = ourtail.c -tcpflood_SOURCES = tcpflood.c chkseq_SOURCES = chkseq.c +tcpflood_SOURCES = tcpflood.c +tcpflood_LDADD = $(SOL_LIBS) + nettester_SOURCES = nettester.c getline.c nettester_LDADD = $(SOL_LIBS) diff --git a/tests/manytcp.sh b/tests/manytcp.sh index b38245df..f0a3eb96 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,5 +1,6 @@ rm -f rsyslog.out.log # work file ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & +sleep 1 echo "rsyslogd started with pid " `cat rsyslog.pid` ./tcpflood 127.0.0.1 13514 1000 20000 sleep 1 -- cgit v1.2.3 From e07b3f380f87166ce055ac4f3253f0367ef5cdce Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 18:51:21 +0200 Subject: working some more on "make distcheck" ... this time I think successfully (at least on Fedora...) --- Makefile.am | 2 +- tests/Makefile.am | 1 + tests/manytcp.sh | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index dfb33339..58e1cfce 100644 --- a/Makefile.am +++ b/Makefile.am @@ -127,5 +127,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-oracle +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout ACLOCAL_AMFLAGS = -I m4 diff --git a/tests/Makefile.am b/tests/Makefile.am index 53a81a93..6d3c8725 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -28,6 +28,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.omod-if-array \ parsertest.sh \ manytcp.sh \ + testsuites/manytcp.conf \ omod-if-array.sh \ cfg.sh diff --git a/tests/manytcp.sh b/tests/manytcp.sh index 3accfb8a..b38245df 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,5 +1,5 @@ rm -f rsyslog.out.log # work file -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -ftestsuites/manytcp.conf & +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & echo "rsyslogd started with pid " `cat rsyslog.pid` ./tcpflood 127.0.0.1 13514 1000 20000 sleep 1 @@ -8,6 +8,8 @@ rm -f work sort < rsyslog.out.log > work ./chkseq work 0 19999 if [ "$?" -ne "0" ]; then + rm -f work rsyslog.out.log echo "sequence error detected" exit 1 fi +rm -f work rsyslog.out.log -- cgit v1.2.3 From 9633ce1afcb3c3e44f969be0503360a4c0c75599 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 19:17:35 +0200 Subject: fixed compile-time problems in im3195 ... however, I did not not a test run due to the lack of existing test drivers and the very low (aka "non-existing" interest from the userbase in the feature). --- plugins/im3195/im3195.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/im3195/im3195.c b/plugins/im3195/im3195.c index 1c2502fe..106da2c8 100644 --- a/plugins/im3195/im3195.c +++ b/plugins/im3195/im3195.c @@ -47,6 +47,7 @@ #include "liblogging/syslogmessage.h" #include "module-template.h" #include "cfsysline.h" +#include "msg.h" #include "errmsg.h" MODULE_TYPE_INPUT @@ -83,7 +84,7 @@ void OnReceive(srAPIObj __attribute__((unused)) *pMyAPI, srSLMGObj* pSLMG) srSLMGGetRawMSG(pSLMG, &pszRawMsg); parseAndSubmitMessage(fromHost, fromHostIP, pszRawMsg, strlen((char*)pszRawMsg), - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_FULL_DELAY, (uchar*)"im3195"); + PARSE_HOSTNAME, eFLOWCTL_FULL_DELAY, (uchar*)"im3195", NULL, 0); } -- cgit v1.2.3 From b9b96fbfc66809532f4ed24d667947fa63ce68f5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 19:38:17 +0200 Subject: removed MSG_NOSIGNAL & provided work-around as this send() option is not supported on Solaris. We now simply ignore SIGPIPE --- tests/tcpflood.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 254e9fd6..fcb68998 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -170,7 +170,7 @@ int sendMessages(void) else socknum = rand() % numConnections; lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); - lenSend = send(sockArray[socknum], buf, lenBuf, MSG_NOSIGNAL); + lenSend = send(sockArray[socknum], buf, lenBuf, 0); if(lenSend != lenBuf) { printf("\r%5.5d\n", i); fflush(stdout); @@ -250,8 +250,17 @@ tcpSend(char *buf, int lenBuf) int main(int argc, char *argv[]) { int ret = 0; + struct sigaction sigAct; static char buf[1024]; + /* on Solaris, we do not HAVE MSG_NOSIGNAL, so for this reason + * we block SIGPIPE (not an issue for this program) + */ + memset(&sigAct, 0, sizeof(sigAct)); + sigemptyset(&sigAct.sa_mask); + sigAct.sa_handler = SIG_IGN; + sigaction(SIGPIPE, &sigAct, NULL); + setvbuf(stdout, buf, _IONBF, 48); if(argc != 5) { -- cgit v1.2.3 From 8d65a9cdd49b812fb86cfd1b33f01c27e995db36 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 14 Apr 2009 14:55:37 +0200 Subject: updated project status --- doc/status.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/status.html b/doc/status.html index dae94884..fff3a6ff 100644 --- a/doc/status.html +++ b/doc/status.html @@ -5,15 +5,15 @@

        This page reflects the status as of 2009-04-03.

        Current Releases

        -

        development: 4.1.5 [2009-03-11] - -change log - -download +

        development: 4.1.6 [2009-04-07] - +change log - +download
        beta: 3.21.11 [2009-04-03] - change log - download

        -

        v3 stable: 3.20.3 [2009-04-02] - change log - +

        v3 stable: 3.20.5 [2009-04-02] - change log - download
        v2 stable: 2.0.6 [2008-08-07] - change log - -- cgit v1.2.3 From 04272876d12488b2039b28683dc53e1c802d303d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 14 Apr 2009 13:52:07 +0200 Subject: implemented $MaxOpenFiles directive and changed testbench ... to utilize it. This work is not yet fully verified to be correct. --- runtime/rsyslog.h | 1 + tests/manytcp.sh | 13 +++++++--- tests/tcpflood.c | 59 +++++++++++++++++++++++++++++++++++-------- tests/testsuites/manytcp.conf | 4 ++- tools/syslogd.c | 28 ++++++++++++++++++++ 5 files changed, 89 insertions(+), 16 deletions(-) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index cea457d8..25ec30fc 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -262,6 +262,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ RS_RET_DUP_FUNC_NAME = -2114, /**< duplicate function name (rainerscript) */ RS_RET_UNKNW_FUNC = -2115, /**< unkown function name (rainerscript) */ + RS_RET_ERR_RLIM_NOFILE = -2116, /**< error setting max. nbr open files process limit */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/tests/manytcp.sh b/tests/manytcp.sh index f0a3eb96..d9b2e9a0 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,13 +1,18 @@ -rm -f rsyslog.out.log # work file +rm -f work rsyslog.out.log rsyslog.out.log.save # work files ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & sleep 1 echo "rsyslogd started with pid " `cat rsyslog.pid` -./tcpflood 127.0.0.1 13514 1000 20000 -sleep 1 +# the config file specifies exactly 1100 connections +./tcpflood 127.0.0.1 13514 1000 40000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 5 # we need this so that rsyslogd can receive all outstanding messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -./chkseq work 0 19999 +./chkseq work 0 39999 if [ "$?" -ne "0" ]; then rm -f work rsyslog.out.log echo "sequence error detected" diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 83f0d1ee..254e9fd6 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -95,21 +95,49 @@ int openConnections(void) sockArray = calloc(numConnections, sizeof(int)); for(i = 0 ; i < numConnections ; ++i) { if(i % 10 == 0) { - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d", i); - write(1, msgBuf, lenMsg); + printf("\r%5.5d", i); + //lenMsg = sprintf(msgBuf, "\r%5.5d", i); + //write(1, msgBuf, lenMsg); } if(openConn(&(sockArray[i])) != 0) { printf("error in trying to open connection i=%d\n", i); return 1; } } - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d open connections\n", i); + lenMsg = sprintf(msgBuf, "\r%5.5d open connections\n", i); write(1, msgBuf, lenMsg); return 0; } +/* we also close all connections because otherwise we may get very bad + * timing for the syslogd - it may not be able to process all incoming + * messages fast enough if we immediately shut down. + * TODO: it may be an interesting excercise to handle that situation + * at the syslogd level, too + * rgerhards, 2009-04-14 + */ +void closeConnections(void) +{ + int i; + char msgBuf[128]; + size_t lenMsg; + + write(1, " close connections", sizeof(" close connections")-1); + for(i = 0 ; i < numConnections ; ++i) { + if(i % 10 == 0) { + lenMsg = sprintf(msgBuf, "\r%5.5d", i); + write(1, msgBuf, lenMsg); + } + close(sockArray[i]); + } + lenMsg = sprintf(msgBuf, "\r%5.5d close connections\n", i); + write(1, msgBuf, lenMsg); + +} + + /* send messages to the tcp connections we keep open. We use * a very basic format that helps identify the message * (via msgnum:: e.g. msgnum:00000001:). This format is suitable @@ -123,13 +151,16 @@ int sendMessages(void) int i; int socknum; int lenBuf; + int lenSend; char buf[2048]; char msgBuf[128]; size_t lenMsg; srand(time(NULL)); /* seed is good enough for our needs */ - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d messages sent", 0); + printf("Sending %d messages.\n", numMsgsToSend); + printf("\r%5.5d messages sent", 0); + lenMsg = sprintf(msgBuf, "\r%5.5d/%5.5d messages sent", 0, numMsgsToSend); write(1, msgBuf, lenMsg); for(i = 0 ; i < numMsgsToSend ; ++i) { if(i < numConnections) @@ -139,18 +170,20 @@ int sendMessages(void) else socknum = rand() % numConnections; lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); - if(send(sockArray[socknum], buf, lenBuf, 0) != lenBuf) { + lenSend = send(sockArray[socknum], buf, lenBuf, MSG_NOSIGNAL); + if(lenSend != lenBuf) { + printf("\r%5.5d\n", i); + fflush(stdout); perror("send test data"); - fprintf(stderr, "send() failed\n"); + printf("send() failed at socket %d, index %d\n", socknum, i); + fflush(stderr); return(1); } if(i % 100 == 0) { - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d", i); - write(1, msgBuf, lenMsg); + printf("\r%5.5d", i); } } - lenMsg = sprintf(msgBuf, "\retb\b\b\b\b%5.5d messages sent\n", i); - write(1, msgBuf, lenMsg); + printf("\r%5.5d messages sent\n", i); return 0; } @@ -219,7 +252,7 @@ int main(int argc, char *argv[]) int ret = 0; static char buf[1024]; - setvbuf(stdout, _IONBF, buf, 48); + setvbuf(stdout, buf, _IONBF, 48); if(argc != 5) { printf("Invalid call of tcpflood\n"); @@ -241,5 +274,9 @@ int main(int argc, char *argv[]) printf("error sending messages\n"); exit(1); } + + //closeConnections(); + printf("End of tcpflood Run\n"); + exit(ret); } diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf index e491cd04..8175732e 100644 --- a/tests/testsuites/manytcp.conf +++ b/tests/testsuites/manytcp.conf @@ -1,7 +1,9 @@ # Test for tcp "flood" testing # rgerhards, 2009-04-08 $ModLoad ../plugins/imtcp/.libs/imtcp -$inputtcpmaxsessions 2000 +$MainMsgQueueTimeoutShutdown 10000 +$MaxOpenFiles 2000 +$InputTCPMaxSessions 1100 $InputTCPServerRun 13514 $ErrorMessagesToStderr off diff --git a/tools/syslogd.c b/tools/syslogd.c index a4f0059b..8c86c12e 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -87,6 +87,7 @@ #include #include #include +#include #include #if HAVE_SYS_TIMESPEC_H @@ -2073,6 +2074,32 @@ static rsRetVal setActionResumeInterval(void __attribute__((unused)) *pVal, int } +/* set the processes max number ob files (upon configuration request) + * 2009-04-14 rgerhards + */ +static rsRetVal setMaxFiles(void __attribute__((unused)) *pVal, int iFiles) +{ + struct rlimit maxFiles; + char errStr[1024]; + DEFiRet; + + maxFiles.rlim_cur = iFiles; + maxFiles.rlim_max = iFiles; + + if(setrlimit(RLIMIT_NOFILE, &maxFiles) < 0) { + /* NOTE: under valgrind, we seem to be unable to extend the size! */ + rs_strerror_r(errno, errStr, sizeof(errStr)); + errmsg.LogError(0, RS_RET_ERR_RLIM_NOFILE, "could not set process file limit to %d: %s [kernel max %ld]", + iFiles, errStr, (long) maxFiles.rlim_max); + ABORT_FINALIZE(RS_RET_ERR_RLIM_NOFILE); + } + dbgprintf("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); + +finalize_it: + RETiRet; +} + + /* set the processes umask (upon configuration request) */ static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) { @@ -2870,6 +2897,7 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"modload", 0, eCmdHdlrCustomHandler, conf.doModLoad, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"includeconfig", 0, eCmdHdlrCustomHandler, conf.doIncludeLine, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"umask", 0, eCmdHdlrFileCreateMode, setUmask, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"maxopenfiles", 0, eCmdHdlrInt, setMaxFiles, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debugprinttemplatelist", 0, eCmdHdlrBinary, NULL, &bDebugPrintTemplateList, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debugprintmodulelist", 0, eCmdHdlrBinary, NULL, &bDebugPrintModuleList, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"debugprintcfsyslinehandlerlist", 0, eCmdHdlrBinary, -- cgit v1.2.3 From 01e5d51c57536acfa8e05d5b22c001a6cf0701d8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 14 Apr 2009 14:36:50 +0200 Subject: added doc for $MaxOpenFiles directive --- ChangeLog | 1 + doc/rsconf1_maxopenfiles.html | 35 +++++++++++++++++++++++++++++++++++ doc/rsyslog_conf.html | 6 +++--- doc/rsyslog_conf_global.html | 18 +++++++++--------- 4 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 doc/rsconf1_maxopenfiles.html diff --git a/ChangeLog b/ChangeLog index 792c108c..3108cdb3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,7 @@ Version 4.3.0 [DEVEL] (rgerhards), 2009-03-?? - improved testbench * added tests for tcp-based reception * added tcp-load test (1000 connections, 20,000 messages) +- added $MaxOpenFiles configuration directive - bugfix: solved potential memory leak in msg processing, could manifest itself in imtcp --------------------------------------------------------------------------- diff --git a/doc/rsconf1_maxopenfiles.html b/doc/rsconf1_maxopenfiles.html new file mode 100644 index 00000000..b6c9cc0e --- /dev/null +++ b/doc/rsconf1_maxopenfiles.html @@ -0,0 +1,35 @@ + + +$MaxOpenFiles - rsyslog.conf file + + +[rsyslog configuration directive overview] + +

        $MaxOpenFiles

        +

        Available Since: 4.3.0

        +

        Type: global configuration directive

        +

        Default: operating system default

        +

        Description:

        +

        Set the maximum number of files that the rsyslog process can have open at any given +time. Note that this includes open tcp sockets, so this setting is the upper limit for +the number of open TCP connections as well. If you expect a large nubmer of concurrent +connections, it is suggested that the number is set to the max number connected plus 1000. +Please note that each dynafile also requires up to 100 open file handles. +

        The setting is similar to running "ulimit -n number-of-files". +

        Please note that depending on permissions and operating system configuration, the +setrlimit() request issued by rsyslog may fail, in which case the previous limit is kept +in effect. Rsyslog will emit a warning message in this case. +

        Sample:

        +

        $MaxOpenFiles 2000

        +

        Bugs:

        +

        For some reason, this settings seems not to work on all platforms. If you experience +problems, please let us know so that we can (hopefully) narrow down the issue. +

        [rsyslog.conf overview] [manual +index] [rsyslog site]

        +

        This documentation is part of the +rsyslog project.
        +Copyright © 2009 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 3 or higher.

        + + diff --git a/doc/rsyslog_conf.html b/doc/rsyslog_conf.html index 852d95b5..6990c6bd 100644 --- a/doc/rsyslog_conf.html +++ b/doc/rsyslog_conf.html @@ -26,7 +26,7 @@ Lines can be continued by specifying a backslash ("\") as the last character of the line. There is a hard-coded maximum line length of 4K. If you need lines larger than that, you need to change compile-time settings inside rsyslog and recompile. -

        Global Directives

        +

        Configuration Directives

        Basic Structure

        Rsyslog supports standard sysklogd's configuration file format and extends it. So in general, you can take a "normal" syslog.conf and @@ -74,9 +74,9 @@ such features is available in rsyslogd, only.

        [rsyslog site]

        This documentation is part of the rsyslog project.
        -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008,2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL -version 2 or higher.

        +version 3 or higher.

        > diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index d011bd2b..3e33f0da 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -1,14 +1,14 @@ -Global Directives - rsyslog.conf +Configuration Directives - rsyslog.conf

        This is a part of the rsyslog.conf documentation.

        back -

        Global Directives

        -

        All global directives need to be specified on a line by their -own and must start with a dollar-sign. Here is a list in alphabetical -order. Follow links for a description.

        -

        Please note that not all directives here are actually global. Some affect -only the next action. This documentation will be changed soon. +

        Configuration Directives

        +

        All configuration directives need to be specified on a line by their +own and must start with a dollar-sign. Note that those starting with +the word "Action" modify the next action and should be specified +in front of it. +

        Here is a list in alphabetical order. Follow links for a description.

        Not all directives have an in-depth description right now. Default values for them are in bold. A more in-depth description will appear as implementation progresses. @@ -180,6 +180,7 @@ instead of UDP (plain TCP syslog, RELP). This resolves the UDP stack size restri
        Note that 2k, the current default, is the smallest size that must be supported in order to be compliant to the upcoming new syslog RFC series.

        • +
      • $MaxOpenFiles
      • $ModDir
      • $ModLoad
      • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, @@ -214,7 +215,6 @@ the value, the less precise the timestamp.
      • $PrivDropToGroupID
      • $PrivDropToUser
      • $PrivDropToUserID
        • -
    • $UMASK

    Where <size_nbr> is specified above, @@ -235,7 +235,7 @@ point of view, "1,,0.0.,.,0" also has the value 1000.

    rsyslog project.
    Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL -version 2 or higher.

    +version 3 or higher.

    -- cgit v1.2.3 From ba5a59128f6559d58cdd4defe46a9db564d3e2c1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 9 Apr 2009 20:48:08 +0200 Subject: cosmetic fix (status message) --- tests/tcpflood.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/tcpflood.c b/tests/tcpflood.c index fcb68998..9c17fd5b 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -160,8 +160,6 @@ int sendMessages(void) printf("Sending %d messages.\n", numMsgsToSend); printf("\r%5.5d messages sent", 0); - lenMsg = sprintf(msgBuf, "\r%5.5d/%5.5d messages sent", 0, numMsgsToSend); - write(1, msgBuf, lenMsg); for(i = 0 ; i < numMsgsToSend ; ++i) { if(i < numConnections) socknum = i; -- cgit v1.2.3 From 5ccec36e8d69230ddbd94d1b63bc9f6518c5ade8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Apr 2009 14:58:46 +0200 Subject: highlighting $DirCreateMode fix --- ChangeLog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ChangeLog b/ChangeLog index 6f8c96ae..ac9d9fd6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,9 @@ Version 4.1.7 [BETA] (rgerhards), 2009-04-?? - bugfix: $InputTCPMaxSessions config directive was accepted, but not honored. This resulted in a fixed upper limit of 200 connections. +- bugfix: the default for $DirCreateMode was 0644, and as such wrong. + It has now been changed to 0700. For some background, please see + http://lists.adiscon.net/pipermail/rsyslog/2009-April/001986.html --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation -- cgit v1.2.3 From 65a85de3d97ab6bc427ea005b75e4b416013de3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 15 Apr 2009 16:53:16 +0200 Subject: Convert to the array-based interface. We'll receive a single statement to be prepared and a batch size. Then, doAction will execute the statement only once per batch hit, making the process much more efficient. This will reduce network and DB server overhead. The downside is that this version cannot be used with rsyslog v3 anymore. If anyone is interested on backporting the module, they should choose all patches up to this one. Better documentation may follow. --- plugins/omoracle/omoracle.c | 203 +++++++++++++++++++++++++++++++++++--------- plugins/omoracle/omoracle.h | 2 + 2 files changed, 164 insertions(+), 41 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index f6679953..02f83551 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -21,9 +21,28 @@ $OmoracleBatchSize: Number of elements to send to the DB on each transaction. + $OmoracleStatement: Statement to be prepared and executed in + batches. Please note that Oracle's prepared statements have their + placeholders as ':identifier', and this module uses the colon to + guess how many placeholders there will be. + All these directives are mandatory. The dbstring can be an Oracle easystring or a DB name, as present in the tnsnames.ora file. + The form of the template is just a list of strings you want + inserted to the DB, for instance: + + $template TestStmt,"%hostname%%msg%" + + Will provide the arguments to a statement like + + $OmoracleStatement \ + insert into foo(hostname,message)values(:host,:message) + + Also note that identifiers to placeholders are arbitrarry. You + need to define the properties on the template in the correct order + you want them passed to the statement! + Author: Luis Fernando Muñoz Mejías @@ -40,6 +59,7 @@ #include #include #include +#include #include "dirty.h" #include "syslogd-types.h" #include "srUtils.h" @@ -63,8 +83,12 @@ struct oracle_batch /* Last element inserted in the buffer. The batch will be * executed when n == size */ int n; - /* Statements to run on this transaction */ - char** statements; + /* Number of arguments the statement takes */ + int arguments; + /* Parameters to pass to the statement on this transaction */ + char*** parameters; + /* Binding parameters */ + OCIBind** bindings; }; typedef struct _instanceData { @@ -80,10 +104,10 @@ typedef struct _instanceData { OCISvcCtx* service; /* Credentials object for the connection. */ OCIAuthInfo* authinfo; - /* Binding parameters, currently unused */ - OCIBind* binding; /* Connection string, kept here for possible retries. */ char* connection; + /* Statement to be prepared. */ + char* txt_statement; /* Batch */ struct oracle_batch batch; } instanceData; @@ -97,6 +121,12 @@ static char* db_user; static char* db_password; /** Batch size. */ static int batch_size; +/** Statement to prepare and execute */ +static char* db_statement; +/** Whether or not the core supports the newer array interface. The + * module is able to work in both modes, but the newer is the + * recommended one for performance reasons. */ +static int array_passing; /** Generic function for handling errors from OCI. @@ -149,9 +179,49 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_ERROR; } +/** Returns the number of bind parameters for the statement given as + * an argument. */ +static int count_bind_parameters(char* p) +{ + int n = 0; + + for (; *p; p++) + if (*p == BIND_MARK) + n++; + dbgprintf ("omoracle statement has %d parameters\n", n); + return n; +} + +/** Prepares the statement, binding all its positional parameters */ +static int prepare_statement(instanceData* pData) +{ + int i; + DEFiRet; + + CHECKERR(pData->error, + OCIStmtPrepare(pData->statement, + pData->error, + pData->txt_statement, + strlen(pData->txt_statement), + OCI_NTV_SYNTAX, OCI_DEFAULT)); + for (i = 0; i < pData->batch.arguments; i++) + CHECKERR(pData->error, + OCIBindByPos(pData->statement, + pData->batch.bindings+i, + pData->error, + i+1, + pData->batch.parameters[i], + sizeof (OCILobLocator*), + SQLT_STR, NULL, NULL, NULL, + 0, 0, OCI_DEFAULT)); +finalize_it: + RETiRet; +} + /* Resource allocation */ BEGINcreateInstance + int i; CODESTARTcreateInstance ASSERT(pData != NULL); @@ -168,12 +238,29 @@ CODESTARTcreateInstance CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->statement), OCI_HTYPE_STMT, 0, NULL)); + pData->txt_statement = strdup(db_statement); + CHKmalloc(pData->txt_statement); + dbgprintf("omoracle will run stored statement: %s\n", + pData->txt_statement); pData->batch.n = 0; pData->batch.size = batch_size; - pData->batch.statements = calloc(pData->batch.size, - sizeof *pData->batch.statements); - CHKmalloc(pData->batch.statements); + pData->batch.arguments = count_bind_parameters(pData->txt_statement); + + /* I know, this can be done with a single malloc() call but this is + * easier to read. :) */ + pData->batch.parameters = malloc(pData->batch.arguments * + sizeof *pData->batch.parameters); + CHKmalloc(pData->batch.parameters); + for (i = 0; i < pData->batch.arguments; i++) { + pData->batch.parameters[i] = calloc(pData->batch.size, + sizeof **pData->batch.parameters); + CHKmalloc(pData->batch.parameters[i]); + } + + pData->batch.bindings = calloc(pData->batch.arguments, + sizeof *pData->batch.bindings); + CHKmalloc(pData->batch.bindings); finalize_it: ENDcreateInstance @@ -183,35 +270,34 @@ ENDcreateInstance static int insert_to_db(instanceData* pData) { DEFiRet; - int i, n; + int i, j, n; + + CHECKERR(pData->error, + OCIStmtExecute(pData->service, + pData->statement, + pData->error, + pData->batch.n, 0, NULL, NULL, OCI_DEFAULT)); - for (i = 0; i < pData->batch.n; i++) { - if (pData->batch.statements[i] == NULL) - continue; - n = strlen(pData->batch.statements[i]); - CHECKERR(pData->error, - OCIStmtPrepare(pData->statement, - pData->error, - pData->batch.statements[i], n, - OCI_NTV_SYNTAX, OCI_DEFAULT)); - CHECKERR(pData->error, - OCIStmtExecute(pData->service, - pData->statement, - pData->error, - 1, 0, NULL, NULL, OCI_DEFAULT)); - free(pData->batch.statements[i]); - pData->batch.statements[i] = NULL; - } CHECKERR(pData->error, OCITransCommit(pData->service, pData->error, 0)); + + for (i = 0; i < pData->batch.arguments; i++) + for (j = 0; j < pData->batch.n; j++) { + free(pData->batch.parameters[i][j]); + pData->batch.parameters[i][j] = NULL; + } + pData->batch.n = 0; finalize_it: + dbgprintf ("omoracle insertion to DB %s\n", iRet == RS_RET_OK ? + "succeeded" : "did not succeed"); RETiRet; } /** Close the session and free anything allocated by createInstance. */ BEGINfreeInstance + int i, j; CODESTARTfreeInstance /* Before actually releasing our resources, let's try to commit @@ -224,9 +310,13 @@ CODESTARTfreeInstance OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); OCIHandleFree(pData->statement, OCI_HTYPE_STMT); free(pData->connection); - while (pData->batch.size--) - free(pData->batch.statements[pData->batch.size]); - free(pData->batch.statements); + for (i = 0; i < pData->batch.arguments; i++) { + for (j = 0; j < pData->batch.size; j++) + free(pData->batch.parameters[i][j]); + free(pData->batch.parameters[i]); + } + free(pData->batch.parameters); + free(pData->batch.bindings); dbgprintf ("omoracle freed all its resources\n"); ENDfreeInstance @@ -253,7 +343,7 @@ CODESTARTtryResume * ... of course I don't know why Oracle might need a full restart... * rgerhards, 2009-03-26 */ - dbgprintf("Attempting to reconnect to DB server\n"); + dbgprintf("omoracle attempting to reconnect to DB server\n"); OCISessionRelease(pData->service, pData->error, NULL, 0, OCI_DEFAULT); OCIHandleFree(pData->service, OCI_HTYPE_SVCCTX); CHECKERR(pData->error, OCISessionGet(pData->environment, pData->error, @@ -261,6 +351,7 @@ CODESTARTtryResume pData->connection, strlen(pData->connection), NULL, 0, NULL, NULL, NULL, OCI_DEFAULT)); + CHKiRet(prepare_statement(pData)); finalize_it: ENDtryResume @@ -296,7 +387,6 @@ CODESTARTisCompatibleWithFeature ENDisCompatibleWithFeature BEGINparseSelectorAct - CODESTARTparseSelectorAct CODE_STD_STRING_REQUESTparseSelectorAct(1); @@ -314,11 +404,12 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); } CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, - OMSR_RQD_TPL_OPT_SQL, " StdFmt")); + OMSR_TPL_AS_ARRAY, " StdFmt")); CHKiRet(createInstance(&pData)); CHKmalloc(pData->connection = strdup(db_name)); CHKiRet(startSession(pData, db_name, db_user, db_password)); - + CHKiRet(prepare_statement(pData)); + dbgprintf ("omoracle module got all its resources allocated " "and connected to the DB\n"); @@ -327,21 +418,23 @@ ENDparseSelectorAct BEGINdoAction int i; - int n; + int n = pData->batch.n; + char **params = (char**) ppString[0]; CODESTARTdoAction - dbgprintf("omoracle attempting to execute statement %s\n", *ppString); - if (pData->batch.n == pData->batch.size) { + if (n == pData->batch.size) { dbgprintf("omoracle batch size limit hit, sending into DB\n"); CHKiRet(insert_to_db(pData)); } - pData->batch.statements[pData->batch.n] = strdup(*ppString); - CHKmalloc(pData->batch.statements[pData->batch.n]); + + for (i = 0; i < pData->batch.arguments && params[i]; i++) { + dbgprintf ("omoracle argument on batch[%d][%d]: %s\n", i, n, params[i]); + pData->batch.parameters[i][n] = strdup(params[i]); + CHKmalloc(pData->batch.parameters[i][n]); + } pData->batch.n++; finalize_it: - dbgprintf ("omoracle %s at executing statement %s\n", - iRet?"did not succeed":"succeeded", *ppString); ENDdoAction BEGINmodExit @@ -373,16 +466,35 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, memset(db_password, 0, n); free(db_password); } - db_name = db_user = db_password = NULL; + if (db_statement != NULL) + free(db_statement); + db_name = db_user = db_password = db_statement = NULL; + RETiRet; +} + +/** As I don't find any handler that reads an entire line, I write my + * own. */ +static int get_db_statement(char** line, char** stmt) +{ + DEFiRet; + + while (isspace(**line)) + (*line)++; + dbgprintf ("Config line: %s\n", *line); + *stmt = strdup(*line); + CHKmalloc(*stmt); + dbgprintf ("Statement: %s\n", *stmt); +finalize_it: RETiRet; } BEGINmodInit() + rsRetVal (*supported_options)(unsigned long *pOpts); + unsigned long opts; CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); - /* CHKiRet(omsdRegCFSLineHdlr((uchar*)"actionomoracle", */ CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); @@ -398,4 +510,13 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclebatchsize", 0, eCmdHdlrInt, NULL, &batch_size, STD_LOADABLE_MODULE_ID)); + CHKiRet(pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &supported_options)); + CHKiRet((*supported_options)(&opts)); + if (!(array_passing = opts & OMSR_TPL_AS_ARRAY)) + ABORT_FINALIZE(RS_RET_ERR); + + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatement", 0, + eCmdHdlrCustomHandler, get_db_statement, + &db_statement, STD_LOADABLE_MODULE_ID)); + ENDmodInit diff --git a/plugins/omoracle/omoracle.h b/plugins/omoracle/omoracle.h index b0e70917..92bcacab 100644 --- a/plugins/omoracle/omoracle.h +++ b/plugins/omoracle/omoracle.h @@ -20,4 +20,6 @@ enum { MAX_BUFSIZE = 2048 }; +#define BIND_MARK ':' + #endif -- cgit v1.2.3 From 24fcd96203c9b8b84a8414cea19dcb3ba989c9ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 15 Apr 2009 16:53:17 +0200 Subject: Fixed a mem leak --- plugins/omoracle/omoracle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 02f83551..12efd61c 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -310,6 +310,7 @@ CODESTARTfreeInstance OCIHandleFree(pData->authinfo, OCI_HTYPE_AUTHINFO); OCIHandleFree(pData->statement, OCI_HTYPE_STMT); free(pData->connection); + free(pData->txt_statement); for (i = 0; i < pData->batch.arguments; i++) { for (j = 0; j < pData->batch.size; j++) free(pData->batch.parameters[i][j]); -- cgit v1.2.3 From f89b761c84270cde71e0a6275ea80bb20f60d2df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 15 Apr 2009 16:53:18 +0200 Subject: Make the counting of bind parameters aware of literals. Literal strings passed in the statement may contain ':', let's not count them. --- plugins/omoracle/omoracle.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 12efd61c..b598192f 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -180,14 +180,21 @@ static int oci_errors(void* handle, ub4 htype, sword status) } /** Returns the number of bind parameters for the statement given as - * an argument. */ + * an argument. It counts the number of appearances of ':', as in + * + * insert into foo(bar, baz) values(:bar, :baz) + * + * while taking in account that string literals must not be parsed. */ static int count_bind_parameters(char* p) { int n = 0; + int enable = 1; for (; *p; p++) - if (*p == BIND_MARK) + if (enable && *p == BIND_MARK ) n++; + else if (*p == '\'') + enable ^= 1; dbgprintf ("omoracle statement has %d parameters\n", n); return n; } @@ -429,7 +436,6 @@ CODESTARTdoAction } for (i = 0; i < pData->batch.arguments && params[i]; i++) { - dbgprintf ("omoracle argument on batch[%d][%d]: %s\n", i, n, params[i]); pData->batch.parameters[i][n] = strdup(params[i]); CHKmalloc(pData->batch.parameters[i][n]); } -- cgit v1.2.3 From 9a897329ec6f80c99ca039f12388961417e0a422 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 15 Apr 2009 16:53:19 +0200 Subject: Add some debugging output --- plugins/omoracle/omoracle.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index b598192f..7199f3e1 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -436,6 +436,7 @@ CODESTARTdoAction } for (i = 0; i < pData->batch.arguments && params[i]; i++) { + dbgprintf("batch[%d][%d]=%s\n", i, n, params[i]); pData->batch.parameters[i][n] = strdup(params[i]); CHKmalloc(pData->batch.parameters[i][n]); } -- cgit v1.2.3 From ca28204f7ba5c5c520b52180e26471e12af83560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 15 Apr 2009 16:53:20 +0200 Subject: Add the callback for OCIBindDynamic. Let's hope it works. --- plugins/omoracle/omoracle.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 7199f3e1..48b97b27 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -179,6 +179,34 @@ static int oci_errors(void* handle, ub4 htype, sword status) return OCI_ERROR; } +/** Callback for OCIBindDynamic. + * + * OCI doesn't insert an array of char* by itself (although it can + * handle arrays of int), so we must either run in batches of size one + * (no way) or bind all parameters with OCI_DATA_AT_EXEC instead of + * OCI_DEFAULT, and then give this function as an argument to + * OCIBindDynamic so that it is able to handle all strings in a single + * server trip. + * + * See the documentation of OCIBindDynamic + * (http://download.oracle.com/docs/cd/B28359_01/appdev.111/b28395/oci16rel003.htm#i444015) + * for more details. + */ +static int __attribute__((unused)) +bind_dynamic (char** in, OCIBind __attribute__((unused))* bind, + int iter, int __attribute__((unused)) idx, + char** out, int* buflen, char* piece, + void** bd) +{ + dbgprintf ("Bound line: %s\n", in[iter]); + *out = in[iter]; + *buflen = sizeof (OCILobLocator*); + *piece = OCI_ONE_PIECE; + *bd = NULL; + return OCI_CONTINUE; +} + + /** Returns the number of bind parameters for the statement given as * an argument. It counts the number of appearances of ':', as in * -- cgit v1.2.3 From 668f9a79fb7268f7935d93249cf283664662996d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Wed, 15 Apr 2009 16:53:21 +0200 Subject: Fixing the batch insertions. Previous versions inserted garbage (the pointer was interpreted as the string itself). It seems inserting arrays of strings is not that easy with OCI. This approach consumes 2KB per entry in the batch, so if you have batches of size 1000 you'll be using 2MB for the batch. This size doesn't change, anyways and the risk of leaking memory is gone. OCI doesn't deal well with batches of strings. :( --- plugins/omoracle/omoracle.c | 63 ++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 48b97b27..ddcb2ffa 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -192,15 +192,15 @@ static int oci_errors(void* handle, ub4 htype, sword status) * (http://download.oracle.com/docs/cd/B28359_01/appdev.111/b28395/oci16rel003.htm#i444015) * for more details. */ -static int __attribute__((unused)) -bind_dynamic (char** in, OCIBind __attribute__((unused))* bind, - int iter, int __attribute__((unused)) idx, - char** out, int* buflen, char* piece, +static int bind_dynamic (char** in, OCIBind __attribute__((unused))* bind, + int iter, int __attribute__((unused)) idx, + char** out, int* buflen, unsigned char* piece, void** bd) { - dbgprintf ("Bound line: %s\n", in[iter]); *out = in[iter]; - *buflen = sizeof (OCILobLocator*); + *buflen = strlen(*out) + 1; + dbgprintf ("omoracle bound line %d, length %d: %s\n", iter, *buflen, + *out); *piece = OCI_ONE_PIECE; *bd = NULL; return OCI_CONTINUE; @@ -239,16 +239,22 @@ static int prepare_statement(instanceData* pData) pData->txt_statement, strlen(pData->txt_statement), OCI_NTV_SYNTAX, OCI_DEFAULT)); - for (i = 0; i < pData->batch.arguments; i++) + for (i = 0; i < pData->batch.arguments; i++) { CHECKERR(pData->error, OCIBindByPos(pData->statement, pData->batch.bindings+i, - pData->error, - i+1, - pData->batch.parameters[i], - sizeof (OCILobLocator*), + pData->error, i+1, NULL, + MAX_BUFSIZE * + sizeof ***pData->batch.parameters, SQLT_STR, NULL, NULL, NULL, - 0, 0, OCI_DEFAULT)); + 0, 0, OCI_DATA_AT_EXEC)); + CHECKERR(pData->error, + OCIBindDynamic(pData->batch.bindings[i], + pData->error, + pData->batch.parameters[i], + bind_dynamic, NULL, NULL)); + } + finalize_it: RETiRet; } @@ -256,7 +262,7 @@ finalize_it: /* Resource allocation */ BEGINcreateInstance - int i; + int i, j; CODESTARTcreateInstance ASSERT(pData != NULL); @@ -284,13 +290,25 @@ CODESTARTcreateInstance /* I know, this can be done with a single malloc() call but this is * easier to read. :) */ - pData->batch.parameters = malloc(pData->batch.arguments * + pData->batch.parameters = calloc(pData->batch.arguments, sizeof *pData->batch.parameters); CHKmalloc(pData->batch.parameters); for (i = 0; i < pData->batch.arguments; i++) { pData->batch.parameters[i] = calloc(pData->batch.size, sizeof **pData->batch.parameters); CHKmalloc(pData->batch.parameters[i]); + for (j = 0; j < pData->batch.size; j++) { + /* Each entry has at most MAX_BUFSIZE bytes + * because OCI doesn't like null-terminated + * strings when operating with batches, and + * the maximum size of each entry must be + * provided when binding + * parameters. MAX_BUFSIZE is long enough for + * usual entries. */ + pData->batch.parameters[i][j] = calloc(MAX_BUFSIZE, + sizeof ***pData->batch.parameters); + CHKmalloc(pData->batch.parameters[i][j]); + } } pData->batch.bindings = calloc(pData->batch.arguments, @@ -305,7 +323,6 @@ ENDcreateInstance static int insert_to_db(instanceData* pData) { DEFiRet; - int i, j, n; CHECKERR(pData->error, OCIStmtExecute(pData->service, @@ -316,12 +333,6 @@ static int insert_to_db(instanceData* pData) CHECKERR(pData->error, OCITransCommit(pData->service, pData->error, 0)); - for (i = 0; i < pData->batch.arguments; i++) - for (j = 0; j < pData->batch.n; j++) { - free(pData->batch.parameters[i][j]); - pData->batch.parameters[i][j] = NULL; - } - pData->batch.n = 0; finalize_it: dbgprintf ("omoracle insertion to DB %s\n", iRet == RS_RET_OK ? @@ -454,19 +465,19 @@ ENDparseSelectorAct BEGINdoAction int i; - int n = pData->batch.n; char **params = (char**) ppString[0]; CODESTARTdoAction - if (n == pData->batch.size) { + if (pData->batch.n == pData->batch.size) { dbgprintf("omoracle batch size limit hit, sending into DB\n"); CHKiRet(insert_to_db(pData)); } for (i = 0; i < pData->batch.arguments && params[i]; i++) { - dbgprintf("batch[%d][%d]=%s\n", i, n, params[i]); - pData->batch.parameters[i][n] = strdup(params[i]); - CHKmalloc(pData->batch.parameters[i][n]); + dbgprintf("batch[%d][%d]=%s\n", i, pData->batch.n, params[i]); + strncpy(pData->batch.parameters[i][pData->batch.n], params[i], + MAX_BUFSIZE); + CHKmalloc(pData->batch.parameters[i][pData->batch.n]); } pData->batch.n++; -- cgit v1.2.3 From 3c886026be31a6af61a1b86773634c7bc4a44d7e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Apr 2009 16:55:25 +0200 Subject: added some doc for omoracle --- doc/Makefile.am | 1 + doc/modules.html | 5 ++- doc/omoracle.html | 78 +++++++++++++++++++++++++++++++++++++++++++ doc/rsyslog_conf_modules.html | 5 +-- 4 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 doc/omoracle.html diff --git a/doc/Makefile.am b/doc/Makefile.am index 3015d6b5..0f3dca70 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -33,6 +33,7 @@ html_files = \ dev_queue.html \ omsnmp.html \ ommysql.html \ + omoracle.html \ omlibdbi.html \ imfile.html \ imtcp.html \ diff --git a/doc/modules.html b/doc/modules.html index 92887508..4eae6db3 100644 --- a/doc/modules.html +++ b/doc/modules.html @@ -4,9 +4,8 @@

    About rsyslog Modules

    -

    Written by - Rainer - Gerhards (2007-07-28)

    +

    Written by +Rainer Gerhards (2007-07-28)

    This document is incomplete. The module interface is also quite incomplete and under development. Do not currently use it! You may want to visit Rainer's blog diff --git a/doc/omoracle.html b/doc/omoracle.html new file mode 100644 index 00000000..40f6360f --- /dev/null +++ b/doc/omoracle.html @@ -0,0 +1,78 @@ + + + +Oracle Database Output Module + + + +rsyslog module reference + +

    Oracle Database Output Module

    +

    Module Name:    omoracle

    +

    Author: Luis Fernando Muñoz Mejías <Luis.Fernando.Munoz.Mejias@cern.ch>

    +

    Available since: : 4.3.0 +

    Status: : contributed module, not maitained by rsyslog core authors +

    Description:

    +

    This module provides native support for logging to Oracle databases. It offers +superior performance over the more generic omlibdbi module. +It also includes a number of enhancements, most importantly prepared statements and +batching, what provides a big performance improvements. +

    +

    Note that this module is maintained by its original author. If you need assistance with it, +it is suggested to post questions to the +rsyslog mailing list. +

    From the header comments of this module: +

    +
+    This is an output module feeding directly to an Oracle
+    database. It uses Oracle Call Interface, a propietary module
+    provided by Oracle.
+
+    Selector lines to be used are of this form:
+
+    :omoracle:;TemplateName
+
+    The module gets its configuration via rsyslog $... directives,
+    namely:
+
+    $OmoracleDBUser: user name to log in on the database.
+
+    $OmoracleDBPassword: password to log in on the database.
+
+    $OmoracleDB: connection string (an Oracle easy connect or a db
+    name as specified by tnsnames.ora)
+
+    $OmoracleBatchSize: Number of elements to send to the DB on each
+    transaction.
+
+    $OmoracleStatement: Statement to be prepared and executed in
+    batches. Please note that Oracle's prepared statements have their
+    placeholders as ':identifier', and this module uses the colon to
+    guess how many placeholders there will be.
+
+    All these directives are mandatory. The dbstring can be an Oracle
+    easystring or a DB name, as present in the tnsnames.ora file.
+
+    The form of the template is just a list of strings you want
+    inserted to the DB, for instance:
+
+    $template TestStmt,"%hostname%%msg%"
+
+    Will provide the arguments to a statement like
+
+    $OmoracleStatement \
+        insert into foo(hostname,message)values(:host,:message)
+
+    Also note that identifiers to placeholders are arbitrarry. You
+    need to define the properties on the template in the correct order
+    you want them passed to the statement!
+
    +

    [rsyslog.conf overview] +[manual index] [rsyslog site]

    +

    This documentation is part of the +rsyslog +project.
    +Copyright © 2008, 2009 by Rainer Gerhards and +Adiscon. +Released under the GNU GPL version 3 or higher.

    + diff --git a/doc/rsyslog_conf_modules.html b/doc/rsyslog_conf_modules.html index a281d9e7..df9abeea 100644 --- a/doc/rsyslog_conf_modules.html +++ b/doc/rsyslog_conf_modules.html @@ -19,6 +19,7 @@ generic database output module (Firebird/Interbase, MS SQL, Sybase, SQLLite, Ingres, Oracle, mSQL)
  • ommail - permits rsyslog to alert folks by mail if something important happens
    • +
  • omoracle - output module for Oracle (native OCI interface)
  • imfile -  input module for text files
  • imrelp - RELP @@ -44,9 +45,9 @@ only available if it has been loaded (using $ModLoad).

    [rsyslog site]

    This documentation is part of the rsyslog project.
    -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL -version 2 or higher.

    +version 3 or higher.

    -- cgit v1.2.3 From 2d5e8ba7cd05a95f897506e51dcc5adb06dbcaa8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 16 Apr 2009 17:26:07 +0200 Subject: added a new error code for too-old rsyslog core which can be emittend when plugin can not load due to missing core functionality. --- plugins/omoracle/omoracle.c | 2 +- runtime/rsyslog.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index ddcb2ffa..71cc8e1f 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -560,7 +560,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &supported_options)); CHKiRet((*supported_options)(&opts)); if (!(array_passing = opts & OMSR_TPL_AS_ARRAY)) - ABORT_FINALIZE(RS_RET_ERR); + ABORT_FINALIZE(RS_RET_RSCORE_TOO_OLD); CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatement", 0, eCmdHdlrCustomHandler, get_db_statement, diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index cea457d8..8e181b9e 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -262,6 +262,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_INVLD_FUNC = -2113, /**< invalid function name for function call (rainerscript) */ RS_RET_DUP_FUNC_NAME = -2114, /**< duplicate function name (rainerscript) */ RS_RET_UNKNW_FUNC = -2115, /**< unkown function name (rainerscript) */ + RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit v1.2.3 From 83c40e8838a2b64a3fdb02196944302533079d2a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 12:37:53 +0200 Subject: some cleanup --- runtime/queue.c | 39 --------------------------------------- 1 file changed, 39 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index c4a0fad2..e5db866c 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -607,28 +607,7 @@ static rsRetVal qDestructLinkedList(qqueue_t __attribute__((unused)) *pThis) static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr) { DEFiRet; - iRet = qqueueAddLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, pUsr); -#if 0 - qLinkedList_t *pEntry; - - ASSERT(pThis != NULL); - if((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - - pEntry->pNext = NULL; - pEntry->pUsr = pUsr; - - if(pThis->tVars.linklist.pRoot == NULL) { - pThis->tVars.linklist.pRoot = pThis->tVars.linklist.pLast = pEntry; - } else { - pThis->tVars.linklist.pLast->pNext = pEntry; - pThis->tVars.linklist.pLast = pEntry; - } - -finalize_it: -#endif RETiRet; } @@ -636,24 +615,6 @@ static rsRetVal qDelLinkedList(qqueue_t *pThis, obj_t **ppUsr) { DEFiRet; iRet = qqueueDelLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, ppUsr); -#if 0 - qLinkedList_t *pEntry; - - ASSERT(pThis != NULL); - ASSERT(pThis->tVars.linklist.pRoot != NULL); - - pEntry = pThis->tVars.linklist.pRoot; - *ppUsr = pEntry->pUsr; - - if(pThis->tVars.linklist.pRoot == pThis->tVars.linklist.pLast) { - pThis->tVars.linklist.pRoot = NULL; - pThis->tVars.linklist.pLast = NULL; - } else { - pThis->tVars.linklist.pRoot = pEntry->pNext; - } - free(pEntry); - -#endif RETiRet; } -- cgit v1.2.3 From 8e536c5b25ca1a7106f541149cf0d76bdf9237da Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 14:21:22 +0200 Subject: highlighted bugfix imported from beta --- ChangeLog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ChangeLog b/ChangeLog index ccd1e7ae..b1d3b8d0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,11 @@ Version 4.1.7 [BETA] (rgerhards), 2009-04-?? - bugfix: the default for $DirCreateMode was 0644, and as such wrong. It has now been changed to 0700. For some background, please see http://lists.adiscon.net/pipermail/rsyslog/2009-April/001986.html +- bugfix: ompgsql did not detect problems in sql command execution + this could cause loss of messages. The handling was correct if the + connection broke, but not if there was a problem with statement + execution. The most probable case for such a case would be invalid + sql inside the template, and this is now much easier to diagnose. --------------------------------------------------------------------------- Version 4.1.6 [DEVEL] (rgerhards), 2009-04-07 - added new "csv" property replacer options to enable simple creation -- cgit v1.2.3 From 9348f8b9c2b5ba806dab8c7337877ab9da3f85b1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 14:33:14 +0200 Subject: preparing for 4.3.0 release --- ChangeLog | 7 ++++++- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 90b351f5..d01f3cb0 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.3.0 [DEVEL] (rgerhards), 2009-03-?? +Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program and feed it (via its stdin) with syslog messages. If the program terminates, it is restarted. @@ -14,6 +14,11 @@ Version 4.3.0 [DEVEL] (rgerhards), 2009-03-?? - added $MaxOpenFiles configuration directive - bugfix: solved potential memory leak in msg processing, could manifest itself in imtcp +- bugfix: ompgsql did not detect problems in sql command execution + this could cause loss of messages. The handling was correct if the + connection broke, but not if there was a problem with statement + execution. The most probable case for such a case would be invalid + sql inside the template, and this is now much easier to diagnose. --------------------------------------------------------------------------- Version 4.1.7 [BETA] (rgerhards), 2009-04-?? - bugfix: $InputTCPMaxSessions config directive was accepted, but not diff --git a/configure.ac b/configure.ac index c7f1d532..6daf3f5a 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.6],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.3.0],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/manual.html b/doc/manual.html index 51271701..ae90f780 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 4.1.6 (devel branch) of rsyslog. +

    This documentation is for version 4.3.0 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From dc777849fcd5c300f229c44daf4278d8b6842b71 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 14:37:41 +0200 Subject: bugfix: missing header (platform compatibility issue) --- tests/tcpflood.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 9c17fd5b..8dbc201b 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3 From 09ca443377ec85364160e965920ab6f56a374d88 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 15:10:06 +0200 Subject: update project status & cleanup removed some warning in imklog compilation, but may not have solved a lurking issue (but placed comment so that we know if something surfaces) --- doc/status.html | 16 ++++++++-------- plugins/imklog/ksym_mod.c | 42 ++++++++++++++++++++++-------------------- plugins/imklog/ksyms.h | 4 ++-- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/doc/status.html b/doc/status.html index fff3a6ff..01a9cf8f 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,22 +2,22 @@ rsyslog status page

    rsyslog status page

    -

    This page reflects the status as of 2009-04-03.

    +

    This page reflects the status as of 2009-04-17.

    Current Releases

    -

    development: 4.1.6 [2009-04-07] - -change log - -download +

    development: 4.3.0 [2009-04-17] - +change log - +download
    beta: 3.21.11 [2009-04-03] - change log - download

    -

    v3 stable: 3.20.5 [2009-04-02] - change log - -download +

    v3 stable: 3.20.6 [2009-04-16] - change log - +download -
    v2 stable: 2.0.6 [2008-08-07] - change log - -download +
    v2 stable: 2.0.7 [2009-04-14] - change log - +download
    v0 and v1 are deprecated and no longer supported. If you absolutely do not like to upgrade, you may consider purchasing a commercial rsyslog support package. Just let us point diff --git a/plugins/imklog/ksym_mod.c b/plugins/imklog/ksym_mod.c index 6e48e89e..be5fdee9 100644 --- a/plugins/imklog/ksym_mod.c +++ b/plugins/imklog/ksym_mod.c @@ -1,9 +1,8 @@ -/* - * ksym_mod.c - functions for building symbol lookup tables for klogd +/* ksym_mod.c - functions for building symbol lookup tables for klogd * Copyright (c) 1995, 1996 Dr. G.W. Wettstein * Copyright (c) 1996 Enjellic Systems Development * Copyright (c) 1998-2007 Martin Schulze - * Copyright (C) 2007-2008 Rainer Gerhards + * Copyright (C) 2007-2009 Rainer Gerhards * * This file is part of rsyslog. * @@ -83,7 +82,6 @@ * Changed llseek() to lseek64() in order to skip a libc warning. */ - /* Includes. */ #include "config.h" #include @@ -112,7 +110,7 @@ #define KSYMS "/proc/kallsyms" static int num_modules = 0; -struct Module *sym_array_modules = (struct Module *) 0; +struct Module *sym_array_modules = (struct Module *) NULL; static int have_modules = 0; @@ -266,7 +264,7 @@ static void FreeModules() } free(sym_array_modules); - sym_array_modules = (struct Module *) 0; + sym_array_modules = (struct Module *) NULL; num_modules = 0; return; } @@ -390,11 +388,11 @@ static int AddSymbol(line) mp->sym_array = (struct sym_table *) realloc(mp->sym_array, \ (mp->num_syms+1) * sizeof(struct sym_table)); - if ( mp->sym_array == (struct sym_table *) 0 ) + if ( mp->sym_array == (struct sym_table *) NULL ) return(0); mp->sym_array[mp->num_syms].name = strdup(p); - if ( mp->sym_array[mp->num_syms].name == (char *) 0 ) + if ( mp->sym_array[mp->num_syms].name == (char *) NULL ) return(0); /* Stuff interesting information into the module. */ @@ -424,15 +422,21 @@ static int AddSymbol(line) * If a match cannot be found a diagnostic string is printed. * If a match is found the pointer to the symbolic name most * closely matching the address is returned. + * + * TODO: We are using int values for the offset, but longs for the value + * values. This may create some trouble in the future (on 64 Bit OS?). + * Anyhow, I have not changed this, because we do not seem to have any + * issue and my understanding of this code is limited (and I don't see + * need to invest more time to dig much deeper). + * rgerhards, 2009-04-17 **************************************************************************/ extern char * LookupModuleSymbol(value, sym) unsigned long value; struct symbol *sym; { - auto int nmod, - nsym; - auto struct sym_table *last; - auto struct Module *mp; + int nmod, nsym; + struct sym_table *last; + struct Module *mp; static char ret[100]; sym->size = 0; @@ -443,8 +447,7 @@ extern char * LookupModuleSymbol(value, sym) for (nmod = 0; nmod < num_modules; ++nmod) { mp = &sym_array_modules[nmod]; - /* - * Run through the list of symbols in this module and + /* Run through the list of symbols in this module and * see if the address can be resolved. */ for(nsym = 1, last = &mp->sym_array[0]; @@ -453,13 +456,12 @@ extern char * LookupModuleSymbol(value, sym) if ( mp->sym_array[nsym].value > value ) { if ( sym->size == 0 || - (value - last->value) < sym->offset || - ( (sym->offset == (value - last->value)) && - (mp->sym_array[nsym].value-last->value) < sym->size ) ) + (int) (value - last->value) < sym->offset || + ( (sym->offset == (int) (value - last->value)) && + (int) (mp->sym_array[nsym].value-last->value) < sym->size ) ) { sym->offset = value - last->value; - sym->size = mp->sym_array[nsym].value - \ - last->value; + sym->size = mp->sym_array[nsym].value - last->value; ret[sizeof(ret)-1] = '\0'; if ( mp->name == NULL ) snprintf(ret, sizeof(ret)-1, @@ -478,5 +480,5 @@ extern char * LookupModuleSymbol(value, sym) return(ret); /* It has been a hopeless exercise. */ - return((char *) 0); + return(NULL); } diff --git a/plugins/imklog/ksyms.h b/plugins/imklog/ksyms.h index b5362ff3..a168947b 100644 --- a/plugins/imklog/ksyms.h +++ b/plugins/imklog/ksyms.h @@ -2,7 +2,7 @@ * Copyright (c) 1995, 1996 Dr. G.W. Wettstein * Copyright (c) 1996 Enjellic Systems Development * Copyright (c) 2004-7 Martin Schulze - * Copyright (c) 2007-2008 Rainer Gerhards + * Copyright (c) 2007-2009 Rainer Gerhards * * This file is part of rsyslog. * @@ -26,7 +26,7 @@ struct symbol { - char *name; + uchar *name; int size; int offset; }; -- cgit v1.2.3 From 889a0a1da8b2fb74b04647a345f64fce6c36708f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 15:19:57 +0200 Subject: some cleanup ... mostly removal of compile-time warnings (thanks to Michael Biebl for suggesting to look after that) --- ChangeLog | 2 ++ plugins/imgssapi/imgssapi.c | 1 - plugins/omgssapi/omgssapi.c | 2 +- runtime/nsd_gtls.c | 2 ++ tools/msggen.c | 2 +- 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index d01f3cb0..48343b5f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,6 @@ --------------------------------------------------------------------------- +Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +--------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program and feed it (via its stdin) with syslog messages. If the program diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index debe935e..b9d7dfe3 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -249,7 +249,6 @@ onErrClose(tcps_sess_t *pSess) static rsRetVal doOpenLstnSocks(tcpsrv_t *pSrv) { - int *pRet = NULL; gsssrv_t *pGSrv; DEFiRet; diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c index e0cc8af6..361f657f 100644 --- a/plugins/omgssapi/omgssapi.c +++ b/plugins/omgssapi/omgssapi.c @@ -444,7 +444,7 @@ CODESTARTdoAction /* error! */ dbgprintf("error forwarding via tcp, suspending\n"); pData->eDestState = eDestFORW_SUSP; - iRet = RS_RET_SUSPENDED; + ABORT_FINALIZE(RS_RET_SUSPENDED); } break; } diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 3a79a015..5786e191 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -82,6 +82,7 @@ static gnutls_certificate_credentials xcred; static gnutls_dh_params dh_params; #ifdef DEBUG +#if 0 /* uncomment, if needed some time again -- DEV Debug only */ /* This defines a log function to be provided to GnuTLS. It hopefully * helps us track down hard to find problems. * rgerhards, 2008-06-20 @@ -90,6 +91,7 @@ static void logFunction(int level, const char *msg) { dbgprintf("GnuTLS log msg, level %d: %s\n", level, msg); } +#endif #endif /* #ifdef DEBUG */ diff --git a/tools/msggen.c b/tools/msggen.c index 7990a3c8..06244c18 100644 --- a/tools/msggen.c +++ b/tools/msggen.c @@ -24,7 +24,7 @@ #include #include -int main(int argc, char *argv[]) +int main(int __attribute__((unused)) argc, char __attribute__((unused)) *argv[]) { int i; -- cgit v1.2.3 From 1fb5cee04dfd4d40e64e26a0c622640781cd06f7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 17 Apr 2009 17:53:33 +0200 Subject: improved testbench - added tests for queue disk-only mode --- ChangeLog | 2 ++ tests/Makefile.am | 4 +++- tests/diskqueue.sh | 31 +++++++++++++++++++++++++++++++ tests/testsuites/diskqueue.conf | 16 ++++++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100755 tests/diskqueue.sh create mode 100644 tests/testsuites/diskqueue.conf diff --git a/ChangeLog b/ChangeLog index 48343b5f..85411722 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +- improved testbench + * added tests for queue disk-only mode (checks disk queue logic) --------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program diff --git a/tests/Makefile.am b/tests/Makefile.am index 87dca985..0f4cbce1 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh +TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh diskqueue.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -27,6 +27,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ parsertest.sh \ + diskqueue.sh \ + testsuites/diskqueue.conf \ manytcp.sh \ testsuites/manytcp.conf \ omod-if-array.sh \ diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh new file mode 100755 index 00000000..6384eb64 --- /dev/null +++ b/tests/diskqueue.sh @@ -0,0 +1,31 @@ +# Test for disk-only queue mode +# This test checks if queue files can be correctly written +# and read back, but it does not test the transition from +# memory to disk mode for DA queues. +# added 2009-04-17 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo testing queue disk-only mode +rm -rf test-spool +mkdir test-spool +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/diskqueue.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +# 20000 messages should be enough - the disk test is slow enough ;) +./tcpflood 127.0.0.1 13514 1 20000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 4 # we need this so that rsyslogd can receive all outstanding messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 19999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log +rm -rf test-spool diff --git a/tests/testsuites/diskqueue.conf b/tests/testsuites/diskqueue.conf new file mode 100644 index 00000000..8851a459 --- /dev/null +++ b/tests/testsuites/diskqueue.conf @@ -0,0 +1,16 @@ +# Test for queue disk mode (see .sh file for details) +# rgerhards, 2009-04-17 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt -- cgit v1.2.3 From aa43d7f83125f20f8efdf4152bdd9a09e7a81495 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 21 Apr 2009 16:32:33 +0200 Subject: doc: added (hopefully) easier to grasp queue explanation --- ChangeLog | 2 + doc/Makefile.am | 16 ++- doc/dataflow.png | Bin 0 -> 24601 bytes doc/direct_queue0.png | Bin 0 -> 2048 bytes doc/direct_queue1.png | Bin 0 -> 2979 bytes doc/direct_queue2.png | Bin 0 -> 4117 bytes doc/direct_queue3.png | Bin 0 -> 4430 bytes doc/direct_queue_directq.png | Bin 0 -> 10075 bytes doc/direct_queue_rsyslog.png | Bin 0 -> 10465 bytes doc/direct_queue_rsyslog2.png | Bin 0 -> 12350 bytes doc/queue_analogy_tv.png | Bin 0 -> 18730 bytes doc/queues.html | 14 ++- doc/queues_analogy.html | 259 ++++++++++++++++++++++++++++++++++++++ doc/src/dataflow.dia | Bin 0 -> 2662 bytes doc/src/direct_queue0.dia | Bin 0 -> 966 bytes doc/src/direct_queue1.dia | Bin 0 -> 1058 bytes doc/src/direct_queue2.dia | Bin 0 -> 1143 bytes doc/src/direct_queue3.dia | Bin 0 -> 1214 bytes doc/src/direct_queue_directq.dia | Bin 0 -> 1705 bytes doc/src/direct_queue_rsyslog.dia | Bin 0 -> 2311 bytes doc/src/direct_queue_rsyslog2.dia | Bin 0 -> 2638 bytes doc/src/queue_analogy_tv.dia | Bin 0 -> 1749 bytes 22 files changed, 286 insertions(+), 5 deletions(-) create mode 100644 doc/dataflow.png create mode 100644 doc/direct_queue0.png create mode 100644 doc/direct_queue1.png create mode 100644 doc/direct_queue2.png create mode 100644 doc/direct_queue3.png create mode 100644 doc/direct_queue_directq.png create mode 100644 doc/direct_queue_rsyslog.png create mode 100644 doc/direct_queue_rsyslog2.png create mode 100644 doc/queue_analogy_tv.png create mode 100644 doc/queues_analogy.html create mode 100644 doc/src/dataflow.dia create mode 100644 doc/src/direct_queue0.dia create mode 100644 doc/src/direct_queue1.dia create mode 100644 doc/src/direct_queue2.dia create mode 100644 doc/src/direct_queue3.dia create mode 100644 doc/src/direct_queue_directq.dia create mode 100644 doc/src/direct_queue_rsyslog.dia create mode 100644 doc/src/direct_queue_rsyslog2.dia create mode 100644 doc/src/queue_analogy_tv.dia diff --git a/ChangeLog b/ChangeLog index 85411722..976d8084 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +- improved doc + * added (hopefully) easier to grasp queue explanation - improved testbench * added tests for queue disk-only mode (checks disk queue logic) --------------------------------------------------------------------------- diff --git a/doc/Makefile.am b/doc/Makefile.am index 0f3dca70..4d9d94ff 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -109,6 +109,20 @@ html_files = \ rsyslog_conf_output.html \ rsyslog_conf_templates.html \ rsyslog_conf_nomatch.html \ + queues_analogy.html \ src/classes.dia -EXTRA_DIST = $(html_files) +grfx_files = \ + direct_queue0.png \ + direct_queue1.png \ + direct_queue2.png \ + direct_queue3.png \ + direct_queue_rsyslog.png \ + direct_queue_rsyslog2.png \ + direct_queue_directq.png \ + dataflow.png \ + queue_analogy_tv.png \ + gssapi.png \ + rsyslog-vers.png + +EXTRA_DIST = $(html_files) $(grfx_files) diff --git a/doc/dataflow.png b/doc/dataflow.png new file mode 100644 index 00000000..fd614d8c Binary files /dev/null and b/doc/dataflow.png differ diff --git a/doc/direct_queue0.png b/doc/direct_queue0.png new file mode 100644 index 00000000..6d1b373f Binary files /dev/null and b/doc/direct_queue0.png differ diff --git a/doc/direct_queue1.png b/doc/direct_queue1.png new file mode 100644 index 00000000..503f8151 Binary files /dev/null and b/doc/direct_queue1.png differ diff --git a/doc/direct_queue2.png b/doc/direct_queue2.png new file mode 100644 index 00000000..cbb99af8 Binary files /dev/null and b/doc/direct_queue2.png differ diff --git a/doc/direct_queue3.png b/doc/direct_queue3.png new file mode 100644 index 00000000..cc49299f Binary files /dev/null and b/doc/direct_queue3.png differ diff --git a/doc/direct_queue_directq.png b/doc/direct_queue_directq.png new file mode 100644 index 00000000..c5d8769d Binary files /dev/null and b/doc/direct_queue_directq.png differ diff --git a/doc/direct_queue_rsyslog.png b/doc/direct_queue_rsyslog.png new file mode 100644 index 00000000..6150222d Binary files /dev/null and b/doc/direct_queue_rsyslog.png differ diff --git a/doc/direct_queue_rsyslog2.png b/doc/direct_queue_rsyslog2.png new file mode 100644 index 00000000..807b064d Binary files /dev/null and b/doc/direct_queue_rsyslog2.png differ diff --git a/doc/queue_analogy_tv.png b/doc/queue_analogy_tv.png new file mode 100644 index 00000000..fedcb558 Binary files /dev/null and b/doc/queue_analogy_tv.png differ diff --git a/doc/queues.html b/doc/queues.html index 41c5865f..4a9509a0 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -1,6 +1,5 @@ - Understanding rsyslog queues back @@ -10,6 +9,13 @@ queue, one part of the system "produces" something while another part "consumes" this something. The "something" is most often syslog messages, but queues may also be used for other purposes.

    +

    This document provides a good insight into technical details, operation modes +and implications. In addition to it, an +rsyslog queue concepts overview document +exists which tries to explain queues with the help of some analogies. This may +probably be a better place to start reading about queues. I assume that once you +have understood that document, the material here will be much easier to grasp +and look much more natural.

    The most prominent example is the main message queue. Whenever rsyslog receives a message (e.g. locally, via UDP, TCP or in whatever else way), it places these messages into the main message queue. Later, it is dequeued by the @@ -18,7 +24,7 @@ front of each action, there is also a queue, which potentially de-couples the filter processing from the actual action (e.g. writing to file, database or forwarding to another host).

    Where are Queues Used?

    -

     Currently, queues are used for the main message queue and for the +

    Currently, queues are used for the main message queue and for the actions.

    There is a single main message queue inside rsyslog. Each input module delivers messages to it. The main message queue worker filters messages based on @@ -354,8 +360,8 @@ save.

    [rsyslog site]

    This documentation is part of the rsyslog project.
    -Copyright © 2008 by Rainer Gerhards and +Copyright © 2008, 2009 by Rainer Gerhards and Adiscon. Released under the GNU GPL -version 2 or higher.

    +version 3 or higher.

    diff --git a/doc/queues_analogy.html b/doc/queues_analogy.html new file mode 100644 index 00000000..1584c66d --- /dev/null +++ b/doc/queues_analogy.html @@ -0,0 +1,259 @@ + + +turning lanes and rsyslog queues - an analogy + +back + +

    Turning Lanes and Rsyslog Queues - an Analogy

    +

    If there is a single object absolutely vital to understanding the way +rsyslog works, this object is queues. Queues offer a variety of services, +including support for multithreading. While there is elaborate in-depth +documentation on the ins and outs of rsyslog queues, +some of the concepts are hard to grasp even for experienced people. I think this +is because rsyslog uses a very high layer of abstraction which includes things +that look quite unnatural, like queues that do not actually queue... +

    With this document, I take a different approach: I will not describe every specific +detail of queue operation but hope to be able to provide the core idea of how +queues are used in rsyslog by using an analogy. I will compare the rsyslog data flow +with real-life traffic flowing at an intersection. +

    But first let's set the stage for the rsyslog part. The graphic below describes +the data flow inside rsyslog: +

    rsyslog data flow +

    Note that there is a video tutorial +available on the data flow. It is not perfect, but may aid in understanding this picture. +

    For our needs, the important fact to know is that messages enter rsyslog on "the +left side" (for example, via UDP), are being preprocessed, put into the +so-called main queue, taken off that queue, be filtered and be placed into one or +several action queues (depending on filter results). They leave rsyslog on "the +right side" where output modules (like the file or database writer) consume them. +

    So there are always two stages where a message (conceptually) is queued - first +in the main queue and later on in n action specific queues (with n being the number of +actions that the message in question needs to be processed by, what is being decided +by the "Filter Engine"). As such, a message will be in at least two queues +during its lifetime (with the exeception of messages being discarded by the queue itself, +but for the purpose of this document, we will ignore that possibility). +

    Also, it is vitally +important to understand that each action has a queue sitting in front of it. +If you have dug into the details of rsyslog configuration, you have probably seen that +a queue mode can be set for each action. And the default queue mode is the so-called +"direct mode", in which "the queue does not actually enqueue data". +That sounds silly, but is not. It is an important abstraction that helps keep the code clean. +

    To understand this, we first need to look at who is the active component. In our data flow, +the active part always sits to the left of the object. For example, the "Preprocessor" +is being called by the inputs and calls itself into the main message queue. That is, the queue +receiver is called, it is passive. One might think that the "Parser & Filter Engine" +is an active component that actively pulls messages from the queue. This is wrong! Actually, +it is the queue that has a pool of worker threads, and these workers pull data from the queue +and then call the passively waiting Parser and Filter Engine with those messages. So the +main message queue is the active part, the Parser and Filter Engine is passive. +

    Let's now try an anlogy analogy for this part: Think about a TV show. The show is produced +in some TV studio, from there sent (actively) to a radio tower. The radio tower passively +receives from the studio and then actively sends out a signal, which is passively received +by your TV set. In our simplified view, we have the following picture: +

    rsyslog queues and TV analogy +

    The lower part of the picture lists the equivalent rsyslog entities, in an abstracted way. +Every queue has a producer (in the above sample the input) and a consumer (in the above sample the Parser +and Filter Engine). Their active and passive functions are equivalent to the TV entities +that are listed on top of the rsyslog entity. For example, a rsyslog consumer can never +actively initate reception of a message in the same way a TV set can not actively +"initiate" a TV show - both can only "handle" (display or process) +what is sent to them. +

    Now let's look at the action queues: here, the active part, the producer, is the +Parser and Filter Engine. The passive part is the Action Processor. The later does any +processing that is necessary to call the output plugin, in particular it processes the template +to create the plugin calling parameters (eiter a string or vector of arguments). From the +action queue's point of view, Action Processor and Output form a single entity. Again, the +TV set analogy holds. The Output does not actively ask the queue for data, but +rater passively waits until the queue itself pushes some data to it. + +

    Armed with this knowledge, we can now look at the way action queue modes work. My analogy here +is a junction, as shown below (note that the colors in the pictures below are not related to +the colors in the pictures above!): +

    +

    This is a very simple real-life traffic case: one road joins another. We look at +traffic on the straight road, here shown by blue and green arrows. Traffic in the +opposing direction is shown in blue. Traffic flows without +any delays as long as nobody takes turns. To be more precise, if the opposing traffic takes +a (right) turn, traffic still continues to flow without delay. However, if a car in the red traffic +flow intend to do a (left, then) turn, the situation changes: +

    +

    The turning car is represented by the green arrow. It can not turn unless there is a gap +in the "blue traffic stream". And as this car blocks the roadway, the remaining +traffic (now shown in red, which should indicate the block condition), +must wait until the "green" car has made its turn. So +a queue will build up on that lane, waiting for the turn to be completed. +Note that in the examples below I do not care that much about the properties of the +opposing traffic. That is, because its structure is not really important for what I intend to +show. Think about the blue arrow as being a traffic stream that most of the time blocks +left-turners, but from time to time has a gap that is sufficiently large for a left-turn +to complete. +

    Our road network designers know that this may be unfortunate, and for more important roads +and junctions, they came up with the concept of turning lanes: +

    +

    Now, the car taking the turn can wait in a special area, the turning lane. As such, +the "straight" traffic is no longer blocked and can flow in parallel to the +turning lane (indicated by a now-green-again arrow). + +

    However, the turning lane offers only finite space. So if too many cars intend to +take a left turn, and there is no gap in the "blue" traffic, we end up with +this well-known situation: +

    +

    The turning lane is now filled up, resulting in a tailback of cars intending to +left turn on the main driving lane. The end result is that "straight" traffic +is again being blocked, just as in our initial problem case without the turning lane. +In essence, the turning lane has provided some relief, but only for a limited amount of +cars. Street system designers now try to weight cost vs. benefit and create (costly) +turning lanes that are sufficiently large to prevent traffic jams in most, but not all +cases. +

    Now let's dig a bit into the mathematical properties of turning lanes. We assume that +cars all have the same length. So, units of cars, the length is alsways one (which is nice, +as we don't need to care about that factor any longer ;)). A turning lane has finite capacity of +n cars. As long as the number of cars wanting to take a turn is less than or eqal +to n, "straigth traffic" is not blocked (or the other way round, traffic +is blocked if at least n + 1 cars want to take a turn!). We can now find an optimal +value for n: it is a function of the probability that a car wants to turn +and the cost of the turning lane +(as well as the probability there is a gap in the "blue" traffic, but we ignore this +in our simple sample). +If we start from some finite upper bound of n, we can decrease +n to a point where it reaches zero. But let's first look at n = 1, in which case exactly +one car can wait on the turning lane. More than one car, and the rest of the traffic is blocked. +Our everyday logic indicates that this is actually the lowest boundary for n. +

    In an abstract view, however, n can be zero and that works nicely. There still can be +n cars at any given time on the turning lane, it just happens that this means there can +be no car at all on it. And, as usual, if we have at least n + 1 cars wanting to turn, +the main traffic flow is blocked. True, but n + 1 = 0 + 1 = 1 so as soon as there is any +car wanting to take a turn, the main traffic flow is blocked (remeber, in all cases, I assume +no sufficently large gaps in the opposing trafic). +

    This is the situation our everyday perception calls "road without turning lane". +In my math model, it is a "road with turning lane of size 0". The subtle difference +is important: my math model guarantees that, in an abstract sense, there always is a turning +lane, it may just be too short. But it exists, even though we don't see it. And now I can +claim that even in my small home village, all roads have turning lanes, which is rather +impressive, isn't it? ;) +

    And now we finally have arrived at rsyslog's queues! Rsyslog action queues exists for +all actions just like all roads in my village have turning lanes! And as in this real-life sample, +it may be hard to see the action queues for that reason. In rsyslog, the "direct" queue +mode is the equivalent to the 0-sized turning lane. And actions queues are the equivalent to turning +lanes in general, with our real-life n being the maximum queue size. +The main traffic line (which sometimes is blocked) is the equivalent to the +main message queue. And the periods without gaps in the opposing traffic are equivalent to +execution time of an action. In a rough sketch, the rsyslog main and action queues look like in the +following picture. +

    +

    We need to read this picture from right to left (otherwise I would need to redo all +the graphics ;)). In action 3, you see a 0-sized turning lane, aka an action queue in "direct" +mode. All other queues are run in non-direct modes, but with different sizes greater than 0. +

    Let us first use our car analogy: +Assume we are in a car on the main lane that wants to take turn into the "action 4" +road. We pass action 1, where a number of cars wait in the turning lane and we pass +action 2, which has a slightly smaller, but still not filled up turning lane. So we pass that +without delay, too. Then we come to "action 4", which has no turning lane. Unfortunately, +the car in front of us wants to turn left into that road, so it blocks the main lane. So, this time +we need to wait. An observer standing on the sidewalk may see that while we need to wait, there are +still some cars in the "action 4" turning lane. As such, even though no new cars can +arrive on the main lane, cars still turn into the "action 4" lane. In other words, +an observer standing in "action 4" road is unable to see that traffic on the main lane +is blocked. +

    Now on to rsyslog: Other than in the real-world traffic example, messages in rsyslog +can - at more or less the +same time - "take turns" into several roads at once. This is done by duplicating the message +if the road has a non-zero-sized "turning lane" - or in rsyslog terms a queue that is +running in any non-direct mode. If so, a deep copy of the message object is made, that placed into +the action queue and then the initial message proceeds on the "main lane". The action +queue then pushes the duplicates through action processing. This is also the reason why a +discard action inside a non-direct queue does not seem to have an effect. Actually, it discards the +copy that was just created, but the original message object continues to flow. +

    +In action 1, we have some entries in the action queue, as we have in action 2 (where the queue is +slightly shorter). As we have seen, new messages pass action one and two almost instantaneously. +However, when a messages reaches action 3, its flow is blocked. Now, message processing must wait +for the action to complete. Processing flow in a direct mode queue is something like a U-turn: + +

    message processing in an rsyslog action queue in direct mode +

    The message starts to execute the action and once this is done, processing flow continues. +In a real-life analogy, this may be the route of a delivery man who needs to drop a parcel +in a side street before he continues driving on the main route. As a side-note, think of what happens +with the rest of the delivery route, at least for today, if the delivery truck has a serious accident +in the side street. The rest of the parcels won't be delivered today, will they? This is exactly how the +discard action works. It drops the message object inside the action and thus the message will no +longer be available for further delivery - but as I said, only if the discard is done in a +direct mode queue (I am stressing this example because it often causes a lot of confusion). +

    Back to the overall scenario. We have seen that messages need to wait for action 3 to +complete. Does this necessarily mean that at the same time no messages can be processed +in action 4? Well, it depends. As in the real-life scenario, action 4 will continue to +receive traffic as long as its action queue ("turn lane") is not drained. In +our drawing, it is not. So action 4 will be executed while messages still wait for action 3 +to be completed. +

    Now look at the overall picture from a slightly different angle: +

    message processing in an rsyslog action queue in direct mode +

    The number of all connected green and red arrows is four - one each for action 1, 2 and 3 +(this one is dotted as action 4 was a special case) and one for the "main lane" as +well as acton 3 (this one contains the sole red arrow). This number is the lower bound for +the number of threads in rsyslog's output system ("right-hand part" of the main message +queue)! Each of the connected arrows is a continous thread and each "turn lane" is +a place where processing is forked onto a new thread. Also, note that in action 3 the processing +is carried out on the main thread, but not in the non-direct queue modes. +

    I have said this is "the lower bound for the number of threads...". This is with +good reason: the main queue may have more than one worker thread (individual action queues +currently do not support this, but could do in the future - there are good reasons for that, too +but exploring why would finally take us away from what we intend to see). Note that you +configure an upper bound for the number of main message queue worker threads. The actual number +varies depending on a lot of operational variables, most importantly the number of messages +inside the queue. The number t_m of actually running threads is within the integer-interval +[0,confLimit] (with confLimit being the operator configured limit, which defaults to 5). +Output plugins may have more than one thread created by themselves. It is quite unusual for an +output plugin to create such threads and so I assume we do not have any of these. +Then, the overall number of threads in rsyslog's filtering and output system is +t_total = t_m + number of actions in non-direct modes. Add the number of +inputs configured to that and you have the total number of threads running in rsyslog at +a given time (assuming again that inputs utilize only one thread per plugin, a not-so-safe +assumption). +

    A quick side-note: I gave the lower bound for t_m as zero, which is somewhat in contrast +to what I wrote at the begin of the last paragraph. Zero is actually correct, because rsyslog +stops all worker threads when there is no work to do. This is also true for the action queues. +So the ultimate lower bound for a rsyslog output system without any work to carry out actually is zero. +But this bound will never be reached when there is continous flow of activity. And, if you are +curios: if the number of workers is zero, the worker wakeup process is actually handled within the +threading context of the "left-hand-side" (or producer) of the queue. After being +started, the worker begins to play the active queue component again. All of this, of course, +can be overridden with configuraton directives. +

    When looking at the threading model, one can simply add n lanes to the main lane but otherwise +retain the traffic analogy. This is a very good description of the actual process (think what this +means to the "turning lanes"; hint: there still is only one per action!). +

    Let's try to do a warp-up: I have hopefully been able to show that in rsyslog, an action +queue "sits in front of" each output plugin. Messages are received and flow, from input +to output, over various stages and two level of queues to the outputs. Actions queues are always +present, but may not easily be visible when in direct mode (where no actual queueing takes place). +The "road junktion with turning lane" analogy well describes the way - and intent - of the various +queue levels in rsyslog. +

    On the output side, the queue is the active component, not the consumer. As such, the consumer +can not ask the queue for anything (like n number of messages) but rather is activated by the queue +itself. As such, a queue somewhat resembles a "living thing" whereas the outputs are +just tools that this "living thing" uses. +

    Note that I left out a couple of subtleties, especially when it comes +to error handling and terminating +a queue (you hopefully have now at least a rough idea why I say "terminating a queue" +and not "terminating an action" - who is the "living thing"?). An action returns +a status to the queue, but it is the queue that ultimately decides which messages can finally be +considered processed and which not. Please note that the queue may even cancel an output right in +the middle of its action. This happens, if configured, if an output needs more than a configured +maximum processing time and is a guard condition to prevent slow outputs from defering a rsyslog +restart for too long. Especially in this case re-queueing and cleanup is not trivial. Also, note that +I did not discuss disk-assisted queue modes. The basic rules apply, but there are some additonal +constraints, especially in regard to the threading model. Transitioning between actual +disk-assisted mode and pure-in-memory-mode (which is done automatically when needed) is also far from +trivial and a real joy for an implementor to work on ;). +

    If you have not done so before, it may be worth reading the +rsyslog queue user's guide, which most importantly lists all +the knobs you can turn to tweak queue operation. +

    [manual index] +[rsyslog.conf] +[rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2009 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 3 or higher.

    + + diff --git a/doc/src/dataflow.dia b/doc/src/dataflow.dia new file mode 100644 index 00000000..3875fc61 Binary files /dev/null and b/doc/src/dataflow.dia differ diff --git a/doc/src/direct_queue0.dia b/doc/src/direct_queue0.dia new file mode 100644 index 00000000..4446619b Binary files /dev/null and b/doc/src/direct_queue0.dia differ diff --git a/doc/src/direct_queue1.dia b/doc/src/direct_queue1.dia new file mode 100644 index 00000000..7a64ea09 Binary files /dev/null and b/doc/src/direct_queue1.dia differ diff --git a/doc/src/direct_queue2.dia b/doc/src/direct_queue2.dia new file mode 100644 index 00000000..b0c394c0 Binary files /dev/null and b/doc/src/direct_queue2.dia differ diff --git a/doc/src/direct_queue3.dia b/doc/src/direct_queue3.dia new file mode 100644 index 00000000..bc477b25 Binary files /dev/null and b/doc/src/direct_queue3.dia differ diff --git a/doc/src/direct_queue_directq.dia b/doc/src/direct_queue_directq.dia new file mode 100644 index 00000000..37fdb44c Binary files /dev/null and b/doc/src/direct_queue_directq.dia differ diff --git a/doc/src/direct_queue_rsyslog.dia b/doc/src/direct_queue_rsyslog.dia new file mode 100644 index 00000000..9a030117 Binary files /dev/null and b/doc/src/direct_queue_rsyslog.dia differ diff --git a/doc/src/direct_queue_rsyslog2.dia b/doc/src/direct_queue_rsyslog2.dia new file mode 100644 index 00000000..c596f39f Binary files /dev/null and b/doc/src/direct_queue_rsyslog2.dia differ diff --git a/doc/src/queue_analogy_tv.dia b/doc/src/queue_analogy_tv.dia new file mode 100644 index 00000000..00fbdeb5 Binary files /dev/null and b/doc/src/queue_analogy_tv.dia differ -- cgit v1.2.3 From 8f1fc598c7f73f5718d0dbc656f7e78c72634a9b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 21 Apr 2009 17:01:29 +0200 Subject: doc status update & typo fixes --- doc/queues_analogy.html | 42 +++++++++++++++++++++--------------------- doc/status.html | 6 +++--- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/doc/queues_analogy.html b/doc/queues_analogy.html index 1584c66d..4f41fa26 100644 --- a/doc/queues_analogy.html +++ b/doc/queues_analogy.html @@ -30,7 +30,7 @@ right side" where output modules (like the file or database writer) consume in the main queue and later on in n action specific queues (with n being the number of actions that the message in question needs to be processed by, what is being decided by the "Filter Engine"). As such, a message will be in at least two queues -during its lifetime (with the exeception of messages being discarded by the queue itself, +during its lifetime (with the exception of messages being discarded by the queue itself, but for the purpose of this document, we will ignore that possibility).

    Also, it is vitally important to understand that each action has a queue sitting in front of it. @@ -46,7 +46,7 @@ is an active component that actively pulls messages from the queue. This is wron it is the queue that has a pool of worker threads, and these workers pull data from the queue and then call the passively waiting Parser and Filter Engine with those messages. So the main message queue is the active part, the Parser and Filter Engine is passive. -

    Let's now try an anlogy analogy for this part: Think about a TV show. The show is produced +

    Let's now try an analogy analogy for this part: Think about a TV show. The show is produced in some TV studio, from there sent (actively) to a radio tower. The radio tower passively receives from the studio and then actively sends out a signal, which is passively received by your TV set. In our simplified view, we have the following picture: @@ -55,16 +55,16 @@ by your TV set. In our simplified view, we have the following picture: Every queue has a producer (in the above sample the input) and a consumer (in the above sample the Parser and Filter Engine). Their active and passive functions are equivalent to the TV entities that are listed on top of the rsyslog entity. For example, a rsyslog consumer can never -actively initate reception of a message in the same way a TV set can not actively +actively initiate reception of a message in the same way a TV set cannot actively "initiate" a TV show - both can only "handle" (display or process) what is sent to them.

    Now let's look at the action queues: here, the active part, the producer, is the -Parser and Filter Engine. The passive part is the Action Processor. The later does any +Parser and Filter Engine. The passive part is the Action Processor. The latter does any processing that is necessary to call the output plugin, in particular it processes the template -to create the plugin calling parameters (eiter a string or vector of arguments). From the +to create the plugin calling parameters (either a string or vector of arguments). From the action queue's point of view, Action Processor and Output form a single entity. Again, the TV set analogy holds. The Output does not actively ask the queue for data, but -rater passively waits until the queue itself pushes some data to it. +rather passively waits until the queue itself pushes some data to it.

    Armed with this knowledge, we can now look at the way action queue modes work. My analogy here is a junction, as shown below (note that the colors in the pictures below are not related to @@ -75,9 +75,9 @@ traffic on the straight road, here shown by blue and green arrows. Traffic in th opposing direction is shown in blue. Traffic flows without any delays as long as nobody takes turns. To be more precise, if the opposing traffic takes a (right) turn, traffic still continues to flow without delay. However, if a car in the red traffic -flow intend to do a (left, then) turn, the situation changes: +flow intends to do a (left, then) turn, the situation changes:

    -

    The turning car is represented by the green arrow. It can not turn unless there is a gap +

    The turning car is represented by the green arrow. It cannot turn unless there is a gap in the "blue traffic stream". And as this car blocks the roadway, the remaining traffic (now shown in red, which should indicate the block condition), must wait until the "green" car has made its turn. So @@ -123,8 +123,8 @@ Our everyday logic indicates that this is actually the lowest boundary for n< n cars at any given time on the turning lane, it just happens that this means there can be no car at all on it. And, as usual, if we have at least n + 1 cars wanting to turn, the main traffic flow is blocked. True, but n + 1 = 0 + 1 = 1 so as soon as there is any -car wanting to take a turn, the main traffic flow is blocked (remeber, in all cases, I assume -no sufficently large gaps in the opposing trafic). +car wanting to take a turn, the main traffic flow is blocked (remember, in all cases, I assume +no sufficiently large gaps in the opposing traffic).

    This is the situation our everyday perception calls "road without turning lane". In my math model, it is a "road with turning lane of size 0". The subtle difference is important: my math model guarantees that, in an abstract sense, there always is a turning @@ -189,9 +189,9 @@ to be completed.

    message processing in an rsyslog action queue in direct mode

    The number of all connected green and red arrows is four - one each for action 1, 2 and 3 (this one is dotted as action 4 was a special case) and one for the "main lane" as -well as acton 3 (this one contains the sole red arrow). This number is the lower bound for +well as action 3 (this one contains the sole red arrow). This number is the lower bound for the number of threads in rsyslog's output system ("right-hand part" of the main message -queue)! Each of the connected arrows is a continous thread and each "turn lane" is +queue)! Each of the connected arrows is a continuous thread and each "turn lane" is a place where processing is forked onto a new thread. Also, note that in action 3 the processing is carried out on the main thread, but not in the non-direct queue modes.

    I have said this is "the lower bound for the number of threads...". This is with @@ -213,22 +213,22 @@ assumption). to what I wrote at the begin of the last paragraph. Zero is actually correct, because rsyslog stops all worker threads when there is no work to do. This is also true for the action queues. So the ultimate lower bound for a rsyslog output system without any work to carry out actually is zero. -But this bound will never be reached when there is continous flow of activity. And, if you are +But this bound will never be reached when there is continuous flow of activity. And, if you are curios: if the number of workers is zero, the worker wakeup process is actually handled within the threading context of the "left-hand-side" (or producer) of the queue. After being started, the worker begins to play the active queue component again. All of this, of course, -can be overridden with configuraton directives. +can be overridden with configuration directives.

    When looking at the threading model, one can simply add n lanes to the main lane but otherwise retain the traffic analogy. This is a very good description of the actual process (think what this means to the "turning lanes"; hint: there still is only one per action!).

    Let's try to do a warp-up: I have hopefully been able to show that in rsyslog, an action queue "sits in front of" each output plugin. Messages are received and flow, from input to output, over various stages and two level of queues to the outputs. Actions queues are always -present, but may not easily be visible when in direct mode (where no actual queueing takes place). -The "road junktion with turning lane" analogy well describes the way - and intent - of the various +present, but may not easily be visible when in direct mode (where no actual queuing takes place). +The "road junction with turning lane" analogy well describes the way - and intent - of the various queue levels in rsyslog.

    On the output side, the queue is the active component, not the consumer. As such, the consumer -can not ask the queue for anything (like n number of messages) but rather is activated by the queue +cannot ask the queue for anything (like n number of messages) but rather is activated by the queue itself. As such, a queue somewhat resembles a "living thing" whereas the outputs are just tools that this "living thing" uses.

    Note that I left out a couple of subtleties, especially when it comes @@ -238,12 +238,12 @@ and not "terminating an action" - who is the "living thing&quo a status to the queue, but it is the queue that ultimately decides which messages can finally be considered processed and which not. Please note that the queue may even cancel an output right in the middle of its action. This happens, if configured, if an output needs more than a configured -maximum processing time and is a guard condition to prevent slow outputs from defering a rsyslog -restart for too long. Especially in this case re-queueing and cleanup is not trivial. Also, note that -I did not discuss disk-assisted queue modes. The basic rules apply, but there are some additonal +maximum processing time and is a guard condition to prevent slow outputs from deferring a rsyslog +restart for too long. Especially in this case re-queuing and cleanup is not trivial. Also, note that +I did not discuss disk-assisted queue modes. The basic rules apply, but there are some additional constraints, especially in regard to the threading model. Transitioning between actual disk-assisted mode and pure-in-memory-mode (which is done automatically when needed) is also far from -trivial and a real joy for an implementor to work on ;). +trivial and a real joy for an implementer to work on ;).

    If you have not done so before, it may be worth reading the rsyslog queue user's guide, which most importantly lists all the knobs you can turn to tweak queue operation. diff --git a/doc/status.html b/doc/status.html index 01a9cf8f..c8608f73 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,7 +2,7 @@ rsyslog status page

    rsyslog status page

    -

    This page reflects the status as of 2009-04-17.

    +

    This page reflects the status as of 2009-04-21.

    Current Releases

    development: 4.3.0 [2009-04-17] - @@ -13,8 +13,8 @@ change log - download

    -

    v3 stable: 3.20.6 [2009-04-16] - change log - -download +

    v3 stable: 3.22.0 [2009-04-21] - change log - +download
    v2 stable: 2.0.7 [2009-04-14] - change log - download -- cgit v1.2.3 From 1f08d919fdd23e225e37cde83024076ad8a1197d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 21 Apr 2009 17:32:51 +0200 Subject: typo fix (important typo) --- doc/queues_analogy.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/queues_analogy.html b/doc/queues_analogy.html index 4f41fa26..e82f9d15 100644 --- a/doc/queues_analogy.html +++ b/doc/queues_analogy.html @@ -148,7 +148,7 @@ mode. All other queues are run in non-direct modes, but with different sizes gre Assume we are in a car on the main lane that wants to take turn into the "action 4" road. We pass action 1, where a number of cars wait in the turning lane and we pass action 2, which has a slightly smaller, but still not filled up turning lane. So we pass that -without delay, too. Then we come to "action 4", which has no turning lane. Unfortunately, +without delay, too. Then we come to "action 3", which has no turning lane. Unfortunately, the car in front of us wants to turn left into that road, so it blocks the main lane. So, this time we need to wait. An observer standing on the sidewalk may see that while we need to wait, there are still some cars in the "action 4" turning lane. As such, even though no new cars can -- cgit v1.2.3 From 029218df5b7fbd6a7e4f945ecd32f84236885f53 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 22 Apr 2009 08:38:25 +0200 Subject: another typo fix... --- doc/queues_analogy.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/queues_analogy.html b/doc/queues_analogy.html index e82f9d15..d7533ad5 100644 --- a/doc/queues_analogy.html +++ b/doc/queues_analogy.html @@ -59,7 +59,7 @@ actively initiate reception of a message in the same way a TV set cannot activel "initiate" a TV show - both can only "handle" (display or process) what is sent to them.

    Now let's look at the action queues: here, the active part, the producer, is the -Parser and Filter Engine. The passive part is the Action Processor. The latter does any +Parser and Filter Engine. The passive part is the Action Processor. The later does any processing that is necessary to call the output plugin, in particular it processes the template to create the plugin calling parameters (either a string or vector of arguments). From the action queue's point of view, Action Processor and Output form a single entity. Again, the -- cgit v1.2.3 From 251e48a34cca7fa7578cf82b3d95557878e409f7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 22 Apr 2009 08:52:13 +0200 Subject: preparing for 4.1.7 --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index fa0c5763..cfee4290 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.1.7 [BETA] (rgerhards), 2009-04-?? +Version 4.1.7 [BETA] (rgerhards), 2009-04-22 - bugfix: $InputTCPMaxSessions config directive was accepted, but not honored. This resulted in a fixed upper limit of 200 connections. - bugfix: the default for $DirCreateMode was 0644, and as such wrong. diff --git a/configure.ac b/configure.ac index 32b27483..5aa64054 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.6],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.1.7],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/manual.html b/doc/manual.html index 51271701..5b9c4b1e 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

    Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

    -

    This documentation is for version 4.1.6 (devel branch) of rsyslog. +

    This documentation is for version 4.1.7 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

    If you like rsyslog, you might -- cgit v1.2.3 From 7667845bd72b6f92eabc975318a4f288a77f2630 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 22 Apr 2009 15:06:45 +0200 Subject: first attempt at dequeueing multiple batches inside the queue ... but this code has serious problems when terminating the queue, also it is far from being optimal. I will commit a series of patches (hopefully) as I am on the path to the final implementation. --- runtime/queue.c | 182 +++++++++++++++++++++++++++++++++--------------------- runtime/queue.h | 4 +- runtime/rsyslog.h | 1 + runtime/wti.c | 17 ++++- runtime/wti.h | 16 ++++- runtime/wtp.c | 4 +- runtime/wtp.h | 4 +- 7 files changed, 149 insertions(+), 79 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index e5db866c..c48eb724 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -8,7 +8,11 @@ * (and in the web doc set on http://www.rsyslog.com/doc). Be sure to read it * if you are getting aquainted to the object. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * NOTE: as of 2009-04-22, I have begin to remove the qqueue* prefix from static + * function names - this makes it really hard to read and does not provide much + * benefit, at least I (now) think so... + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -63,12 +67,13 @@ DEFobjCurrIf(glbl) /* forward-definitions */ rsRetVal qqueueChkPersist(qqueue_t *pThis); static rsRetVal qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); -static rsRetVal qqueueRateLimiter(qqueue_t *pThis); +static rsRetVal RateLimiter(qqueue_t *pThis); static int qqueueChkStopWrkrDA(qqueue_t *pThis); +static rsRetVal GetDeqMaxAtOnce(qqueue_t *pThis, int *pVal); static int qqueueIsIdleDA(qqueue_t *pThis); -static rsRetVal qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave); -static rsRetVal qqueueConsumerCancelCleanup(void *arg1, void *arg2); -static rsRetVal qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex); +static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave); +static rsRetVal ConsumerCancelCleanup(void *arg1, void *arg2); +static rsRetVal UngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex); /* some constants for queuePersist () */ #define QUEUE_CHECKPOINT 1 @@ -369,9 +374,11 @@ qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) CHKiRet(wtpConstruct (&pThis->pWtpDA)); CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); + // MULTIQUEUE: TODO: this should be DA-specific! + CHKiRet(wtpSetpfGetDeqMaxAtOnce (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqMaxAtOnce)); CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleDA)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerDA)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) qqueueConsumerCancelCleanup)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerDA)); + CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) ConsumerCancelCleanup)); CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueStartDA)); CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueTurnOffDAMode)); CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); @@ -721,7 +728,7 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) while(iUngottenObjs > 0) { /* fill the queue from disk */ CHKiRet(obj.Deserialize((void*) &pUsr, (uchar*)"msg", psQIF, NULL, NULL)); - qqueueUngetObj(pThis, pUsr, MUTEX_ALREADY_LOCKED); + UngetObj(pThis, pUsr, MUTEX_ALREADY_LOCKED); --iUngottenObjs; /* one less */ } @@ -922,7 +929,7 @@ static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis, __attribute_ * rgerhards, 2008-01-20 */ static rsRetVal -qqueueUngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex) +UngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; @@ -1266,6 +1273,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->pConsumer = pConsumer; pThis->iNumWorkerThreads = iWorkerThreads; pThis->iDeqtWinToHr = 25; /* disable time-windowed dequeuing by default */ + pThis->iDeqMaxAtOnce = 8; /* conservative default, should still provide good performance */ pThis->pszFilePrefix = NULL; pThis->qType = qType; @@ -1315,7 +1323,7 @@ finalize_it: * rgerhards, 2008-01-16 */ static rsRetVal -qqueueConsumerCancelCleanup(void *arg1, void *arg2) +ConsumerCancelCleanup(void *arg1, void *arg2) { DEFiRet; @@ -1327,7 +1335,7 @@ qqueueConsumerCancelCleanup(void *arg1, void *arg2) if(pUsr != NULL) { /* make sure the data element is not lost */ dbgoprint((obj_t*) pThis, "cancelation cleanup handler consumer called, we need to unget one user data element\n"); - CHKiRet(qqueueUngetObj(pThis, pUsr, LOCK_MUTEX)); + CHKiRet(UngetObj(pThis, pUsr, LOCK_MUTEX)); } finalize_it: @@ -1376,38 +1384,68 @@ finalize_it: } +/* dequeue as many user points as are available, until we hit the configured + * upper limit of pointers. + * This must only be called when the queue mutex is LOOKED, otherwise serious + * malfunction will happen. + */ +static inline rsRetVal +DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *iRemainingQueueSize) +{ + int nDequeued; + int iQueueSize; + void *pUsr; + rsRetVal localRet; + DEFiRet; + + nDequeued = 0; + do { +dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); + CHKiRet(qqueueDel(pThis, &pUsr)); + qqueueChkPersist(pThis); /* is is questionable if we should really need to call this every time... */ + iQueueSize = qqueueGetOverallQueueSize(pThis); + + /* check if we should discard this element */ + localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr); + if(localRet == RS_RET_QUEUE_FULL) + continue; + else if(localRet != RS_RET_OK) + ABORT_FINALIZE(localRet); + + /* all well, use this element */ + pWti->paUsrp->pUsrp[nDequeued++] = pUsr; + } while(iQueueSize > 0 && nDequeued < pThis->iDeqMaxAtOnce); + + //bRunsDA = pThis->bRunsDA; /* cache this for after mutex release */ + pWti->paUsrp->nElem = nDequeued; + *iRemainingQueueSize = iQueueSize; + +finalize_it: + RETiRet; +} + + /* dequeue the queued object for the queue consumers. * rgerhards, 2008-10-21 + * I made a radical change - we now dequeue multiple elements, and store these objects in + * an array of user pointers. We expect that this increases performance. + * rgerhards, 2009-04-22 */ static rsRetVal -qqueueDequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +DequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) { DEFiRet; - void *pUsr; - int iQueueSize; - int bRunsDA; /* cache for early mutex release */ + int iQueueSize = 0; /* keep the compiler happy... */ - /* dequeue element (still protected from mutex) */ - iRet = qqueueDel(pThis, &pUsr); - qqueueChkPersist(pThis); - iQueueSize = qqueueGetOverallQueueSize(pThis); /* cache this for after mutex release */ - bRunsDA = pThis->bRunsDA; /* cache this for after mutex release */ - - /* We now need to save the user pointer for the cancel cleanup handler, BUT ONLY - * if we could successfully obtain a user pointer. Otherwise, we would bring the - * cancel cleanup handler into big troubles (and we did ;)). Note that we can - * NOT set the variable further below, as this may lead to an object leak. We - * may get cancelled before we reach that part of the code, so the only - * solution is to do it here. -- rgerhards, 2008-02-27 - */ - if(iRet == RS_RET_OK) { - pWti->pUsrp = pUsr; - } + /* dequeue element batch (still protected from mutex) */ + iRet = DequeueConsumableElements(pThis, pWti, &iQueueSize); /* awake some flow-controlled sources if we can do this right now */ /* TODO: this could be done better from a performance point of view -- do it only if * we have someone waiting for the condition (or only when we hit the watermark right * on the nail [exact value]) -- rgerhards, 2008-03-14 + * now that we dequeue batches of pointers, this is much less an issue... + * rgerhards, 2009-04-22 */ if(iQueueSize < pThis->iFullDlyMrk) { pthread_cond_broadcast(&pThis->belowFullDlyWtrMrk); @@ -1417,37 +1455,16 @@ qqueueDequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) pthread_cond_broadcast(&pThis->belowLightDlyWtrMrk); } - /* rgerhards, 2008-09-30: I reversed the order of cond_signal und mutex_unlock - * as of the pthreads recommendation on predictable scheduling behaviour. I don't see - * any problems caused by this, but I add this comment in case some will be seen - * in the next time. - */ pthread_cond_signal(&pThis->notFull); d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); /* WE ARE NO LONGER PROTECTED BY THE MUTEX */ - /* do actual processing (the lengthy part, runs in parallel) - * If we had a problem while dequeing, we do not call the consumer, - * but we otherwise ignore it. This is in the hopes that it will be - * self-healing. However, this is really not a good thing. - * rgerhards, 2008-01-03 - */ - if(iRet != RS_RET_OK) - FINALIZE; - - /* we are running in normal, non-disk-assisted mode do a quick check if we need to drain the queue. - * In DA mode, we do not discard any messages as we assume the disk subsystem is fast enough to - * provide real-time creation of spool files. - * Note: It is OK to use the cached iQueueSize here, because it does not hurt if it is slightly wrong. - */ - CHKiRet(qqueueChkDiscardMsg(pThis, iQueueSize, bRunsDA, pUsr)); - -finalize_it: if(iRet != RS_RET_OK && iRet != RS_RET_DISCARDMSG) { dbgoprint((obj_t*) pThis, "error %d dequeueing element - ignoring, but strange things " "may happen\n", iRet); } + RETiRet; } @@ -1490,7 +1507,7 @@ finalize_it: * but you get the idea from the code above. */ static rsRetVal -qqueueRateLimiter(qqueue_t *pThis) +RateLimiter(qqueue_t *pThis) { DEFiRet; int iDelay; @@ -1553,15 +1570,18 @@ qqueueRateLimiter(qqueue_t *pThis) * rgerhards, 2008-01-21 */ static rsRetVal -qqueueConsumerReg(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +ConsumerReg(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) { DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave)); - CHKiRet(pThis->pConsumer(pThis->pUsr, pWti->pUsrp)); + CHKiRet(DequeueConsumable(pThis, pWti, iCancelStateSave)); +// MULTIQUEUE: here we need to iterate through array! - or better pass it as whole? ... probably + int i; + for(i = 0 ; i < pWti->paUsrp->nElem ; i++) + CHKiRet(pThis->pConsumer(pThis->pUsr, pWti->paUsrp->pUsrp[i])); /* we now need to check if we should deliberately delay processing a bit * and, if so, do that. -- rgerhards, 2008-01-30 @@ -1587,15 +1607,19 @@ finalize_it: * rgerhards, 2008-01-14 */ static rsRetVal -qqueueConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +ConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) { DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(qqueueDequeueConsumable(pThis, pWti, iCancelStateSave)); - CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->pUsrp)); + CHKiRet(DequeueConsumable(pThis, pWti, iCancelStateSave)); +// MULTIQUEUE: + int i; + for(i = 0 ; i < pWti->paUsrp->nElem ; i++) + CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->paUsrp->pUsrp[i])); + finalize_it: dbgoprint((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); @@ -1653,12 +1677,27 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) * the DA queue */ static int -qqueueChkStopWrkrReg(qqueue_t *pThis) +ChkStooWrkrReg(qqueue_t *pThis) { return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && qqueueGetOverallQueueSize(pThis) == 0); } +/* return the configured "deq max at once" interval + * rgerhards, 2009-04-22 + */ +static rsRetVal +GetDeqMaxAtOnce(qqueue_t *pThis, int *pVal) +{ + DEFiRet; + assert(pVal != NULL); +RUNLOG_VAR("%d", pThis->iDeqMaxAtOnce); // MULTIQUEUE: delete this when done + + *pVal = pThis->iDeqMaxAtOnce; + + RETiRet; +} + /* must only be called when the queue mutex is locked, else results * are not stable! DA queue version */ @@ -1673,7 +1712,7 @@ qqueueIsIdleDA(qqueue_t *pThis) * are not stable! Regular queue version */ static int -qqueueIsIdleReg(qqueue_t *pThis) +IsIdleReg(qqueue_t *pThis) { #if 0 /* enable for performance testing */ int ret; @@ -1701,7 +1740,7 @@ qqueueIsIdleReg(qqueue_t *pThis) * I am telling this, because I, too, always get confused by those... */ static rsRetVal -qqueueRegOnWrkrShutdown(qqueue_t *pThis) +RegOnWrkrShutdown(qqueue_t *pThis) { DEFiRet; @@ -1722,7 +1761,7 @@ qqueueRegOnWrkrShutdown(qqueue_t *pThis) * hook to indicate in the parent queue (if we are a child) that we are not done yet. */ static rsRetVal -qqueueRegOnWrkrStartup(qqueue_t *pThis) +RegOnWrkrStartup(qqueue_t *pThis) { DEFiRet; @@ -1788,13 +1827,14 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:Reg", obj.GetName((obj_t*) pThis)); CHKiRet(wtpConstruct (&pThis->pWtpReg)); CHKiRet(wtpSetDbgHdr (pThis->pWtpReg, pszBuf, lenBuf)); - CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRateLimiter)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrReg)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleReg)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) qqueueConsumerReg)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void*pWti))qqueueConsumerCancelCleanup)); - CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrStartup)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) qqueueRegOnWrkrShutdown)); + CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RateLimiter)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) ChkStooWrkrReg)); + CHKiRet(wtpSetpfGetDeqMaxAtOnce (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqMaxAtOnce)); + CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) IsIdleReg)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerReg)); + CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void*pWti))ConsumerCancelCleanup)); + CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RegOnWrkrStartup)); + CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RegOnWrkrShutdown)); CHKiRet(wtpSetpmutUsr (pThis->pWtpReg, pThis->mut)); CHKiRet(wtpSetpcondBusy (pThis->pWtpReg, &pThis->notEmpty)); CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpReg, pThis->iNumWorkerThreads)); diff --git a/runtime/queue.h b/runtime/queue.h index a267862d..7ecb9294 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -54,6 +54,7 @@ typedef struct qWrkThrd_s { pthread_mutex_t mut; } qWrkThrd_t; /* type for queue worker threads */ + /* the queue object */ typedef struct queue_s { BEGINobjInstance; @@ -84,6 +85,7 @@ typedef struct queue_s { int toActShutdown; /* timeout for long-running action shutdown in ms */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ int toEnq; /* enqueue timeout */ + int iDeqMaxAtOnce; /* max number of elements that shall be dequeued at once */ /* rate limiting settings (will be expanded) */ int iDeqSlowdown; /* slow down dequeue by specified nbr of microseconds */ /* end rate limiting */ @@ -97,7 +99,7 @@ typedef struct queue_s { * applied to detect user configuration errors (and tell me how should we detect what * the user really wanted...). -- rgerhards, 2008-04-02 */ - /* ane dequeue time window */ + /* end dequeue time window */ rsRetVal (*pConsumer)(void *,void*); /* user-supplied consumer function for dequeued messages */ /* calling interface for pConsumer: arg1 is the global user pointer from this structure, arg2 is the * user pointer that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 is pointer diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 026fbbed..74ea5270 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -60,6 +60,7 @@ /* define some base data types */ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ +typedef struct aUsrp_s aUsrp_t; typedef struct thrdInfo thrdInfo_t; typedef struct obj_s obj_t; typedef struct filed selector_t;/* TODO: this so far resides in syslogd.c, think about modularization */ diff --git a/runtime/wti.c b/runtime/wti.c index 544bffa7..f50b3894 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -201,6 +201,9 @@ CODESTARTobjDestruct(wti) pthread_cond_destroy(&pThis->condExitDone); pthread_mutex_destroy(&pThis->mut); + if(pThis->paUsrp != NULL) + free(pThis->paUsrp); + if(pThis->pszDbgHdr != NULL) free(pThis->pszDbgHdr); ENDobjDestruct(wti) @@ -209,6 +212,7 @@ ENDobjDestruct(wti) /* Standard-Constructor for the wti object */ BEGINobjConstruct(wti) /* be sure to specify the object type also in END macro! */ + pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); pthread_cond_init(&pThis->condExitDone, NULL); pthread_mutex_init(&pThis->mut, NULL); @@ -222,15 +226,21 @@ rsRetVal wtiConstructFinalize(wti_t *pThis) { DEFiRet; + int iDeqMaxAtOnce; ISOBJ_TYPE_assert(pThis, wti); dbgprintf("%s: finalizing construction of worker instance data\n", wtiGetDbgHdr(pThis)); /* initialize our thread instance descriptor */ - pThis->pUsrp = NULL; pThis->tCurrCmd = eWRKTHRD_STOPPED; + /* we now alloc the array for user pointers. We obtain the max from the queue itself. */ + CHKiRet(pThis->pWtp->pfGetDeqMaxAtOnce(pThis->pWtp->pUsr, &iDeqMaxAtOnce)); + CHKmalloc(pThis->paUsrp = calloc(1, sizeof(aUsrp_t))); + CHKmalloc(pThis->paUsrp->pUsrp = calloc((size_t)iDeqMaxAtOnce, sizeof(void*))); + +finalize_it: RETiRet; } @@ -314,7 +324,8 @@ wtiWorkerCancelCleanup(void *arg) DBGPRINTF("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis)); /* call user supplied handler (that one e.g. requeues the element) */ - pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->pUsrp); +// MULTIQUEUE: need to change here! + pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->paUsrp->pUsrp[0]); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); d_pthread_mutex_lock(&pWtp->mut); @@ -366,7 +377,7 @@ wtiWorker(wti_t *pThis) ISOBJ_TYPE_assert(pWtp, wtp); dbgSetThrdName(pThis->pszDbgHdr); - pThis->pUsrp = NULL; + pThis->paUsrp->nElem = 0; /* flag no elements present */ // MULTIQUEUE: do we really need this any longer (cnacel handeler)? pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); diff --git a/runtime/wti.h b/runtime/wti.h index 6b60b833..85c98fe6 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -1,6 +1,6 @@ /* Definition of the worker thread instance (wti) class. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 by Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -28,13 +28,25 @@ #include "wtp.h" #include "obj.h" +/* the user pointer array object + * This object is used to dequeue multiple user pointers which are than handed over + * to processing. The size of elements is fixed after queue creation, but may be + * modified by config variables (better said: queue properties). + * rgerhards, 2009-04-22 + */ +struct aUsrp_s { + int nElem; /* actual number of element in this entry */ + obj_t **pUsrp; /* actual elements (array!) */ +}; + + /* the worker thread instance class */ typedef struct wti_s { BEGINobjInstance; int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ pthread_t thrdID; /* thread ID */ qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */ - obj_t *pUsrp; /* pointer to an object meaningful for current user pointer (e.g. queue pUsr data elemt) */ + aUsrp_t *paUsrp; /* pointer to an object array meaningful for current user pointer (e.g. queue pUsr data elemt) */ wtp_t *pWtp; /* my worker thread pool (important if only the work thread instance is passed! */ pthread_cond_t condExitDone; /* signaled when the thread exit is done (once per thread existance) */ pthread_mutex_t mut; diff --git a/runtime/wtp.c b/runtime/wtp.c index 04eb974f..e1966099 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -88,6 +88,7 @@ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! pthread_cond_init(&pThis->condThrdTrm, NULL); /* set all function pointers to "not implemented" dummy so that we can safely call them */ pThis->pfChkStopWrkr = NotImplementedDummy; + pThis->pfGetDeqMaxAtOnce = NotImplementedDummy; pThis->pfIsIdle = NotImplementedDummy; pThis->pfDoWork = NotImplementedDummy; pThis->pfOnIdle = NotImplementedDummy; @@ -117,7 +118,7 @@ wtpConstructFinalize(wtp_t *pThis) */ if((pThis->pWrkr = malloc(sizeof(wti_t*) * pThis->iNumWorkerThreads)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - + for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { CHKiRet(wtiConstruct(&pThis->pWrkr[i])); pWti = pThis->pWrkr[i]; @@ -584,6 +585,7 @@ DEFpropSetMethPTR(wtp, pmutUsr, pthread_mutex_t) DEFpropSetMethPTR(wtp, pcondBusy, pthread_cond_t) DEFpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)) DEFpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)) +DEFpropSetMethFP(wtp, pfGetDeqMaxAtOnce, rsRetVal(*pVal)(void*, int*)) DEFpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int)) DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)) DEFpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)) diff --git a/runtime/wtp.h b/runtime/wtp.h index b9cb07c5..5894000a 100644 --- a/runtime/wtp.h +++ b/runtime/wtp.h @@ -67,10 +67,11 @@ typedef struct wtp_s { int bThrdStateChanged; /* at least one thread state has changed if 1 */ /* end sync variables */ /* user objects */ - void *pUsr; /* pointer to user object */ + void *pUsr; /* pointer to user object (in this case, the queue the wtp belongs to) */ pthread_mutex_t *pmutUsr; pthread_cond_t *pcondBusy; /* condition the user will signal "busy again, keep runing" on (awakes worker) */ rsRetVal (*pfChkStopWrkr)(void *pUsr, int); + rsRetVal (*pfGetDeqMaxAtOnce)(void *pUsr, int*); /* obtains max dequeue count from queue config */ rsRetVal (*pfRateLimiter)(void *pUsr); rsRetVal (*pfIsIdle)(void *pUsr, int); rsRetVal (*pfDoWork)(void *pUsr, void *pWti, int); @@ -104,6 +105,7 @@ int wtpGetCurNumWrkr(wtp_t *pThis, int bLockMutex); PROTOTYPEObjClassInit(wtp); PROTOTYPEpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)); PROTOTYPEpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)); +PROTOTYPEpropSetMethFP(wtp, pfGetDeqMaxAtOnce, rsRetVal(*pVal)(void*, int*)); PROTOTYPEpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int)); PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)); PROTOTYPEpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)); -- cgit v1.2.3 From e4b3f6d287d74b34d27b4e296c33cb3f1294a58c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 22 Apr 2009 16:39:58 +0200 Subject: now batches are handed down to the actual consumer ... but the action consumer does not do anything really intelligent with them. But the DA consumer is already done, as is the main message queue consumer. --- action.c | 30 ++++++++++++++++++++++++++++-- runtime/queue.c | 30 +++++++++++++++++------------- runtime/queue.h | 9 ++++----- tools/syslogd.c | 26 +++++++++++++++++--------- 4 files changed, 66 insertions(+), 29 deletions(-) diff --git a/action.c b/action.c index 03073153..8395642c 100644 --- a/action.c +++ b/action.c @@ -42,10 +42,11 @@ #include "cfsysline.h" #include "srUtils.h" #include "errmsg.h" +#include "wti.h" #include "datetime.h" /* forward definitions */ -rsRetVal actionCallDoAction(action_t *pAction, msg_t *pMsg); +rsRetVal actionCallDoActionMULTIQUEUE(action_t *pAction, aUsrp_t*); /* object static data (once for all instances) */ /* TODO: make this an object! DEFobjStaticHelpers -- rgerhards, 2008-03-05 */ @@ -255,7 +256,8 @@ actionConstructFinalize(action_t *pThis) * to be run on multiple threads. So far, this is forbidden by the interface * spec. -- rgerhards, 2008-01-30 */ - CHKiRet(qqueueConstruct(&pThis->pQueue, ActionQueType, 1, iActionQueueSize, (rsRetVal (*)(void*,void*))actionCallDoAction)); + CHKiRet(qqueueConstruct(&pThis->pQueue, ActionQueType, 1, iActionQueueSize, + (rsRetVal (*)(void*,aUsrp_t*))actionCallDoActionMULTIQUEUE)); obj.SetName((obj_t*) pThis->pQueue, pszQName); /* ... set some properties ... */ @@ -415,6 +417,7 @@ rsRetVal actionDbgPrint(action_t *pThis) } +//MULTIQUEUE: think about these two functions below /* call the DoAction output plugin entry point * rgerhards, 2008-01-28 */ @@ -527,6 +530,29 @@ finalize_it: #pragma GCC diagnostic warning "-Wempty-body" +/* receive an array of to-process user pointers and submit them + * for processing. + * rgerhards, 2009-04-22 + */ +rsRetVal +actionCallDoActionMULTIQUEUE(action_t *pAction, aUsrp_t *paUsrp) +{ + int i; + msg_t *pMsg; + DEFiRet; + + assert(paUsrp != NULL); + + for(i = 0 ; i < paUsrp->nElem ; i++) { + pMsg = (msg_t*) paUsrp->pUsrp[i]; +dbgprintf("actionCall..MULTIQUEUE: i: %d, pMsg: %p\n", i, pMsg); + CHKiRet(actionCallDoAction(pAction, pMsg)); + } +finalize_it: + RETiRet; +} + + /* call the HUP handler for a given action, if such a handler is defined. The * action mutex is locked, because the HUP handler most probably needs to modify * some internal state information. diff --git a/runtime/queue.c b/runtime/queue.c index c48eb724..ea8567ea 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -899,6 +899,8 @@ static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis) static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) { + aUsrp_t aUsrp; + obj_t *pMsgp; DEFiRet; ASSERT(pThis != NULL); @@ -908,8 +910,13 @@ static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) * mode the consumer probably has a lot to convey (which get's lost in the other modes * because they are asynchronous. But direct mode is deliberately synchronous. * rgerhards, 2008-02-12 + * We use our knowledge about the aUsrp_t structure below, but without that, we + * pay a too-large performance toll... -- rgerhards, 2009-04-22 */ - iRet = pThis->pConsumer(pThis->pUsr, pUsr); + pMsgp = (obj_t*) pUsr; + aUsrp.nElem = 1; /* there always is only one in direct mode */ + aUsrp.pUsrp = &pMsgp; + iRet = pThis->pConsumer(pThis->pUsr, &aUsrp); RETiRet; } @@ -959,7 +966,7 @@ UngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex) * rgerhards, 2008-01-29 */ static rsRetVal -qqueueGetUngottenObj(qqueue_t *pThis, obj_t **ppUsr) +GetUngottenObj(qqueue_t *pThis, obj_t **ppUsr) { DEFiRet; @@ -1015,7 +1022,7 @@ qqueueDel(qqueue_t *pThis, void *pUsr) * losing the whole process because it loops... -- rgerhards, 2008-01-03 */ if(pThis->iUngottenObjs > 0) { - iRet = qqueueGetUngottenObj(pThis, (obj_t**) pUsr); + iRet = GetUngottenObj(pThis, (obj_t**) pUsr); } else { iRet = pThis->qDel(pThis, pUsr); ATOMIC_DEC(pThis->iQueueSize); @@ -1243,7 +1250,7 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) * to modify some parameters before the queue is actually started. */ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)) + int iMaxQueueSize, rsRetVal (*pConsumer)(void*,aUsrp_t*)) { DEFiRet; qqueue_t *pThis; @@ -1578,14 +1585,12 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) ISOBJ_TYPE_assert(pWti, wti); CHKiRet(DequeueConsumable(pThis, pWti, iCancelStateSave)); -// MULTIQUEUE: here we need to iterate through array! - or better pass it as whole? ... probably - int i; - for(i = 0 ; i < pWti->paUsrp->nElem ; i++) - CHKiRet(pThis->pConsumer(pThis->pUsr, pWti->paUsrp->pUsrp[i])); + CHKiRet(pThis->pConsumer(pThis->pUsr, pWti->paUsrp)); /* we now need to check if we should deliberately delay processing a bit * and, if so, do that. -- rgerhards, 2008-01-30 */ +//TODO: MULTIQUEUE: the following setting is no longer correct - need to think about how to do that... if(pThis->iDeqSlowdown) { dbgoprint((obj_t*) pThis, "sleeping %d microseconds as requested by config params\n", pThis->iDeqSlowdown); @@ -1597,7 +1602,7 @@ finalize_it: } -/* This is a special consumer to feed the disk-queue in disk-assited mode. +/* This is a special consumer to feed the disk-queue in disk-assisted mode. * When active, our own queue more or less acts as a memory buffer to the disk. * So this consumer just needs to drain the memory queue and submit entries * to the disk queue. The disk queue will then call the actual consumer from @@ -1609,18 +1614,17 @@ finalize_it: static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) { + int i; DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); CHKiRet(DequeueConsumable(pThis, pWti, iCancelStateSave)); -// MULTIQUEUE: - int i; + /* iterate over returned results and enqueue them in DA queue */ for(i = 0 ; i < pWti->paUsrp->nElem ; i++) CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->paUsrp->pUsrp[i])); - finalize_it: dbgoprint((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); RETiRet; @@ -1944,7 +1948,7 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) * to the regular files. -- rgerhards, 2008-01-29 */ while(pThis->iUngottenObjs > 0) { - CHKiRet(qqueueGetUngottenObj(pThis, &pUsr)); + CHKiRet(GetUngottenObj(pThis, &pUsr)); CHKiRet((objSerialize(pUsr))(pUsr, psQIF)); objDestruct(pUsr); } diff --git a/runtime/queue.h b/runtime/queue.h index 7ecb9294..4fb57d07 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -100,11 +100,10 @@ typedef struct queue_s { * the user really wanted...). -- rgerhards, 2008-04-02 */ /* end dequeue time window */ - rsRetVal (*pConsumer)(void *,void*); /* user-supplied consumer function for dequeued messages */ + rsRetVal (*pConsumer)(void *,aUsrp_t*); /* user-supplied consumer function for dequeued messages */ /* calling interface for pConsumer: arg1 is the global user pointer from this structure, arg2 is the - * user pointer that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 is pointer - * to message) - * rgerhards, 2008-01-28 + * user pointer array that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 + * is pointer to an array of message message pointers) */ /* type-specific handlers (set during construction) */ rsRetVal (*qConstruct)(struct queue_s *pThis); @@ -185,7 +184,7 @@ rsRetVal qqueueStart(qqueue_t *pThis); rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)); + int iMaxQueueSize, rsRetVal (*pConsumer)(void*,aUsrp_t*)); PROTOTYPEObjClassInit(qqueue); PROTOTYPEpropSetMeth(qqueue, iPersistUpdCnt, int); PROTOTYPEpropSetMeth(qqueue, iDeqtWinFromHr, int); diff --git a/tools/syslogd.c b/tools/syslogd.c index 8c86c12e..f48fa759 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -129,6 +129,7 @@ #include "omfile.h" #include "omdiscard.h" #include "threads.h" +#include "wti.h" #include "queue.h" #include "stream.h" #include "conf.h" @@ -1202,22 +1203,29 @@ processMsg(msg_t *pMsg) /* The consumer of dequeued messages. This function is called by the * queue engine on dequeueing of a message. It runs on a SEPARATE - * THREAD. - * Please note: the message object is destructed by the queue itself! + * THREAD. It receives an array of pointers, which it must iterate + * over. We do not do any further batching, as this is of no benefit + * for the main queue. */ static rsRetVal -msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) +msgConsumer(void __attribute__((unused)) *notNeeded, aUsrp_t *paUsrp) { + int i; + msg_t *pMsg; DEFiRet; - msg_t *pMsg = (msg_t*) pUsr; - assert(pMsg != NULL); + assert(paUsrp != NULL); - if((pMsg->msgFlags & NEEDS_PARSING) != 0) { - parseMsg(pMsg); + for(i = 0 ; i < paUsrp->nElem ; i++) { + pMsg = (msg_t*) paUsrp->pUsrp[i]; +dbgprintf("msgConsumer..MULTIQUEUE: i: %d, pMsg: %p\n", i, pMsg); + if((pMsg->msgFlags & NEEDS_PARSING) != 0) { + parseMsg(pMsg); + } + processMsg(pMsg); + msgDestruct(&pMsg); } - processMsg(pMsg); - msgDestruct(&pMsg); +dbgprintf("DONE msgConsumer..MULTIQUEUE:\n"); RETiRet; } -- cgit v1.2.3 From 50174408b02cd0e5553c5e92a73cc43fb1364db9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 22 Apr 2009 18:37:56 +0200 Subject: added test for DA queue mode (in main msg queue) - needs more work The problem is that the rsyslog engine writes messages too quickly, so that the queue never enters DA mode. We still have some look contemption. One possible solution (hopefully) is to finally move the parser code out of imtcp and onto the next thread. Need to address these issues first, then come back to the test case. --- tests/Makefile.am | 4 ++- tests/da-mainmsg-q.sh | 63 ++++++++++++++++++++++++++++++++++++++ tests/tcpflood.c | 15 +++++---- tests/testsuites/da-mainmsg-q.conf | 20 ++++++++++++ 4 files changed, 95 insertions(+), 7 deletions(-) create mode 100755 tests/da-mainmsg-q.sh create mode 100644 tests/testsuites/da-mainmsg-q.conf diff --git a/tests/Makefile.am b/tests/Makefile.am index 0f4cbce1..b4509dee 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh diskqueue.sh +TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh da-mainmsg-q.sh diskqueue.sh manytcp.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -29,6 +29,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ parsertest.sh \ diskqueue.sh \ testsuites/diskqueue.conf \ + da-mainmsg-q.sh \ + testsuites/da-mainmsg-q.conf \ manytcp.sh \ testsuites/manytcp.conf \ omod-if-array.sh \ diff --git a/tests/da-mainmsg-q.sh b/tests/da-mainmsg-q.sh new file mode 100755 index 00000000..dc091e49 --- /dev/null +++ b/tests/da-mainmsg-q.sh @@ -0,0 +1,63 @@ +# Test for DA mode on the main message queue +# This test checks if DA mode operates correctly. To do so, +# it uses a small in-memory queue size, so that DA mode is initiated +# rather soon, and disk spooling used. There is some uncertainty (based +# on machine speeds), but in general the test should work rather well. +# We add a few messages after the initial run, just so that we can +# check everything recovers from DA mode correctly. +# added 2009-04-22 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo "THIS TEST DOES NOT YET WORK RELIABLY!" +echo "testing main message queue in DA mode (going to disk)" +rm -f work rsyslog.out.log +rm -rf test-spool +mkdir test-spool +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/da-mainmsg-q.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +# +# part1: send first 50 messages (in memory, only) +# +./tcpflood 127.0.0.1 13514 1 50 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +ls -l test-spool +sleep 1 # we need this so that rsyslogd can receive all outstanding messages +# +# part 2: send bunch of messages. This should trigger DA mode +# +# 20000 messages should be enough - the disk test is slow enough ;) +./tcpflood 127.0.0.1 13514 2 20000 50 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +ls -l test-spool +sleep 5 # we need this so that rsyslogd can receive all outstanding messages +# +# send another handful +# +ls -l test-spool +./tcpflood 127.0.0.1 13514 1 50 20050 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 1 # we need this so that rsyslogd can receive all outstanding messages +# +# clean up and check test result +# +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 20099 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log +rm -rf test-spool diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 8dbc201b..2ca796ca 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -6,6 +6,7 @@ * argv[2] target port * argv[3] number of connections * argv[4] number of messages to send (connection is random) + * argv[5] initial message number (optional) * * Part of the testbench for rsyslog. * @@ -51,6 +52,7 @@ static int targetPort; static int numMsgsToSend; /* number of messages to send */ static int numConnections; /* number of connections to create */ static int *sockArray; /* array of sockets to use */ +static int msgNum = 0; /* initial message number to start with */ /* open a single tcp connection @@ -154,8 +156,6 @@ int sendMessages(void) int lenBuf; int lenSend; char buf[2048]; - char msgBuf[128]; - size_t lenMsg; srand(time(NULL)); /* seed is good enough for our needs */ @@ -168,19 +168,20 @@ int sendMessages(void) socknum = i - (numMsgsToSend - numConnections); else socknum = rand() % numConnections; - lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); + lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", msgNum); lenSend = send(sockArray[socknum], buf, lenBuf, 0); if(lenSend != lenBuf) { printf("\r%5.5d\n", i); fflush(stdout); perror("send test data"); - printf("send() failed at socket %d, index %d\n", socknum, i); + printf("send() failed at socket %d, index %d, msgNum %d\n", socknum, i, msgNum); fflush(stderr); return(1); } if(i % 100 == 0) { printf("\r%5.5d", i); } + ++msgNum; } printf("\r%5.5d messages sent\n", i); @@ -262,9 +263,9 @@ int main(int argc, char *argv[]) setvbuf(stdout, buf, _IONBF, 48); - if(argc != 5) { + if(argc != 5 && argc != 6) { printf("Invalid call of tcpflood\n"); - printf("Usage: tcpflood target-host target-port num-connections num-messages\n"); + printf("Usage: tcpflood target-host target-port num-connections num-messages [initial msgnum]\n"); exit(1); } @@ -272,6 +273,8 @@ int main(int argc, char *argv[]) targetPort = atoi(argv[2]); numConnections = atoi(argv[3]); numMsgsToSend = atoi(argv[4]); + if(argc == 6) + msgNum = atoi(argv[5]); if(openConnections() != 0) { printf("error opening connections\n"); diff --git a/tests/testsuites/da-mainmsg-q.conf b/tests/testsuites/da-mainmsg-q.conf new file mode 100644 index 00000000..2d0e5b92 --- /dev/null +++ b/tests/testsuites/da-mainmsg-q.conf @@ -0,0 +1,20 @@ +# Test for DA mode in main message queue (see .sh file for details) +# rgerhards, 2009-04-22 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk assisted mode +$WorkDirectory test-spool +$MainMsgQueueSize 100 # this *should* trigger moving on to DA mode... +$MainMsgQueueHighWatermark 80 +$MainMsgQueueLowWatermark 40 +$MainMsgQueueFilename mainq +$MainMsgQueueType linkedlist +$MainMsgQueueDeqzezeSlowdown 1000 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt -- cgit v1.2.3 From feddb7ea18769399ce53e3958d5c4a0b68e964bc Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 23 Apr 2009 11:19:32 +0200 Subject: improving debugging info a bit --- runtime/queue.c | 6 ++++-- tests/testsuites/da-mainmsg-q.conf | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index ea8567ea..4e37a0c2 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1818,9 +1818,11 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ /* call type-specific constructor */ CHKiRet(pThis->qConstruct(pThis)); /* this also sets bIsDA */ - dbgoprint((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, qsize %d, child %d starting\n", + dbgoprint((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, qsize %d, child %d, " + "full delay %d, light delay %d starting\n", pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize, - qqueueGetOverallQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1); + qqueueGetOverallQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1, + pThis->iFullDlyMrk, pThis->iLightDlyMrk); if(pThis->qType == QUEUETYPE_DIRECT) FINALIZE; /* with direct queues, we are already finished... */ diff --git a/tests/testsuites/da-mainmsg-q.conf b/tests/testsuites/da-mainmsg-q.conf index 2d0e5b92..3465d13b 100644 --- a/tests/testsuites/da-mainmsg-q.conf +++ b/tests/testsuites/da-mainmsg-q.conf @@ -8,7 +8,9 @@ $ErrorMessagesToStderr off # set spool locations and switch queue to disk assisted mode $WorkDirectory test-spool -$MainMsgQueueSize 100 # this *should* trigger moving on to DA mode... +$MainMsgQueueSize 200 # this *should* trigger moving on to DA mode... +# note: we must set QueueSize sufficiently high, so that 70% (light delay mark) +# is high enough above HighWatermark! $MainMsgQueueHighWatermark 80 $MainMsgQueueLowWatermark 40 $MainMsgQueueFilename mainq -- cgit v1.2.3 From f8d9aad08222f59ba2d27437c1e2369896452aa1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 23 Apr 2009 11:45:50 +0200 Subject: bugfix: compile problems in im3195 --- ChangeLog | 1 + plugins/im3195/im3195.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index af38162e..0cc57c81 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,7 @@ Version 4.1.8 [BETA] (rgerhards), 2009-04-?? - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs +- bugfix: compile problems in im3195 --------------------------------------------------------------------------- Version 4.1.7 [BETA] (rgerhards), 2009-04-22 - bugfix: $InputTCPMaxSessions config directive was accepted, but not diff --git a/plugins/im3195/im3195.c b/plugins/im3195/im3195.c index 1c2502fe..106da2c8 100644 --- a/plugins/im3195/im3195.c +++ b/plugins/im3195/im3195.c @@ -47,6 +47,7 @@ #include "liblogging/syslogmessage.h" #include "module-template.h" #include "cfsysline.h" +#include "msg.h" #include "errmsg.h" MODULE_TYPE_INPUT @@ -83,7 +84,7 @@ void OnReceive(srAPIObj __attribute__((unused)) *pMyAPI, srSLMGObj* pSLMG) srSLMGGetRawMSG(pSLMG, &pszRawMsg); parseAndSubmitMessage(fromHost, fromHostIP, pszRawMsg, strlen((char*)pszRawMsg), - MSG_PARSE_HOSTNAME, NOFLAG, eFLOWCTL_FULL_DELAY, (uchar*)"im3195"); + PARSE_HOSTNAME, eFLOWCTL_FULL_DELAY, (uchar*)"im3195", NULL, 0); } -- cgit v1.2.3 From 5c0aeae8ab1f344a022d586dc26c5d78203f7e0b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 23 Apr 2009 12:50:07 +0200 Subject: added $MainMsgQueueDequeueBatchSize and $ActionQueueDequeueBatchSize configuration directives --- ChangeLog | 2 ++ action.c | 4 ++++ doc/queues.html | 27 +++++++++++++++++++++++++++ doc/rsyslog_conf_global.html | 2 ++ runtime/queue.c | 22 ++++++++++------------ runtime/queue.h | 3 ++- runtime/wti.c | 6 +++--- runtime/wtp.c | 4 ++-- runtime/wtp.h | 4 ++-- tests/testsuites/da-mainmsg-q.conf | 2 ++ tools/syslogd.c | 4 ++++ 11 files changed, 60 insertions(+), 20 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1ad99be1..ccc8e2c5 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +- added $MainMsgQueueDequeueBatchSize and $ActionQueueDequeueBatchSize + configuration directives --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? - improved doc diff --git a/action.c b/action.c index 8395642c..be3c7556 100644 --- a/action.c +++ b/action.c @@ -64,6 +64,7 @@ static int bActionRepMsgHasMsg = 0; /* last messsage repeated... has msg fragme /* main message queue and its configuration parameters */ static queueType_t ActionQueType = QUEUETYPE_DIRECT; /* type of the main message queue above */ static int iActionQueueSize = 1000; /* size of the main message queue above */ +static int iActionQueueDeqBatchSize = 16; /* batch size for action queues */ static int iActionQHighWtrMark = 800; /* high water mark for disk-assisted queues */ static int iActionQLowWtrMark = 200; /* low water mark for disk-assisted queues */ static int iActionQDiscardMark = 9800; /* begin to discard messages */ @@ -144,6 +145,7 @@ actionResetQueueParams(void) ActionQueType = QUEUETYPE_DIRECT; /* type of the main message queue above */ iActionQueueSize = 1000; /* size of the main message queue above */ + iActionQueueDeqBatchSize = 16; /* default batch size */ iActionQHighWtrMark = 800; /* high water mark for disk-assisted queues */ iActionQLowWtrMark = 200; /* low water mark for disk-assisted queues */ iActionQDiscardMark = 9800; /* begin to discard messages */ @@ -272,6 +274,7 @@ actionConstructFinalize(action_t *pThis) qqueueSetpUsr(pThis->pQueue, pThis); setQPROP(qqueueSetsizeOnDiskMax, "$ActionQueueMaxDiskSpace", iActionQueMaxDiskSpace); + setQPROP(qqueueSetiDeqBatchSize, "$ActionQueueDequeueBatchSize", iActionQueueDeqBatchSize); setQPROP(qqueueSetMaxFileSize, "$ActionQueueFileSize", iActionQueMaxFileSize); setQPROPstr(qqueueSetFilePrefix, "$ActionQueueFileName", pszActionQFName); setQPROP(qqueueSetiPersistUpdCnt, "$ActionQueueCheckpointInterval", iActionQPersistUpdCnt); @@ -857,6 +860,7 @@ actionAddCfSysLineHdrl(void) CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszActionQFName, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesize", 0, eCmdHdlrInt, NULL, &iActionQueueSize, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuedequeuebatchsize", 0, eCmdHdlrInt, NULL, &iActionQueueDeqBatchSize, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuemaxdiskspace", 0, eCmdHdlrSize, NULL, &iActionQueMaxDiskSpace, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuehighwatermark", 0, eCmdHdlrInt, NULL, &iActionQHighWtrMark, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuelowwatermark", 0, eCmdHdlrInt, NULL, &iActionQLowWtrMark, NULL)); diff --git a/doc/queues.html b/doc/queues.html index 4a9509a0..f063e87c 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -332,6 +332,33 @@ in this regard - it was just not requested so far. So if you need more fine-grained control, let us know and we'll probably implement it. There are two configuration directives, both should be used together or results are unpredictable:" $<object>QueueDequeueTimeBegin <hour>" and "$<object>QueueDequeueTimeEnd <hour>". The hour parameter must be specified in 24-hour format (so 10pm is 22). A use case for this parameter can be found in the rsyslog wiki.

    +

    Performance

    +

    The locking involved with maintaining the queue has a potentially large +performance impact. How large this is, and if it exists at all, depends much on +the configuration and actual use case. However, the queue is able to work on +so-called "batches" when dequeueing data elements. With batches, +multiple data elements are dequeued at once (with a single locking call). +The queue dequeues all available elements up to a configured upper +limit (<object>DequeueBatchSize <number>). It is important +to note that the actual upper limit is dictated by availability. The queue engine +will never wait for a batch to fill. So even if a high upper limit is configured, +batches may consist of fewer elements, even just one, if there are no more elements +waiting in the queue. +

    Batching +can improve performance considerably. Note, however, that it affects the +order in which messages are passed to the queue worker threads, as each worker +now receive as batch of messages. Also, the larger the batch size and the higher +the maximum number of permitted worker threads, the more main memory is needed. +For a busy server, large batch sizes (around 1,000 or even more elements) may be useful. +Please note that with batching, the main memory must hold BatchSize * NumOfWorkers +objects in memory (worst-case scenario), even if running in disk-only mode. So if you +use the default 5 workers at the main message queue and set the batch size to 1,000, you need +to be prepared that the main message queue holds up to 5,000 messages in main memory +in addition to the configured queue size limits! +

    The queue object's default maximum batch size +is eight, but there exists different defaults for the actual parts of +rsyslog processing that utilize queues. So you need to check these object's +defaults.

    Terminating Queues

    Terminating a process sounds easy, but can be complex. Terminating a running queue is in fact the most complex operation a queue diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 3e33f0da..af05715d 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -56,6 +56,7 @@ default template for UDP and plain TCP forwarding action

  • $ActionGSSForwardDefaultTemplate [templateName] - sets a new default template for GSS-API forwarding action
  • $ActionQueueCheckpointInterval <number>
    • +
  • $ActionQueueDequeueBatchSize <number> [default 16]
  • $ActionQueueDequeueSlowdown <number> [number is timeout in microseconds (1000000us is 1sec!), default 0 (no delay). Simple rate-limiting!]
    • @@ -125,6 +126,7 @@ not recommended for use with rsyslog. To do a full restart, simply stop and star compatibility reasons. If it is set to "off", a HUP will only close open files. This is a much quicker action and usually the only one that is needed e.g. for log rotation. It is recommended to set the setting to "off".
  • $IncludeConfig
  • MainMsgQueueCheckpointInterval <number>
    • +
  • $MainMsgQueueDequeueBatchSize <number> [default 32]
  • $MainMsgQueueDequeueSlowdown <number> [number is timeout in microseconds (1000000us is 1sec!), default 0 (no delay). Simple rate-limiting!]
    • diff --git a/runtime/queue.c b/runtime/queue.c index c5f9df81..f3d3fe71 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -69,7 +69,7 @@ rsRetVal qqueueChkPersist(qqueue_t *pThis); static rsRetVal qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); static rsRetVal RateLimiter(qqueue_t *pThis); static int qqueueChkStopWrkrDA(qqueue_t *pThis); -static rsRetVal GetDeqMaxAtOnce(qqueue_t *pThis, int *pVal); +static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal); static int qqueueIsIdleDA(qqueue_t *pThis); static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave); static rsRetVal ConsumerCancelCleanup(void *arg1, void *arg2); @@ -375,7 +375,7 @@ qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); // MULTIQUEUE: TODO: this should be DA-specific! - CHKiRet(wtpSetpfGetDeqMaxAtOnce (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqMaxAtOnce)); + CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleDA)); CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerDA)); CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) ConsumerCancelCleanup)); @@ -1280,7 +1280,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->pConsumer = pConsumer; pThis->iNumWorkerThreads = iWorkerThreads; pThis->iDeqtWinToHr = 25; /* disable time-windowed dequeuing by default */ - pThis->iDeqMaxAtOnce = 8; /* conservative default, should still provide good performance */ + pThis->iDeqBatchSize = 8; /* conservative default, should still provide good performance */ pThis->pszFilePrefix = NULL; pThis->qType = qType; @@ -1421,7 +1421,7 @@ dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); /* all well, use this element */ pWti->paUsrp->pUsrp[nDequeued++] = pUsr; - } while(iQueueSize > 0 && nDequeued < pThis->iDeqMaxAtOnce); + } while(iQueueSize > 0 && nDequeued < pThis->iDeqBatchSize); //bRunsDA = pThis->bRunsDA; /* cache this for after mutex release */ pWti->paUsrp->nElem = nDequeued; @@ -1691,14 +1691,11 @@ ChkStooWrkrReg(qqueue_t *pThis) * rgerhards, 2009-04-22 */ static rsRetVal -GetDeqMaxAtOnce(qqueue_t *pThis, int *pVal) +GetDeqBatchSize(qqueue_t *pThis, int *pVal) { DEFiRet; assert(pVal != NULL); -RUNLOG_VAR("%d", pThis->iDeqMaxAtOnce); // MULTIQUEUE: delete this when done - - *pVal = pThis->iDeqMaxAtOnce; - + *pVal = pThis->iDeqBatchSize; RETiRet; } @@ -1819,10 +1816,10 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ CHKiRet(pThis->qConstruct(pThis)); /* this also sets bIsDA */ dbgoprint((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, qsize %d, child %d, " - "full delay %d, light delay %d starting\n", + "full delay %d, light delay %d, deq batch size %d starting\n", pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize, qqueueGetOverallQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1, - pThis->iFullDlyMrk, pThis->iLightDlyMrk); + pThis->iFullDlyMrk, pThis->iLightDlyMrk, pThis->iDeqBatchSize); if(pThis->qType == QUEUETYPE_DIRECT) FINALIZE; /* with direct queues, we are already finished... */ @@ -1835,7 +1832,7 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ CHKiRet(wtpSetDbgHdr (pThis->pWtpReg, pszBuf, lenBuf)); CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RateLimiter)); CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) ChkStooWrkrReg)); - CHKiRet(wtpSetpfGetDeqMaxAtOnce (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqMaxAtOnce)); + CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) IsIdleReg)); CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerReg)); CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void*pWti))ConsumerCancelCleanup)); @@ -2301,6 +2298,7 @@ DEFpropSetMeth(qqueue, iMinMsgsPerWrkr, int) DEFpropSetMeth(qqueue, bSaveOnShutdown, int) DEFpropSetMeth(qqueue, pUsr, void*) DEFpropSetMeth(qqueue, iDeqSlowdown, int) +DEFpropSetMeth(qqueue, iDeqBatchSize, int) DEFpropSetMeth(qqueue, sizeOnDiskMax, int64) diff --git a/runtime/queue.h b/runtime/queue.h index 4fb57d07..8a60254b 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -85,7 +85,7 @@ typedef struct queue_s { int toActShutdown; /* timeout for long-running action shutdown in ms */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ int toEnq; /* enqueue timeout */ - int iDeqMaxAtOnce; /* max number of elements that shall be dequeued at once */ + int iDeqBatchSize; /* max number of elements that shall be dequeued at once */ /* rate limiting settings (will be expanded) */ int iDeqSlowdown; /* slow down dequeue by specified nbr of microseconds */ /* end rate limiting */ @@ -202,6 +202,7 @@ PROTOTYPEpropSetMeth(qqueue, bSaveOnShutdown, int); PROTOTYPEpropSetMeth(qqueue, pUsr, void*); PROTOTYPEpropSetMeth(qqueue, iDeqSlowdown, int); PROTOTYPEpropSetMeth(qqueue, sizeOnDiskMax, int64); +PROTOTYPEpropSetMeth(qqueue, iDeqBatchSize, int); #define qqueueGetID(pThis) ((unsigned long) pThis) #endif /* #ifndef QUEUE_H_INCLUDED */ diff --git a/runtime/wti.c b/runtime/wti.c index f50b3894..df1ea0ed 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -226,7 +226,7 @@ rsRetVal wtiConstructFinalize(wti_t *pThis) { DEFiRet; - int iDeqMaxAtOnce; + int iDeqBatchSize; ISOBJ_TYPE_assert(pThis, wti); @@ -236,9 +236,9 @@ wtiConstructFinalize(wti_t *pThis) pThis->tCurrCmd = eWRKTHRD_STOPPED; /* we now alloc the array for user pointers. We obtain the max from the queue itself. */ - CHKiRet(pThis->pWtp->pfGetDeqMaxAtOnce(pThis->pWtp->pUsr, &iDeqMaxAtOnce)); + CHKiRet(pThis->pWtp->pfGetDeqBatchSize(pThis->pWtp->pUsr, &iDeqBatchSize)); CHKmalloc(pThis->paUsrp = calloc(1, sizeof(aUsrp_t))); - CHKmalloc(pThis->paUsrp->pUsrp = calloc((size_t)iDeqMaxAtOnce, sizeof(void*))); + CHKmalloc(pThis->paUsrp->pUsrp = calloc((size_t)iDeqBatchSize, sizeof(void*))); finalize_it: RETiRet; diff --git a/runtime/wtp.c b/runtime/wtp.c index e1966099..8bb55cf7 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -88,7 +88,7 @@ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! pthread_cond_init(&pThis->condThrdTrm, NULL); /* set all function pointers to "not implemented" dummy so that we can safely call them */ pThis->pfChkStopWrkr = NotImplementedDummy; - pThis->pfGetDeqMaxAtOnce = NotImplementedDummy; + pThis->pfGetDeqBatchSize = NotImplementedDummy; pThis->pfIsIdle = NotImplementedDummy; pThis->pfDoWork = NotImplementedDummy; pThis->pfOnIdle = NotImplementedDummy; @@ -585,7 +585,7 @@ DEFpropSetMethPTR(wtp, pmutUsr, pthread_mutex_t) DEFpropSetMethPTR(wtp, pcondBusy, pthread_cond_t) DEFpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)) DEFpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)) -DEFpropSetMethFP(wtp, pfGetDeqMaxAtOnce, rsRetVal(*pVal)(void*, int*)) +DEFpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)) DEFpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int)) DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)) DEFpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)) diff --git a/runtime/wtp.h b/runtime/wtp.h index 5894000a..88bd9197 100644 --- a/runtime/wtp.h +++ b/runtime/wtp.h @@ -71,7 +71,7 @@ typedef struct wtp_s { pthread_mutex_t *pmutUsr; pthread_cond_t *pcondBusy; /* condition the user will signal "busy again, keep runing" on (awakes worker) */ rsRetVal (*pfChkStopWrkr)(void *pUsr, int); - rsRetVal (*pfGetDeqMaxAtOnce)(void *pUsr, int*); /* obtains max dequeue count from queue config */ + rsRetVal (*pfGetDeqBatchSize)(void *pUsr, int*); /* obtains max dequeue count from queue config */ rsRetVal (*pfRateLimiter)(void *pUsr); rsRetVal (*pfIsIdle)(void *pUsr, int); rsRetVal (*pfDoWork)(void *pUsr, void *pWti, int); @@ -105,7 +105,7 @@ int wtpGetCurNumWrkr(wtp_t *pThis, int bLockMutex); PROTOTYPEObjClassInit(wtp); PROTOTYPEpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)); PROTOTYPEpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)); -PROTOTYPEpropSetMethFP(wtp, pfGetDeqMaxAtOnce, rsRetVal(*pVal)(void*, int*)); +PROTOTYPEpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)); PROTOTYPEpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int)); PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)); PROTOTYPEpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)); diff --git a/tests/testsuites/da-mainmsg-q.conf b/tests/testsuites/da-mainmsg-q.conf index 3465d13b..b55643c0 100644 --- a/tests/testsuites/da-mainmsg-q.conf +++ b/tests/testsuites/da-mainmsg-q.conf @@ -16,6 +16,8 @@ $MainMsgQueueLowWatermark 40 $MainMsgQueueFilename mainq $MainMsgQueueType linkedlist $MainMsgQueueDeqzezeSlowdown 1000 +# ucomment, as we now have an issue (finally the test case works ;)) +#$MainMsgQueueDequeueBatchSize 80 $template outfmt,"%msg:F,58:2%\n" $template dynfile,"rsyslog.out.log" # trick to use relative path names! diff --git a/tools/syslogd.c b/tools/syslogd.c index f48fa759..5b795755 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -304,6 +304,7 @@ static int iMainMsgQtoWrkShutdown = 60000; /* timeout for worker thread shutdo static int iMainMsgQWrkMinMsgs = 100; /* minimum messages per worker needed to start a new one */ static int iMainMsgQDeqSlowdown = 0; /* dequeue slowdown (simple rate limiting) */ static int64 iMainMsgQueMaxDiskSpace = 0; /* max disk space allocated 0 ==> unlimited */ +static int iMainMsgQueDeqBatchSize = 32; /* dequeue batch size */ static int bMainMsgQSaveOnShutdown = 1; /* save queue on shutdown (when DA enabled)? */ static int iMainMsgQueueDeqtWinFromHr = 0; /* hour begin of time frame when queue is to be dequeued */ static int iMainMsgQueueDeqtWinToHr = 25; /* hour begin of time frame when queue is to be dequeued */ @@ -370,6 +371,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bMainMsgQSaveOnShutdown = 1; MainMsgQueType = QUEUETYPE_FIXED_ARRAY; iMainMsgQueMaxDiskSpace = 0; + iMainMsgQueDeqBatchSize = 32; glbliActionResumeRetryCount = 0; return RS_RET_OK; @@ -2508,6 +2510,7 @@ init(void) setQPROP(qqueueSetMaxFileSize, "$MainMsgQueueFileSize", iMainMsgQueMaxFileSize); setQPROP(qqueueSetsizeOnDiskMax, "$MainMsgQueueMaxDiskSpace", iMainMsgQueMaxDiskSpace); + setQPROP(qqueueSetiDeqBatchSize, "$MainMsgQueueDequeueBatchSize", iMainMsgQueDeqBatchSize); setQPROPstr(qqueueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName); setQPROP(qqueueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt); setQPROP(qqueueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown ); @@ -2888,6 +2891,7 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuedequeueslowdown", 0, eCmdHdlrInt, NULL, &iMainMsgQDeqSlowdown, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueueworkerthreadminimummessages", 0, eCmdHdlrInt, NULL, &iMainMsgQWrkMinMsgs, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuemaxfilesize", 0, eCmdHdlrSize, NULL, &iMainMsgQueMaxFileSize, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuedequeuebatchsize", 0, eCmdHdlrSize, NULL, &iMainMsgQueDeqBatchSize, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuemaxdiskspace", 0, eCmdHdlrSize, NULL, &iMainMsgQueMaxDiskSpace, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesaveonshutdown", 0, eCmdHdlrBinary, NULL, &bMainMsgQSaveOnShutdown, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuedequeuetimebegin", 0, eCmdHdlrInt, NULL, &iMainMsgQueueDeqtWinFromHr, NULL)); -- cgit v1.2.3 From 42c3dcfc1ca71814e62763338a24eae8c8463069 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 23 Apr 2009 13:39:11 +0200 Subject: performance enhancement: imtcp calls parser no longer on input thread but rather inside on of the potentially many main msg queue worker threads (an enhancement scheduled for all input plugins where this is possible) --- ChangeLog | 4 ++++ tcps_sess.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/ChangeLog b/ChangeLog index 1ad99be1..98f59e9e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,6 +6,10 @@ Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? * added tests for queue disk-only mode (checks disk queue logic) - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs +- performance enhancemnt: imtcp calls parser no longer on input thread + but rather inside on of the potentially many main msg queue worker + threads (an enhancement scheduled for all input plugins where this is + possible) --------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program diff --git a/tcps_sess.c b/tcps_sess.c index d0edc018..2a3ec0df 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -43,6 +43,7 @@ #include "errmsg.h" #include "netstrm.h" #include "msg.h" +#include "datetime.h" /* static data */ @@ -50,6 +51,7 @@ DEFobjStaticHelpers DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) DEFobjCurrIf(netstrm) +DEFobjCurrIf(datetime) static int iMaxLine; /* maximum size of a single message */ @@ -191,6 +193,52 @@ SetUsrP(tcps_sess_t *pThis, void *pUsr) } +/* This is a helper for submitting the message to the rsyslog core. + * It does some common processing, including resetting the various + * state variables to a "processed" state. + * Note that this function is also called if we had a buffer overflow + * due to a too-long message. So far, there is no indication this + * happened and it may be worth thinking about different handling + * of this case (what obviously would require a change to this + * function or some related code). + * rgerhards, 2009-04-23 + */ +static rsRetVal +doSubmitMessage(tcps_sess_t *pThis) +{ + msg_t *pMsg; + struct syslogTime stTime; + time_t ttGenTime; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, tcps_sess); + + //TODO: if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { + datetime.getCurrTime(&stTime, &ttGenTime); + //} + /* we now create our own message object and submit it to the queue */ + CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); + /* first trim the buffer to what we have actually received */ + CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); + memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); + pMsg->iLenRawMsg = pThis->iMsg; + MsgSetInputName(pMsg, (char*)pThis->pSrv->pszInputName); + MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); + pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; + pMsg->bParseHOSTNAME = 1; + MsgSetRcvFrom(pMsg, (char*)pThis->fromHost); + CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); + CHKiRet(submitMsg(pMsg)); + +finalize_it: + /* reset status variables */ + pThis->bAtStrtOfFram = 1; + pThis->iMsg = 0; + + RETiRet; +} + + /* This should be called before a normal (non forced) close * of a TCP session. This function checks if there is any unprocessed * message left in the TCP stream. Such a message is probably a @@ -230,9 +278,7 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); - pThis->bAtStrtOfFram = 1; + doSubmitMessage(pThis); } finalize_it: @@ -313,9 +359,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); - pThis->iMsg = 0; + doSubmitMessage(pThis); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -326,9 +370,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(( (c == '\n') || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); - pThis->iMsg = 0; + doSubmitMessage(pThis); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -345,9 +387,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - parseAndSubmitMessage(pThis->fromHost, pThis->fromHostIP, pThis->pMsg, pThis->iMsg, - PARSE_HOSTNAME, eFLOWCTL_LIGHT_DELAY, pThis->pSrv->pszInputName, NULL, 0); - pThis->iMsg = 0; + doSubmitMessage(pThis); pThis->inputState = eAtStrtFram; } } @@ -432,6 +472,7 @@ CODESTARTObjClassExit(tcps_sess) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(datetime, CORE_COMPONENT); ENDObjClassExit(tcps_sess) @@ -443,6 +484,7 @@ BEGINObjClassInit(tcps_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE c /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); iMaxLine = glbl.GetMaxLine(); /* get maximum size we currently support */ -- cgit v1.2.3 From 6c5264159c099ddc4d06590508980ee53a83b67b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 23 Apr 2009 14:29:41 +0200 Subject: fixing a small (newly-introduced) memory leak ... plus simplifying free() calls after agreement on mailing list that we no longer need to check if the pointer is non-NULL --- plugins/imtcp/imtcp.c | 12 ++++-------- runtime/queue.c | 15 +++++---------- runtime/wti.c | 8 +++----- runtime/wtp.c | 3 +-- 4 files changed, 13 insertions(+), 25 deletions(-) diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 5a8a62f6..9883fa89 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -245,14 +245,10 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus iTCPSessMax = 200; iStrmDrvrMode = 0; iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; - if(pszInputName != NULL) { - free(pszInputName); - pszInputName = NULL; - } - if(pszStrmDrvrAuthMode != NULL) { - free(pszStrmDrvrAuthMode); - pszStrmDrvrAuthMode = NULL; - } + free(pszInputName); + pszInputName = NULL; + free(pszStrmDrvrAuthMode); + pszStrmDrvrAuthMode = NULL; return RS_RET_OK; } diff --git a/runtime/queue.c b/runtime/queue.c index f3d3fe71..a5feef3d 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -490,9 +490,7 @@ static rsRetVal qDestructFixedArray(qqueue_t *pThis) ASSERT(pThis != NULL); queueDrain(pThis); /* discard any remaining queue entries */ - - if(pThis->tVars.farray.pBuf != NULL) - free(pThis->tVars.farray.pBuf); + free(pThis->tVars.farray.pBuf); RETiRet; } @@ -2063,11 +2061,8 @@ CODESTARTobjDestruct(qqueue) /* type-specific destructor */ iRet = pThis->qDestruct(pThis); - if(pThis->pszFilePrefix != NULL) - free(pThis->pszFilePrefix); - - if(pThis->pszSpoolDir != NULL) - free(pThis->pszSpoolDir); + free(pThis->pszFilePrefix); + free(pThis->pszSpoolDir); ENDobjDestruct(qqueue) @@ -2081,8 +2076,8 @@ qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix) { DEFiRet; - if(pThis->pszFilePrefix != NULL) - free(pThis->pszFilePrefix); + free(pThis->pszFilePrefix); + pThis->pszFilePrefix = NULL; if(pszPrefix == NULL) /* just unset the prefix! */ ABORT_FINALIZE(RS_RET_OK); diff --git a/runtime/wti.c b/runtime/wti.c index df1ea0ed..32eab57c 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -201,11 +201,9 @@ CODESTARTobjDestruct(wti) pthread_cond_destroy(&pThis->condExitDone); pthread_mutex_destroy(&pThis->mut); - if(pThis->paUsrp != NULL) - free(pThis->paUsrp); - - if(pThis->pszDbgHdr != NULL) - free(pThis->pszDbgHdr); + free(pThis->paUsrp->pUsrp); + free(pThis->paUsrp); + free(pThis->pszDbgHdr); ENDobjDestruct(wti) diff --git a/runtime/wtp.c b/runtime/wtp.c index 8bb55cf7..a23e85f9 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -152,8 +152,7 @@ CODESTARTobjDestruct(wtp) pthread_mutex_destroy(&pThis->mut); pthread_mutex_destroy(&pThis->mutThrdShutdwn); - if(pThis->pszDbgHdr != NULL) - free(pThis->pszDbgHdr); + free(pThis->pszDbgHdr); ENDobjDestruct(wtp) -- cgit v1.2.3 From 8159d0a117837ed573e80cf86f4f013bf9534ab2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 23 Apr 2009 16:02:42 +0200 Subject: fixed abort condition in DA mode --- runtime/queue.c | 3 +-- runtime/wti.c | 1 - runtime/wtp.c | 2 +- tests/da-mainmsg-q.sh | 1 - tests/testsuites/da-mainmsg-q.conf | 3 --- 5 files changed, 2 insertions(+), 8 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index a5feef3d..c2df928b 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -374,8 +374,7 @@ qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) CHKiRet(wtpConstruct (&pThis->pWtpDA)); CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); - // MULTIQUEUE: TODO: this should be DA-specific! - CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); + CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleDA)); CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerDA)); CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) ConsumerCancelCleanup)); diff --git a/runtime/wti.c b/runtime/wti.c index 32eab57c..346ef7aa 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -210,7 +210,6 @@ ENDobjDestruct(wti) /* Standard-Constructor for the wti object */ BEGINobjConstruct(wti) /* be sure to specify the object type also in END macro! */ - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); pthread_cond_init(&pThis->condExitDone, NULL); pthread_mutex_init(&pThis->mut, NULL); diff --git a/runtime/wtp.c b/runtime/wtp.c index a23e85f9..9891a55c 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -78,7 +78,7 @@ wtpGetDbgHdr(wtp_t *pThis) /* Not implemented dummy function for constructor */ -static rsRetVal NotImplementedDummy() { return RS_RET_OK; } +static rsRetVal NotImplementedDummy() { return RS_RET_NOT_IMPLEMENTED; } /* Standard-Constructor for the wtp object */ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! */ diff --git a/tests/da-mainmsg-q.sh b/tests/da-mainmsg-q.sh index dc091e49..2ea6278e 100755 --- a/tests/da-mainmsg-q.sh +++ b/tests/da-mainmsg-q.sh @@ -7,7 +7,6 @@ # check everything recovers from DA mode correctly. # added 2009-04-22 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 -echo "THIS TEST DOES NOT YET WORK RELIABLY!" echo "testing main message queue in DA mode (going to disk)" rm -f work rsyslog.out.log rm -rf test-spool diff --git a/tests/testsuites/da-mainmsg-q.conf b/tests/testsuites/da-mainmsg-q.conf index b55643c0..fbda27d4 100644 --- a/tests/testsuites/da-mainmsg-q.conf +++ b/tests/testsuites/da-mainmsg-q.conf @@ -15,9 +15,6 @@ $MainMsgQueueHighWatermark 80 $MainMsgQueueLowWatermark 40 $MainMsgQueueFilename mainq $MainMsgQueueType linkedlist -$MainMsgQueueDeqzezeSlowdown 1000 -# ucomment, as we now have an issue (finally the test case works ;)) -#$MainMsgQueueDequeueBatchSize 80 $template outfmt,"%msg:F,58:2%\n" $template dynfile,"rsyslog.out.log" # trick to use relative path names! -- cgit v1.2.3 From 10bab38993ae6853d7e23c6f6bd44eb0ed69e001 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 27 Apr 2009 15:40:54 +0200 Subject: begin implementation of new transactional output module interface code is not complete, error cases are not handled. --- ChangeLog | 4 ++ action.c | 11 ++- doc/dev_oplugins.html | 179 ++++++++++++++++++++++++++++++++++++++++++---- plugins/ompgsql/ompgsql.c | 22 ++++++ runtime/module-template.h | 72 ++++++++++++++++++- runtime/modules.c | 29 +++++++- runtime/modules.h | 2 + runtime/rsyslog.h | 11 +++ 8 files changed, 313 insertions(+), 17 deletions(-) diff --git a/ChangeLog b/ChangeLog index 016fa051..f17a4f10 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ - added $MainMsgQueueDequeueBatchSize and $ActionQueueDequeueBatchSize configuration directives +- implemented a new transactional output module interface which provides + superior performance (for databases potentially far superior performance) +- increased ompgsql performance by adapting to new transactional + output module interface --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? - improved doc diff --git a/action.c b/action.c index be3c7556..476e072a 100644 --- a/action.c +++ b/action.c @@ -466,8 +466,7 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) d_pthread_mutex_lock(&pAction->mutActExec); pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); pthread_setcancelstate(iCancelStateSave, NULL); - do { - /* on first invocation, this if should never be true. We just put it at the top + do { /* on first invocation, this if should never be true. We just put it at the top * of the loop so that processing (and code) is simplified. This code is actually * triggered on the 2nd+ invocation. -- rgerhards, 2008-01-30 */ @@ -490,6 +489,10 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) if(bCallAction) { /* call configured action */ + /* MULTIQUEUE: TODO: and this now gets us in trouble. If it was suspended, we can + * assume (and must so) that the action did not succeed. So we now need to redo all + * those messages from the batch that are not yet processed. + */ iRet = pAction->pMod->mod.om.doAction(ppMsgs, pMsg->msgFlags, pAction->pModData); if(iRet == RS_RET_SUSPENDED) { dbgprintf("Action requested to be suspended, done that.\n"); @@ -546,11 +549,15 @@ actionCallDoActionMULTIQUEUE(action_t *pAction, aUsrp_t *paUsrp) assert(paUsrp != NULL); + if(pAction->pMod->mod.om.beginTransaction != NULL) + CHKiRet(pAction->pMod->mod.om.beginTransaction(pAction->pModData)); for(i = 0 ; i < paUsrp->nElem ; i++) { pMsg = (msg_t*) paUsrp->pUsrp[i]; dbgprintf("actionCall..MULTIQUEUE: i: %d, pMsg: %p\n", i, pMsg); CHKiRet(actionCallDoAction(pAction, pMsg)); } + if(pAction->pMod->mod.om.endTransaction != NULL) + CHKiRet(pAction->pMod->mod.om.endTransaction(pAction->pModData)); finalize_it: RETiRet; } diff --git a/doc/dev_oplugins.html b/doc/dev_oplugins.html index 5bfc974c..2e195028 100644 --- a/doc/dev_oplugins.html +++ b/doc/dev_oplugins.html @@ -144,19 +144,172 @@ array-passing capability not blindly be used.     In such cases, we can not guar plugin from segfaulting and if the plugin (as currently always) is run within rsyslog's process space, that results in a segfault for rsyslog. So do not do this.

    Batching of Messages

    -

    With the current plugin interface, each message is passed via a separate call to the plugin. -This is annoying and costs performance in some uses cases (primarily for database outputs). -However, that's the way it (currently) is, no easy way around it. There are some ideas -to implement batching capabilities inside the rsyslog core, but without that the only -resort is to do it inside your plugin yourself. You are not prohibited from doing so. -There are some consequences, though: most importantly, the rsyslog core is no longer -intersted in messages that it passed to a plugin. As such, it will not try to make sure -the message is not lost before it was ultimately processed (because rsyslog, due to -doAction() returning successfully, thinks the message *was* ultimately processed). -

    When the rsyslog core receives batching capabilities, this will be implemented in -a way that is fully compatible to the existing plugin interface. While we have not yet -thought about the implementation, that will probably mean that some new interfaces -or options be used to turn on batching capabilities. +

    Starting with rsyslog 4.3.x, batching of output messages is supported. Previously, only +a single-message interface was supported. +

    With the single message plugin interface, each message is passed via a separate call to the plugin. +Most importantly, the rsyslog engine assumes that each call to the plugin is a complete transaction +and as such assumes that messages be properly commited after the plugin returns to the engine. +

    With the batching interface, rsyslog employs something along the line of +"transactions". Obviously, the rsyslog core can not make non-transactional outputs +to be fully transactional. But what it can is support that the output tells the core which +messages have been commited by the output and which not yet. The core can than take care +of those uncommited messages when problems occur. For example, if a plugin has received +50 messages but not yet told the core that it commited them, and then returns an error state, the +core assumes that all these 50 messages were not written to the output. The core then +requeues all 50 messages and does the usual retry processing. Once the output plugin tells the +core that it is ready again to accept messages, the rsyslog core will provide it with these 50 +not yet commited messages again (actually, at this point, the rsyslog core no longer knows that +it is re-submiting the messages). If, in contrary, the plugin had told rsyslog that 40 of these 50 +messages were commited (before it failed), then only 10 would have been requeued and resubmitted. +

    In order to provide an efficient implementation, there are some (mild) constraints in that +transactional model: first of all, rsyslog itself specifies the ultimate transaction boundaries. +That is, it tells the plugin when a transaction begins and when it must finish. The plugin +is free to commit messages in between, but it must commit all work done when the core +tells it that the transaction ends. All messages passed in between a begin and end transaction +notification are called a batch of messages. They are passed in one by one, just as without +transaction support. Note that batch sizes are variable within the range of 1 to a user configured +maximum limit. Most importantly, that means that plugins may receive batches of single messages, +so they are required to commit each message individually. If the plugin tries to be "smarter" +than the rsyslog engine and does not commit messages in those cases (for example), the plugin +puts message stream integrity at risk: once rsyslog has notified the plugin of transacton end, +it discards all messages as it considers them committed and save. If now something goes wrong, +the rsyslog core does not try to recover lost messages (and keep in mind that "goes wrong" +includes such uncontrollable things like connection loss to a database server). So it is +highly recommended to fully abide to the plugin interface details, even though you may +think you can do it better. The second reason for that is that the core engine will +have configuration settings that enable the user to tune commit rate to their use-case +specific needs. And, as a relief: why would rsyslog ever decide to use batches of one? +There is a trivial case and that is when we have very low activity so that no queue of +messages builds up, in which case it makes sense to commit work as it arrives. +(As a side-note, there are some valid cases where a timeout-based commit feature makes sense. +This is also under evaluation and, once decided, the core will offer an interface plus a way +to preserve message stream integrity for properly-crafted plugins). +

    The second restriction is that if a plugin makes commits in between (what is perfectly +legal) those commits must be in-order. So if a commit is made for message ten out of 50, +this means that messages one to nine are also commited. It would be possible to remove +this restriction, but we have decided to deliberately introduce it to simpify things. +

    Output Plugin Transaction Interface

    +

    In order to keep compatible with existing output plugins (and because it introduces +no complexity), the transactional plugin interface is build on the traditional +non-transactional one. Well... actually the traditional interface was transactional +since its introduction, in the sense that each message was processed in its own +transaction. +

    So the current doAction() entry point can be considered to have this +structure (from the transactional interface point of view): +

    
+doAction()
+    {
+    beginTransaction()
+    ProcessMessage()
+    endTransaction()
+    }
+
    +

    For the transactional interface, we now move these implicit beginTransaction() +and endTransaction(() call out of the message processing body, resulting is such +a structure: +

    
+beginTransaction()
+    {
+    /* prepare for transaction */
+    }
+
+doAction()
+    {
+    ProcessMessage()
+    /* maybe do partial commits */
+    }
+
+endTransaction()
+    {
+    /* commit (rest of) batch */
+    }
+
    +

    And this calling structure actually is the transactional interface! It is as simple as this. +For the new interface, the core calls a beginTransaction() entry point inside the +plugin at the start of the batch. Similarly, the core call endTransaction() at the +end of the batch. The plugin must implement these entry points according to its needs. +

    But how does the core know when to use the old or the new calling interface? This is rather +easy: when loading a plugin, the core queries the plugin for the beginTransaction() +and endTransaction() entry points. If the plugin supports these, the new interface is +used. If the plugin does not support them, the old interface is used and rsyslog implies that +a commit is done after each message. Note that there is no special "downlevel" handling +necessary to support this. In the case of the non-transactional interface, rsyslog considers +each completed call to doAction as partial commit up to the current message. +So implementation inside the core is very straightforward. +

    Actually, we recommend that the transactional entry points only be defined by those +plugins that actually need them. All others should not define them in which case +the default commit behaviour inside rsyslog will apply (thus removing complexity from the +plugin). +

    In order to support partial commits, special return codes must be defined for +doAction. All those return codes mean that processing completed successfully. +But they convey additional information about the commit status as follows: +

    + + + + + + + + + + + + + +
    RS_RET_OKThe record and all previous inside the batch has been commited. +Note: this definition is what makes integrating plugins without the +transaction being/end calls so easy - this is the traditional "success" return +state and if every call returns it, there is no need for actually calling +endTransaction(), because there is no transaction open).
    RS_RET_DEFER_COMMITThe record has been processed, but is not yet commited. This is the +expected state for transactional-aware plugins.
    RS_RET_PREVIOUS_COMMITTEDThe previous record inside the batch has been committed, but the +current one not yet. This state is introduced to support sources that fill up +buffers and commit once a buffer is completely filled. That may occur halfway +in the next record, so it may be important to be able to tell the +engine the everything up to the previouos record is commited
    +

    Note that the typical calling cycle is beginTransaction(), +followed by n times +doAction() followed by endTransaction(). However, if either +beginTransaction() or doAction() return back an error state +(including RS_RET_SUSPENDED), then the transaction is considered aborted. In result, the +remaining calls in this cycle (e.g. endTransaction()) are never made and a +new cycle (starting with beginTransaction() is begun when processing resumes. +So an output plugin must expect and handle those partial cycles gracefully. +

    The question remains how can a plugin know if the core supports batching? +First of all, even if the engine would not know it, the plugin would return with RS_RET_DEFER_COMMIT, +what then would be treated as an error by the engine. This would effectively disable the +output, but cause no further harm (but may be harm enough in itself). +

    The real solution is to enable the plugin to query the rsyslog core if this feature is +supported or not. At the time of the introduction of batching, no such query-interface +exists. So we introduce it with that release. What the means is if a rsyslog core can +not provide this query interface, it is a core that was build before batching support +was available. So the absence of a query interface indicates that the transactional +interface is not available. One might now be tempted the think there is no need to do +the actual check, but is is recommended to ask the rsyslog engine explicitely if +the transactional interface is present and will be honored. This enables us to +create versions in the future which have, for whatever reason we do not yet know, no +support for this interface. +

    The logic to do these checks is contained in the INITChkCoreFeature macro, +which can be used as follows: +

    
+INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING);
+
    +

    Here, bCoreSupportsBatching is a plugin-defined integer which after execution is +1 if batches (and thus the transational interface) is supported and 0 otherwise. +CORE_FEATURE_BATCHING is the feature we are interested in. Future versions of rsyslog +may contain additional feature-test-macros (you can see all of them in +./runtime/rsyslog.h). +

    Note that the ompsql output plugin supports transactional mode in a hybrid way and +thus can be considered good example code. + +

    Open Issues

    +
      +
    • Processing errors handling +
    • reliable re-queue during error handling and queue termination +
    + + +

    Licensing

    From the rsyslog point of view, plugins constitute separate projects. As such, we think plugins are not required to be compatible with GPLv3. However, this is diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index 6daac1c7..003cf6a8 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -170,6 +170,9 @@ tryExec(uchar *pszCmd, instanceData *pData) int bHadError = 0; /* try insert */ +BEGINfunc +RUNLOG_VAR("%p", pData->f_hpgsql); +RUNLOG_VAR("%s", pszCmd); pgRet = PQexec(pData->f_hpgsql, (char*)pszCmd); execState = PQresultStatus(pgRet); if(execState != PGRES_COMMAND_OK && execState != PGRES_TUPLES_OK) { @@ -178,6 +181,7 @@ tryExec(uchar *pszCmd, instanceData *pData) } PQclear(pgRet); +ENDfunc return(bHadError); } @@ -230,6 +234,14 @@ CODESTARTtryResume } ENDtryResume + +BEGINbeginTransaction +CODESTARTbeginTransaction +dbgprintf("ompgsql: beginTransaction\n"); + iRet = writePgSQL((uchar*) "begin", pData); /* TODO: make user-configurable */ +ENDbeginTransaction + + BEGINdoAction CODESTARTdoAction dbgprintf("\n"); @@ -237,6 +249,13 @@ CODESTARTdoAction ENDdoAction +BEGINendTransaction +CODESTARTendTransaction + iRet = writePgSQL((uchar*) "commit;", pData); /* TODO: make user-configurable */ +dbgprintf("ompgsql: endTransaction\n"); +ENDendTransaction + + BEGINparseSelectorAct int iPgSQLPropErr = 0; CODESTARTparseSelectorAct @@ -314,6 +333,7 @@ ENDmodExit BEGINqueryEtryPt CODESTARTqueryEtryPt CODEqueryEtryPt_STD_OMOD_QUERIES +CODEqueryEtryPt_TXIF_OMOD_QUERIES /* we support the transactional interface! */ ENDqueryEtryPt @@ -322,6 +342,8 @@ CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); + INITChkCoreFeature(bCoreSupportsBatching, CORE_FEATURE_BATCHING); + DBGPRINTF("ompgsql: %susing transactional output interface.\n", bCoreSupportsBatching ? "" : "not "); ENDmodInit /* vi:set ai: */ diff --git a/runtime/module-template.h b/runtime/module-template.h index 6f7d877c..3e963199 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -39,7 +39,8 @@ #define DEF_OMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA \ - DEFobjCurrIf(obj) + DEFobjCurrIf(obj) \ + static __attribute__((unused)) int bCoreSupportsBatching; #define DEF_IMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA \ DEFobjCurrIf(obj) @@ -160,6 +161,37 @@ static rsRetVal isCompatibleWithFeature(syslogFeature __attribute__((unused)) eF RETiRet;\ } + +/* beginTransaction() + * introduced in v4.3.3 -- rgerhards, 2009-04-27 + */ +#define BEGINbeginTransaction \ +static rsRetVal beginTransaction(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTbeginTransaction /* currently empty, but may be extended */ + +#define ENDbeginTransaction \ + RETiRet;\ +} + + +/* endTransaction() + * introduced in v4.3.3 -- rgerhards, 2009-04-27 + */ +#define BEGINendTransaction \ +static rsRetVal endTransaction(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTendTransaction /* currently empty, but may be extended */ + +#define ENDendTransaction \ + RETiRet;\ +} + + /* doAction() */ #define BEGINdoAction \ @@ -324,6 +356,18 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ *pEtryPoint = tryResume;\ } + +/* the following definition is queryEtryPt block that must be added + * if an output module supports the transactional interface. + * rgerhards, 2009-04-27 + */ +#define CODEqueryEtryPt_TXIF_OMOD_QUERIES \ + else if(!strcmp((char*) name, "beginTransaction")) {\ + *pEtryPoint = beginTransaction;\ + } else if(!strcmp((char*) name, "endTransaction")) {\ + *pEtryPoint = endTransaction;\ + } + /* the following definition is the standard block for queryEtryPt for INPUT * modules. This can be used if no specific handling (e.g. to cover version * differences) is needed. @@ -393,6 +437,32 @@ finalize_it:\ } +/* now come some check functions, which enable a standard way of obtaining feature + * information from the core. feat is the to-be-tested feature and featVar is a + * variable that receives the result (0-not support, 1-supported). + * This must be a macro, so that it is put into the output's code. Otherwise, we + * would need to rely on a library entry point, which is what we intend to avoid ;) + * rgerhards, 2009-04-27 + */ +#define INITChkCoreFeature(featVar, feat) \ +{ \ + rsRetVal MACRO_Ret; \ + rsRetVal (*pQueryCoreFeatureSupport)(int*, unsigned); \ + int bSupportsIt; \ + featVar = 0; \ + MACRO_Ret = pHostQueryEtryPt((uchar*)"queryCoreFeatureSupport", &pQueryCoreFeatureSupport); \ + if(MACRO_Ret == RS_RET_OK) { \ + /* found entry point, so let's see if core supports it */ \ + CHKiRet((*pQueryCoreFeatureSupport)(&bSupportsIt, feat)); \ + if(bSupportsIt) \ + featVar = 1; \ + } else if(MACRO_Ret != RS_RET_ENTRY_POINT_NOT_FOUND) { \ + ABORT_FINALIZE(MACRO_Ret); /* Something else went wrong, what is not acceptable */ \ + } \ +} + + + /* definitions for host API queries */ #define CODEmodInit_QueryRegCFSLineHdlr \ CHKiRet(pHostQueryEtryPt((uchar*)"regCfSysLineHdlr", &omsdRegCFSLineHdlr)); diff --git a/runtime/modules.c b/runtime/modules.c index 9fdb48e7..024c1c9a 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -207,19 +207,38 @@ static void moduleDestruct(modInfo_t *pThis) } +/* This enables a module to query the core for specific features. + * rgerhards, 2009-04-22 + */ +static rsRetVal queryCoreFeatureSupport(int *pBool, unsigned uFeat) +{ + DEFiRet; + + if((pBool == NULL)) + ABORT_FINALIZE(RS_RET_PARAM_ERROR); + + *pBool = (uFeat & CORE_FEATURE_BATCHING) ? 1 : 0; + +finalize_it: + RETiRet; +} + + /* The following function is the queryEntryPoint for host-based entry points. * Modules may call it to get access to core interface functions. Please note * that utility functions can be accessed via shared libraries - at least this * is my current shool of thinking. * Please note that the implementation as a query interface allows to take * care of plug-in interface version differences. -- rgerhards, 2007-07-31 + * ... but often it better not to use a new interface. So we now add core + * functions here that a plugin may request. -- rgerhards, 2009-04-22 */ static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)()) { DEFiRet; if((name == NULL) || (pEtryPoint == NULL)) - return RS_RET_PARAM_ERROR; + ABORT_FINALIZE(RS_RET_PARAM_ERROR); if(!strcmp((char*) name, "regCfSysLineHdlr")) { *pEtryPoint = regCfSysLineHdlr; @@ -227,6 +246,8 @@ static rsRetVal queryHostEtryPt(uchar *name, rsRetVal (**pEtryPoint)()) *pEtryPoint = objGetObjInterface; } else if(!strcmp((char*) name, "OMSRgetSupportedTplOpts")) { *pEtryPoint = OMSRgetSupportedTplOpts; + } else if(!strcmp((char*) name, "queryCoreFeatureSupport")) { + *pEtryPoint = queryCoreFeatureSupport; } else { *pEtryPoint = NULL; /* to be on the safe side */ ABORT_FINALIZE(RS_RET_ENTRY_POINT_NOT_FOUND); @@ -400,6 +421,12 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"tryResume", &pNew->tryResume)); /* try load optional interfaces */ localRet = (*pNew->modQueryEtryPt)((uchar*)"doHUP", &pNew->doHUP); + if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + ABORT_FINALIZE(localRet); + localRet = (*pNew->modQueryEtryPt)((uchar*)"beginTransaction", &pNew->mod.om.beginTransaction); + if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + ABORT_FINALIZE(localRet); + localRet = (*pNew->modQueryEtryPt)((uchar*)"endTransaction", &pNew->mod.om.endTransaction); if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) ABORT_FINALIZE(localRet); break; diff --git a/runtime/modules.h b/runtime/modules.h index 372529ee..e33bbbe1 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -110,7 +110,9 @@ typedef struct modInfo_s { struct {/* data for output modules */ /* below: perform the configured action */ + rsRetVal (*beginTransaction)(void*); rsRetVal (*doAction)(uchar**, unsigned, void*); + rsRetVal (*endTransaction)(void*); rsRetVal (*parseSelectorAct)(uchar**, void**,omodStringRequest_t**); } om; struct { /* data for library modules */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 74ea5270..3b2dff62 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -58,6 +58,15 @@ #endif +/* the rsyslog core provides information about present feature to plugins + * asking it. Below are feature-test macros which must be used to query + * features. Note that this must be powers of two, so that multiple queries + * can be combined. -- rgerhards, 2009-04-27 + */ +#define CORE_FEATURE_BATCHING 1 +/*#define CORE_FEATURE_whatever 2 ... and so on ... */ + + /* define some base data types */ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ typedef struct aUsrp_s aUsrp_t; @@ -268,6 +277,8 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_FORK = -2118, /**< error during fork() */ RS_RET_ERR_WRITE_PIPE = -2119, /**< error writing to pipe */ RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ + RS_RET_DEFER_COMMIT = -2121, /**< output plugin status: not yet committed (an OK state!) */ + RS_RET_PREVIOUS_COMMITTED = -2122, /**< output plugin status: previous record was committed (an OK state!) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ -- cgit v1.2.3 From 2debe89434872b68f36cc4282d8a02039847c605 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 29 Apr 2009 17:07:01 +0200 Subject: minor cleanup --- action.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/action.c b/action.c index 476e072a..928b30dc 100644 --- a/action.c +++ b/action.c @@ -500,7 +500,8 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) } } - } while(iRet == RS_RET_SUSPENDED && (pAction->iResumeRetryCount == -1 || iRetries < pAction->iResumeRetryCount)); /* do...while! */ + } while( iRet == RS_RET_SUSPENDED + && (pAction->iResumeRetryCount == -1 || iRetries < pAction->iResumeRetryCount)); /* do...while! */ if(iRet == RS_RET_DISABLE_ACTION) { dbgprintf("Action requested to be disabled, done that.\n"); -- cgit v1.2.3 From e65a30ddaaf027855b57355fe35b88b97053ac99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Tue, 28 Apr 2009 21:14:50 +0200 Subject: Add licensing information. I'm not sure if GPLv3 contemplates the ability to link to proprietary software, if it was previous work. I explicitly allow linking to OCI. --- plugins/omoracle/omoracle.c | 4 ++++ plugins/omoracle/omoracle.h | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 71cc8e1f..3a1f141f 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -43,6 +43,10 @@ need to define the properties on the template in the correct order you want them passed to the statement! + This file is licensed under the terms of the GPL version 3 or, at + your choice, any later version. Exceptionally (perhaps), you are + allowed to link to the Oracle Call Interface in your derived work + Author: Luis Fernando Muñoz Mejías diff --git a/plugins/omoracle/omoracle.h b/plugins/omoracle/omoracle.h index 92bcacab..0ff879b3 100644 --- a/plugins/omoracle/omoracle.h +++ b/plugins/omoracle/omoracle.h @@ -3,6 +3,12 @@ This module needs OCI to be installed (on Red Hat-like systems this is usually the oracle-instantclient-devel RPM). + This file is part of rsyslog. + + This file is licensed under the terms of the GPL version 3 or, at + your choice, any later version. Exceptionally (perhaps), you are + allowed to link to the Oracle Call Interface in your derived work + Author: Luis Fernando Muñoz Mejías */ #ifndef __OMORACLEH__ -- cgit v1.2.3 From 65a69831e955fa32b23e8f25d3ba67b1e3058e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Tue, 28 Apr 2009 21:14:51 +0200 Subject: Add the $OmoracleBatchItemSize directive This directive controls the amount of memory needed for properties in the batch. Users should specify the largest value they expect in the statement. As per Rainer's comment: on MAX_BUFSIZE: I'd tend to make this configurable, because with RFC5424 messages can be much longer and RFC5425 now recommends a minimum maximum size of 8K. So we let users to choose. Maybe we need a sensible default value to make users' lifes easier? Also, the old non-vector based interface is not supported anymore. I broke it already when moving to this stage. --- plugins/omoracle/omoracle.c | 46 ++++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index 3a1f141f..d6349d89 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -21,6 +21,13 @@ $OmoracleBatchSize: Number of elements to send to the DB on each transaction. + $OmoracleBatchItemSize: Number of characters each property may + have. Make it as big as the longest value you expect for *any* + property in the sentence. For instance, if you expect 5 arguments + to the statement, 4 have 10 bytes and the 5th may be up to 3KB, + then specify $OmoracleBatchItemSize 3072. Please, remember to + leave space to the trailing \0!! + $OmoracleStatement: Statement to be prepared and executed in batches. Please note that Oracle's prepared statements have their placeholders as ':identifier', and this module uses the colon to @@ -89,6 +96,8 @@ struct oracle_batch int n; /* Number of arguments the statement takes */ int arguments; + /** Maximum size of each parameter */ + int param_size; /* Parameters to pass to the statement on this transaction */ char*** parameters; /* Binding parameters */ @@ -125,12 +134,10 @@ static char* db_user; static char* db_password; /** Batch size. */ static int batch_size; +/** Size of each element in the batch. */ +static int batch_item_size; /** Statement to prepare and execute */ static char* db_statement; -/** Whether or not the core supports the newer array interface. The - * module is able to work in both modes, but the newer is the - * recommended one for performance reasons. */ -static int array_passing; /** Generic function for handling errors from OCI. @@ -248,8 +255,7 @@ static int prepare_statement(instanceData* pData) OCIBindByPos(pData->statement, pData->batch.bindings+i, pData->error, i+1, NULL, - MAX_BUFSIZE * - sizeof ***pData->batch.parameters, + pData->batch.param_size, SQLT_STR, NULL, NULL, NULL, 0, 0, OCI_DATA_AT_EXEC)); CHECKERR(pData->error, @@ -290,6 +296,7 @@ CODESTARTcreateInstance pData->batch.n = 0; pData->batch.size = batch_size; + pData->batch.param_size = batch_item_size * sizeof ***pData->batch.parameters; pData->batch.arguments = count_bind_parameters(pData->txt_statement); /* I know, this can be done with a single malloc() call but this is @@ -302,15 +309,14 @@ CODESTARTcreateInstance sizeof **pData->batch.parameters); CHKmalloc(pData->batch.parameters[i]); for (j = 0; j < pData->batch.size; j++) { - /* Each entry has at most MAX_BUFSIZE bytes - * because OCI doesn't like null-terminated - * strings when operating with batches, and - * the maximum size of each entry must be - * provided when binding - * parameters. MAX_BUFSIZE is long enough for - * usual entries. */ - pData->batch.parameters[i][j] = calloc(MAX_BUFSIZE, - sizeof ***pData->batch.parameters); + /* Each entry has at most + * pData->batch.param_size bytes because OCI + * doesn't like null-terminated strings when + * operating with batches, and the maximum + * size of each entry must be provided when + * binding parameters. pData->batch.param_size + * is long enough for usual entries. */ + pData->batch.parameters[i][j] = malloc(pData->batch.param_size); CHKmalloc(pData->batch.parameters[i][j]); } } @@ -480,7 +486,7 @@ CODESTARTdoAction for (i = 0; i < pData->batch.arguments && params[i]; i++) { dbgprintf("batch[%d][%d]=%s\n", i, pData->batch.n, params[i]); strncpy(pData->batch.parameters[i][pData->batch.n], params[i], - MAX_BUFSIZE); + pData->batch.param_size); CHKmalloc(pData->batch.parameters[i][pData->batch.n]); } pData->batch.n++; @@ -520,6 +526,7 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, if (db_statement != NULL) free(db_statement); db_name = db_user = db_password = db_statement = NULL; + batch_size = batch_item_size = 0; RETiRet; } @@ -549,6 +556,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr((uchar*) "resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoracledbuser", 0, eCmdHdlrGetWord, NULL, &db_user, STD_LOADABLE_MODULE_ID)); @@ -563,11 +571,15 @@ CODEmodInit_QueryRegCFSLineHdlr STD_LOADABLE_MODULE_ID)); CHKiRet(pHostQueryEtryPt((uchar*)"OMSRgetSupportedTplOpts", &supported_options)); CHKiRet((*supported_options)(&opts)); - if (!(array_passing = opts & OMSR_TPL_AS_ARRAY)) + if (!(opts & OMSR_TPL_AS_ARRAY)) ABORT_FINALIZE(RS_RET_RSCORE_TOO_OLD); CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatement", 0, eCmdHdlrCustomHandler, get_db_statement, &db_statement, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclebatchitemsize", 0, + eCmdHdlrInt, NULL, + &batch_item_size, STD_LOADABLE_MODULE_ID)); + ENDmodInit -- cgit v1.2.3 From c35ce31aed4d873ed9564332374fbf82d7ff187d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Tue, 28 Apr 2009 21:14:52 +0200 Subject: Replace get_db_statement by a template. Instead of reading a complete line, we'll use a template and delegate in the core to read such template. Then, all omoracle has to do is to find that template and use it as the prepared statement. I'm not sure if this is the correct approach, though. It has to dig too much into rsyslog's structures... txt_statement is stored in a private area, so that we don't mess too much with rsyslog's internals (I still don't feel comfortable with this much digging into template structures). --- plugins/omoracle/omoracle.c | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index d6349d89..d1b3b955 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -28,10 +28,11 @@ then specify $OmoracleBatchItemSize 3072. Please, remember to leave space to the trailing \0!! - $OmoracleStatement: Statement to be prepared and executed in - batches. Please note that Oracle's prepared statements have their - placeholders as ':identifier', and this module uses the colon to - guess how many placeholders there will be. + $OmoracleStatementTemplate: Name of the template containing the + statement to be prepared and executed in batches. Please note that + Oracle's prepared statements have their placeholders as + ':identifier', and this module uses the colon to guess how many + placeholders there will be. All these directives are mandatory. The dbstring can be an Oracle easystring or a DB name, as present in the tnsnames.ora file. @@ -273,6 +274,7 @@ finalize_it: /* Resource allocation */ BEGINcreateInstance int i, j; + struct template* tpl; CODESTARTcreateInstance ASSERT(pData != NULL); @@ -289,14 +291,16 @@ CODESTARTcreateInstance CHECKENV(pData->environment, OCIHandleAlloc(pData->environment, (void*) &(pData->statement), OCI_HTYPE_STMT, 0, NULL)); - pData->txt_statement = strdup(db_statement); + tpl = tplFind(db_statement, strlen(db_statement)); + pData->txt_statement = strdup(tpl->pEntryRoot->data.constant.pConstant); CHKmalloc(pData->txt_statement); dbgprintf("omoracle will run stored statement: %s\n", pData->txt_statement); pData->batch.n = 0; pData->batch.size = batch_size; - pData->batch.param_size = batch_item_size * sizeof ***pData->batch.parameters; + pData->batch.param_size = batch_item_size * + sizeof ***pData->batch.parameters; pData->batch.arguments = count_bind_parameters(pData->txt_statement); /* I know, this can be done with a single malloc() call but this is @@ -439,7 +443,6 @@ finalize_it: BEGINisCompatibleWithFeature CODESTARTisCompatibleWithFeature /* Right now, this module is compatible with nothing. */ - dbgprintf ("***** OMORACLE ***** At isCompatibleWithFeature\n"); iRet = RS_RET_INCOMPATIBLE; ENDisCompatibleWithFeature @@ -465,6 +468,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1); CHKiRet(createInstance(&pData)); CHKmalloc(pData->connection = strdup(db_name)); CHKiRet(startSession(pData, db_name, db_user, db_password)); + CHKiRet(prepare_statement(pData)); dbgprintf ("omoracle module got all its resources allocated " @@ -530,22 +534,6 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, RETiRet; } -/** As I don't find any handler that reads an entire line, I write my - * own. */ -static int get_db_statement(char** line, char** stmt) -{ - DEFiRet; - - while (isspace(**line)) - (*line)++; - dbgprintf ("Config line: %s\n", *line); - *stmt = strdup(*line); - CHKmalloc(*stmt); - dbgprintf ("Statement: %s\n", *stmt); -finalize_it: - RETiRet; -} - BEGINmodInit() rsRetVal (*supported_options)(unsigned long *pOpts); unsigned long opts; @@ -574,8 +562,8 @@ CODEmodInit_QueryRegCFSLineHdlr if (!(opts & OMSR_TPL_AS_ARRAY)) ABORT_FINALIZE(RS_RET_RSCORE_TOO_OLD); - CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatement", 0, - eCmdHdlrCustomHandler, get_db_statement, + CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclestatementtemplate", 0, + eCmdHdlrGetWord, NULL, &db_statement, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar*) "omoraclebatchitemsize", 0, -- cgit v1.2.3 From 9823c73d1d07e50e6bec7f7c02c88d61d6955526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luis=20Fernando=20Mu=C3=B1oz=20Mej=C3=ADas?= Date: Tue, 28 Apr 2009 21:14:53 +0200 Subject: Make it recover from errors on insertions. If the database rejected some entry, making the statement fail on it, the batch was not cleaned and the same values were retried over and over, causing a cascade of failures and a denial of service. We use now OCI_BATCH_ERRORS so that everything valid in the batch is inserted, and rejected values can be discarded. --- plugins/omoracle/omoracle.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/omoracle/omoracle.c b/plugins/omoracle/omoracle.c index d1b3b955..331b7dd4 100644 --- a/plugins/omoracle/omoracle.c +++ b/plugins/omoracle/omoracle.c @@ -342,13 +342,12 @@ static int insert_to_db(instanceData* pData) OCIStmtExecute(pData->service, pData->statement, pData->error, - pData->batch.n, 0, NULL, NULL, OCI_DEFAULT)); + pData->batch.n, 0, NULL, NULL, + OCI_BATCH_ERRORS)); - CHECKERR(pData->error, - OCITransCommit(pData->service, pData->error, 0)); - - pData->batch.n = 0; finalize_it: + pData->batch.n = 0; + OCITransCommit(pData->service, pData->error, 0); dbgprintf ("omoracle insertion to DB %s\n", iRet == RS_RET_OK ? "succeeded" : "did not succeed"); RETiRet; -- cgit v1.2.3 From bbb0d7c9d1b63dfd78f0ab33fd014909b20bf785 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 30 Apr 2009 19:59:32 +0200 Subject: added action processing message state diagram --- doc/action-call.dot | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 doc/action-call.dot diff --git a/doc/action-call.dot b/doc/action-call.dot new file mode 100644 index 00000000..86c6834d --- /dev/null +++ b/doc/action-call.dot @@ -0,0 +1,33 @@ +// This file is part of rsyslog. +// +// rsyslog action call state diagram +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot action-call.dot -Tpng >action-call.png + +digraph G { + label="\n\nrsyslog message states during action processing\nhttp://www.rsyslog.com"; + //fontsize=20; + + ok [label="ready for processing" color="green"]; + mpf [label="message permanent failure" color="red"]; + tf [label="temporary failure"] + cPen [label="commit pending"]; + com [label="committed" color="red"]; + + tf -> tf [label="retry fails, i < n"]; + tf -> mpf [label="retry fails, i = n"]; + tf -> ok [label="retry succeeds"]; + ok -> com [label="doAction RS_RET_OK"]; + ok -> cPen [label="doAction COMMIT_PENDING"]; + ok -> tf [label="doAction RS_RET_SUSPENDED"]; + ok -> mpf [label="doAction RS_RET_DISABLED"]; + cPen -> com [label="endTransaction RS_RET_OK"]; + cPen -> tf [label="endTransaction _SUSPENDED"]; + + //{rank=same; tf cPen} + {rank=same; com mpf} +} -- cgit v1.2.3 From 68877497a131d5b7c5b1588b771a623fc0ad41c1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 7 May 2009 10:44:46 +0200 Subject: first shot at action state machine implemention (untested) I am commiting it so that the code is visible, but will no begin with the test environment. --- action.c | 462 +++++++++++++++++++++++++++++++++++++++--------------- action.h | 15 +- runtime/conf.c | 2 +- runtime/modules.c | 25 ++- 4 files changed, 372 insertions(+), 132 deletions(-) diff --git a/action.c b/action.c index 928b30dc..9ad05a9f 100644 --- a/action.c +++ b/action.c @@ -45,6 +45,8 @@ #include "wti.h" #include "datetime.h" +#define NO_TIME_PROVIDED 0 /* indicate we do not provide any cached time */ + /* forward definitions */ rsRetVal actionCallDoActionMULTIQUEUE(action_t *pAction, aUsrp_t*); @@ -310,87 +312,223 @@ finalize_it: } -/* set an action back to active state -- rgerhards, 2007-08-02 + +/* set the global resume interval + */ +rsRetVal actionSetGlobalResumeInterval(int iNewVal) +{ + glbliActionResumeInterval = iNewVal; + return RS_RET_OK; +} + + +/* returns the action state name in human-readable form + * returned string must not be modified. + * rgerhards, 2009-05-07 */ -static rsRetVal actionResume(action_t *pThis) +static uchar *getActStateName(action_t *pThis) +{ + switch(pThis->eState) { + case ACT_STATE_RDY: + return (uchar*) "rdy"; + case ACT_STATE_ITX: + return (uchar*) "itx"; + case ACT_STATE_RTRY: + return (uchar*) "rtry"; + case ACT_STATE_SUSP: + return (uchar*) "susp"; + case ACT_STATE_DIED: + return (uchar*) "died"; + case ACT_STATE_COMM: + return (uchar*) "comm"; + default: + return (uchar*) "ERROR/UNKNWON"; + } +} + + +/* returns a suitable return code based on action state + * rgerhards, 2009-05-07 + */ +static rsRetVal getReturnCode(action_t *pThis) { DEFiRet; ASSERT(pThis != NULL); - pThis->bSuspended = 0; + switch(pThis->eState) { + case ACT_STATE_RDY: + iRet = RS_RET_OK; + break; + case ACT_STATE_ITX: + if(pThis->bHadAutoCommit) { + pThis->bHadAutoCommit = 0; /* auto-reset */ + iRet = RS_RET_PREVIOUS_COMMITTED; + } else { + iRet = RS_RET_DEFER_COMMIT; + } + break; + case ACT_STATE_RTRY: + iRet = RS_RET_SUSPENDED; + break; + case ACT_STATE_SUSP: + iRet = RS_RET_SUSPENDED; + break; + case ACT_STATE_DIED: + iRet = RS_RET_DISABLE_ACTION; + break; + default: + DBGPRINTF("Invalid action engine state %d, program error\n", + (int) pThis->eState); + iRet = RS_RET_ERR; + break; + } RETiRet; } -/* set the global resume interval +/* Handles the transient commit state. So far, this is + * mostly a dummy... + * rgerhards, 2007-08-02 */ -rsRetVal actionSetGlobalResumeInterval(int iNewVal) +static void actionCommitted(action_t *pThis) { - glbliActionResumeInterval = iNewVal; - return RS_RET_OK; + pThis->eState = ACT_STATE_RDY; + DBGPRINTF("Action has committed.\n"); +} + + +/* Disable action, this means it will never again be usable + * until rsyslog is reloaded. Use only as a last resort, but + * depends on output module. + * rgerhards, 2007-08-02 + */ +static void actionDisable(action_t *pThis) +{ + pThis->eState = ACT_STATE_DIED; + DBGPRINTF("Action requested to be disabled, done that.\n"); +} + + +/* Suspend action, this involves changing the acton state as well + * as setting the next retry time. + * if we have more than 10 retries, we prolong the + * retry interval. If something is really stalled, it will + * get re-tried only very, very seldom - but that saves + * CPU time. TODO: maybe a config option for that? + * rgerhards, 2007-08-02 + */ +static inline void actionSuspend(action_t *pThis, time_t ttNow) +{ + if(ttNow == NO_TIME_PROVIDED) + time(&ttNow); + pThis->eState = ACT_STATE_SUSP; + pThis->ttResumeRtry = ttNow + pThis->iResumeInterval * (pThis->iNbrResRtry / 10 + 1); + DBGPRINTF("Action requested to be suspended, done that, retry=%d\n", (int) pThis->ttResumeRtry); } -/* suspend an action -- rgerhards, 2007-08-02 +/* actually do retry processing. Note that the function receives a timestamp so + * that we do not need to call the (expensive) time() API. + * Note that we do the full retry processing here, doing the configured number of + * iterations. + * rgerhards, 2009-05-07 */ -static rsRetVal actionSuspend(action_t *pThis, time_t tNow) +static rsRetVal actionDoRetry(action_t *pThis, time_t ttNow) { + int iRetries; + int iSleepPeriod; DEFiRet; ASSERT(pThis != NULL); - pThis->bSuspended = 1; - pThis->ttResumeRtry = tNow + pThis->iResumeInterval; - pThis->iNbrResRtry = 0; /* tell that we did not yet retry to resume */ + + iRetries = 0; + while(pThis->eState == ACT_STATE_RTRY) { + iRet = pThis->pMod->tryResume(pThis->pModData); + if(iRet == RS_RET_SUSPENDED) { + /* max retries reached? */ + if((pThis->iResumeRetryCount != -1 && iRetries >= pThis->iResumeRetryCount)) { + actionSuspend(pThis, ttNow); + } else { + ++pThis->iNbrResRtry; + ++iRetries; + iSleepPeriod = pThis->iResumeInterval; + ttNow += iSleepPeriod; /* not truly exact, but sufficiently... */ + srSleep(iSleepPeriod, 0); + } + } else if(iRet == RS_RET_DISABLE_ACTION) { + actionDisable(pThis); + } + } + + if(pThis->eState == ACT_STATE_RDY) { + pThis->iNbrResRtry = 0; + } RETiRet; } /* try to resume an action -- rgerhards, 2007-08-02 - * returns RS_RET_OK if resumption worked, RS_RET_SUSPEND if the - * action is still suspended. + * changed to new action state engine -- rgerhards, 2009-05-07 */ static rsRetVal actionTryResume(action_t *pThis) { DEFiRet; - time_t ttNow; + time_t ttNow = NO_TIME_PROVIDED; ASSERT(pThis != NULL); - /* for resume handling, we must always obtain a fresh timestamp. We used - * to use the action timestamp, but in this case we will never reach a - * point where a resumption is actually tried, because the action timestamp - * is always in the past. So we can not avoid doing a fresh time() call - * here. -- rgerhards, 2009-03-18 - */ - time(&ttNow); /* cache "now" */ - - /* first check if it is time for a re-try */ - if(ttNow > pThis->ttResumeRtry) { - iRet = pThis->pMod->tryResume(pThis->pModData); - if(iRet == RS_RET_SUSPENDED) { - /* set new tryResume time */ - ++pThis->iNbrResRtry; - /* if we have more than 10 retries, we prolong the - * retry interval. If something is really stalled, it will - * get re-tried only very, very seldom - but that saves - * CPU time. TODO: maybe a config option for that? - * rgerhards, 2007-08-02 - */ - pThis->ttResumeRtry = ttNow + pThis->iResumeInterval * (pThis->iNbrResRtry / 10 + 1); + if(pThis->eState == ACT_STATE_SUSP) { + /* if we are suspended, we need to check if the timeout expired. + * for this handling, we must always obtain a fresh timestamp. We used + * to use the action timestamp, but in this case we will never reach a + * point where a resumption is actually tried, because the action timestamp + * is always in the past. So we can not avoid doing a fresh time() call + * here. -- rgerhards, 2009-03-18 + */ + time(&ttNow); /* cache "now" */ + if(ttNow > pThis->ttResumeRtry) { + pThis->eState = ACT_STATE_RTRY; /* back to retries */ } - } else { - /* it's too early, we are still suspended --> indicate this */ - iRet = RS_RET_SUSPENDED; } - if(iRet == RS_RET_OK) - actionResume(pThis); + if(pThis->eState == ACT_STATE_RTRY) { + if(ttNow == NO_TIME_PROVIDED) /* use cached result if we have it */ + time(&ttNow); + CHKiRet(actionDoRetry(pThis, ttNow)); + } + + DBGPRINTF("actionTryResume: action state: %s, next retry (if applicable): %u [now %u]\n", + getActStateName(pThis), (unsigned) pThis->ttResumeRtry, (unsigned) ttNow); + +finalize_it: + RETiRet; +} + + +/* prepare an action for performing work. This involves trying to recover it, + * depending on its current state. + * rgerhards, 2009-05-07 + */ +static rsRetVal actionPrepare(action_t *pThis) +{ + DEFiRet; + + assert(pThis != NULL); + if(pThis->eState == ACT_STATE_RTRY) { + CHKiRet(actionTryResume(pThis)); + } - dbgprintf("actionTryResume: iRet: %d, next retry (if applicable): %u [now %u]\n", - iRet, (unsigned) pThis->ttResumeRtry, (unsigned) ttNow); + /* if we are now ready, we initialize the transaction and advance + * action state accordingly + */ + if(pThis->eState == ACT_STATE_RDY) { + CHKiRet(pThis->pMod->mod.om.beginTransaction(pThis->pModData)); + pThis->eState = ACT_STATE_ITX; + } +finalize_it: RETiRet; } @@ -407,12 +545,11 @@ rsRetVal actionDbgPrint(action_t *pThis) dbgprintf("\n\tInstance data: 0x%lx\n", (unsigned long) pThis->pModData); dbgprintf("\tRepeatedMsgReduction: %d\n", pThis->f_ReduceRepeated); dbgprintf("\tResume Interval: %d\n", pThis->iResumeInterval); - dbgprintf("\tSuspended: %d", pThis->bSuspended); - if(pThis->bSuspended) { - dbgprintf(" next retry: %u, number retries: %d", (unsigned) pThis->ttResumeRtry, pThis->iNbrResRtry); + if(pThis->eState == ACT_STATE_SUSP) { + dbgprintf("\tresume next retry: %u, number retries: %d", + (unsigned) pThis->ttResumeRtry, pThis->iNbrResRtry); } - dbgprintf("\n"); - dbgprintf("\tDisabled: %d\n", !pThis->bEnabled); + dbgprintf("\tState: %s\n", getActStateName(pThis)); dbgprintf("\tExec only when previous is suspended: %d\n", pThis->bExecWhenPrevSusp); dbgprintf("\n"); @@ -420,25 +557,16 @@ rsRetVal actionDbgPrint(action_t *pThis) } -//MULTIQUEUE: think about these two functions below -/* call the DoAction output plugin entry point - * rgerhards, 2008-01-28 +/* prepare the calling parameters for doAction() + * rgerhards, 2009-05-07 */ -#pragma GCC diagnostic ignored "-Wempty-body" -rsRetVal -actionCallDoAction(action_t *pAction, msg_t *pMsg) +static rsRetVal prepareDoActionParams(action_t *pAction, msg_t *pMsg, uchar ***pppMsgs) { - DEFiRet; - int iRetries; + uchar **ppMsgs = *pppMsgs; int i; - int iArr; - int iSleepPeriod; - int bCallAction; - int iCancelStateSave; - uchar **ppMsgs; /* array of message pointers for doAction */ + DEFiRet; ASSERT(pAction != NULL); - /* create the array for doAction() message pointers */ if((ppMsgs = calloc(pAction->iNumTpls, sizeof(uchar *))) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -456,87 +584,154 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) default:assert(0); /* software bug if this happens! */ } } - iRetries = 0; - /* We now must guard the output module against execution by multiple threads. The - * plugin interface specifies that output modules must not be thread-safe (except - * if they notify us they are - functionality not yet implemented...). - * rgerhards, 2008-01-30 - */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pAction->mutActExec); - pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); - pthread_setcancelstate(iCancelStateSave, NULL); - do { /* on first invocation, this if should never be true. We just put it at the top - * of the loop so that processing (and code) is simplified. This code is actually - * triggered on the 2nd+ invocation. -- rgerhards, 2008-01-30 - */ - if(iRet == RS_RET_SUSPENDED) { - /* ok, this calls for our retry logic... */ - ++iRetries; - iSleepPeriod = pAction->iResumeInterval; - srSleep(iSleepPeriod, 0); - } - /* first check if we are suspended and, if so, retry */ - if(actionIsSuspended(pAction)) { - iRet = actionTryResume(pAction); - if(iRet == RS_RET_OK) - bCallAction = 1; - else - bCallAction = 0; - } else { - bCallAction = 1; - } - if(bCallAction) { - /* call configured action */ - /* MULTIQUEUE: TODO: and this now gets us in trouble. If it was suspended, we can - * assume (and must so) that the action did not succeed. So we now need to redo all - * those messages from the batch that are not yet processed. - */ - iRet = pAction->pMod->mod.om.doAction(ppMsgs, pMsg->msgFlags, pAction->pModData); - if(iRet == RS_RET_SUSPENDED) { - dbgprintf("Action requested to be suspended, done that.\n"); - actionSuspend(pAction, getActNow(pAction)); - } - } +finalize_it: + *pppMsgs = ppMsgs; + RETiRet; +} - } while( iRet == RS_RET_SUSPENDED - && (pAction->iResumeRetryCount == -1 || iRetries < pAction->iResumeRetryCount)); /* do...while! */ - if(iRet == RS_RET_DISABLE_ACTION) { - dbgprintf("Action requested to be disabled, done that.\n"); - pAction->bEnabled = 0; /* that's it... */ - } - - pthread_cleanup_pop(1); /* unlock mutex */ +/* cleanup doAction calling parameters + * rgerhards, 2009-05-07 + */ +static rsRetVal cleanupDoActionParams(action_t *pAction, uchar ***pppMsgs) +{ + uchar **ppMsgs = *pppMsgs; + int i; + int iArr; + DEFiRet; -finalize_it: - /* cleanup */ + ASSERT(pAction != NULL); for(i = 0 ; i < pAction->iNumTpls ; ++i) { if(ppMsgs[i] != NULL) { switch(pAction->eParamPassing) { case ACT_ARRAY_PASSING: iArr = 0; while(((char **)ppMsgs[i])[iArr] != NULL) - d_free(((char **)ppMsgs[i])[iArr++]); - d_free(ppMsgs[i]); + free(((char **)ppMsgs[i])[iArr++]); + free(ppMsgs[i]); break; case ACT_STRING_PASSING: - d_free(ppMsgs[i]); + free(ppMsgs[i]); break; default: assert(0); } } } - d_free(ppMsgs); - msgDestruct(&pMsg); /* we are now finished with the message */ + free(ppMsgs); + *pppMsgs = NULL; RETiRet; } -#pragma GCC diagnostic warning "-Wempty-body" +/* call the DoAction output plugin entry point + * Performance note: we build the action parameters here in this function. That + * means we do it while we hold the action look, potentially reducing concurrency + * (especially if the action queue is run in DIRECT mode). As an alternative, we + * may generate all params for the batch as whole before aquiring the action. However, + * that requires more memory, for large batches potentially a lot of memory. So for the + * time being, I am doing it here - the performance hit should be very minor and may even + * not be a hit because we may gain CPU cache locality gains with the "fewer memory" + * approach (I'd say that is rater likely). + * rgerhards, 2008-01-28 + */ +rsRetVal +actionCallDoAction(action_t *pThis, msg_t *pMsg) +{ + uchar **ppMsgs; /* array of message pointers for doAction */ + DEFiRet; + + ASSERT(pThis != NULL); + ISOBJ_TYPE_assert(pMsg, msg); + + CHKiRet(prepareDoActionParams(pThis, pMsg, &ppMsgs)); + + pThis->bHadAutoCommit = 0; + iRet = pThis->pMod->mod.om.doAction(ppMsgs, pMsg->msgFlags, pThis->pModData); + switch(iRet) { + case RS_RET_OK: + actionCommitted(pThis); + break; + case RS_RET_DEFER_COMMIT: + /* we are done, action state remains the same */ + break; + case RS_RET_PREVIOUS_COMMITTED: + /* action state remains the same, but we had a commit. */ + pThis->bHadAutoCommit = 1; + break; + case RS_RET_SUSPENDED: + actionSuspend(pThis, NO_TIME_PROVIDED); + break; + case RS_RET_DISABLE_ACTION: + actionDisable(pThis); + break; + default:/* permanent failure of this message - no sense in retrying. This is + * not yet handled (but easy TODO) + */ + FINALIZE; + } + iRet = getReturnCode(pThis); + +finalize_it: + cleanupDoActionParams(pThis, &ppMsgs); /* iRet ignored! */ + + RETiRet; +} + + +/* finish processing a batch. Most importantly, that means we commit if we + * need to do so. + * rgerhards, 2008-01-28 + */ +static rsRetVal +finishBatch(action_t *pThis) +{ + DEFiRet; + + ASSERT(pThis != NULL); + + if(pThis->eState == ACT_STATE_RDY) + FINALIZE; /* nothing to do */ + + CHKiRet(actionPrepare(pThis)); + if(pThis->eState == ACT_STATE_ITX) { + iRet = pThis->pMod->mod.om.endTransaction(pThis->pModData); + switch(iRet) { + case RS_RET_OK: + actionCommitted(pThis); + break; + case RS_RET_SUSPENDED: + actionSuspend(pThis, NO_TIME_PROVIDED); + break; + case RS_RET_DISABLE_ACTION: + actionDisable(pThis); + break; + case RS_RET_DEFER_COMMIT: + DBGPRINTF("output plugin error: endTransaction() returns RS_RET_DEFER_COMMIT " + "- ignored\n"); + actionCommitted(pThis); + break; + case RS_RET_PREVIOUS_COMMITTED: + DBGPRINTF("output plugin error: endTransaction() returns RS_RET_PREVIOUS_COMMITTED " + "- ignored\n"); + actionCommitted(pThis); + break; + default:/* permanent failure of this message - no sense in retrying. This is + * not yet handled (but easy TODO) + */ + FINALIZE; + } + } + iRet = getReturnCode(pThis); + +finalize_it: + RETiRet; +} + + +#pragma GCC diagnostic ignored "-Wempty-body" /* receive an array of to-process user pointers and submit them * for processing. * rgerhards, 2009-04-22 @@ -545,23 +740,36 @@ rsRetVal actionCallDoActionMULTIQUEUE(action_t *pAction, aUsrp_t *paUsrp) { int i; + int iCancelStateSave; msg_t *pMsg; DEFiRet; assert(paUsrp != NULL); - if(pAction->pMod->mod.om.beginTransaction != NULL) - CHKiRet(pAction->pMod->mod.om.beginTransaction(pAction->pModData)); + /* We now must guard the output module against execution by multiple threads. The + * plugin interface specifies that output modules must not be thread-safe (except + * if they notify us they are - functionality not yet implemented...). + * rgerhards, 2008-01-30 + */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + d_pthread_mutex_lock(&pAction->mutActExec); + pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); + pthread_setcancelstate(iCancelStateSave, NULL); + for(i = 0 ; i < paUsrp->nElem ; i++) { pMsg = (msg_t*) paUsrp->pUsrp[i]; dbgprintf("actionCall..MULTIQUEUE: i: %d, pMsg: %p\n", i, pMsg); CHKiRet(actionCallDoAction(pAction, pMsg)); + msgDestruct(&pMsg); /* TODO: change: we are now finished with the message */ } - if(pAction->pMod->mod.om.endTransaction != NULL) - CHKiRet(pAction->pMod->mod.om.endTransaction(pAction->pModData)); + iRet = finishBatch(pAction); + + pthread_cleanup_pop(1); /* unlock mutex */ + finalize_it: RETiRet; } +#pragma GCC diagnostic warning "-Wempty-body" /* call the HUP handler for a given action, if such a handler is defined. The @@ -799,8 +1007,8 @@ actionCallAction(action_t *pAction, msg_t *pMsg) * should check from time to time if affairs have improved. * rgerhards, 2007-07-24 */ - if(pAction->bEnabled == 0) { - ABORT_FINALIZE(RS_RET_OK); + if(pAction->eState == ACT_STATE_DIED) { + ABORT_FINALIZE(RS_RET_DISABLE_ACTION); } pAction->tActNow = -1; /* we do not yet know our current time (clear prev. value) */ @@ -983,7 +1191,7 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques dbgprintf("module is incompatible with RepeatedMsgReduction - turned off\n"); pAction->f_ReduceRepeated = 0; } - pAction->bEnabled = 1; /* action is enabled */ + pAction->eState = ACT_STATE_RDY; /* action is enabled */ if(bSuspended) actionSuspend(pAction, time(NULL)); /* "good" time call, only during init and unavoidable */ diff --git a/action.h b/action.h index f2706af6..69d6713c 100644 --- a/action.h +++ b/action.h @@ -36,6 +36,15 @@ extern int glbliActionResumeRetryCount; +typedef enum { + ACT_STATE_DIED = 0, /* action permanently failed and now disabled - MUST BE ZEO! */ + ACT_STATE_RDY = 1, /* action ready, waiting for new transaction */ + ACT_STATE_ITX = 2, /* transaction active, waiting for new data or commit */ + ACT_STATE_COMM = 3, /* transaction finished (a transient state) */ + ACT_STATE_RTRY = 4, /* failure occured, trying to restablish ready state */ + ACT_STATE_SUSP = 5 /* suspended due to failure (return fail until timeout expired) */ +} action_state_t; + /* the following struct defines the action object data structure */ struct action_s { @@ -45,8 +54,10 @@ struct action_s { time_t tLastExec; /* time this action was last executed */ int bExecWhenPrevSusp;/* execute only when previous action is suspended? */ int iSecsExecOnceInterval; /* if non-zero, minimum seconds to wait until action is executed again */ - short bEnabled; /* is the related action enabled (1) or disabled (0)? */ - short bSuspended; /* is the related action temporarily suspended? */ + action_state_t eState; /* current state of action */ + int bHadAutoCommit; /* did an auto-commit happen during doAction()? */ + //short bEnabled; /* is the related action enabled (1) or disabled (0)? */ + //short bSuspended; /* is the related action temporarily suspended? */ time_t ttResumeRtry; /* when is it time to retry the resume? */ int iResumeInterval;/* resume interval for this action */ int iResumeRetryCount;/* how often shall we retry a suspended action? (-1 --> eternal) */ diff --git a/runtime/conf.c b/runtime/conf.c index 27ab8bb4..c776ef46 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -1080,7 +1080,7 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction) dbgprintf("module is incompatible with RepeatedMsgReduction - turned off\n"); pAction->f_ReduceRepeated = 0; } - pAction->bEnabled = 1; /* action is enabled */ + pAction->eState = ACT_STATE_RDY; /* action is enabled */ iNbrActions++; /* one more active action! */ } break; diff --git a/runtime/modules.c b/runtime/modules.c index 024c1c9a..bfd87a71 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -68,6 +68,21 @@ static modInfo_t *pLoadedModulesLast = NULL; /* tail-pointer */ uchar *pModDir = NULL; /* read-only after startup */ +/* we provide a set of dummy functions for output modules that do not support the + * transactional interface. As they do not do this, they commit each message they + * receive, and as such the dummies can always return RS_RET_OK without causing + * harm. This simplifies things as in action processing we do not need to check + * if the transactional entry points exist. + */ +static rsRetVal dummyBeginTransaction() +{ + return RS_RET_OK; +} +static rsRetVal dummyEndTransaction() +{ + return RS_RET_OK; +} + #ifdef DEBUG /* we add some home-grown support to track our users (and detect who does not free us). In * the long term, this should probably be migrated into debug.c (TODO). -- rgerhards, 2008-03-11 @@ -423,11 +438,17 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ localRet = (*pNew->modQueryEtryPt)((uchar*)"doHUP", &pNew->doHUP); if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) ABORT_FINALIZE(localRet); + localRet = (*pNew->modQueryEtryPt)((uchar*)"beginTransaction", &pNew->mod.om.beginTransaction); - if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + pNew->mod.om.beginTransaction = dummyBeginTransaction; + else if(localRet != RS_RET_OK) ABORT_FINALIZE(localRet); + localRet = (*pNew->modQueryEtryPt)((uchar*)"endTransaction", &pNew->mod.om.endTransaction); - if(localRet != RS_RET_OK && localRet != RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + if(localRet == RS_RET_MODULE_ENTRY_POINT_NOT_FOUND) + pNew->mod.om.beginTransaction = dummyEndTransaction; + else if(localRet != RS_RET_OK) ABORT_FINALIZE(localRet); break; case eMOD_LIB: -- cgit v1.2.3 From 9e1bb31a4dc20d79515a19c85f2a5fec6a3d0c21 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 7 May 2009 13:37:25 +0200 Subject: fixed some bugs & added testing helpers The action state machine now works correctly and has been verified a few hand-picked cases. I am missing automatted tests, though, this is not easy to achive... Anyhow, I've improved omtesting, so that it can be used in such automatted tests. --- action.c | 107 +++++++++++++++++++++++------ plugins/omtesting/omtesting.c | 156 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 231 insertions(+), 32 deletions(-) diff --git a/action.c b/action.c index 9ad05a9f..829ef1eb 100644 --- a/action.c +++ b/action.c @@ -387,14 +387,31 @@ static rsRetVal getReturnCode(action_t *pThis) } +/* set the action to a new state + * rgerhards, 2007-08-02 + */ +static inline void actionSetState(action_t *pThis, action_state_t newState) +{ + pThis->eState = newState; + DBGPRINTF("Action %p transitioned to state: %s\n", pThis, getActStateName(pThis)); +} + /* Handles the transient commit state. So far, this is * mostly a dummy... * rgerhards, 2007-08-02 */ static void actionCommitted(action_t *pThis) { - pThis->eState = ACT_STATE_RDY; - DBGPRINTF("Action has committed.\n"); + actionSetState(pThis, ACT_STATE_RDY); +} + + +/* set action to "rtry" state. + * rgerhards, 2007-08-02 + */ +static void actionRetry(action_t *pThis) +{ + actionSetState(pThis, ACT_STATE_RTRY); } @@ -405,8 +422,7 @@ static void actionCommitted(action_t *pThis) */ static void actionDisable(action_t *pThis) { - pThis->eState = ACT_STATE_DIED; - DBGPRINTF("Action requested to be disabled, done that.\n"); + actionSetState(pThis, ACT_STATE_DIED); } @@ -422,9 +438,9 @@ static inline void actionSuspend(action_t *pThis, time_t ttNow) { if(ttNow == NO_TIME_PROVIDED) time(&ttNow); - pThis->eState = ACT_STATE_SUSP; pThis->ttResumeRtry = ttNow + pThis->iResumeInterval * (pThis->iNbrResRtry / 10 + 1); - DBGPRINTF("Action requested to be suspended, done that, retry=%d\n", (int) pThis->ttResumeRtry); + actionSetState(pThis, ACT_STATE_SUSP); + DBGPRINTF("earliest retry=%d\n", (int) pThis->ttResumeRtry); } @@ -442,10 +458,15 @@ static rsRetVal actionDoRetry(action_t *pThis, time_t ttNow) ASSERT(pThis != NULL); +RUNLOG_STR("actionDoRetry():"); iRetries = 0; while(pThis->eState == ACT_STATE_RTRY) { iRet = pThis->pMod->tryResume(pThis->pModData); - if(iRet == RS_RET_SUSPENDED) { + if(iRet == RS_RET_OK) { + actionSetState(pThis, ACT_STATE_RDY); +RUNLOG_STR("tryResume succeeded"); + } else if(iRet == RS_RET_SUSPENDED) { +RUNLOG_STR("still suspended");; /* max retries reached? */ if((pThis->iResumeRetryCount != -1 && iRetries >= pThis->iResumeRetryCount)) { actionSuspend(pThis, ttNow); @@ -479,6 +500,7 @@ static rsRetVal actionTryResume(action_t *pThis) ASSERT(pThis != NULL); +RUNLOG_STR("actionTryResume()"); if(pThis->eState == ACT_STATE_SUSP) { /* if we are suspended, we need to check if the timeout expired. * for this handling, we must always obtain a fresh timestamp. We used @@ -489,7 +511,7 @@ static rsRetVal actionTryResume(action_t *pThis) */ time(&ttNow); /* cache "now" */ if(ttNow > pThis->ttResumeRtry) { - pThis->eState = ACT_STATE_RTRY; /* back to retries */ + actionSetState(pThis, ACT_STATE_RTRY); /* back to retries */ } } @@ -515,6 +537,7 @@ static rsRetVal actionPrepare(action_t *pThis) { DEFiRet; +RUNLOG_STR("actionPrepare()"); assert(pThis != NULL); if(pThis->eState == ACT_STATE_RTRY) { CHKiRet(actionTryResume(pThis)); @@ -525,7 +548,7 @@ static rsRetVal actionPrepare(action_t *pThis) */ if(pThis->eState == ACT_STATE_RDY) { CHKiRet(pThis->pMod->mod.om.beginTransaction(pThis->pModData)); - pThis->eState = ACT_STATE_ITX; + actionSetState(pThis, ACT_STATE_ITX); } finalize_it: @@ -646,6 +669,7 @@ actionCallDoAction(action_t *pThis, msg_t *pMsg) ASSERT(pThis != NULL); ISOBJ_TYPE_assert(pMsg, msg); + DBGPRINTF("entering actionCalldoAction(), state: %s\n", getActStateName(pThis)); CHKiRet(prepareDoActionParams(pThis, pMsg, &ppMsgs)); pThis->bHadAutoCommit = 0; @@ -662,7 +686,7 @@ actionCallDoAction(action_t *pThis, msg_t *pMsg) pThis->bHadAutoCommit = 1; break; case RS_RET_SUSPENDED: - actionSuspend(pThis, NO_TIME_PROVIDED); + actionRetry(pThis); break; case RS_RET_DISABLE_ACTION: actionDisable(pThis); @@ -681,6 +705,29 @@ finalize_it: } +/* process a message + * this readies the action and then calls doAction() + * rgerhards, 2008-01-28 + */ +rsRetVal +actionProcessMessage(action_t *pThis, msg_t *pMsg) +{ + DEFiRet; + + ASSERT(pThis != NULL); + ISOBJ_TYPE_assert(pMsg, msg); + +RUNLOG_STR("inside actionProcessMsg()"); + CHKiRet(actionPrepare(pThis)); + if(pThis->eState == ACT_STATE_ITX) + CHKiRet(actionCallDoAction(pThis, pMsg)); + + iRet = getReturnCode(pThis); +finalize_it: + RETiRet; +} + + /* finish processing a batch. Most importantly, that means we commit if we * need to do so. * rgerhards, 2008-01-28 @@ -703,7 +750,7 @@ finishBatch(action_t *pThis) actionCommitted(pThis); break; case RS_RET_SUSPENDED: - actionSuspend(pThis, NO_TIME_PROVIDED); + actionRetry(pThis); break; case RS_RET_DISABLE_ACTION: actionDisable(pThis); @@ -731,6 +778,33 @@ finalize_it: } +/* receive an array of to-process user pointers and submit them + * for processing. + * rgerhards, 2009-04-22 + */ +rsRetVal +actionCallDoActionMULTIQUEUEprocessing(action_t *pAction, aUsrp_t *paUsrp) +{ + int i; + msg_t *pMsg; + rsRetVal localRet; + DEFiRet; + + assert(paUsrp != NULL); + + for(i = 0 ; i < paUsrp->nElem ; i++) { + pMsg = (msg_t*) paUsrp->pUsrp[i]; +dbgprintf("actionCall..MULTIQUEUE: i: %d, pMsg: %p\n", i, pMsg); + localRet = actionProcessMessage(pAction, pMsg); + dbgprintf("action call returned %d\n", localRet); + msgDestruct(&pMsg); /* TODO: change: we are now finished with the message */ + CHKiRet(localRet); + } + iRet = finishBatch(pAction); + +finalize_it: + RETiRet; +} #pragma GCC diagnostic ignored "-Wempty-body" /* receive an array of to-process user pointers and submit them * for processing. @@ -739,9 +813,7 @@ finalize_it: rsRetVal actionCallDoActionMULTIQUEUE(action_t *pAction, aUsrp_t *paUsrp) { - int i; int iCancelStateSave; - msg_t *pMsg; DEFiRet; assert(paUsrp != NULL); @@ -756,17 +828,10 @@ actionCallDoActionMULTIQUEUE(action_t *pAction, aUsrp_t *paUsrp) pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); pthread_setcancelstate(iCancelStateSave, NULL); - for(i = 0 ; i < paUsrp->nElem ; i++) { - pMsg = (msg_t*) paUsrp->pUsrp[i]; -dbgprintf("actionCall..MULTIQUEUE: i: %d, pMsg: %p\n", i, pMsg); - CHKiRet(actionCallDoAction(pAction, pMsg)); - msgDestruct(&pMsg); /* TODO: change: we are now finished with the message */ - } - iRet = finishBatch(pAction); + iRet = actionCallDoActionMULTIQUEUEprocessing(pAction, paUsrp); pthread_cleanup_pop(1); /* unlock mutex */ -finalize_it: RETiRet; } #pragma GCC diagnostic warning "-Wempty-body" diff --git a/plugins/omtesting/omtesting.c b/plugins/omtesting/omtesting.c index 411bcf88..8f6cdbe5 100644 --- a/plugins/omtesting/omtesting.c +++ b/plugins/omtesting/omtesting.c @@ -22,7 +22,7 @@ * NOTE: read comments in module-template.h to understand how this file * works! * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -46,12 +46,14 @@ #include #include #include +#include #include #include #include #include "dirty.h" #include "syslogd-types.h" #include "module-template.h" +#include "cfsysline.h" MODULE_TYPE_OUTPUT @@ -59,9 +61,18 @@ MODULE_TYPE_OUTPUT */ DEF_OMOD_STATIC_DATA +static int bEchoStdout = 0; /* echo non-failed messages to stdout */ + typedef struct _instanceData { + enum { MD_SLEEP, MD_FAIL, MD_RANDFAIL, MD_ALWAYS_SUSPEND } + mode; + int bEchoStdout; int iWaitSeconds; int iWaitUSeconds; /* milli-seconds (one million of a second, just to make sure...) */ + int iCurrCallNbr; + int iFailFrequency; + int iResumeAfter; + int iCurrRetries; } instanceData; BEGINcreateInstance @@ -85,19 +96,106 @@ CODESTARTisCompatibleWithFeature ENDisCompatibleWithFeature -BEGINtryResume -CODESTARTtryResume -ENDtryResume +/* implement "fail" command in retry processing */ +static rsRetVal doFailOnResume(instanceData *pData) +{ + DEFiRet; -BEGINdoAction -CODESTARTdoAction + dbgprintf("fail retry curr %d, max %d\n", pData->iCurrRetries, pData->iResumeAfter); + if(++pData->iCurrRetries == pData->iResumeAfter) { + iRet = RS_RET_OK; + } else { + iRet = RS_RET_SUSPENDED; + } + + RETiRet; +} + + +/* implement "fail" command */ +static rsRetVal doFail(instanceData *pData) +{ + DEFiRet; + + dbgprintf("fail curr %d, frquency %d\n", pData->iCurrCallNbr, pData->iFailFrequency); + if(pData->iCurrCallNbr++ % pData->iFailFrequency == 0) { + pData->iCurrRetries = 0; + iRet = RS_RET_SUSPENDED; + } + + RETiRet; +} + + +/* implement "sleep" command */ +static rsRetVal doSleep(instanceData *pData) +{ + DEFiRet; struct timeval tvSelectTimeout; dbgprintf("sleep(%d, %d)\n", pData->iWaitSeconds, pData->iWaitUSeconds); tvSelectTimeout.tv_sec = pData->iWaitSeconds; tvSelectTimeout.tv_usec = pData->iWaitUSeconds; /* milli seconds */ select(0, NULL, NULL, NULL, &tvSelectTimeout); - //dbgprintf(":omtesting: end doAction(), iRet %d\n", iRet); + RETiRet; +} + + +/* implement "randomfail" command */ +static rsRetVal doRandFail(void) +{ + DEFiRet; + if((rand() >> 4) < (RAND_MAX >> 5)) { /* rougly same probability */ + iRet = RS_RET_OK; + dbgprintf("omtesting randfail: succeeded this time\n"); + } else { + iRet = RS_RET_SUSPENDED; + dbgprintf("omtesting randfail: failed this time\n"); + } + RETiRet; +} + + +BEGINtryResume +CODESTARTtryResume + dbgprintf("omtesting tryResume() called\n"); + switch(pData->mode) { + case MD_SLEEP: + break; + case MD_FAIL: + iRet = doFailOnResume(pData); + break; + case MD_RANDFAIL: + iRet = doRandFail(); + break; + case MD_ALWAYS_SUSPEND: + iRet = RS_RET_SUSPENDED; + } + dbgprintf("omtesting tryResume() returns iRet %d\n", iRet); +ENDtryResume + + +BEGINdoAction +CODESTARTdoAction + dbgprintf("omtesting received msg '%s'\n", ppString[0]); + switch(pData->mode) { + case MD_SLEEP: + iRet = doSleep(pData); + break; + case MD_FAIL: + iRet = doFail(pData); + break; + case MD_RANDFAIL: + iRet = doRandFail(); + case MD_ALWAYS_SUSPEND: + iRet = RS_RET_SUSPENDED; + } + + if(iRet == RS_RET_OK && pData->bEchoStdout) { + fprintf(stdout, "%s", ppString[0]); + fflush(stdout); + } + dbgprintf(":omtesting: end doAction(), iRet %d\n", iRet); ENDdoAction @@ -113,7 +211,7 @@ BEGINparseSelectorAct int i; uchar szBuf[1024]; CODESTARTparseSelectorAct -CODE_STD_STRING_REQUESTparseSelectorAct(0) +CODE_STD_STRING_REQUESTparseSelectorAct(1) /* code here is quick and dirty - if you like, clean it up. But keep * in mind it is just a testing aid ;) -- rgerhards, 2007-12-31 */ @@ -135,6 +233,7 @@ CODE_STD_STRING_REQUESTparseSelectorAct(0) if(isspace(*p)) ++p; + dbgprintf("omtesting command: '%s'\n", szBuf); if(!strcmp((char*) szBuf, "sleep")) { /* parse seconds */ for(i = 0 ; *p && !isspace(*p) && ((unsigned) i < sizeof(szBuf) - 1) ; ++i) { @@ -152,12 +251,43 @@ CODE_STD_STRING_REQUESTparseSelectorAct(0) if(isspace(*p)) ++p; pData->iWaitUSeconds = atoi((char*) szBuf); - } - /* once there are other modes, here is the spot to add it! */ - else { + pData->mode = MD_SLEEP; + } else if(!strcmp((char*) szBuf, "fail")) { + /* "fail fail-freqency resume-after" + * fail-frequency specifies how often doAction() fails + * resume-after speicifes how fast tryResume() should come back with success + * all numbers being "times called" + */ + /* parse fail-frequence */ + for(i = 0 ; *p && !isspace(*p) && ((unsigned) i < sizeof(szBuf) - 1) ; ++i) { + szBuf[i] = *p++; + } + szBuf[i] = '\0'; + if(isspace(*p)) + ++p; + pData->iFailFrequency = atoi((char*) szBuf); + /* parse resume-after */ + for(i = 0 ; *p && !isspace(*p) && ((unsigned) i < sizeof(szBuf) - 1) ; ++i) { + szBuf[i] = *p++; + } + szBuf[i] = '\0'; + if(isspace(*p)) + ++p; + pData->iResumeAfter = atoi((char*) szBuf); + pData->iCurrCallNbr = 1; + pData->mode = MD_FAIL; + } else if(!strcmp((char*) szBuf, "randfail")) { + pData->mode = MD_RANDFAIL; + } else if(!strcmp((char*) szBuf, "always_suspend")) { + pData->mode = MD_ALWAYS_SUSPEND; + } else { dbgprintf("invalid mode '%s', doing 'sleep 1 0' - fix your config\n", szBuf); } + pData->bEchoStdout = bEchoStdout; + CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (uchar*)"RSYSLOG_TraditionalForwardFormat")); + CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -177,6 +307,10 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(omsdRegCFSLineHdlr((uchar *)"actionomtestingechostdout", 0, eCmdHdlrBinary, NULL, + &bEchoStdout, STD_LOADABLE_MODULE_ID)); + /* we seed the random-number generator in any case... */ + srand(time(NULL)); ENDmodInit /* * vi:set ai: -- cgit v1.2.3 From 9a41bcee58eb95635d1038ae91b3164ecb4b8da8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 8 May 2009 10:43:35 +0200 Subject: added design document --- doc/design.tex | 397 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 doc/design.tex diff --git a/doc/design.tex b/doc/design.tex new file mode 100644 index 00000000..b07ab3c3 --- /dev/null +++ b/doc/design.tex @@ -0,0 +1,397 @@ +\documentclass[a4paper,10pt]{article} +\usepackage{amsmath} +\usepackage{amsfonts} +\usepackage{amssymb} +\usepackage{graphicx} +\usepackage{listings} +\usepackage{algorithm,algorithmic} + +\newcommand{\IN}{\mathbb{N}} +\newcommand{\MM}{\mathfrak{M}} +\newcommand{\QQ}{\mathfrak{Q}} +\newcommand{\AAA}{\mathfrak{A}} +\title{Rsyslog Design and Internals} +\author{Rainer Gerhards} + +\begin{document} + +\maketitle + +\begin{abstract} +This paper describes rsyslog design and internals. It is created to facilitate a discussion about the implementation of "batched queue processing". As such, it does not describe the full design of rsyslog but rather those elements that are relevant to queues. However, the document may be expanded in the future. +\end{abstract} + +\section{Preliminaries} +\subsection{Notational Conventions} +In general, in rsyslog there exists single objects $o$, which are used to build larger sets $O$, which form a superset $\mathfrak{O}$ of all those objects that exist at a given time inside a running instance of rsyslog. As seen above, single objects are always described by lower case letters ($o$), larger sets by upper case letters ($O$) and the ``all-sets'' in gothic letters ($\mathfrak{O}$). Often, objects $O_i, i \in \IN, i \le |\mathfrak{O}|$ partition $\mathfrak{O}$, but this is not necessarily the case. +\section{Objects} +%\subsection{State Sets} +Be +$$S_m = \{ new, ready, comitPend, discardable \}$$ +the totally ordered set of message states and be +$$S_p = \{ ready, actTempFail, actPermFail, comitPend, comit\}$$ +the totally ordered set of message processing states (inside a batch). States may be contained in many objects $o$. I write $S(o)$ to denote the state that is associated with the object $o$. +\emph{Is one a subset of the other?} + +A \emph{message} +$$m = (m_s, \ldots), m_s \in S_m$$ +is a tuple of attributes. Be the set $\MM$ composed of all messages that exist at a given time inside rsyslog. + +% Be a processable message +$$p = (m, p_s), m \in \MM : \sigma_m(m) \geq ready, s_p \in S_p$$ +an ordered pair. + +An \emph{action} +$$a = (a_C, a_\psi)$$ +is an ordered pair of a tuple of configuration attributes $a_C$, and a tuple of processing functions $a_\psi$. Be the set $\AAA$ composed of all actions that exist in rsyslog after the configuration file has been processed. + +An \emph{action unit} +$$u = (f, a_1, \ldots, a_n), a_i \in \AAA \text{ for } i \in \IN, i \le n$$ +is a tuple consisting of a filter function $f$ and $n \in \IN$ actions. \emph{Does rsyslog still support nonsense action units with $n=0$? - check!} + +A \emph{queue} +$$Q = (\phi_q, M), M \subseteq \MM$$ +is a totally ordered finite set of messages where $\phi_q$ is the associated order relation\footnote{In actual programming, different types of queues exist and, among others, they differ in how $\phi_q$ is defined.}. There exist an upper bound on the cardinality of $M$, which is user-configured. If we need to obtain the set of message from a queue, we write $M(Q)$. + +Be $\QQ = \{Q_m, Q_1, Q_2, \ldots, Q_{|\AAA|}\}$ the set of all queues that exist inside rsyslog after the configuration file has been processed, with $|\QQ| = |\AAA| + 1$. + +Be +$$M_0 = \MM \setminus \bigcup_{i=1}^{|\QQ|} M_i$$ +the set of non-queued messages. This set is partioned into the set $M_{0_u} = \{m \in M_0: \sigma_s(m) < ready\}$ the set of messages not yet fully created and the set $M_{0_r} = M_0 \setminus M_{0_u}$ of messages that are ready for processing but not currently enqueued into any queue. + +Be $b_b$ a user-configurable upper bound of the cardinality of any batch greater than one inside the system and be +$$B = \{ (m, s_p) | m \in M(Q) \}$$ +a finite set that contains a subset of processable messages from a single queue. The actual cardinality $b = |B|$ is depending on the cardinality of $M$ at the time of batch creation and is $1 \leq b \leq \text{max}(b_b, |M(Q)|)$. + +\section{Processing} +\subsection{Object States} +Various objects keep state. Some of these objects, like messages, batches and actions seem to share state. However, thinking about shared state leads to very complex setup. As such, state is modelled for each object $o$ individually. Instead, the state function $S_O(o)$ can be used to obtain an obtain an individual objects state. That state can be used to modify the state diagrams of the other objects with which relationships exist. + +\subsubsection{Actions} +Actions are provided by output plugins. An action enables the engine to write messages to some destination. It is important to note that ``destination'' is a very broad abstraction. A destination may be a file inside a local or remote file system, a database table or a remote syslog server in another network. + +Actions are transactional in the following sense: more than one message can be submitted to an action. The action does not necessarily process the submitted messages unless the caller ends the transaction. However, the action itself may also end the transaction and notify the caller. This is \emph{not} considered an error condition and \emph{must} be handeled gracefully by the caller. If an transaction aborts, the caller \emph{must} assume that none of the elements submitted since the begin of transaction have been processed. The action will try to backout anything that was already processed at the time the transaction failed. However, not all outputs work on actually transactional destination. As such, an action is permitted not to backout incomplete interim results. As such, after a transaction abort, some message duplication may occur. We call this the \emph{relaxed integrity condition} for actions. + +An output transaction is started by calling \emph{beginTransaction()} either explicitely or implicitely by a call to \emph{doAction()} without calling \emph{beginTransaction()} before. Then, one or more calls to \emph{doAction()} follow. When the caller intends to finish the transaction, it calls \emph{endTransaction()}. However, the transaction may also be terminated from the action itself in response to a \emph{doAction()} call. + +Mathematical, an action transaction builds a totally orderred set of uncommitted messages $M_u$. The order relation is defined over the sequence in which messages are being provided to \emph{doAction()}. At any time a commit is attempted, the full set $M_u$ is committed and may either succeeed completely or not at all (in the sense of the relaxed integrity condition described above). + +A commit is attempted when +\begin{enumerate} +\item the caller decides to call \emph{endTransaction()} +\item or earlier if the action decides it needs to commit now (e.g. because of buffers filling up). +\end{enumerate} + +In the seconds case, the action may decide to commit all message but the current one or all (this is depending on action logic). So if the action decideds to commit a transaction before the caller calls \emph{endTransaction()}, a set of commited messages $M_c$ is build and $M_u$ is modified. Be $n$ the $n$-th iterated \emph{doAction()} call and $m_n$ the current message of this call, then the sets are build as follows: + +\begin{algorithm} +%\caption{} +\begin{algorithmic} +\IF{action commits $m_n$} + \STATE $M_c = M_u \cup m_n$ + \STATE $M_u = \emptyset$ +\ELSE + \STATE $M_c = M_u$ + \STATE $M_u = \{ m_n\}$ +\ENDIF +\end{algorithmic} +\end{algorithm} + +In other words, if anything is committed early, it is always the full set $M_u$, with or without the current message. The caller needs to know which messages are already commited. As \emph{doAction()} finishes one transaction and starts a new one in a single call, we can not use action state the let the caller know this happened. So we use our above finding and just convey back if the transacton is still continuing or the current message or all others before it were committed. The caller must then act accordingly. Please note that when an error happens, the whole transaction must still be considered failed. As such, ``partial commit'' states need not to be mixed with failure states. + +Please note that the above method leaves a small potential issue unaddressed: if the action does an early commit of $M_u \setminus m_n$, an error happens when adding $m_n$ to the new $M_u$ (like running out of resources), the action would need to convey both the successful transaction as well as the failure state. This is not possible with the current interface. We could use callbacks to provide such notification, but this complicates the code. So, if that situaton arises, the action must temporarily buffer the error condition and convey it as part of either the next \emph{doAction()} call or during \emph{endTransation()} processing. This can be done, for example, by advancing its internal state accordingly. + +The state set for a actions is defined as follows: +$$ +S_A = \{ rdy, itx, comm, rtry, susp, died \} +$$ + +With the semantics of the various states being the following: + +\begin{center} +\begin{tabular}{|l|l|} \hline + State & Semantics \\\hline + rdy & ready, waiting for transaction begin\\ + itx & in transaction, accept more data \\ + comm & transaction finished \\ + rtry & action failed but may be able to recover \\ + susp & action currently defunctional until timeout expires \\ + died & unrecoverable error condition occured, no longer usable \\\hline +\end{tabular} +\end{center} + +In the associated state diagram below, we do not include the \emph{died} state, because it is entered whenever a totally unrecoverable error state may occur. This is a very exceptional incident (which most output plugins do not even support), so we have kept the diagram simple. + +\includegraphics[scale=0.5]{action_state.jpeg} + +\emph{Note well} that the state diagram describes the action state. It does \emph{not} describe the transaction state. While action- and transaction state are closely related to each other, they are different entities. + +The return code of \emph{doAction()} and \emph{endTransaction()} is used to convey the transaction state. As such, it is a function of the actions's current state after processing the request. The mapping is as shown below: + +\begin{center} +\begin{tabular}{|l|l|} \hline + State & Return Code (RS\_RET\_\ldots)\\\hline + rdy & OK \\ + itx & COMMITTED (if there was an auto-commit without $m_n$)\\ + & DEFER\_COMMIT (if there was no auto-commit)\\ + comm & internal state, not to be exposed to upper layer \\ + rtry & SUSPENDED \emph{(new code needed)} \\ + susp & SUSPENDED \\ + died & DISABLED \\\hline +\end{tabular} +\end{center} + +For the rest of this document, let's assume there is a function \emph{getReturnCode()} that implements this mapping. + +It is important to think about how retries are handled. There is a user-configured per-action upper number of retries $C_r$ and retry interval $C_i$. In \emph{rsyslog v3}, there is no concept of output transactions. As such, only single messages are processed. When a temporary action failure occurs, the action is re-tried $C_r$ times, where the action processing thread is waiting in a \emph{sleep()} $C_i$ operating system API call\footnote{a suitable API is used, not \emph{sleep()} itself}. If the action succeeds during the retry processing, everything continues as usual. If it does not succeed, two things happen: +\begin{itemize} +\item the message is flagged as ``action permanent failure'' (what may trigger backup processing) +\item the action is actually suspended for $C_i$ seconds +\end{itemize} +If then a new message is sent to the action, and $C_i$ seconds have not yet elapsed, the action is flagged as having failed without being re-tried again\footnote{During the analysis for this paper, it was seen that actually $C_r$ retries are attempted in v3, but each of them will never actually re-try the action. This is a software bug, which does not cause any harm and thus will not be fixed in v3. The new implementation in v4 will obviously not inherit this problem}. This is done in an effort to reduce resource utilization and prevent the system from slowing down e.g. by too-many retries to a remote server that went offline. + +With transactional output mode in \emph{rsyslog v4}, the logic above can no longer work. First of all, retrying single actions does not help, because all of the current transaction needs to be resubmitted. As such, the upper layers need to be notified of failure. Then, they need to resubmit the batch. In that design, the lower layer needs to return immediately after detecting the failure. Recovery handling is now to be done when the next transaction is started. However, we must make sure that we do not do excessive retries. So retry processing is only to be carried out if it was not tried less than $C_i$ seconds ago. + +The required functionality can be implemeted by a \emph{prepareAction} function that readies the action for processing if there is need to do so. That function is then called in all entry points before anything else is done. Then, actual processing is carried out and the resulting action state be used to generate the return code for the upper-layer caller. Find below a rough pseudocode to do so: + +\lstset{language=python} +\begin{lstlisting} +def prepareAction(): + if state == rtry: + try recovery (adjust state accordingly) + if state == rdy: + beginTransaction() [output plugin] + +def processMessage(message): + prepareAction() + if state == itx + doAction(message) [output plugin] + return getReturnCode() + +def doEndTransaction(): + prepareAction() + if state == itx + endTransaction(); [output plugin] + return getReturnCode() +\end{lstlisting} + +\subsection{Output Subsystem Layers} +The rsyslog engine is organized in layers, where each layer is represented by the dominating object: + +\begin{center} +\includegraphics[scale=0.75]{rsyslog_output_layers.jpeg} +\end{center} + +If looking at the data flow, a queue dequeues batches of messages, which are than run through a generic action system and put into output plugins. Note that on the batch layer, only batches are supported as units of work, whereas the action layer is message-oriented but supports transactions of multiple messages. This is done by indicating when a transaction necessarily needs to end (that point being the end of batch from the batch layer). + +The plugins can be written by third parties and are roughly comparable to minidrivers. The generic action system provides all complexity of action processing wheras the output plugin provides a limited set of callbacks that enable the generic framework to talk to the actual destination system. As such, writing outputs is a very simple task. However, rsyslog does not limit the creation of very complex outputs, which may be able to offer superior performance for some destinations. + +\subsection{Output Failure} +\subsubsection{Cases} +When an output action is called, it may encounter a failure condition. In general, there are two different cases: +\begin{enumerate} +\item action caused failures +\item message-content caused failures +\end{enumerate}. + +Failures rooted in the action are things like broken network connections, file systems run out of space or database servers that are down. Most importantly, the failure is not related to message content. As such, it is appropriate to retry the action with the same message until it finally succeeds (assuming that someone restores the system in question to proper operation). We can not expect that the problem is cleared just by discarding the current message and re-trying with the next one. + +In my view, action caused failures are the far majority of all failures. For rsyslog versions 3 and below, all rsyslog-provided plugins consider failures to be action-caused and thus potentially recoverable by simple retry. With the only exception being fatal error conditions that render the whole action unusable. + +David Lang pointed out, that there may also exist error conditions that are not caused by the action (or the subsystem it talks to) itself, but rather by message data. He provided the following samples where message content can cause permanent issues with action execution: + +\begin{itemize} +\item unicode text causing grief +\item dynafile hits a read-only file +\item basicly data-driven things that trigger bugs in the message delivery +mechanism in some form. +\end{itemize} + +As David Lang said ``In an ideal world these would never happen, but for most output types I can think of some form of corrupt input that could cause that message to fail.''. +So this class of failure conditions actually exists. No matter how often the action retry mechanism is called, it will never succeeds (one may argue that the read-only dynafile is fixable, but we could replace that sample with an invalidly generated filename). The proper cure for these actions is to find the offending one and discard it. + +In conclusion, actions need to return different error states for these two different types of failures. Traditionally, RS\_RET\_SUSPENDED is returned when an action specific failure is hit. Most existing plugins also do this if a message-related failure occured, simply because they did not yet know that this situation exists. However, plugins also return different error codes, and at least these can be treated to mean message-permanent failures. To support this, a change to plugins is still required, because many simple return SUSPENDED state if anything went wrong (replacing the real error condition with SUSPENDED). A dedicated PROBABLE\_INVALID\_MSG return state is probably useful so that an output plugin can convey back that it consideres the message to be bad. On the other hand, this implies that the plugin must try to detect those, what means that the developer must think about all potential message-causes problems. That approach can be considered unreliable and as such it may be better not to provide such a dedicted state. + +\subsubsection{Handling of Failures} +In spite of the two different failure cases, different handling is needed for them. The action-based failure cases can and must be handled on the action level. As transactions abort when a failure occurs, support from the upper ``batch layer'' is necessary in order to handle resending batches of messages. + +For message-caused failure cases, the offending message must be found and then be discarded. A complexity here is that while a failure-causing message is being searched for, an action-based failure might occur. In that case, first the action-based failure condition must be solved, before the search for the problem message can continue. + +One approach might be that when the action-layer conveys back an action-caused failure (SUSPENDED), the batch layer knows that it simply needs to restart the full transaction (but not start an ``invalid message search''). If a message-based error condition is conveyed back, the batch system can not restart the full batch. Instead, it needs to enter search mode, where it creates partitions of the original batch, and calls itself recursively (at least in theory) on each of the subsets. + +Then, the same handling applies until either a failing message has been found or all messages have been successfully processed. Note that in the recursive step, action-based failures are recovered by full batch resubmits. This solves the above-mentioned complexity in a consistent way. + +If a binary-search-like method is used to detect failing records\footnote{This was originally suggested by David Lang.}, recursion may not really be an issue, as the recursion depth is limited to $\log_2 |B|$ where $B$ is the message batch. + +A message-caused failure can be rooted in one or more messages. One important question is if it is expected that the failure is caused by a single or multiple messages. Both is possible, so it is a question of probability. If we assume that it is more probable that a single messages causes the problems, it is useful to immediately return back to full batch submission of transactions once a problem-causing message has been identified. But then, if there are multiple problem-causing messages inside the batch, we may need many more iterations. + +If, on the other hand, we assume that it is more probable that multiple messages cause problems, it may make sense to keep resubmitting only subsets of the batch. However, then the performance is suboptimal if actually only one message was problematic. A solution might be to pick a compromise, e.g. first assume that a single message is problematic, but assume the opposite as soon as a second message with problems has been found. + +A potential algorithm for processing $n \le |B|$ messages from batch $B$ is described below. In the pseudocode, a ``processable'' message is one that neither is already committed nor had a permanent failure with this action. The term ``mpf'' means ``message permanent failure'' for this action (this will later be described in a batch state set). + +\begin{small} +\lstset{language=python} +\begin{lstlisting} +def submitBatch(B, n): + foreach processable message in + (first [at most] n messages of batch): + call processMessage + if action-caused failure: + retry full batch + if action-caused permanent failure: + mark all n messages as mpf + return + if auto-commit: + mark commited messages in batch as committed + if message-caused failure: + if n == 1: + mark message as mpf + return + else: + call submitBatch(B, n/2) + call submitBatch(B, n/2) +\end{lstlisting} +\end{small} + +After submitBatch() has completed, all messages are either committed or in mpf state. + +Note that an action-caused permanent failure occurs if an action-caused failure can not be resolved with the operator-configured number of retries. It will never occur if the user configured infinite retries. While an action is suspended, all calls will result in an action-caused permanent failure. Please keep in mind that these will be resubmitted to any backup actions inside the action unit, so the action's ability to cause permanent failure states is vital for a number of use cases (backup syslog server, to name just one). + +Batch processing inside an action unit thus can follow these strucuture: + +\begin{algorithm} +\caption{processBatch(B)} +\begin{algorithmic} +\FORALL{action $a$ in action unit} + \IF{execute action only on messages that failed before} + \STATE $n = |\text{messages in batch in mpf state}|$ + \STATE change mpf state back to ready + \ELSE + \STATE $n = |B \setminus \text{msgs with state discard}|$ + \STATE change all message states $\ne$ discard to ready + \ENDIF + \IF{$n >0$ } + \STATE call submitBatch(B, n) for action $a$ + \ENDIF +\ENDFOR +\end{algorithmic} +\end{algorithm} + +\paragraph{Why is it Important to differentiate the failure cases?} +This text originates from the mailing list and must be merged in. I provide it in the form it is, so it will not be forgotten (plus, it conveys the information). + +One may think that it is not necessary to differentiate between action-caused and message-caused failures. However, not doing so introduces subtle issues, because +then you either + +A) do not need the batch logic at all (because the action is configured for +infinite retries) + +Or + +B) you loose many messages if the action is not configured for infinite +retries and you have a longer-duration outage e.g. on a database server. +Let's say it is offline for a couple of hours, then you lose almost +everything in that period + +To prevent this, you need two different retry methods. + +One may argue that it is hard to differentiate between the two failure cases. This is correct. Buit I think it mostly depends on the quality of the output module. + +First of all, ``mostly'' implies that there may be some other cases, where it +really is impossible to differentiate between the two. In that case, I would +treat the issue as an action-caused failure. There are two reasons for this: + +1) rsyslog v3 currently does this always and not even a single person +complained about that so far. This is an empiric argument, and it does not +mean it caused problems. But it carries the co-notation that this seems not +to be too bad. + +2) If we would treat it as message-caused failure, we would no longer be able +to handle extended outages of destination systems, which I consider a vitally +important feature. + +When weighing the two, I know of lots of people who rely on 2), in sharp +contrast to knowig noone having problems with 1). So my conclusion is that it is +less problematic to define an otherwise undefinable failure reason to be +action-caused. Even more so as I assume this problem only exists in the +minority of cases. + +Now back to the quality of the output module: thinking about databases, their +API is usually very good at conveying back if there was a SQL error or a +connection abort. So while a SQL error may also be an indication of a +configuration problem, I would strongly tend to treat it is a being +message-caused. This is under the assumption that any reasonable responsive +admin will hopefully test his configuration at least once before turning it +into production. And config SQL errors should manifest immediately, so I +expect these to be fixed before a configuration runs in production. So it is +the chore of the output module to interpret the return code it received from +its API and decide whether this is more likely action-caused or +message-caused. For database outputs, I would assume that it is always easy +to classify failures that can only be action-caused, especially in the +dominating case of a failed network connection or a failed server. + +For other outputs it may not be as easy. But, for example, all stream network +outputs can detect a broken connection, so this also is a sure fit. + +For dynafiles, it really depends on how hard the output module is tries to differentiate +between the two failure cases. But I think you can go great length here, too. +Especially if you do not only look at the create() return code, but, iff a +failure occurs, you do more API calls to find out the cause. + +So I think the remaining problem is small enough to cause not too much issues +(and if so, they are unavoidable in any case). In conclusion, the two failure states are not only necessary, but can sufficiently sure enough be detected. + +\subsection{Random Topics} +I have begun to gather material from the mailing list in this section, because I feel it may be useful for others as well. Right now, the information is well hidden in the mailing list archives and there may be value in combining it all in one place. + +Due to the nature of this material, there is no specific organization between the subchapters and also formatting and language doesn't deny its rooting in the mailing list. + +\subsection{Reliability of Message Dequeueing} +A batch is actually dequeued when it is taken off a queue. So if at that point we +have a system power failure (for whatever reason), the messages are lost. +While the rsyslog engine intends to be very reliable, it is not a complete +transactional system. A slight risk remains. For this, you need to understand +what happens when the batch is processed. I assume that we have no sudden, +untrappable process termination. Then, if a batch cannot be processed, it is +returned back to the top of queue. This is not yet implemented, but is how +single messages (which you can think of an abstraction of a batch in the +current code) are handled. If, for example, the engine shuts down, but an +action takes longer than the configured shutdown timeout, the action is +cancelled and the queue engine reclaims the unprocessed messages. They go +into a special area inside the .qi file and are placed on top of the queue +once the engine restarts. + +The only case where this not work is sudden process termination. I see two +cases: + +a) a fatal software bug +We cannot really address this. Even if the messages were remaining in the +queue until finally processed, a software bug (maybe an invalid pointer) may +affect the queue structures at large, possibly even at the risk of total loss +of all data inside that queue. So this is an inevitable risk. + +b) sudden power fail +... which can and should be mitigated at another level + +One may argue that there also is + +c) admin error +e.g, kill -9 rsyslogd +Here a fully transactional queue will probably help. + +However, I do not think that the risk involved justifies a far more complex +fully transactional implementation of the queue object. Some risk always +remains (what in the disaster case, even with a fully transactional queue?). + +And it is so complex to let the messages stay in queue because it is complex +to work with such messages and disk queues. It would also cost a lot of +performance, especially when done reliably (need to sync). We would then need +to touch each element at least four times, twice as much as currently. Also, +the hybrid disk/memory queues become very, very complex. There are more +complexities around this, I just wanted to tell the most obvious. + +So, all in all, the idea is that messages are dequeued, processed and put +back to the queue (think: ungetc()) when something goes wrong. Reasonable +(but not more) effort is made to prevent message loss while the messages are +in unprocessed state outside of the queue. +\end{document} -- cgit v1.2.3 From 2ca67f83aa9c52568a5347b600361e698322a337 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 8 May 2009 17:56:16 +0200 Subject: worked on rsyslog design internals paper --- doc/action_state.dot | 33 ++++++++++ doc/batch_state.dot | 28 +++++++++ doc/design.tex | 174 ++++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 197 insertions(+), 38 deletions(-) create mode 100644 doc/action_state.dot create mode 100644 doc/batch_state.dot diff --git a/doc/action_state.dot b/doc/action_state.dot new file mode 100644 index 00000000..d56d9da0 --- /dev/null +++ b/doc/action_state.dot @@ -0,0 +1,33 @@ +// This file is part of rsyslog. +// +// rsyslog message state diagram +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot file.dot -Tpng >file.png + +digraph msgState { + compound=true; nodesep=1.0 + //label="\n\nrsyslog action transaction states\nhttp://www.rsyslog.com"; + //fontsize=20; + + rdy [label="ready" group="main"]; + itx [label="in Tx" group="main"]; + comm [label="commit"] + rtry [label="retry"] + susp [label="suspended"] + + rdy -> itx [label="transaction begins"] + itx -> itx [label="success"] + itx -> comm [label="commit\n(caller or auto)"] + itx -> rtry [label="error"] + comm -> rdy [label="success"] + comm -> rtry [label="error"] + rtry -> rdy [label="recovered"] + rtry -> susp [label="could not\nrecover"] + susp -> rtry [label="timeout expired"] + + {rank=same; comm rtry} +} diff --git a/doc/batch_state.dot b/doc/batch_state.dot new file mode 100644 index 00000000..0dd48b47 --- /dev/null +++ b/doc/batch_state.dot @@ -0,0 +1,28 @@ +// This file is part of rsyslog. +// +// rsyslog batch state diagram +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot file.dot -Tpng >file.png + +digraph msgState { + compound=true; nodesep=1.0 + //label="\n\nrsyslog batch states\nhttp://www.rsyslog.com"; + rankdir=LR + + rdy [label="ready"]; + bad [label="message-caused\nfailure"]; + sub [label="submitted"] + disc [label="discarded" color="red"] + + rdy -> sub [label="submitted to action"] + rdy -> bad [label="permanent fail"] + rdy -> disc [label="action requests discarding"] + sub -> rdy [label="next action or\naction-caused failure"] + bad -> rdy [label="next action"] + + //{rank=same; comm rtry } +} diff --git a/doc/design.tex b/doc/design.tex index b07ab3c3..25e870a2 100644 --- a/doc/design.tex +++ b/doc/design.tex @@ -7,9 +7,12 @@ \usepackage{algorithm,algorithmic} \newcommand{\IN}{\mathbb{N}} -\newcommand{\MM}{\mathfrak{M}} -\newcommand{\QQ}{\mathfrak{Q}} -\newcommand{\AAA}{\mathfrak{A}} +\newcommand{\MM}{\mathcal{M}} +\newcommand{\QQ}{\mathcal{Q}} +\newcommand{\AAA}{\mathcal{A}} +%\newcommand{\MM}{\mathfrak{M}} +%\newcommand{\QQ}{\mathfrak{Q}} +%\newcommand{\AAA}{\mathfrak{A}} \title{Rsyslog Design and Internals} \author{Rainer Gerhards} @@ -21,47 +24,112 @@ This paper describes rsyslog design and internals. It is created to facilitate a discussion about the implementation of "batched queue processing". As such, it does not describe the full design of rsyslog but rather those elements that are relevant to queues. However, the document may be expanded in the future. \end{abstract} +\tableofcontents + \section{Preliminaries} \subsection{Notational Conventions} -In general, in rsyslog there exists single objects $o$, which are used to build larger sets $O$, which form a superset $\mathfrak{O}$ of all those objects that exist at a given time inside a running instance of rsyslog. As seen above, single objects are always described by lower case letters ($o$), larger sets by upper case letters ($O$) and the ``all-sets'' in gothic letters ($\mathfrak{O}$). Often, objects $O_i, i \in \IN, i \le |\mathfrak{O}|$ partition $\mathfrak{O}$, but this is not necessarily the case. +In general, in rsyslog there exists single objects $o$, which are used to build larger sets $O$, which form a superset $\mathcal{O}$ of all those objects that exist at a given time inside a running instance of rsyslog. As seen above, single objects are always described by lower case letters ($o$), larger sets by upper case letters ($O$) and the ``all-sets'' in caligraphic letters ($\mathcal{O}$). Often, objects $O_i, i \in \IN, i \le |\mathcal{O}|$ partition $\mathcal{O}$, but this is not necessarily the case. \section{Objects} -%\subsection{State Sets} -Be -$$S_m = \{ new, ready, comitPend, discardable \}$$ -the totally ordered set of message states and be -$$S_p = \{ ready, actTempFail, actPermFail, comitPend, comit\}$$ -the totally ordered set of message processing states (inside a batch). States may be contained in many objects $o$. I write $S(o)$ to denote the state that is associated with the object $o$. -\emph{Is one a subset of the other?} - -A \emph{message} -$$m = (m_s, \ldots), m_s \in S_m$$ -is a tuple of attributes. Be the set $\MM$ composed of all messages that exist at a given time inside rsyslog. - -% Be a processable message -$$p = (m, p_s), m \in \MM : \sigma_m(m) \geq ready, s_p \in S_p$$ -an ordered pair. - -An \emph{action} -$$a = (a_C, a_\psi)$$ -is an ordered pair of a tuple of configuration attributes $a_C$, and a tuple of processing functions $a_\psi$. Be the set $\AAA$ composed of all actions that exist in rsyslog after the configuration file has been processed. -An \emph{action unit} -$$u = (f, a_1, \ldots, a_n), a_i \in \AAA \text{ for } i \in \IN, i \le n$$ -is a tuple consisting of a filter function $f$ and $n \in \IN$ actions. \emph{Does rsyslog still support nonsense action units with $n=0$? - check!} +\subsection{Plugins} +Plugins provide code potentially written by a third party to extend rsyslog. + +Conceptually, a plugin is a tupel of callable functions $(\phi_1, \phi_2, \ldots)$ which implement an interface. There are three different types of plugins: input, output and libraray. The plugin type denotes the primary interface implemented by the plugin. Additional interfaces may be implemented\footnote{This is not yet done in plugins, but is possible and assumed to be done at a later point in time}. + +In the context of this paper, the output plugin interface is most important. It implements three entry points: + +\paragraph{doAction()} +is used to submit messages to the output plugin. The entry point may or may not commit the messages to their ultimate destination. + +\paragraph{beginTransaction()} +is used to inform the plugin that a new transaction begins. It must prepare for processing. + +\paragraph{endTransaction()} +is indicated that the upper layer \emph{needs} to close the transaction. If there is any uncommited data left, it must be commited or rolled back. + +Every instance of an output plugin is guaranteed \emph{not} to be called concurrently by multiple threads. Further, no context switch will happen between calls to $doAction()$ and $endTransaction()$. + +\subsection{State Sets} +Several object have associated state based on a specific state set. These state sets are described together with the objects. + +As a general rule, individual state is associated with all intances $o$ of a class of objects. This state is called the object's \marginpar{state component} \emph{state component} $s$. If we want to obtain an object's state, we write $S(o)$. Please note that $S(o)$ is only defined for those objects that have a state component. + +\subsection{Messages} +A message $m$ represents a a single syslog message inside the system. It is a tuple of attributes. Some of these attributes directly orginate from the message content, some others are meta-information taken from the context. For example, there is an meta-attribute ``time of reception'' which conveys when the message was received by rsyslog's input subsystem. We do not list attributes here, as there are many and it is not of importance which exactly they are. + +The set $\MM$ is composed of all messages that exist at a given time inside rsyslog. -A \emph{queue} -$$Q = (\phi_q, M), M \subseteq \MM$$ -is a totally ordered finite set of messages where $\phi_q$ is the associated order relation\footnote{In actual programming, different types of queues exist and, among others, they differ in how $\phi_q$ is defined.}. There exist an upper bound on the cardinality of $M$, which is user-configured. If we need to obtain the set of message from a queue, we write $M(Q)$. +\subsection{Queue} +A queue +$$Q = (C, \Phi, M)$$ +is a tupel of a set of configuration parameters $C$, a set of callbacks $\Phi$ and a set of messages $M \subseteq \MM$. + +If we need to obtain the set of message from a queue, we write $M(Q)$. The elements of the set of configuration parameters are written as $C_{param}$ where $param$ is an abbreviation of the parameter's meaning. To obtain a specific parameter from a queue, we write $C_{param}(Q)$. The most important elements of $C$ are: + +\paragraph{$C_{type}$} which denotes the queue implementation type. Most importantly, this selects from a set of queue drivers (for example disk-only or in-memory driver), which affects the basic operation of the queue instance. + +\paragraph{$C_{mMsg}$} which denotes the upper bound on the cardinality of $M$. + +\paragraph{$C_{mBatch}$} which denotes the upper bound of the cardinality of message batches created for this queue. Be $\QQ = \{Q_m, Q_1, Q_2, \ldots, Q_{|\AAA|}\}$ the set of all queues that exist inside rsyslog after the configuration file has been processed, with $|\QQ| = |\AAA| + 1$. -Be -$$M_0 = \MM \setminus \bigcup_{i=1}^{|\QQ|} M_i$$ -the set of non-queued messages. This set is partioned into the set $M_{0_u} = \{m \in M_0: \sigma_s(m) < ready\}$ the set of messages not yet fully created and the set $M_{0_r} = M_0 \setminus M_{0_u}$ of messages that are ready for processing but not currently enqueued into any queue. +Then +$$M_0 = \MM \setminus \bigcup_{i=1}^{|\QQ|} Q_i(M)$$ +\marginpar{at-risk-set}is the set of non-queued messages. The messages have either never been enqueued or have been dequeued but not finally been processed. This set represents the messages that may potentially be lost during an unclean shutdown of rsyslogd. This is why I call this set the ``\emph{at-risk-set}''. + + +\subsection{Batches} +A batch represents multiple processable messages. It is a unit of processing inside rsyslog's output system. Batches are used to dequeue a number of messages from a queue and then submit them to the lower action layer. Batches are natural \emph{transaction boundaries}, in the sense that multiple output transactions may be done on the messages inside a batch, but each transaction must end at the end of the batch. A batch is always associated to a specific queue $Q$. + +A batch +$$B = (b_1, b_2, \ldots, b_n )$$ +is a $n$-tuple of \marginpar{processable message}processable messages +$$b = (m, s)$$ +which are an ordered pair of a message $m$ and an associated processing state $s$. To denote the $n$-th message inside the batch, we write $m(b_n)$, to denote the status component of the $n$-th message, we write $S(b_n)$. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.4]{batch_state.jpeg} +\end{center} +\caption{batch message processing states} +\label{fig_batchmsg_states} +\end{figure} + +The state set for the processing states is defined as follows: +$$ +S_B = \{ rdy, bad, sub, disc \} +$$ + +With the semantics of the various states being the following: + +\begin{center} +\begin{tabular}{|l|l|} \hline + State & Semantics \\\hline + rdy & ready for processing\\ + bad & this message triggered an unrecoverable failure in action\\ + & processing and must not be resubmitted to this action\\ + sub & message submitted for processsing, result yet unknown \\ + disc & action sucessfully processed, but must not be submitted \\ + & to any further action in action unit \\\hline +\end{tabular} +\end{center} +The associated state diagram is shown in figure \ref{fig_batchmsg_states} on page \pageref{fig_batchmsg_states}. + +Batch sizes vary. The actual cardinality is a function of the cardinality of $M(Q)$ at the time of batch creation and the queue configuration: + +$$1 \leq |B| \leq \max(C_{mBatch}(Q), |M(Q)|)$$ + +\subsection{Action Unit} +An action unit +$$u = (f, a_1, \ldots, a_n), a_i \in \AAA \text{ for } i \in \IN, i \le n$$ +is a tuple consisting of a filter function $f$ and $n \in \IN$ actions. \emph{Does rsyslog still support nonsense action units with $n=0$? - check!} + +\subsection{Action} +An action +$$a = (a_C, a_\psi)$$ +is an ordered pair of a tuple of configuration attributes $a_C$, and a tuple of processing functions $a_\psi$. Be the set $\AAA$ composed of all actions that exist in rsyslog after the configuration file has been processed. -Be $b_b$ a user-configurable upper bound of the cardinality of any batch greater than one inside the system and be -$$B = \{ (m, s_p) | m \in M(Q) \}$$ -a finite set that contains a subset of processable messages from a single queue. The actual cardinality $b = |B|$ is depending on the cardinality of $M$ at the time of batch creation and is $1 \leq b \leq \text{max}(b_b, |M(Q)|)$. \section{Processing} \subsection{Object States} @@ -120,9 +188,15 @@ With the semantics of the various states being the following: \end{tabular} \end{center} -In the associated state diagram below, we do not include the \emph{died} state, because it is entered whenever a totally unrecoverable error state may occur. This is a very exceptional incident (which most output plugins do not even support), so we have kept the diagram simple. +In the associated state diagram in figure \ref{fig_action_states}, we do not include the \emph{died} state, because it is entered whenever a totally unrecoverable error state may occur. This is a very exceptional incident (which most output plugins do not even support), so we have kept the diagram simple. +\begin{figure} +\begin{center} \includegraphics[scale=0.5]{action_state.jpeg} +\end{center} +\caption{Action State Diagram} +\label{fig_action_states} +\end{figure} \emph{Note well} that the state diagram describes the action state. It does \emph{not} describe the transaction state. While action- and transaction state are closely related to each other, they are different entities. @@ -178,9 +252,10 @@ def doEndTransaction(): \subsection{Output Subsystem Layers} The rsyslog engine is organized in layers, where each layer is represented by the dominating object: -\begin{center} +\begin{figure} \includegraphics[scale=0.75]{rsyslog_output_layers.jpeg} -\end{center} +\label{rsyslog output layers} +\end{figure} If looking at the data flow, a queue dequeues batches of messages, which are than run through a generic action system and put into output plugins. Note that on the batch layer, only batches are supported as units of work, whereas the action layer is message-oriented but supports transactions of multiple messages. This is done by indicating when a transaction necessarily needs to end (that point being the end of batch from the batch layer). @@ -394,4 +469,27 @@ So, all in all, the idea is that messages are dequeued, processed and put back to the queue (think: ungetc()) when something goes wrong. Reasonable (but not more) effort is made to prevent message loss while the messages are in unprocessed state outside of the queue. + +\paragraph{More reliable can actually be less reliable} +On the rsyslog mailing list, we had a discussion about how reliable rsyslog should be. It circles about a small potential window of message loss in the case of sudden power failure. Rsyslog can be configured to put all messages into a disk queue (instead of main memory), so these messages survive such a powerfail condition. However, messages dequeued and scheduled for processing during the power outage may be lost. + +I now consider a case where we have bursty UDP traffic and rsyslog is configured to use a disk-only queue (which obviously is much slower than an in-memory queue). Looking at processing speeds, the max burst rate is limited by using an ultra-reliable queue. To avoid using UDP messages, a second instance could be run that uses an in-memory queue and forwards received messages to the one in ultra-reliable mode (that is with the disk-only queue). So that second instance queues in memory until the (slower) reliable rsyslogd can now accept the message and put it into the reliable queue. Let's say that you have a burst of $r$ messages and that from these burst only $r/2$ can be enqueued (because the ultra reliable queue is so slow). So you lose $r/2$ messages. + +Now consider the case that you run rsyslog with just a reliable queue, one that is kept in memory but not able to cover the power failure scenario. Obviously, all messages in that queue are lost when power fails (or almost all to be precise). However, that system has a much broader bandwidth. So with it, there would never have been r messages inside the queue, because that system has a much higher sustained message rate (and thus the burst causes much less of trouble). Let's say the system is just twice as fast in this setup (I guess it usually would be *much* faster). Than, it would be able to process all r records. + +In that scenario, the ultra-reliable system loses $r/2$ messages, whereas the somewhat more "unreliable" system loses none - by virtue of being able to process messages as they arrive. + +Now extend that picture to messages residing inside the OS buffers or even those that are still queued in their sources because a stream transport blocked sending them. + +I know that each detail of this picture can be argued at length about. + +However, my opinion is that there is no "ultra-reliable" system in life, only various probabilities in losing messages. These probabilities often depend on each other, what makes calculating them very hard to impossible. Still, the probability of message loss in the system at large is just the product of the probabilities in each of its components. And reliability is just the inverse of that probability. + +This is where *I* conclude that it can make sense to permit a system to lose some messages under certain circumstances, if that influences the overall probability calculation towards the desired end result. In that sense, I tend to think that a fast, memory-queuing rsyslogd instance can be much more reliable compared to one that is configured as being ultra-reliable, where the rest of the system at large is badly influenced by this (the scenario above). + +However, I also know that for regulatory requirements, you often seem to need to prove that a system may not lose messages once it has received them, even at the cost of an overall increased probability of message loss. + +My view of reliability is much the same as my view of security: there is no such thing as "being totally secure", you can just reduce the probability that something bad happens. The worst thing in security is someone who thinks he is "totally secure" and as such is no longer actively looking at potential issues. + +The same I see for reliability. There is no thing like "being totally reliable" and it is a really bad idea to think you could ever be. Knowing this, one may begin to think about how to decrease the overall probability of message loss AND think about what rate is acceptable (and what to do with these cases, e.g. "how can they hurt"). \end{document} -- cgit v1.2.3 From 1e4f374d7d087fac8fbe807545ea1ed3f44c80e1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 11 May 2009 09:38:29 +0200 Subject: doc: added "overall picture" of what rsyslog is --- doc/design.tex | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/design.tex b/doc/design.tex index 25e870a2..c9bfcdfc 100644 --- a/doc/design.tex +++ b/doc/design.tex @@ -10,9 +10,6 @@ \newcommand{\MM}{\mathcal{M}} \newcommand{\QQ}{\mathcal{Q}} \newcommand{\AAA}{\mathcal{A}} -%\newcommand{\MM}{\mathfrak{M}} -%\newcommand{\QQ}{\mathfrak{Q}} -%\newcommand{\AAA}{\mathfrak{A}} \title{Rsyslog Design and Internals} \author{Rainer Gerhards} @@ -29,8 +26,10 @@ This paper describes rsyslog design and internals. It is created to facilitate a \section{Preliminaries} \subsection{Notational Conventions} In general, in rsyslog there exists single objects $o$, which are used to build larger sets $O$, which form a superset $\mathcal{O}$ of all those objects that exist at a given time inside a running instance of rsyslog. As seen above, single objects are always described by lower case letters ($o$), larger sets by upper case letters ($O$) and the ``all-sets'' in caligraphic letters ($\mathcal{O}$). Often, objects $O_i, i \in \IN, i \le |\mathcal{O}|$ partition $\mathcal{O}$, but this is not necessarily the case. -\section{Objects} +\section{Overall Design} +From a high-level prespective, rsyslogd is ``just'' a high-performance message router. It accepts messages from various sources, applies user-configured filters to them, and routes potentially transformed messages to destinations based on these filters. +\section{Objects} \subsection{Plugins} Plugins provide code potentially written by a third party to extend rsyslog. @@ -62,7 +61,7 @@ The set $\MM$ is composed of all messages that exist at a given time inside rsys \subsection{Queue} A queue $$Q = (C, \Phi, M)$$ -is a tupel of a set of configuration parameters $C$, a set of callbacks $\Phi$ and a set of messages $M \subseteq \MM$. +is a triplet of a set of configuration parameters $C$, a set of callbacks $\Phi$ and a set of messages $M \subseteq \MM$. If we need to obtain the set of message from a queue, we write $M(Q)$. The elements of the set of configuration parameters are written as $C_{param}$ where $param$ is an abbreviation of the parameter's meaning. To obtain a specific parameter from a queue, we write $C_{param}(Q)$. The most important elements of $C$ are: -- cgit v1.2.3 From 8e430258fdc9b0577ea8e54dae21cc5942f90104 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 11 May 2009 17:38:33 +0200 Subject: added capability to draw configuration graphs - added $GenerateConfigGraph configuration command which can be used to generate nice-looking (and very informative) rsyslog configuration graphs. - added $ActionName configuration directive (currently only used for graph generation, but may find other uses) --- ChangeLog | 13 ++- action.c | 11 +- action.h | 1 + doc/Makefile.am | 3 + doc/rsconf1_generateconfiggraph.html | 121 ++++++++++++++++++++++ doc/rsyslog_conf_global.html | 3 + doc/rsyslog_confgraph_complex.conf | 108 +++++++++++++++++++ doc/rsyslog_confgraph_complex.png | Bin 0 -> 143204 bytes doc/rsyslog_confgraph_std.conf | 79 ++++++++++++++ doc/rsyslog_confgraph_std.png | Bin 0 -> 167756 bytes doc/troubleshoot.html | 9 ++ runtime/rsyslog.h | 1 + tools/syslogd.c | 194 ++++++++++++++++++++++++++++++++++- 13 files changed, 530 insertions(+), 13 deletions(-) create mode 100644 doc/rsconf1_generateconfiggraph.html create mode 100644 doc/rsyslog_confgraph_complex.conf create mode 100644 doc/rsyslog_confgraph_complex.png create mode 100644 doc/rsyslog_confgraph_std.conf create mode 100644 doc/rsyslog_confgraph_std.png diff --git a/ChangeLog b/ChangeLog index 98f59e9e..d37b8e0b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,15 +1,20 @@ --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +- performance enhancemnt: imtcp calls parser no longer on input thread + but rather inside on of the potentially many main msg queue worker + threads (an enhancement scheduled for all input plugins where this is + possible) +- added $GenerateConfigGraph configuration command which can be used + to generate nice-looking (and very informative) rsyslog configuration + graphs. +- added $ActionName configuration directive (currently only used for + graph generation, but may find other uses) - improved doc * added (hopefully) easier to grasp queue explanation - improved testbench * added tests for queue disk-only mode (checks disk queue logic) - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs -- performance enhancemnt: imtcp calls parser no longer on input thread - but rather inside on of the potentially many main msg queue worker - threads (an enhancement scheduled for all input plugins where this is - possible) --------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program diff --git a/action.c b/action.c index 03073153..51620fce 100644 --- a/action.c +++ b/action.c @@ -60,6 +60,7 @@ static int glbliActionResumeInterval = 30; int glbliActionResumeRetryCount = 0; /* how often should suspended actions be retried? */ static int bActionRepMsgHasMsg = 0; /* last messsage repeated... has msg fragment in it */ +static uchar *pszActionName; /* short name for the action */ /* main message queue and its configuration parameters */ static queueType_t ActionQueType = QUEUETYPE_DIRECT; /* type of the main message queue above */ static int iActionQueueSize = 1000; /* size of the main message queue above */ @@ -163,8 +164,7 @@ actionResetQueueParams(void) glbliActionResumeRetryCount = 0; /* I guess it is smart to reset this one, too */ - if(pszActionQFName != NULL) - d_free(pszActionQFName); + d_free(pszActionQFName); pszActionQFName = NULL; /* prefix for the main message queue file */ RETiRet; @@ -191,8 +191,8 @@ rsRetVal actionDestruct(action_t *pThis) SYNC_OBJ_TOOL_EXIT(pThis); pthread_mutex_destroy(&pThis->mutActExec); - if(pThis->ppTpl != NULL) - d_free(pThis->ppTpl); + d_free(pThis->pszName); + d_free(pThis->ppTpl); d_free(pThis); RETiRet; @@ -829,6 +829,7 @@ actionAddCfSysLineHdrl(void) { DEFiRet; + CHKiRet(regCfSysLineHdlr((uchar *)"actionname", 0, eCmdHdlrGetWord, NULL, &pszActionName, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszActionQFName, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesize", 0, eCmdHdlrInt, NULL, &iActionQueueSize, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuemaxdiskspace", 0, eCmdHdlrSize, NULL, &iActionQueMaxDiskSpace, NULL)); @@ -881,6 +882,8 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques CHKiRet(actionConstruct(&pAction)); /* create action object first */ pAction->pMod = pMod; pAction->pModData = pModData; + pAction->pszName = pszActionName; + pszActionName = NULL; /* free again! */ pAction->bExecWhenPrevSusp = bActExecWhenPrevSusp; pAction->iSecsExecOnceInterval = iActExecOnceInterval; pAction->iExecEveryNthOccur = iActExecEveryNthOccur; diff --git a/action.h b/action.h index f2706af6..2a1487a5 100644 --- a/action.h +++ b/action.h @@ -72,6 +72,7 @@ struct action_s { */ qqueue_t *pQueue; /* action queue */ SYNC_OBJ_TOOL; /* required for mutex support */ + uchar *pszName; /* action name (for documentation) */ pthread_mutex_t mutActExec; /* mutex to guard actual execution of doAction for single-threaded modules */ }; typedef struct action_s action_t; diff --git a/doc/Makefile.am b/doc/Makefile.am index 4d9d94ff..0703b8fc 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -78,6 +78,7 @@ html_files = \ rsconf1_filecreatemode.html \ rsconf1_filegroup.html \ rsconf1_fileowner.html \ + rsconf1_generateconfiggraph.html \ rsconf1_gssforwardservicename.html \ rsconf1_gsslistenservicename.html \ rsconf1_gssmode.html \ @@ -113,6 +114,8 @@ html_files = \ src/classes.dia grfx_files = \ + rsyslog_confgraph_complex.png\ + rsyslog_confgraph_std.png \ direct_queue0.png \ direct_queue1.png \ direct_queue2.png \ diff --git a/doc/rsconf1_generateconfiggraph.html b/doc/rsconf1_generateconfiggraph.html new file mode 100644 index 00000000..0b18463a --- /dev/null +++ b/doc/rsconf1_generateconfiggraph.html @@ -0,0 +1,121 @@ + + +rsyslog.conf file + + +back + +

    $GenerateConfigGraph

    +

    Type: global configuration directive

    +

    Default:

    +

    Available Since: 4.3.1

    +

    Description:

    +

    This directive permits to create (hopefully) good-looking visualizations of rsyslogd's +configuration. It does not affect rsyslog operation. If the directive is specified multiple +times, all but the last are ignored. If it is specified, a graph is created. This happens +both during a regular startup as well a config check run. It is recommended to include +this directive only for documentation purposes and remove it from a production +configuraton. +

    The graph is not drawn by rsyslog itself. Instead, it uses the great open source tool +Graphviz to do the actual drawing. This has at least +two advantages: +

      +
    • the graph drawing support code in rsyslog is extremly slim and without overhead +
    • the user may change or further annotate the generated file, thus potentially +improving his documentation +
    +The drawback, of course, is that you need to run Graphviz once you have generated +the control file with rsyslog. Fortunately, the process to do so is rather easy: +
      +
    1. add "$GenerateConfigGraph /path/to/file.dot" to rsyslog.conf (from now on, I +will call the file just file.dot). Optionally, add "$ActionName" statement +in front of those actions that you like to use friendly names with. If you do +this, keep the names short. +
    2. run rsyslog at least once (either in regular or configuration check mode) +
    3. remember to remove the $GenerateConfigGraph directive when you no longer need it (or +comment it out) +
    4. change your working directory to where you place the dot file +
    5. if you would like to edit the rsyslog-generated file, now is the time to do so +
    6. do "dot -Tpng file.dot > file.png" +
    7. remember that you can use "convert -resize 50% file.png resized.png" if +dot's output is too large (likely) or too small. Resizing can be especially useful if +you intend to get a rough overview over your configuration. +
    +After completing these steps, you should have a nice graph of your configuration. Details +are missing, but that is exactly the point. At the start of the graph is always (at least +in this version, could be improved) a node called "inputs" in a tripple hexagon +shape. This represents all inputs active in the system (assuming you have defined some, +what the current version does not check). Next comes the main queue. It is given in a +hexagon shape. That shape indicates that a queue is peresent and used to de-couple +the inbound from the outbound part of the graph. In technical terms, here is a +threading boundary. Action with "real" queues (other than in direct mode) +also utilize this shape. For actions, notice that a "hexagon action" creates +a deep copy of the message. As such, a "discard hexagon action" actually does +nothing, because it duplicates the message and then discards the duplicate. +At the end of the diagram, you always see a "discard" action. This indicates +that rsyslog discards messages which have been run through all available rules. +

    Edges are labeled with information about when they are taken. For filters, the type of +filter, but not any specifics, are given. It is also indicated if no filter is +applied in the configuration file (by using a "*.*" selector). Edges without +labels are unconditionally taken. The actions themselfs are labeled with the name of +the output module that handles them. If provided, the name given via +"ActionName" is used instead. No further details are provided. +

    If there is anything in red, this should draw your attention. In this case, rsyslogd +has detected something that does not look quite right. A typical example is a discard +action which is followed by some other actions in an action unit. Even though something +may be red, it can be valid - rsyslogd's graph generator does not yet check each and +every speciality, so the configuration may just cover a very uncommon case. +

    Now let's look at some examples. The graph below was generated on a fairly standard +Fedora rsyslog.conf file. It had only the usually commented-out last forwarding action +activated: +

    +rsyslog configuration graph for a default fedora rsyslog.conf +

    This is the typical structure for a simple rsyslog configuration. There are a couple of +actions, each guarded by a filter. Messages run from top to bottom and control branches +whenever a filter evaluates to true. As there is no discard action, all messages will +run through all filters and discarded in the system default discard action right after +all configured actions. +

    +

    A more complex example can be seen in the next graph. This is a configuration I +created for testing the graph-creation features, so it contains a little bit of +everything. However, real-world configurations can look quite complex, too (and I +wouldn't say this one is very complex): +

    + +

    +

    Here, we have a user-defined discard action. You can immediately see this because +processing branches after the first "builtin-file" action. Those messages +where the filter evaluates to true for will never run through the left-hand action +branch. However, there is also a configuration error present: there are two more +actions (now shown red) after the discard action. As the message is discarded, these will +never be executed. Note that the discard branch contains no further filters. This is +because these actions are all part of the same action unit, which is guarded only by +an entry filter. The same is present a bit further down at the node labeled +"write_system_log_2". This note has one more special feature, that is label +was set via "ActionName", thus is does not have standard form (the same +happened to the node named "Forward" right at the top of the diagram. +Inside this diagram, the "Forward" node is executed asynchonously on its own +queue. All others are executed synchronously. +

    Configuration graphs are useful for documenting a setup, but are also a great +troubleshooting resource. It is important to +remember that these graphs are generated +from rsyslogd's in-memory action processing structures. You can not get closer +to understanding on how rsyslog interpreted its configuration files. +So if the graph does not look +what you intended to do, there is probably something worng in rsyslog.conf. +

    If something is not working as expected, but you do not spot the error immediately, +I recommend to generate a graph and zoom it so that you see all of it in one great picture. +You may not be able to read anything, but the structure should look good to you and +so you can zoom into those areas that draw your attention. +

    Sample:

    +

    $DirOwner /path/to/graphfile-file.dot

    + +

    [rsyslog.conf overview] [manual +index] [rsyslog site]

    +

    This documentation is part of the +rsyslog project.
    +Copyright © 2009 by Rainer Gerhards and +Adiscon. Released under the GNU GPL +version 2 or higher.

    + + diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 3e33f0da..43eacc43 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -18,6 +18,8 @@ many parameter settings modify queue parameters. If in doubt, use the default, it is usually well-chosen and applicable in most cases.

    • $ActionExecOnlyWhenPreviousIsSuspended
      • +
    • $ActionName <a_single_word> - used primarily for documentation, e.g. when +generating a configuration graph. Available sice 4.3.1.
    • $ActionExecOnlyOnceEveryInterval <seconds> - execute action only if the last execute is at last <seconds> seconds in the past (more info in ommail, @@ -116,6 +118,7 @@ default 60000 (1 minute)]
    • $FileCreateMode
    • $FileGroup
    • $FileOwner
      • +
    • $GenerateConfigGraph
    • $GssForwardServiceName
    • $GssListenServiceName
    • $GssMode
      • diff --git a/doc/rsyslog_confgraph_complex.conf b/doc/rsyslog_confgraph_complex.conf new file mode 100644 index 00000000..3d7ec0a3 --- /dev/null +++ b/doc/rsyslog_confgraph_complex.conf @@ -0,0 +1,108 @@ +$DebugPrintTemplateList off +$DebugPrintCfSysLineHandlerList off +$DebugPrintModuleList off +#$ResetConfigVariables +$ErrorMessagesToStderr off +$ModLoad /home/rger/proj/rsyslog/plugins/imuxsock/.libs/imuxsock.so +#$ModLoad /home/rger/proj/rsyslog/plugins/imklog/.libs/imklog +#$ModLoad /home/rger/proj/rsyslog/plugins/imtcp/.libs/imtcp +$ModLoad /home/rger/proj/rsyslog/plugins/imtcp/.libs/imtcp +$ModLoad /home/rger/proj/rsyslog/plugins/imudp/.libs/imudp +$ModLoad /home/rger/proj/rsyslog/plugins/omstdout/.libs/omstdout +$ModLoad /home/rger/proj/rsyslog/plugins/omprog/.libs/omprog +$ModLoad /home/rger/proj/rsyslog/plugins/omtesting/.libs/omtesting +#$ModLoad /home/rger/proj/rsyslog/plugins/ommail/.libs/ommail +# +# +# PGSQL testing +$ModLoad /home/rger/proj/rsyslog/plugins/ompgsql/.libs/ompgsql.so +$template pgfmt,"insert into SystemEvents (Message, Facility, FromHost, Priority, DeviceReportedTime, ReceivedAt, InfoUnitID, SysLogTag) values ('%msg%', %syslogfacility%, '%HOSTNAME%', %syslogpriority%, '%timereported:::date-pgsql%', '%timegenerated:::date-pgsql%', %iut%, '%syslogtag%');",STDSQL +#$ActionQueueType linkedlist +#*.* :ompgsql:127.0.0.1,rsyslog,postgres,;pgfmt + +#$ActionOMStdoutArrayInterface on +#*.* :omstdout: + +$ActionResumeInterval 4 +$ActionResumeRetryCount 3 +$ActionQueueType LinkedList # run asynchronously +$ActionName Forward to 172.19.3.9 +*.* @@172.19.3.9:10514 +#*.* :omtesting:randfail +#*.* :omtesting:always_suspend +#*.* :omtesting:fail 2 2 + +#$UDPServerTimeRequery 10 +$UDPServerRun 514 +$inputtcpmaxsessions 2000 +$InputTCPServerRun 12514 + +#$PrivDropToUser rger +#$InputTCPServerInputName tcp/514 +#$InputTCPServerAddtlFrameDelimiter 10 +#$InputTCPServerRun 514 +#$AllowedSender UDP,127.0.0.1/32 +#$AllowedSender TCP,127.0.0.1/32 + +$PreserveFQDN off + +#$HUPisRestart on + +#$MainMsgQueueType direct +$MainMsgQueueType linkedlist +$MainMsgQueueDequeueBatchSize 200 +#$MainMsgQueueWorkerTimeoutThreadShutdown -1 + +#---- test DA mode +# set spool locations and switch queue to disk assisted mode +$WorkDirectory spool +$MainMsgQueueSize 200 # this *should* trigger moving on to DA mode... +# note: we must set QueueSize sufficiently high, so that 70% (light delay mark) +# is high enough above HighWatermark! +$MainMsgQueueHighWatermark 80 +$MainMsgQueueLowWatermark 40 +$MainMsgQueueFilename mainq +$MainMsgQueueType linkedlist +# ucomment, as we now have an issue (finally the test case works ;)) +#$MainMsgQueueDequeueBatchSize 80 +#---- end test DA mode + +#$template test,"%timereported:::date-rfc3339%,%timereported:::date-mysql%,%timereported:::date-subseconds%, %timegenerated:::date-mysql%, %timegenerated:::date-subseconds%, msg: %msg%\n" +#$template db,"re: '%msg:R,ERE,1,FIELD:dsn=([0-9]+\.[0-9]+\.[0-9])--end%', msg: '%msg%'\n" +#$template db,"re: '%msg:R,ERE,1,ZERO:dsn=([0-9]+\.[0-9]+\.[0-9])--end%', msg: '%msg%'\n" +#$template DEBUG,"Debug line with all properties:\nFROMHOST: '%FROMHOST%', fromhost-ip: '%fromhost-ip%, HOSTNAME: '%HOSTNAME%', PRI: %PRI%,\nsyslogtag '%syslogtag%', programname: '%programname%', APP-NAME: '%APP-NAME%', PROCID: '%PROCID%', MSGID: '%MSGID%',\nTIMESTAMP: '%TIMESTAMP%', STRUCTURED-DATA: '%STRUCTURED-DATA%',\nmsg: '%msg%'\nescaped msg: '%msg:::drop-cc%'\nrawmsg: '%rawmsg%'\n\n" +$template csv,"%syslogtag:::csv%,%msg:::upppercase,csv%,%msg%\n" +*.* -/home/rger/proj/rsyslog/logfile +kern.* -/home/rger/proj/rsyslog/logfile +$ActionExecOnlyWhenPreviousIsSuspended on +& -/tmp/xyz/uuu +$ActionExecOnlyWhenPreviousIsSuspended off +& ~ +& -/tmp/xyz/uuu2 +& -/tmp/xyz/uuu3 + + +#$template dynfile,"/home/rger/proj/rsyslog/test-%syslogtag%" +#*.* -?dynfile +#:msg, ereregex, "test|tast" /home/rger/proj/rsyslog/ere +#if strlen($syslogtag & strlen($msg)) > 10 then /home/rger/proj/rsyslog/longlog +#if strlen($msg) > 10 then /home/rger/proj/rsyslog/longlog +#if tolower($msg) contains 'test' then /home/rger/proj/rsyslog/longlog +#if $msg contains 'test' then /home/rger/proj/rsyslog/longlog + +#$ActionOMProgBinary /home/rger/proj/rsyslog/consumer +#*.* :omprog: + +#$actionresumeretryCount -1 +#$actionResumeInterval 4 +#$template dynfile,"/mnt2/logs/logfile.log" +#*.* /mnt2/logs/logfile.log +#if $msg contains 'test' then ?dynfile +#*.* ?dynfile +:msg, contains, "test " /tmpo/sdafsdf + +$ActionName write_system_log_2 +if $msg == 'test' then /tmpo/sdafsdf2 +& /tmpo/234234 +*.* @@(o,z9)172.19.3.21:10514 +$GenerateConfigGraph /home/rger/proj/rsyslog/rsyslog.dot diff --git a/doc/rsyslog_confgraph_complex.png b/doc/rsyslog_confgraph_complex.png new file mode 100644 index 00000000..21c04c57 Binary files /dev/null and b/doc/rsyslog_confgraph_complex.png differ diff --git a/doc/rsyslog_confgraph_std.conf b/doc/rsyslog_confgraph_std.conf new file mode 100644 index 00000000..64c9a18a --- /dev/null +++ b/doc/rsyslog_confgraph_std.conf @@ -0,0 +1,79 @@ +#rsyslog v3 config file + +# if you experience problems, check +# http://www.rsyslog.com/troubleshoot for assistance + +#### MODULES #### + +$ModLoad imuxsock.so # provides support for local system logging (e.g. via logger command) +$ModLoad imklog.so # provides kernel logging support (previously done by rklogd) +#$ModLoad immark.so # provides --MARK-- message capability + +# Provides UDP syslog reception +#$ModLoad imudp.so +#$UDPServerRun 514 + +# Provides TCP syslog reception +#$ModLoad imtcp.so +#$InputTCPServerRun 514 + + +#### GLOBAL DIRECTIVES #### + +# Use default timestamp format +$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat + +# File syncing capability is disabled by default. This feature is usually not required, +# not useful and an extreme performance hit +#$ActionFileEnableSync on + + +#### RULES #### + +# Log all kernel messages to the console. +# Logging much else clutters up the screen. +#kern.* /dev/console + +# Log anything (except mail) of level info or higher. +# Don't log private authentication messages! +*.info;mail.none;authpriv.none;cron.none /var/log/messages + +# The authpriv file has restricted access. +authpriv.* /var/log/secure + +# Log all the mail messages in one place. +mail.* -/var/log/maillog + + +# Log cron stuff +cron.* /var/log/cron + +# Everybody gets emergency messages +*.emerg * + +# Save news errors of level crit and higher in a special file. +uucp,news.crit /var/log/spooler + +# Save boot messages also to boot.log +local7.* /var/log/boot.log + + + +# ### begin forwarding rule ### +# The statement between the begin ... end define a SINGLE forwarding +# rule. They belong together, do NOT split them. If you create multiple +# forwarding rules, duplicate the whole block! +# Remote Logging (we use TCP for reliable delivery) +# +# An on-disk queue is created for this action. If the remote host is +# down, messages are spooled to disk and sent when it is up again. +#$WorkDirectory /var/spppl/rsyslog # where to place spool files +#$ActionQueueFileName fwdRule1 # unique name prefix for spool files +#$ActionQueueMaxDiskSpace 1g # 1gb space limit (use as much as possible) +#$ActionQueueSaveOnShutdown on # save messages to disk on shutdown +$ActionQueueType LinkedList # run asynchronously +#$ActionResumeRetryCount -1 # infinite retries if host is down +# remote host is: name/ip:port, e.g. 192.168.0.1:514, port optional +*.* @@remote-host:514 +# ### end of the forwarding rule ### +$GenerateConfigGraph /home/rger/proj/rsyslog/rsyslog.dot diff --git a/doc/rsyslog_confgraph_std.png b/doc/rsyslog_confgraph_std.png new file mode 100644 index 00000000..655a7f82 Binary files /dev/null and b/doc/rsyslog_confgraph_std.png differ diff --git a/doc/troubleshoot.html b/doc/troubleshoot.html index e655c2ef..cb4367f6 100644 --- a/doc/troubleshoot.html +++ b/doc/troubleshoot.html @@ -28,6 +28,15 @@ mode can be used in parallel to a running instance of rsyslogd.

      /path/to/rsyslogd -f/path/to/config-file -N1

      You should also specify other options you usually give (like -c3 and whatever else). Any problems experienced are reported to stderr [aka "your screen" (if not redirected)]. +

      Configuration Graphs +

      Starting with rsyslog 4.3.1, the +"$GenerateConfigGraph" +command is supported, a very valuable troubleshooting tool. It permits to +generate a graph of how rsyslogd understood its configuration file. It is assumed that +many configuration issues can easily be detected just by looking at the configuration graph. +Full details of how to generate the graphs, and what to look for can be found in the +"$GenerateConfigGraph" +manual page.

      Asking for Help

      If you can't find the answer yourself, you should look at these places for community help. diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 026fbbed..fd5a5371 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -267,6 +267,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_FORK = -2118, /**< error during fork() */ RS_RET_ERR_WRITE_PIPE = -2119, /**< error writing to pipe */ RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ + RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 8c86c12e..3a751a30 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -284,6 +284,7 @@ static int gidDropPriv = 0; /* group-id to which priveleges should be dropped to extern int errno; +static uchar *pszConfDAGFile = NULL; /* name of config DAG file, non-NULL means generate one */ /* main message queue and its configuration parameters */ static qqueue_t *pMsgQueue = NULL; /* the main message queue */ static int iMainMsgQueueSize = 10000; /* size of the main message queue above */ @@ -1939,10 +1940,9 @@ static void doDie(int sig) static void freeAllDynMemForTermination(void) { - if(pszMainMsgQFName != NULL) - free(pszMainMsgQFName); - if(pModDir != NULL) - free(pModDir); + free(pszMainMsgQFName); + free(pModDir); + free(pszConfDAGFile); } @@ -2210,6 +2210,184 @@ static void freeSelectors(void) } +/* helper to generateConfigDAG, to print out all actions via + * the llExecFunc() facility. + * rgerhards, 2007-08-02 + */ +struct dag_info { + FILE *fp; /* output file */ + int iActUnit; /* current action unit number */ + int iAct; /* current action in unit */ + int bDiscarded; /* message discarded (config error) */ + }; +DEFFUNC_llExecFunc(generateConfigDAGAction) +{ + action_t *pAction; + uchar *pszModName; + uchar *pszVertexName; + struct dag_info *pDagInfo; + DEFiRet; + + pDagInfo = (struct dag_info*) pParam; + pAction = (action_t*) pData; + + pszModName = module.GetStateName(pAction->pMod); + + /* vertex */ + if(pAction->pszName == NULL) { + if(!strcmp((char*)pszModName, "builtin-discard")) + pszVertexName = (uchar*)"discard"; + else + pszVertexName = pszModName; + } else { + pszVertexName = pAction->pszName; + } + + fprintf(pDagInfo->fp, "\tact%d_%d\t\t[label=\"%s\"%s%s]\n", + pDagInfo->iActUnit, pDagInfo->iAct, pszVertexName, + pDagInfo->bDiscarded ? " style=dotted color=red" : "", + (pAction->pQueue->qType == QUEUETYPE_DIRECT) ? "" : " shape=hexagon" + ); + + /* edge */ + if(pDagInfo->iAct == 0) { + } else { + fprintf(pDagInfo->fp, "\tact%d_%d -> act%d_%d[%s%s]\n", + pDagInfo->iActUnit, pDagInfo->iAct - 1, + pDagInfo->iActUnit, pDagInfo->iAct, + pDagInfo->bDiscarded ? " style=dotted color=red" : "", + pAction->bExecWhenPrevSusp ? " label=\"only if\\nsuspended\"" : "" ); + } + + /* check for discard */ + if(!strcmp((char*) pszModName, "builtin-discard")) { + fprintf(pDagInfo->fp, "\tact%d_%d\t\t[shape=box]\n", + pDagInfo->iActUnit, pDagInfo->iAct); + pDagInfo->bDiscarded = 1; + } + + + ++pDagInfo->iAct; + + RETiRet; +} + + +/* create config DAG + * This functions takes a rsyslog config and produces a .dot file for use + * with graphviz (http://www.graphviz.org). This is done in an effort to + * document, and also potentially troubleshoot, configurations. Plus, I + * consider it a nice feature to explain some concepts. Note that the + * current version only produces a graph with relatively little information. + * This is a foundation that may be later expanded (if it turns out to be + * useful enough). + * rgerhards, 2009-05-11 + */ +static rsRetVal +generateConfigDAG(uchar *pszDAGFile) +{ + selector_t *f; + FILE *fp; + int iActUnit = 1; + int bHasFilter = 0; /* filter associated with this action unit? */ + int bHadFilter; + int i; + struct dag_info dagInfo; + char *pszFilterName; + char szConnectingNode[64]; + DEFiRet; + + assert(pszDAGFile != NULL); + + if((fp = fopen((char*) pszDAGFile, "w")) == NULL) { + logmsgInternal(NO_ERRCODE, LOG_SYSLOG|LOG_INFO, (uchar*) + "configuraton graph output file could not be opened, none generated", 0); + ABORT_FINALIZE(RS_RET_FILENAME_INVALID); + } + + dagInfo.fp = fp; + + /* from here on, we assume writes go well. This here is a really + * unimportant utility function and if something goes wrong, it has + * almost no effect. So let's not overdo this... + */ + fprintf(fp, "# graph created by rsyslog " VERSION "\n\n" + "# use the dot tool from http://www.graphviz.org to visualize!\n" + "digraph rsyslogConfig {\n" + "\tinputs [shape=tripleoctagon]\n" + "\tinputs -> act0_0\n" + "\tact0_0 [label=\"main\\nqueue\" shape=hexagon]\n" + /*"\tmainq -> act1_0\n"*/ + ); + strcpy(szConnectingNode, "act0_0"); + dagInfo.bDiscarded = 0; + + for(f = Files; f != NULL ; f = f->f_next) { + /* BSD-Style filters are currently ignored */ + bHadFilter = bHasFilter; + if(f->f_filter_type == FILTER_PRI) { + bHasFilter = 0; + for (i = 0; i <= LOG_NFACILITIES; i++) + if (f->f_filterData.f_pmask[i] != 0xff) { + bHasFilter = 1; + break; + } + } else { + bHasFilter = 1; + } + + /* we know we have a filter, so it can be false */ + switch(f->f_filter_type) { + case FILTER_PRI: + pszFilterName = "pri filter"; + break; + case FILTER_PROP: + pszFilterName = "property filter"; + break; + case FILTER_EXPR: + pszFilterName = "script filter"; + break; + } + + /* write action unit node */ + if(bHasFilter) { + fprintf(fp, "\t%s -> act%d_end\t[label=\"%s:\\nfalse\"]\n", + szConnectingNode, iActUnit, pszFilterName); + fprintf(fp, "\t%s -> act%d_0\t[label=\"%s:\\ntrue\"]\n", + szConnectingNode, iActUnit, pszFilterName); + fprintf(fp, "\tact%d_end\t\t\t\t[shape=point]\n", iActUnit); + snprintf(szConnectingNode, sizeof(szConnectingNode), "act%d_end", iActUnit); + } else { + fprintf(fp, "\t%s -> act%d_0\t[label=\"no filter\"]\n", + szConnectingNode, iActUnit); + snprintf(szConnectingNode, sizeof(szConnectingNode), "act%d_0", iActUnit); + } + + /* draw individual nodes */ + dagInfo.iActUnit = iActUnit; + dagInfo.iAct = 0; + dagInfo.bDiscarded = 0; + llExecFunc(&f->llActList, generateConfigDAGAction, &dagInfo); /* actions */ + + /* finish up */ + if(bHasFilter && !dagInfo.bDiscarded) { + fprintf(fp, "\tact%d_%d -> %s\n", + iActUnit, dagInfo.iAct - 1, szConnectingNode); + } + + ++iActUnit; + } + + fprintf(fp, "\t%s -> act%d_0\n", szConnectingNode, iActUnit); + fprintf(fp, "\tact%d_0\t\t[label=discard shape=box]\n" + "}\n", iActUnit); + fclose(fp); + +finalize_it: + RETiRet; +} + + /* helper to dbPrintInitInfo, to print out all actions via * the llExecFunc() facility. * rgerhards, 2007-08-02 @@ -2223,6 +2401,7 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction) RETiRet; } + /* print debug information as part of init(). This pretty much * outputs the whole config of rsyslogd. I've moved this code * out of init() to clean it somewhat up. @@ -2230,7 +2409,7 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction) */ static void dbgPrintInitInfo(void) { - register selector_t *f; + selector_t *f; int iSelNbr = 1; int i; @@ -2467,6 +2646,10 @@ init(void) } } + /* check if we need to generate a config DAG and, if so, do that */ + if(pszConfDAGFile != NULL) + generateConfigDAG(pszConfDAGFile); + /* we are done checking the config - now validate if we should actually run or not. * If not, terminate. -- rgerhards, 2008-07-25 */ @@ -2903,6 +3086,7 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"debugprintcfsyslinehandlerlist", 0, eCmdHdlrBinary, NULL, &bDebugPrintCfSysLineHandlerList, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"moddir", 0, eCmdHdlrGetWord, NULL, &pModDir, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"generateconfiggraph", 0, eCmdHdlrGetWord, NULL, &pszConfDAGFile, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"errormessagestostderr", 0, eCmdHdlrBinary, NULL, &bErrMsgToStderr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"maxmessagesize", 0, eCmdHdlrSize, setMaxMsgSize, NULL, NULL)); -- cgit v1.2.3 From d1985c4a65feb5caaea855517431ed564a13ed4c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 11 May 2009 22:00:48 +0200 Subject: doc: given some concrete advise on the common %hostname% content problem --- doc/troubleshoot.html | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/troubleshoot.html b/doc/troubleshoot.html index cb4367f6..a8855fd4 100644 --- a/doc/troubleshoot.html +++ b/doc/troubleshoot.html @@ -18,7 +18,14 @@ is a rsyslog-doc package, that often needs to be installed separately.

      Malformed Messages and Message Properties

      A common trouble source are ill-formed syslog messages, which lead to to all sorts of interesting problems, including malformed hostnames and dates. -Read the quoted guide to find relief. +Read the quoted guide to find relief. A common symptom is that the %HOSTNAME% property is +used for generating dynafile names, but some glibberish shows up. This is caused by the +malformed syslog messages, so be sure to read the +guide if you face that problem. Just let me add that the +common work-around is to use %FROMHOST% or %FROMHOST-IP% instead. These do not take the +hostname from the message, but rather use the host that sent the message (taken from +the socket layer). Of course, this does not work over NAT or relay chains, where the +only cure is to make sure senders emit well-formed messages.

      Configuration Problems

      Rsyslog 3.21.1 and above has been enhanced to support extended configuration checking. It offers a special command line switch (-N1) that puts it into "config verfication mode". -- cgit v1.2.3 From bb79e96dc300fa5a2182e7c047afb3b15c5dc870 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 12 May 2009 15:27:40 +0200 Subject: moving to a cleaner implementation of batches ... now that we know what we need from a theoretical POV. --- action.c | 25 +++++++++++---------- runtime/Makefile.am | 1 + runtime/batch.h | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/debug.c | 14 +++++++----- runtime/queue.c | 49 +++++++++++++++++++++++++---------------- runtime/queue.h | 5 +++-- runtime/rsyslog.h | 2 ++ runtime/wti.c | 10 ++++----- runtime/wti.h | 14 ++---------- tools/syslogd.c | 9 ++++---- 10 files changed, 132 insertions(+), 60 deletions(-) create mode 100644 runtime/batch.h diff --git a/action.c b/action.c index 509ad749..b12eda6e 100644 --- a/action.c +++ b/action.c @@ -42,13 +42,14 @@ #include "cfsysline.h" #include "srUtils.h" #include "errmsg.h" +#include "batch.h" #include "wti.h" #include "datetime.h" #define NO_TIME_PROVIDED 0 /* indicate we do not provide any cached time */ /* forward definitions */ -rsRetVal actionCallDoActionMULTIQUEUE(action_t *pAction, aUsrp_t*); +static rsRetVal actionCallDoActionMULTIQUEUE(action_t *pAction, batch_t *pBatch); /* object static data (once for all instances) */ /* TODO: make this an object! DEFobjStaticHelpers -- rgerhards, 2008-03-05 */ @@ -261,7 +262,7 @@ actionConstructFinalize(action_t *pThis) * spec. -- rgerhards, 2008-01-30 */ CHKiRet(qqueueConstruct(&pThis->pQueue, ActionQueType, 1, iActionQueueSize, - (rsRetVal (*)(void*,aUsrp_t*))actionCallDoActionMULTIQUEUE)); + (rsRetVal (*)(void*, batch_t*))actionCallDoActionMULTIQUEUE)); obj.SetName((obj_t*) pThis->pQueue, pszQName); /* ... set some properties ... */ @@ -782,19 +783,19 @@ finalize_it: * for processing. * rgerhards, 2009-04-22 */ -rsRetVal -actionCallDoActionMULTIQUEUEprocessing(action_t *pAction, aUsrp_t *paUsrp) +static rsRetVal +actionCallDoActionMULTIQUEUEprocessing(action_t *pAction, batch_t *pBatch) { int i; msg_t *pMsg; rsRetVal localRet; DEFiRet; - assert(paUsrp != NULL); + assert(pBatch != NULL); - for(i = 0 ; i < paUsrp->nElem ; i++) { - pMsg = (msg_t*) paUsrp->pUsrp[i]; -dbgprintf("actionCall..MULTIQUEUE: i: %d, pMsg: %p\n", i, pMsg); + for(i = 0 ; i < pBatch->nElem ; i++) { + pMsg = (msg_t*) pBatch->pElem[i].pUsrp; +dbgprintf("actionCall..MULTIQUEUE: i: %d/%d, pMsg: %p\n", i, pBatch->nElem, pMsg); localRet = actionProcessMessage(pAction, pMsg); dbgprintf("action call returned %d\n", localRet); msgDestruct(&pMsg); /* TODO: change: we are now finished with the message */ @@ -810,13 +811,13 @@ finalize_it: * for processing. * rgerhards, 2009-04-22 */ -rsRetVal -actionCallDoActionMULTIQUEUE(action_t *pAction, aUsrp_t *paUsrp) +static rsRetVal +actionCallDoActionMULTIQUEUE(action_t *pAction, batch_t *pBatch) { int iCancelStateSave; DEFiRet; - assert(paUsrp != NULL); + assert(pBatch != NULL); /* We now must guard the output module against execution by multiple threads. The * plugin interface specifies that output modules must not be thread-safe (except @@ -828,7 +829,7 @@ actionCallDoActionMULTIQUEUE(action_t *pAction, aUsrp_t *paUsrp) pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); pthread_setcancelstate(iCancelStateSave, NULL); - iRet = actionCallDoActionMULTIQUEUEprocessing(pAction, paUsrp); + iRet = actionCallDoActionMULTIQUEUEprocessing(pAction, pBatch); pthread_cleanup_pop(1); /* unlock mutex */ diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 2f0a1aa0..f0fb1cdd 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -8,6 +8,7 @@ librsyslog_la_SOURCES = \ rsyslog.c \ rsyslog.h \ atomic.h \ + batch.h \ syslogd-types.h \ module-template.h \ obj-types.h \ diff --git a/runtime/batch.h b/runtime/batch.h new file mode 100644 index 00000000..cb40cf42 --- /dev/null +++ b/runtime/batch.h @@ -0,0 +1,63 @@ +/* Definition of the batch_t data structure. + * I am not sure yet if this will become a full-blown object. For now, this header just + * includes the object definition and is not accompanied by code. + * + * Copyright 2009 by Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#ifndef BATCH_H_INCLUDED +#define BATCH_H_INCLUDED + +/* enum for batch states. Actually, we violate a layer here, in that we assume that a batch is used + * for action processing. So far, this seems acceptable, the status is simply ignored inside the + * main message queue. But over time, it could potentially be useful to split the two. + * rgerhad, 2009-05-12 + */ +typedef enum { + BATCH_STATE_RDY = 0, /* object ready for processing */ + BATCH_STATE_BAD = 1, /* unrecoverable failure while processing, do NOT resubmit to same action */ + BATCH_STATE_SUB = 2, /* message submitted for processing, outcome yet unkonwn */ + BATCH_STATE_DISC = 3, /* discarded - processed OK, but do not submit to any other action */ +} batch_state_t; + + +/* an object inside a batch, including any information (state!) needed for it to "life". + */ +struct batch_obj_s { + obj_t *pUsrp; /* pointer to user object (most often message) */ + batch_state_t state; /* associated state */ +}; + +/* the batch + * This object is used to dequeue multiple user pointers which are than handed over + * to processing. The size of elements is fixed after queue creation, but may be + * modified by config variables (better said: queue properties). + * Note that a "user pointer" in rsyslog context so far always is a message + * object. We stick to the more generic term because queues may potentially hold + * other types of objects, too. + * rgerhards, 2009-05-12 + */ +struct batch_s { + int nElem; /* actual number of element in this entry */ + batch_obj_t *pElem; /* batch elements */ +}; + +#endif /* #ifndef BATCH_H_INCLUDED */ diff --git a/runtime/debug.c b/runtime/debug.c index 4ee90226..4f45a1e3 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -1050,7 +1050,9 @@ int dbgEntrFunc(dbgFuncDB_t **ppFuncDB, const char *file, const char *func, int /* when we reach this point, we have a fully-initialized FuncDB! */ ATOMIC_INC(pFuncDB->nTimesCalled); if(bLogFuncFlow && dbgPrintNameIsInList((const uchar*)pFuncDB->file, printNameFileRoot)) - dbgprintf("%s:%d: %s: enter\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); + if(strcmp(pFuncDB->file, "stringbuf.c")) { /* TODO: make configurable */ + dbgprintf("%s:%d: %s: enter\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); + } if(pThrd->stackPtr >= (int) (sizeof(pThrd->callStack) / sizeof(dbgFuncDB_t*))) { dbgprintf("%s:%d: %s: debug module: call stack for this thread full, suspending call tracking\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); @@ -1080,10 +1082,12 @@ void dbgExitFunc(dbgFuncDB_t *pFuncDB, int iStackPtrRestore, int iRet) dbgFuncDBPrintActiveMutexes(pFuncDB, "WARNING: mutex still owned by us as we exit function, mutex: ", pthread_self()); if(bLogFuncFlow && dbgPrintNameIsInList((const uchar*)pFuncDB->file, printNameFileRoot)) { - if(iRet == RS_RET_NO_IRET) - dbgprintf("%s:%d: %s: exit: (no iRet)\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); - else - dbgprintf("%s:%d: %s: exit: %d\n", pFuncDB->file, pFuncDB->line, pFuncDB->func, iRet); + if(strcmp(pFuncDB->file, "stringbuf.c")) { /* TODO: make configurable */ + if(iRet == RS_RET_NO_IRET) + dbgprintf("%s:%d: %s: exit: (no iRet)\n", pFuncDB->file, pFuncDB->line, pFuncDB->func); + else + dbgprintf("%s:%d: %s: exit: %d\n", pFuncDB->file, pFuncDB->line, pFuncDB->func, iRet); + } } pThrd->stackPtr = iStackPtrRestore; if(pThrd->stackPtr < 0) { diff --git a/runtime/queue.c b/runtime/queue.c index c2df928b..c3a8e9d4 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -65,7 +65,7 @@ DEFobjStaticHelpers DEFobjCurrIf(glbl) /* forward-definitions */ -rsRetVal qqueueChkPersist(qqueue_t *pThis); +static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates); static rsRetVal qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); static rsRetVal RateLimiter(qqueue_t *pThis); static int qqueueChkStopWrkrDA(qqueue_t *pThis); @@ -896,8 +896,8 @@ static rsRetVal qDestructDirect(qqueue_t __attribute__((unused)) *pThis) static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) { - aUsrp_t aUsrp; - obj_t *pMsgp; + batch_t singleBatch; + batch_obj_t batchObj; DEFiRet; ASSERT(pThis != NULL); @@ -907,17 +907,19 @@ static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) * mode the consumer probably has a lot to convey (which get's lost in the other modes * because they are asynchronous. But direct mode is deliberately synchronous. * rgerhards, 2008-02-12 - * We use our knowledge about the aUsrp_t structure below, but without that, we + * We use our knowledge about the batch_t structure below, but without that, we * pay a too-large performance toll... -- rgerhards, 2009-04-22 */ - pMsgp = (obj_t*) pUsr; - aUsrp.nElem = 1; /* there always is only one in direct mode */ - aUsrp.pUsrp = &pMsgp; - iRet = pThis->pConsumer(pThis->pUsr, &aUsrp); + batchObj.state = BATCH_STATE_RDY; + batchObj.pUsrp = (obj_t*) pUsr; + singleBatch.nElem = 1; /* there always is only one in direct mode */ + singleBatch.pElem = &batchObj; + iRet = pThis->pConsumer(pThis->pUsr, &singleBatch); RETiRet; } + static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis, __attribute__((unused)) void **out) { return RS_RET_OK; @@ -1247,7 +1249,7 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) * to modify some parameters before the queue is actually started. */ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*,aUsrp_t*)) + int iMaxQueueSize, rsRetVal (*pConsumer)(void*, batch_t*)) { DEFiRet; qqueue_t *pThis; @@ -1402,11 +1404,12 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *iRemainingQueueSize rsRetVal localRet; DEFiRet; + /* this is the place to destruct the old messages and pull them off the queue - MULTI-DEQUEUE */ + nDequeued = 0; do { dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); CHKiRet(qqueueDel(pThis, &pUsr)); - qqueueChkPersist(pThis); /* is is questionable if we should really need to call this every time... */ iQueueSize = qqueueGetOverallQueueSize(pThis); /* check if we should discard this element */ @@ -1417,11 +1420,15 @@ dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); ABORT_FINALIZE(localRet); /* all well, use this element */ - pWti->paUsrp->pUsrp[nDequeued++] = pUsr; + pWti->batch.pElem[nDequeued].pUsrp = pUsr; + pWti->batch.pElem[nDequeued].state = BATCH_STATE_RDY; + ++nDequeued; } while(iQueueSize > 0 && nDequeued < pThis->iDeqBatchSize); + qqueueChkPersist(pThis, nDequeued); /* it is sufficient to persist only when the bulk of work is done */ + //bRunsDA = pThis->bRunsDA; /* cache this for after mutex release */ - pWti->paUsrp->nElem = nDequeued; + pWti->batch.nElem = nDequeued; *iRemainingQueueSize = iQueueSize; finalize_it: @@ -1582,7 +1589,7 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) ISOBJ_TYPE_assert(pWti, wti); CHKiRet(DequeueConsumable(pThis, pWti, iCancelStateSave)); - CHKiRet(pThis->pConsumer(pThis->pUsr, pWti->paUsrp)); + CHKiRet(pThis->pConsumer(pThis->pUsr, &pWti->batch)); /* we now need to check if we should deliberately delay processing a bit * and, if so, do that. -- rgerhards, 2008-01-30 @@ -1619,8 +1626,8 @@ ConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) CHKiRet(DequeueConsumable(pThis, pWti, iCancelStateSave)); /* iterate over returned results and enqueue them in DA queue */ - for(i = 0 ; i < pWti->paUsrp->nElem ; i++) - CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->paUsrp->pUsrp[i])); + for(i = 0 ; i < pWti->batch.nElem ; i++) + CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->batch.pElem[i].pUsrp)); finalize_it: dbgoprint((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); @@ -1974,17 +1981,21 @@ finalize_it: /* check if we need to persist the current queue info. If an - * error occurs, thus should be ignored by caller (but we still + * error occurs, this should be ignored by caller (but we still * abide to our regular call interface)... * rgerhards, 2008-01-13 + * nUpdates is the number of updates since the last call to this function. + * It may be > 1 due to batches. -- rgerhards, 2009-05-12 */ -rsRetVal qqueueChkPersist(qqueue_t *pThis) +static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates) { DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); + assert(nUpdates > 0); - if(pThis->iPersistUpdCnt && ++pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { + pThis->iUpdsSincePersist += nUpdates; + if(pThis->iPersistUpdCnt && pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { qqueuePersist(pThis, QUEUE_CHECKPOINT); pThis->iUpdsSincePersist = 0; } @@ -2199,7 +2210,7 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* and finally enqueue the message */ CHKiRet(qqueueAdd(pThis, pUsr)); - qqueueChkPersist(pThis); + qqueueChkPersist(pThis, 1); finalize_it: if(pThis->qType != QUEUETYPE_DIRECT) { diff --git a/runtime/queue.h b/runtime/queue.h index 8a60254b..4a5f16a1 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -27,6 +27,7 @@ #include #include "obj.h" #include "wtp.h" +#include "batch.h" #include "stream.h" /* queue types */ @@ -100,7 +101,7 @@ typedef struct queue_s { * the user really wanted...). -- rgerhards, 2008-04-02 */ /* end dequeue time window */ - rsRetVal (*pConsumer)(void *,aUsrp_t*); /* user-supplied consumer function for dequeued messages */ + rsRetVal (*pConsumer)(void *,batch_t*); /* user-supplied consumer function for dequeued messages */ /* calling interface for pConsumer: arg1 is the global user pointer from this structure, arg2 is the * user pointer array that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 * is pointer to an array of message message pointers) @@ -184,7 +185,7 @@ rsRetVal qqueueStart(qqueue_t *pThis); rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*,aUsrp_t*)); + int iMaxQueueSize, rsRetVal (*pConsumer)(void*,batch_t*)); PROTOTYPEObjClassInit(qqueue); PROTOTYPEpropSetMeth(qqueue, iPersistUpdCnt, int); PROTOTYPEpropSetMeth(qqueue, iDeqtWinFromHr, int); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index ee941b2b..53a510b3 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -95,6 +95,8 @@ typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct tcpsrv_s tcpsrv_t; typedef struct vmstk_s vmstk_t; +typedef struct batch_obj_s batch_obj_t; +typedef struct batch_s batch_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ /* some universal 64 bit define... */ diff --git a/runtime/wti.c b/runtime/wti.c index 346ef7aa..c3fa127e 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -201,8 +201,7 @@ CODESTARTobjDestruct(wti) pthread_cond_destroy(&pThis->condExitDone); pthread_mutex_destroy(&pThis->mut); - free(pThis->paUsrp->pUsrp); - free(pThis->paUsrp); + free(pThis->batch.pElem); free(pThis->pszDbgHdr); ENDobjDestruct(wti) @@ -234,8 +233,7 @@ wtiConstructFinalize(wti_t *pThis) /* we now alloc the array for user pointers. We obtain the max from the queue itself. */ CHKiRet(pThis->pWtp->pfGetDeqBatchSize(pThis->pWtp->pUsr, &iDeqBatchSize)); - CHKmalloc(pThis->paUsrp = calloc(1, sizeof(aUsrp_t))); - CHKmalloc(pThis->paUsrp->pUsrp = calloc((size_t)iDeqBatchSize, sizeof(void*))); + CHKmalloc(pThis->batch.pElem = calloc((size_t)iDeqBatchSize, sizeof(batch_obj_t*))); finalize_it: RETiRet; @@ -322,7 +320,7 @@ wtiWorkerCancelCleanup(void *arg) /* call user supplied handler (that one e.g. requeues the element) */ // MULTIQUEUE: need to change here! - pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->paUsrp->pUsrp[0]); + pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->batch.pElem[0].pUsrp); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); d_pthread_mutex_lock(&pWtp->mut); @@ -374,7 +372,7 @@ wtiWorker(wti_t *pThis) ISOBJ_TYPE_assert(pWtp, wtp); dbgSetThrdName(pThis->pszDbgHdr); - pThis->paUsrp->nElem = 0; /* flag no elements present */ // MULTIQUEUE: do we really need this any longer (cnacel handeler)? + pThis->batch.nElem = 0; /* flag no elements present */ // MULTIQUEUE: do we really need this any longer (cnacel handeler)? pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); diff --git a/runtime/wti.h b/runtime/wti.h index 85c98fe6..0990941e 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -27,17 +27,7 @@ #include #include "wtp.h" #include "obj.h" - -/* the user pointer array object - * This object is used to dequeue multiple user pointers which are than handed over - * to processing. The size of elements is fixed after queue creation, but may be - * modified by config variables (better said: queue properties). - * rgerhards, 2009-04-22 - */ -struct aUsrp_s { - int nElem; /* actual number of element in this entry */ - obj_t **pUsrp; /* actual elements (array!) */ -}; +#include "batch.h" /* the worker thread instance class */ @@ -46,11 +36,11 @@ typedef struct wti_s { int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ pthread_t thrdID; /* thread ID */ qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */ - aUsrp_t *paUsrp; /* pointer to an object array meaningful for current user pointer (e.g. queue pUsr data elemt) */ wtp_t *pWtp; /* my worker thread pool (important if only the work thread instance is passed! */ pthread_cond_t condExitDone; /* signaled when the thread exit is done (once per thread existance) */ pthread_mutex_t mut; int bShutdownRqtd; /* shutdown for this thread requested? 0 - no , 1 - yes */ + batch_t batch; /* pointer to an object array meaningful for current user pointer (e.g. queue pUsr data elemt) */ uchar *pszDbgHdr; /* header string for debug messages */ } wti_t; diff --git a/tools/syslogd.c b/tools/syslogd.c index 866c0173..7ee5dbd7 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -138,6 +138,7 @@ #include "datetime.h" #include "parser.h" #include "sysvar.h" +#include "batch.h" /* definitions for objects we access */ DEFobjCurrIf(obj) @@ -1211,16 +1212,16 @@ processMsg(msg_t *pMsg) * for the main queue. */ static rsRetVal -msgConsumer(void __attribute__((unused)) *notNeeded, aUsrp_t *paUsrp) +msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch) { int i; msg_t *pMsg; DEFiRet; - assert(paUsrp != NULL); + assert(pBatch != NULL); - for(i = 0 ; i < paUsrp->nElem ; i++) { - pMsg = (msg_t*) paUsrp->pUsrp[i]; + for(i = 0 ; i < pBatch->nElem ; i++) { + pMsg = (msg_t*) pBatch->pElem[i].pUsrp; dbgprintf("msgConsumer..MULTIQUEUE: i: %d, pMsg: %p\n", i, pMsg); if((pMsg->msgFlags & NEEDS_PARSING) != 0) { parseMsg(pMsg); -- cgit v1.2.3 From fbb040b411ee564e4a4bbaf53ec342236929324f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 12 May 2009 15:58:11 +0200 Subject: one astrisk too much - and we have a segfault... - fixed ;) --- runtime/wti.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/wti.c b/runtime/wti.c index c3fa127e..7029dcfd 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -233,7 +233,7 @@ wtiConstructFinalize(wti_t *pThis) /* we now alloc the array for user pointers. We obtain the max from the queue itself. */ CHKiRet(pThis->pWtp->pfGetDeqBatchSize(pThis->pWtp->pUsr, &iDeqBatchSize)); - CHKmalloc(pThis->batch.pElem = calloc((size_t)iDeqBatchSize, sizeof(batch_obj_t*))); + CHKmalloc(pThis->batch.pElem = calloc((size_t)iDeqBatchSize, sizeof(batch_obj_t))); finalize_it: RETiRet; -- cgit v1.2.3 From e2b229868955a6f6a6380273314d0d90ddad1273 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 12 May 2009 17:57:04 +0200 Subject: action batch processing implemented ... passed initial tests, but of course more are needed --- action.c | 156 +++++++++++++++++++++++++++++++++++++++++++++--------- runtime/batch.h | 4 +- runtime/rsyslog.h | 1 + 3 files changed, 136 insertions(+), 25 deletions(-) diff --git a/action.c b/action.c index b12eda6e..0253e1b6 100644 --- a/action.c +++ b/action.c @@ -49,7 +49,7 @@ #define NO_TIME_PROVIDED 0 /* indicate we do not provide any cached time */ /* forward definitions */ -static rsRetVal actionCallDoActionMULTIQUEUE(action_t *pAction, batch_t *pBatch); +static rsRetVal processBatchMain(action_t *pAction, batch_t *pBatch); /* object static data (once for all instances) */ /* TODO: make this an object! DEFobjStaticHelpers -- rgerhards, 2008-03-05 */ @@ -262,7 +262,7 @@ actionConstructFinalize(action_t *pThis) * spec. -- rgerhards, 2008-01-30 */ CHKiRet(qqueueConstruct(&pThis->pQueue, ActionQueType, 1, iActionQueueSize, - (rsRetVal (*)(void*, batch_t*))actionCallDoActionMULTIQUEUE)); + (rsRetVal (*)(void*, batch_t*))processBatchMain)); obj.SetName((obj_t*) pThis->pQueue, pszQName); /* ... set some properties ... */ @@ -372,10 +372,8 @@ static rsRetVal getReturnCode(action_t *pThis) iRet = RS_RET_SUSPENDED; break; case ACT_STATE_SUSP: - iRet = RS_RET_SUSPENDED; - break; case ACT_STATE_DIED: - iRet = RS_RET_DISABLE_ACTION; + iRet = RS_RET_ACTION_FAILED; break; default: DBGPRINTF("Invalid action engine state %d, program error\n", @@ -459,15 +457,12 @@ static rsRetVal actionDoRetry(action_t *pThis, time_t ttNow) ASSERT(pThis != NULL); -RUNLOG_STR("actionDoRetry():"); iRetries = 0; while(pThis->eState == ACT_STATE_RTRY) { iRet = pThis->pMod->tryResume(pThis->pModData); if(iRet == RS_RET_OK) { actionSetState(pThis, ACT_STATE_RDY); -RUNLOG_STR("tryResume succeeded"); } else if(iRet == RS_RET_SUSPENDED) { -RUNLOG_STR("still suspended");; /* max retries reached? */ if((pThis->iResumeRetryCount != -1 && iRetries >= pThis->iResumeRetryCount)) { actionSuspend(pThis, ttNow); @@ -501,7 +496,6 @@ static rsRetVal actionTryResume(action_t *pThis) ASSERT(pThis != NULL); -RUNLOG_STR("actionTryResume()"); if(pThis->eState == ACT_STATE_SUSP) { /* if we are suspended, we need to check if the timeout expired. * for this handling, we must always obtain a fresh timestamp. We used @@ -522,8 +516,10 @@ RUNLOG_STR("actionTryResume()"); CHKiRet(actionDoRetry(pThis, ttNow)); } - DBGPRINTF("actionTryResume: action state: %s, next retry (if applicable): %u [now %u]\n", - getActStateName(pThis), (unsigned) pThis->ttResumeRtry, (unsigned) ttNow); + if(Debug && (pThis->eState == ACT_STATE_RTRY ||pThis->eState == ACT_STATE_SUSP)) { + dbgprintf("actionTryResume: action state: %s, next retry (if applicable): %u [now %u]\n", + getActStateName(pThis), (unsigned) pThis->ttResumeRtry, (unsigned) ttNow); + } finalize_it: RETiRet; @@ -538,11 +534,8 @@ static rsRetVal actionPrepare(action_t *pThis) { DEFiRet; -RUNLOG_STR("actionPrepare()"); assert(pThis != NULL); - if(pThis->eState == ACT_STATE_RTRY) { - CHKiRet(actionTryResume(pThis)); - } + CHKiRet(actionTryResume(pThis)); /* if we are now ready, we initialize the transaction and advance * action state accordingly @@ -779,40 +772,155 @@ finalize_it: } -/* receive an array of to-process user pointers and submit them - * for processing. - * rgerhards, 2009-04-22 +/* try to submit a partial batch of elements. + * rgerhards, 2009-05-12 */ static rsRetVal -actionCallDoActionMULTIQUEUEprocessing(action_t *pAction, batch_t *pBatch) +tryDoAction(action_t *pAction, batch_t *pBatch, int *pnElem) { int i; + int iElemProcessed; + int iCommittedUpTo; msg_t *pMsg; rsRetVal localRet; DEFiRet; assert(pBatch != NULL); + assert(pnElem != NULL); - for(i = 0 ; i < pBatch->nElem ; i++) { + i = pBatch->iDoneUpTo; /* all messages below that index are processed */ + iElemProcessed = 0; + iCommittedUpTo = i; + while(iElemProcessed <= *pnElem && i < pBatch->nElem) { pMsg = (msg_t*) pBatch->pElem[i].pUsrp; -dbgprintf("actionCall..MULTIQUEUE: i: %d/%d, pMsg: %p\n", i, pBatch->nElem, pMsg); + dbgprintf("submitBatch: i:%d, batch size %d, to process %d, pMsg: %p\n", i, pBatch->nElem, *pnElem, pMsg);//remove later! localRet = actionProcessMessage(pAction, pMsg); dbgprintf("action call returned %d\n", localRet); + if(localRet == RS_RET_OK) { + /* mark messages as committed */ + while(iCommittedUpTo < i) { + pBatch->pElem[iCommittedUpTo++].state = BATCH_STATE_COMM; + } + } else if(localRet == RS_RET_PREVIOUS_COMMITTED) { + /* mark messages as committed */ + while(iCommittedUpTo < i - 1) { + pBatch->pElem[iCommittedUpTo++].state = BATCH_STATE_COMM; + } + pBatch->pElem[i].state = BATCH_STATE_SUB; + } else if(localRet == RS_RET_PREVIOUS_COMMITTED) { + pBatch->pElem[i].state = BATCH_STATE_SUB; + } else { + iRet = localRet; + FINALIZE; + } + ++i; + ++iElemProcessed; + } + +finalize_it: + if(pBatch->iDoneUpTo != iCommittedUpTo) { + *pnElem += iCommittedUpTo - pBatch->iDoneUpTo; + pBatch->iDoneUpTo = iCommittedUpTo; + } + RETiRet; +} + + +/* submit a batch for actual action processing. + * The first nElem elements are processed. This function calls itself + * recursively if it needs to handle errors. + * rgerhards, 2009-05-12 + */ +static rsRetVal +submitBatch(action_t *pAction, batch_t *pBatch, int nElem) +{ + int i; + int bDone; + rsRetVal localRet; + DEFiRet; + + assert(pBatch != NULL); + + bDone = 0; + do { + localRet = tryDoAction(pAction, pBatch, &nElem); + if( localRet == RS_RET_OK + || localRet == RS_RET_PREVIOUS_COMMITTED + || localRet == RS_RET_DEFER_COMMIT) { + /* try commit transaction, once done, we can simply do so as if + * that return state was returned from tryDoAction(). + */ + localRet = finishBatch(pAction); + } + + if( localRet == RS_RET_OK + || localRet == RS_RET_PREVIOUS_COMMITTED + || localRet == RS_RET_DEFER_COMMIT) { + bDone = 1; + } else if(localRet == RS_RET_SUSPENDED) { + ; /* do nothing, this will retry the full batch */ + } else if(localRet == RS_RET_ACTION_FAILED) { + /* in this case, the whole batch can not be processed */ + for(i = 0 ; i < nElem ; ++i) { + pBatch->pElem[++pBatch->iDoneUpTo].state = BATCH_STATE_BAD; + } + bDone = 1; + } else { + if(nElem == 1) { + pBatch->pElem[++pBatch->iDoneUpTo].state = BATCH_STATE_BAD; + bDone = 1; + } else { + /* retry with half as much. Depth is log_2 batchsize, so recursion is not too deep */ + submitBatch(pAction, pBatch, nElem / 2); + submitBatch(pAction, pBatch, nElem - (nElem / 2)); + bDone = 1; + } + } + } while(!bDone); /* do .. while()! */ + + RETiRet; +} + + +/* receive a batch and process it. This includes retry handling. + * rgerhards, 2009-05-12 + */ +static rsRetVal +processAction(action_t *pAction, batch_t *pBatch) +{ + int i; + msg_t *pMsg; + rsRetVal localRet; + DEFiRet; + + assert(pBatch != NULL); + + pBatch->iDoneUpTo = 0; + /* TODO: think about action batches, must be handled at upper layer! + * MULTIQUEUE + */ + localRet = submitBatch(pAction, pBatch, pBatch->nElem); + CHKiRet(localRet); + + /* this must be moved away - up into the dequeue part of the queue, I guess, but that's for another day */ + for(i = 0 ; i < pBatch->nElem ; i++) { + pMsg = (msg_t*) pBatch->pElem[i].pUsrp; msgDestruct(&pMsg); /* TODO: change: we are now finished with the message */ - CHKiRet(localRet); } iRet = finishBatch(pAction); finalize_it: RETiRet; } + + #pragma GCC diagnostic ignored "-Wempty-body" /* receive an array of to-process user pointers and submit them * for processing. * rgerhards, 2009-04-22 */ static rsRetVal -actionCallDoActionMULTIQUEUE(action_t *pAction, batch_t *pBatch) +processBatchMain(action_t *pAction, batch_t *pBatch) { int iCancelStateSave; DEFiRet; @@ -829,7 +937,7 @@ actionCallDoActionMULTIQUEUE(action_t *pAction, batch_t *pBatch) pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); pthread_setcancelstate(iCancelStateSave, NULL); - iRet = actionCallDoActionMULTIQUEUEprocessing(pAction, pBatch); + iRet = processAction(pAction, pBatch); pthread_cleanup_pop(1); /* unlock mutex */ diff --git a/runtime/batch.h b/runtime/batch.h index cb40cf42..fcbbafce 100644 --- a/runtime/batch.h +++ b/runtime/batch.h @@ -35,7 +35,8 @@ typedef enum { BATCH_STATE_RDY = 0, /* object ready for processing */ BATCH_STATE_BAD = 1, /* unrecoverable failure while processing, do NOT resubmit to same action */ BATCH_STATE_SUB = 2, /* message submitted for processing, outcome yet unkonwn */ - BATCH_STATE_DISC = 3, /* discarded - processed OK, but do not submit to any other action */ + BATCH_STATE_COMM = 3, /* message successfully commited */ + BATCH_STATE_DISC = 4, /* discarded - processed OK, but do not submit to any other action */ } batch_state_t; @@ -57,6 +58,7 @@ struct batch_obj_s { */ struct batch_s { int nElem; /* actual number of element in this entry */ + int iDoneUpTo; /* all messages below this index have state other than RDY */ batch_obj_t *pElem; /* batch elements */ }; diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 53a510b3..25f9eefe 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -281,6 +281,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ RS_RET_DEFER_COMMIT = -2121, /**< output plugin status: not yet committed (an OK state!) */ RS_RET_PREVIOUS_COMMITTED = -2122, /**< output plugin status: previous record was committed (an OK state!) */ + RS_RET_ACTION_FAILED = -2122, /**< action failed and is now suspended (consider this permanent for the time being) */ RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ /* RainerScript error messages (range 1000.. 1999) */ -- cgit v1.2.3 From 73b16a5d7703078a46d960bd8922d2ae3a662769 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 13 May 2009 13:22:25 +0200 Subject: added test for property replacer field functionality to testbench plus some cosmetic changes --- runtime/obj.c | 4 +--- tests/Makefile.am | 12 +++++++++++- tools/syslogd.c | 5 ++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/runtime/obj.c b/runtime/obj.c index 2a9df9ed..20b918eb 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -205,9 +205,7 @@ DestructObjSelf(obj_t *pThis) DEFiRet; ISOBJ_assert(pThis); - if(pThis->pszName != NULL) { - free(pThis->pszName); - } + free(pThis->pszName); RETiRet; } diff --git a/tests/Makefile.am b/tests/Makefile.am index b4509dee..087fa6af 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh da-mainmsg-q.sh diskqueue.sh manytcp.sh +TESTS = $(TESTRUNS) cfg.sh fieldtest.sh parsertest.sh omod-if-array.sh da-mainmsg-q.sh diskqueue.sh manytcp.sh TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -18,7 +18,15 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ err1.rstest \ NoExistFile.cfgtest \ testsuites/parse1.conf \ + testsuites/field1.conf \ testsuites/1.parse1 \ + testsuites/2.parse1 \ + testsuites/3.parse1 \ + testsuites/date1.parse1 \ + testsuites/date2.parse1 \ + testsuites/date3.parse1 \ + testsuites/date4.parse1 \ + testsuites/date5.parse1 \ testsuites/rfc3164.parse1 \ testsuites/rfc5424-1.parse1 \ testsuites/rfc5424-2.parse1 \ @@ -26,7 +34,9 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rfc5424-4.parse1 \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ + testsuites/1.field1-if-array \ parsertest.sh \ + fieldtest.sh \ diskqueue.sh \ testsuites/diskqueue.conf \ da-mainmsg-q.sh \ diff --git a/tools/syslogd.c b/tools/syslogd.c index 7ee5dbd7..cae07811 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1155,6 +1155,7 @@ DEFFUNC_llExecFunc(processMsgDoActions) ABORT_FINALIZE(RS_RET_OK); } + /* MULTIQUEUE: look at this below! (I say: batch states!) */ iRetMod = actionCallAction(pAction, pDoActData->pMsg); if(iRetMod == RS_RET_DISCARDMSG) { ABORT_FINALIZE(RS_RET_DISCARDMSG); @@ -1170,7 +1171,9 @@ finalize_it: } -/* Process (consume) a received message. Calls the actions configured. +/* Process (consume) a received message from the main queue. Here, messages are + * filtered and those where the filter evaluates to true are passed to the action + * queue for further processing. * rgerhards, 2005-10-13 */ static void -- cgit v1.2.3 From 4a8c02870a55e19c1bebfae5cb70d1ec5aa7c203 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 13 May 2009 16:00:15 +0200 Subject: moved user object destruction to queue itself So far, the consumer was responsible for destroying objects. However, this does not work well with ultra-reliable queues. This is the first move to support them. --- action.c | 1 - runtime/msg.h | 2 +- runtime/queue.c | 40 +++++++++++++++++++++++++++++++++++++--- tools/syslogd.c | 1 - 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/action.c b/action.c index 0253e1b6..424cb00b 100644 --- a/action.c +++ b/action.c @@ -905,7 +905,6 @@ processAction(action_t *pAction, batch_t *pBatch) /* this must be moved away - up into the dequeue part of the queue, I guess, but that's for another day */ for(i = 0 ; i < pBatch->nElem ; i++) { pMsg = (msg_t*) pBatch->pElem[i].pUsrp; - msgDestruct(&pMsg); /* TODO: change: we are now finished with the message */ } iRet = finishBatch(pAction); diff --git a/runtime/msg.h b/runtime/msg.h index c8350626..14148441 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,7 +51,7 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; -short bDoLock; /* use the mutex? */ + short bDoLock; /* use the mutex? */ pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ diff --git a/runtime/queue.c b/runtime/queue.c index c3a8e9d4..6bea338a 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1390,13 +1390,47 @@ finalize_it: } +/* Delete a batch of processed user objects from the queue, which includes + * destructing the objects themself. The pointer piRemainingQueu + * rgerhards, 2009-05-13 + */ +static inline rsRetVal +DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch) +{ + int i; + void *pUsr; + DEFiRet; + + /* this is the place to destruct the old messages and pull them off the queue - MULTI-DEQUEUE */ + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pBatch != NULL); + + /* if the queue runs in DA mode, the DA worker already deleted the message. But + * in regular mode, we need to do it ourselfs. We differentiate between the two cases, + * because it is actually the easiest way to handle the destruct-Problem in a simple + * and pUsrp-Type agnostic way (else we would need an objAddRef() generic function). + */ + if(!pThis->bRunsDA) { + for(i = 0 ; i < pBatch->nElem ; ++i) { + /* TODO: pull msgs off the queue (not yet necessary) */ + pUsr = pBatch->pElem[i].pUsrp; + objDestruct(pUsr); + } + } + pBatch->nElem = 0; + + RETiRet; +} + + /* dequeue as many user points as are available, until we hit the configured * upper limit of pointers. * This must only be called when the queue mutex is LOOKED, otherwise serious * malfunction will happen. */ static inline rsRetVal -DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *iRemainingQueueSize) +DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSize) { int nDequeued; int iQueueSize; @@ -1405,6 +1439,7 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *iRemainingQueueSize DEFiRet; /* this is the place to destruct the old messages and pull them off the queue - MULTI-DEQUEUE */ + DeleteProcessedBatch(pThis, &pWti->batch); nDequeued = 0; do { @@ -1427,9 +1462,8 @@ dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); qqueueChkPersist(pThis, nDequeued); /* it is sufficient to persist only when the bulk of work is done */ - //bRunsDA = pThis->bRunsDA; /* cache this for after mutex release */ pWti->batch.nElem = nDequeued; - *iRemainingQueueSize = iQueueSize; + *piRemainingQueueSize = iQueueSize; finalize_it: RETiRet; diff --git a/tools/syslogd.c b/tools/syslogd.c index cae07811..88a588e9 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1230,7 +1230,6 @@ dbgprintf("msgConsumer..MULTIQUEUE: i: %d, pMsg: %p\n", i, pMsg); parseMsg(pMsg); } processMsg(pMsg); - msgDestruct(&pMsg); } dbgprintf("DONE msgConsumer..MULTIQUEUE:\n"); -- cgit v1.2.3 From 20d4eb3b476e45f0c7757d71d9fbaa9706fe7edb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 14 May 2009 18:52:55 +0200 Subject: worked on rsyslog design document (queue design enhancement) --- doc/design.tex | 224 +++++++++++++++++++++++++++++++++++- doc/rsyslog_queue_pointers.jpeg | Bin 0 -> 9226 bytes doc/rsyslog_queue_pointers2.jpeg | Bin 0 -> 20917 bytes doc/src/rsyslog_queue_pointers.dia | Bin 0 -> 1657 bytes doc/src/rsyslog_queue_pointers2.dia | Bin 0 -> 2911 bytes 5 files changed, 218 insertions(+), 6 deletions(-) create mode 100644 doc/rsyslog_queue_pointers.jpeg create mode 100644 doc/rsyslog_queue_pointers2.jpeg create mode 100644 doc/src/rsyslog_queue_pointers.dia create mode 100644 doc/src/rsyslog_queue_pointers2.dia diff --git a/doc/design.tex b/doc/design.tex index c9bfcdfc..2927c517 100644 --- a/doc/design.tex +++ b/doc/design.tex @@ -5,13 +5,17 @@ \usepackage{graphicx} \usepackage{listings} \usepackage{algorithm,algorithmic} +\usepackage{float} + +\pagestyle{headings} \newcommand{\IN}{\mathbb{N}} \newcommand{\MM}{\mathcal{M}} \newcommand{\QQ}{\mathcal{Q}} \newcommand{\AAA}{\mathcal{A}} \title{Rsyslog Design and Internals} -\author{Rainer Gerhards} +\author{Rainer Gerhards\\ +rgerhards@adiscon.com} \begin{document} @@ -24,16 +28,40 @@ This paper describes rsyslog design and internals. It is created to facilitate a \tableofcontents \section{Preliminaries} +\subsection{On the Use of English} +\begin{quotation} +\begin{flushright} +I ventured to write this book in English because ... \\ +it will be more easily read in poor English, \\ +than in good German by 90\% of my intended readers. \\ +--- HANS J. STETTER, Analysis of Discretization Methods for \\ +Ordinary Differential Equations (1973) +\end{flushright} +\end{quotation} + +There is not much I could add to Mr. Stetter's thought, except, maybe, that the number to quote probably tends more to 99\% in this case than to the 90\% Mr. Stetter notes. So please pardon those errors in language use that I have not yet been able to fix or even see. Suggestions for corrections and improvements are always welcome. \subsection{Notational Conventions} In general, in rsyslog there exists single objects $o$, which are used to build larger sets $O$, which form a superset $\mathcal{O}$ of all those objects that exist at a given time inside a running instance of rsyslog. As seen above, single objects are always described by lower case letters ($o$), larger sets by upper case letters ($O$) and the ``all-sets'' in caligraphic letters ($\mathcal{O}$). Often, objects $O_i, i \in \IN, i \le |\mathcal{O}|$ partition $\mathcal{O}$, but this is not necessarily the case. +\subsection{Definitions} +\subsubsection{Audit Grade} +In the context of this document, ``audit grade'' means that a subsystem never loses a message that it has taken responsibility for, not even in cases of sudden power failures. The only limit in this restriction is that a subsystem does not guarantee message survival if the subsytem at large is being destroyed (e.g. during a disaster) or some of its components are not of audit-grade. This draws a fine limitation on the audit-grade of a subsystem. + +For example, the rsyslog queue subsystem receives messages and acknowledges them to the submitter (e.g. an input), when they have been enqueued in the storage system. If the queue system is configured to provide audit-grade operation\footnote{Audit-grade queue operation is considerably slower than regular operations, as such this mode is not enabled by default. Most installations will never need a completely audit-grade queue}, the queue relies on the storage subsystem to work properly. If, for example, a disk read error occurs, the message may no longer be readable from the disk and as such is lost. The root cause here is that the disk subsystem was not of audit grade, because it otherwise would not have lost the message. So in this case the queue code is of audit grade, but the one of its components, the disk subsytem, was not. So the overall system is not of audit grade. + +To simplify talking about the audit-gradness of several subsytems, we assume that all of their subsystems are also of audit grade. In an actual deployment, however, this means the the system designer must carefully select audit-grade subsystems. Overlooking a single non-audit-grade component will make the whole system of not audit grade quality. + +Please note that it can be rather tricky to ensure a complete system is of audit grade. A border case is main memory integrity. Even with error-correcting memory, there may situations arise where a memory error occurs (probably due to a very unlikely series of well-hitting cosmic rays) that is unrecoverable. At this point, system integrity is at risk. The only real solution is to immediately shut down the system and restart it (without giving any process a chance to execute). Note, however, that in an extreme view, an operating system routine that does so can also be considered dangerous, as memory in use by this routine might be affected by the malfunction. We could extend this scenario and further complicate it, but that goes beyond the scope of this paper. The example was primarily meant to show how subtle audit-grade reliability is. + +In rsyslog, we currently use a slightly \marginpar{duplication\\permitted}relaxed consistency condition for message integrity inside an audit-grade subsystem. While we do not accept message loss, we permit slight message \emph{duplication}, but only in exceptional cases. This is permitted because, with proper message generation, the dulication problem can be easily fixed at the end-to-end layer. For example, the original sender can include a UUID, which can be used to sort out duplicates at the final destination. Insisting on not allowing duplication complicates matters and is often impossible with today's logging protocols. So, for the time being, we aim at this relaxed criteria, which is hard enough to achive. After we have achieved that goal, we may further try to solve the duplicaton problem. Some hooks already exist. But we do not guarantee such an effort will be made any time soon. + \section{Overall Design} From a high-level prespective, rsyslogd is ``just'' a high-performance message router. It accepts messages from various sources, applies user-configured filters to them, and routes potentially transformed messages to destinations based on these filters. \section{Objects} \subsection{Plugins} Plugins provide code potentially written by a third party to extend rsyslog. -Conceptually, a plugin is a tupel of callable functions $(\phi_1, \phi_2, \ldots)$ which implement an interface. There are three different types of plugins: input, output and libraray. The plugin type denotes the primary interface implemented by the plugin. Additional interfaces may be implemented\footnote{This is not yet done in plugins, but is possible and assumed to be done at a later point in time}. +Conceptually, a plugin is a tuple of callable functions $(\phi_1, \phi_2, \ldots)$ which implement an interface. There are three different types of plugins: input, output and library. The plugin type denotes the primary interface implemented by the plugin. Additional interfaces may be implemented\footnote{This is not yet done in plugins, but is possible and assumed to be done at a later point in time}. In the context of this paper, the output plugin interface is most important. It implements three entry points: @@ -51,7 +79,7 @@ Every instance of an output plugin is guaranteed \emph{not} to be called concurr \subsection{State Sets} Several object have associated state based on a specific state set. These state sets are described together with the objects. -As a general rule, individual state is associated with all intances $o$ of a class of objects. This state is called the object's \marginpar{state component} \emph{state component} $s$. If we want to obtain an object's state, we write $S(o)$. Please note that $S(o)$ is only defined for those objects that have a state component. +As a general rule, individual state is associated with all instances $o$ of a class of objects. This state is called the object's \marginpar{state component} \emph{state component} $s$. If we want to obtain an object's state, we write $S(o)$. Please note that $S(o)$ is only defined for those objects that have a state component. \subsection{Messages} A message $m$ represents a a single syslog message inside the system. It is a tuple of attributes. Some of these attributes directly orginate from the message content, some others are meta-information taken from the context. For example, there is an meta-attribute ``time of reception'' which conveys when the message was received by rsyslog's input subsystem. We do not list attributes here, as there are many and it is not of importance which exactly they are. @@ -83,7 +111,7 @@ A batch represents multiple processable messages. It is a unit of processing ins A batch $$B = (b_1, b_2, \ldots, b_n )$$ -is a $n$-tuple of \marginpar{processable message}processable messages +is a $n$-tuple of \marginpar{processable\\message}processable messages $$b = (m, s)$$ which are an ordered pair of a message $m$ and an associated processing state $s$. To denote the $n$-th message inside the batch, we write $m(b_n)$, to denote the status component of the $n$-th message, we write $S(b_n)$. @@ -137,11 +165,11 @@ Various objects keep state. Some of these objects, like messages, batches and ac \subsubsection{Actions} Actions are provided by output plugins. An action enables the engine to write messages to some destination. It is important to note that ``destination'' is a very broad abstraction. A destination may be a file inside a local or remote file system, a database table or a remote syslog server in another network. -Actions are transactional in the following sense: more than one message can be submitted to an action. The action does not necessarily process the submitted messages unless the caller ends the transaction. However, the action itself may also end the transaction and notify the caller. This is \emph{not} considered an error condition and \emph{must} be handeled gracefully by the caller. If an transaction aborts, the caller \emph{must} assume that none of the elements submitted since the begin of transaction have been processed. The action will try to backout anything that was already processed at the time the transaction failed. However, not all outputs work on actually transactional destination. As such, an action is permitted not to backout incomplete interim results. As such, after a transaction abort, some message duplication may occur. We call this the \emph{relaxed integrity condition} for actions. +Actions are transactional in the following sense: more than one message can be submitted to an action. The action does not necessarily process the submitted messages unless the caller ends the transaction. However, the action itself may also end the transaction and notify the caller. This is \emph{not} considered an error condition and \emph{must} be handled gracefully by the caller. If a transaction aborts, the caller \emph{must} assume that none of the elements submitted since the begin of transaction have been processed. The action will try to backout anything that was already processed at the time the transaction failed. However, not all outputs work on actually transactional destination. As such, an action is permitted not to backout incomplete interim results. As such, after a transaction abort, some message duplication may occur. We call this the \emph{relaxed integrity condition} for actions. An output transaction is started by calling \emph{beginTransaction()} either explicitely or implicitely by a call to \emph{doAction()} without calling \emph{beginTransaction()} before. Then, one or more calls to \emph{doAction()} follow. When the caller intends to finish the transaction, it calls \emph{endTransaction()}. However, the transaction may also be terminated from the action itself in response to a \emph{doAction()} call. -Mathematical, an action transaction builds a totally orderred set of uncommitted messages $M_u$. The order relation is defined over the sequence in which messages are being provided to \emph{doAction()}. At any time a commit is attempted, the full set $M_u$ is committed and may either succeeed completely or not at all (in the sense of the relaxed integrity condition described above). +Mathematically, an action transaction builds a totally ordered set of uncommitted messages $M_u$. The order relation is defined over the sequence in which messages are being provided to \emph{doAction()}. At any time a commit is attempted, the full set $M_u$ is committed and may either succeeed completely or not at all (in the sense of the relaxed integrity condition described above). A commit is attempted when \begin{enumerate} @@ -491,4 +519,188 @@ However, I also know that for regulatory requirements, you often seem to need to My view of reliability is much the same as my view of security: there is no such thing as "being totally secure", you can just reduce the probability that something bad happens. The worst thing in security is someone who thinks he is "totally secure" and as such is no longer actively looking at potential issues. The same I see for reliability. There is no thing like "being totally reliable" and it is a really bad idea to think you could ever be. Knowing this, one may begin to think about how to decrease the overall probability of message loss AND think about what rate is acceptable (and what to do with these cases, e.g. "how can they hurt"). + +\paragraph{Different Use Cases} +As David Lang pointed out, there exist different use cases for different levels of reliability. Most importantly, there exist use cases that do not demand very high throughput but rather ultra-realiability of the queue system. Here, ultra-reliability is just another word for the queue being of ``audit-grade''. Even if the queue provides audit-grade, the overall system is only then of audit-grade when all other components - most notably the transport protocols spoken by the inputs and outputs - are also of audit-grade. Most importantly, this means that an audit-grade system purely based on the IETF syslog protocol series can not be build. + +Used together with truly reliable protocols \emph{and} senders that block processing until a final acknowledgement has been received, an audit-grade system can potentially build based on rsyslog. To do so, an audit-grade queue subsystem is required, which is not present in releases less than 4.1.? (most importantly, v2 and v3 do not provide this capability). + +\subsection{Audit-Grade Queue Operations} +\subsubsection{Perquisites} +Audit-grade queue operations certain perquisites: +\begin{itemize} +\item rsyslog engine is of version 4.1.? or greater +\item disk-only queue type +\item checkpoint interval set to 1 +\item queue is configured to not permit losing any messages\footnote{The queue has several settings that can be used to fine-tune situations in which it may discard messages intentionally. All of these must be turned off. Most importantly, that means the producer is blocked for an infinite time if the queue is full.} +\item queue consumer must also be of audit-grade +\end{itemize} +Only when these prequisites are met, queue operation can be considered of being audit-grade. Note that when message loss in case of sudden power failure and similar incidents is acceptable, neither disk-only queues nore a checkpoint interval of 1 is necessary. Such a configuration can also be build with rsyslog v3, which is up to that level. + +Note that in the sections below we describe the implementation in broader terms. Most importantly, we do not restrict ourselves to disk-only queue storage drivers. This is important, because it simplifies design and opens the capability to introduce new, possibly faster-performing, queue storage drivers in the future. + +But it is important to keep in mind that a concrete queue is only of audit-grade if it matches all the perquisites given here, most importantly with the right configuration. + +\subsubsection{Implementation} +Messages are enqueued by the queue producer (either an input module or the main message queue's consumer). The enqueue operation is completed only when the message has been successfully accepted by the queue storage driver. Then and only then the producer is permitted to remove the queue from its own storage system. A rough sketch is given in algorithm \ref{alg_q_enq}. + +\begin{algorithm} +\caption{enqueueObject($o$)} +\begin{algorithmic} +\label{alg_q_enq} +\STATE lock queue mutex +\WHILE{queue is !ready for enqueue} + \STATE wait on queue to become ready +\ENDWHILE +\STATE call queue store driver to add $o$ +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + +The dequeue-operation is more complex. We must ensure that each object stays in the queue until it is finally processed. Hereby, an object is finally processed, when processing of it has been completed. Remeber that to enhance performance objects are dequeued in batches of many. So at any given time, multiple messages may be processed, but not necessarily have finally completed doing so. If another worker thread then tries to obtain a new batch for processing, those ``in-process'' message must not be handed out a second time. Also, if a sudden power loss occurs during processing, queue operation must restart at the point of last commit. This means that all ``in-process'' messages need to be changed back to ``no processed'' state and be restarted again. In those cases the (acceptable) slight message duplication can occur. + +In our design, we differentiate between ``logical'' and ``physical'' dequeuing of batches. If a batch is generated for processing, it is logically dequeued --- in the sense that no other batch generating request will be able to receive another copy of these messages. If no exceptional situation happens, those messages will be processed and thus can be considered consumed under normal circumstances. + +However, actual deletion from the physical queue storage happens only after the batch is fully processed. At this point, all objects have been acknowledged by their destinations, which now have the responsibility for the object's survival. Consequently, we can delete them from the queue store. This process we call ``physical dequeue''. A first idea is given in algorithm \ref{alg_pdeq_batch_1} (remember that $O(b)$ contains all objects within the given batch $b$). + +\begin{algorithm} +\caption{physDequeueBatch($b$), first approach} +\begin{algorithmic} +\label{alg_pdeq_batch_1} +\STATE lock queue mutex +\FORALL{$o \in O(b)$} + \STATE find $o$ in queue storage + \STATE remove $o$ and keep queue structures intact +\ENDFOR +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + +This algorithm is simple, but requires searching the queue store for the object to be dequeued -- a potentially lengthy operation. However, we can improve the searching process if we know more about the inner structure of batch objects. It seems appropriate to logically dequeue objects in queue-sequential order. A drawback of doing so is that we must prevent other worker threads from trying to dequeue concurrently. This is not really a drawback. We need to guard dequeue operations by a mutex in any case, because otherwise internal structures can not be kept consistent. Practical experience and testing have shown that many small dequeue operations cause a lot of locking contention and as such badly affect performance. So it actually is a welcome enhancement to aquire the queue lock only once for the whole batch dequeue operation. As dequeing is a comperatively fast operation, the lock is not held for extended periods of time. + +A first approach to this functionality is shown in algorithm \ref{alg_ldeq_batch_1}. Note that $C_mBatch$ is the configured maximum number of elements inside a batch, $i$ is an index to address the objects inside the batch. + +\begin{figure}[h] +\begin{center} +\includegraphics[scale=0.6]{rsyslog_queue_pointers.jpeg} +\end{center} +\caption{\textbf{Queue Store Pointers}: boxes represent queue entries, colored boxes entries with objects. Objects in green are unprocessed, in blue are logically but not physicalled dequeued and those in gray are physically dequeued. White indicates not yet used entries. Gray objects may be overwritten at any time. Their entries are actually free, we have used the gray color primarily to indicate there once existed objects. Each queue pointer points to the next entry to process.} +\label{fig_queue_ptr} +\end{figure} + +\begin{algorithm} +\caption{logicDequeueBatch($b$)} +\begin{algorithmic} +\label{alg_ldeq_batch_1} +\STATE lock queue mutex +\STATE $0 \to i$ +\WHILE{while queue non-empty and $i < C_mBatch$} + \STATE obtain next obj $o$ from queue store + \STATE advance logical dequeue position + \STATE put $o$ into batch +\ENDWHILE +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} + +A key concept is somewhat hidden in \marginpar{queue pointers} \emph{advance logical dequeue position}. Each queue store is purely sequential, with objects being enqueued at one ``end'' of the store and dequeued at the other. Of course, each queue store has only finite capacity, but we ignore this to explain the overall picture. A queue can be implemented by two pointers: one that points to the tail of the queue, where new messages are enqueued and one that points to the head of it, where new messages are dequeued. The idea is now to duplicate the dequeue pointer and split it into one for logical and one for physical dequeueing. Figure \ref{fig_queue_ptr} shows this three-pointer approach. Now, we can simple advance either the logical or physical dequeue pointer, depending on operation, and do not need to find the first dequeue position inside the queue store. The logical dequeue pointer always points at it. This mode can be implemented with all currently existing queue storage drivers (but the sequential disk driver may need to use a second file handle or stream object instead of two pointers). + +This makes an efficient implementation of algorithm \ref{alg_ldeq_batch_1} possible: when it logically dequeues, it just needs to advance the logical dequeue pointer. So the algorithm executes in $O(n)$ time where $n$ specifies the number of elements to dequeue with an upper bound of $C_mBatch$. + +\begin{figure}[h] +\begin{center} +\includegraphics[scale=0.6]{rsyslog_queue_pointers2.jpeg} +\end{center} +\caption{\textbf{Physically Dequeueing Messages}: In this sample, we have two batches. With multiple workers, they may be physically dequeued at any time.} +\label{fig_queue_ptr_deq} +\end{figure} + +Furthermore, we can also improve algorithm \ref{alg_pdeq_batch_1}: Consider that each batch is logically dequeued as an atomic operation. That means all batch objects form a sequential subset of the queue. Figure \ref{fig_queue_ptr_deq} shows the situation when two batches have been logically dequeued. So the costly ``find'' operation now needs to be carried out only once at the beginning of the batch. As all other objects are sequential, once we have found the batch begin inside the queue, we can simply physically dequeue the $|b|$ elements in queue-sequential order after it. So the cost of the find operation can be reduced from $O(|b|)$ to $O(1)$. + +We can even reduce the remaining cost of the find operation. If the batch to be physically dequeued is right at the queue's head (as as ``B1'' in the figure), the find immediately terminates with the first element and incurs no cost at all. The situation is different if the batch is not at the queue head, ``B2'' is an example for that (assuming that ``B1'' has not yet been dequeued). We would now still need to search over the objects that are not part of the batch and can then finally get to the object at the head of the batch in question. For queue storage drivers that support random access to queue elements, storing a simple pointer to the batches' queue head element further improves the situation and enables $O(1)$ access to the queue element. This is indicated by the dotted lines in figure \ref{fig_queue_ptr_deq}. Once the head of the queue has been found, two things can happen (depending on the capabilities of the queue storage driver): + +\begin{enumerate} +\item the head element can be flagged as ``this and next $n$ elements are deleted'' +\item all elements are actually deleted +\end{enumerate} + +Note that a mixed form is also possible (and probably useful for our \emph{singly} linked list storage driver: there, some $n'$ elements be actually deleted and the head element is flagged as ``this and next $n - n'$ elements are deleted''. Note that in the linked-list case, all but the first elements can be deleted with ease, so probably just the head would stay inside the queue. Note that removing elements off the queue, where possible, is useful because it frees resources. On a busy system, freeing messages as soon as possible can prevent message loss (in non-audit-grade setup) or system slowdown. So it should be done when possible. + +If we have a purely sequential queue storage driver (currently the sequential disk driver), finding and updating the head element is not an option. Even in this case, we can observe that the batch at the actual physical dequeue pointer will eventually be submitted for dequeuing. So a route to take is to create a list of elements that can be deleted as soon as the physical dequeue pointer reaches any of these elements. We call this the \marginpar{to-delete list}``to-delete list''. To facilitate processing, this list must be ordered in sequence of logical dequeing. This information may not be available from the storage subsystem itself, but it can easily be generated. To do so, a strictly monotonically increasing counter is kept with each logical dequeue operation and stored as part of the batch\footnote{As this must be done via the usual computer-implemented modular arithmetic, we must be careful that we do not see repetion of values because of overflows. Each day has $60 \cdot 60 \cot 24 = 86,400$ seconds (ignoring the subleties of UTC). Now let's assume that we have a moderately-busy system with 1,000 messages per second. We further assume, to be on the save side, that each message is processed inside its own batch. So we have $86,400,000$ batches per day. If we now use a typical $32$-bit integer for generating the batch IDs, we the unique range will be used up after +$$\frac{2^{32}}{8640000} \approx 497 \text{ days}$$ +days of uninterrupted rsyslog operation. While this sounds somewhat save, it goes down to approximately 10 days of messages are submitted at rate of 50,000 messages per second (which is high, but not unheared of). So it is strongly advised to use 64 bits, which we consider to be save, because for our 1,000 messages per second the range would be exhausted only after +$$\frac{2^{64}}{8640000} \approx 2.135 \cdot 10^{11} \text{ days}$$ +which equals approximately $584,500,000$ \emph{years}. So even at a rate of one million messages per second, the range would be sufficient for over 500,000 years of continuos operations -- that should be far sufficient.} +An example: let us assume that ``B2'' was submitted for physical dequeueing first. Then, the head of ``B2'' is not at the queue's physical dequeue pointer. As such, no action can be carried out immediately. So the batch head pointer is stored into a ``to be deleted'' list. Processing continues. Some time later, batch ``B1'' is submitted for deletion. Now, the head pointer is at the head of the physical dequeue list, as such all batch elements are dequeued. Then, the ``to be deleted'' list is checked, and ``B2'' is found in it. Now, ``B2'' is at the head of the (new) physical dequeue pointer and can also be removed. So, ultimately, all messages are physically dequeued. This is more formally describe in algorithm \ref{alg_phys_deq_seq_store}. In that pseudocode, we made a simplification by always putting the to be deleted batch in the ``to-delete'' list, which then enables us to use somewhat more generic code to carry out the work. + +Note that there is a price to pay for deletions via the ``to-delete'' list: if a sudden power failure happens during processing, the set of duplicate messages is increased. For example, if power fails after ``B2'' has been fully processed and scheduled for deletion, but \emph{before ``B1'' is also submitted for deletion}, ``B2'' will be reprocessed after recovery. This would not happen if ``B2'' would have been removed from the queue. + +\begin{algorithm} +\caption{deleteBatch($b$)} +\begin{algorithmic} +\label{alg_phys_deq_seq_store} +\REQUIRE queue mutex is locked by caller +\STATE enqueue $b.head, |b|$ in ``to-delete'' list $D$ +\COMMENT ``to-delete'' list must be in order of logical dequeue +\WHILE{$D.head = Q.pysDeqPtr$} + \FOR{$|b|$ elements} + \STATE delete element at queue head + \STATE move $q.pysDeqPtr$ + \ENDFOR + \STATE remove head of ``to-delete'' list +\ENDWHILE +\end{algorithmic} +\end{algorithm} + +\paragraph{Warp-Up of Queue Delete Operations} +When evaluating which route to take, the ``to-delete'' list approach looks elegant for all cases. The negative side effect of potentially increased message duplication currently does not even exist: today, the sequential disk queue storage driver permits only a single worker thread and thus there will always only be one thread at a time. Even if we remove that limitation, message duplication could not be avoided, as stated in the algorithm description above. What remains are the other queue storage drivers. however, they operate in-memory, so message duplication will not happen simply because all messages will be lost on sudden fatal failure. The advantage of limited message duplication only exists in the so-far hypothetical case of a random-access, audit-grade disk queue storage driver. Thus, the decision could be postponed unless that happens (if it ever does). + +From a code complexity point of view, the ``to-delete'' list approch is definitely advantagous. Not only because of the reduced number of algorithms required. We also do not need to maintain unique batch IDs and all the logic associated with them. + +The other aspect to look at is memory consumption. Assuming that we delete the actual objects, just not their containers inside the queue, extra memory consumption is not really that worse. More importantly, currently only the linked-list queue storage driver can benefit at all, because it is the only driver capable of deleting queue entries in mid-queue. All others, including the array memory driver, do not have this capability. + +From a performance point of view, the ``to delete'' list approach looks approximately as good as the others, with some mild better performance for some storage drivers for a non-``to delete'' list approach. This can be mitigated, especially if the potentially somewhat-costly maintenance of the ``to-delete'' list is slightly optimized and the algorithm actually checks if the to be deleted batch is right at the queues delete pointer position. The improved code simplicity, together with current CPU's code caching, may even result in an otherwise not expected speedup. + +In conclusion, we will implement the ``to-delte'' list approach on the queue layer (above the queue storage drivers). However, we will leave the window open to permit overwriting it with queue storage driver specific functionality. How to do this will not be specified now, as there is currently no need and we do not even know if there ever will be. However, we retain the discussion on the various modes as well as the relevant algorithmic discussions and data structurs inside this paper so that it is readily available should need arise. We also think this is important so that everybody later knows that the decision was made based on good argument and not by accident (we consider this useful in another design enhancement attempt). + + +\subsubsection{Queue Stores} +Currently, rsyslog supports three different types of queue store drivers: + +\begin{itemize} +\item memory array +\item memory linked list +\item disk sequential file +\end{itemize} + +They all provide an abstracted sequential queue store as shown in figure \ref{fig_queue_ptr} on page \pageref{fig_queue_ptr}. + +Obviously, some differences exist. Most importantly, the disk sequential file driver does \emph{not} support more than one queue worker thread (in order to prevent excessive disk activity and the subtle issues with rewriting parts of sequential files). So if this driver is used, the queue automatically limits itself to a maximum of one worker thread (even if user configuration settings + +Different queue store drivers have different properties: + +\begin{tabular}{|l||l|l|l|}\hline + & array & linked list & seqential file \\ \hline +pointer type & integer index & memory address & file number and \\ + & & & offset within file \\ \hline +physical access & random & random & sequential \\ \hline +remove middle & no & yes & no \\ +elements & & & \\ \hline +access to $n$-th& $O(1)$, index:& $O(n)$, follow & not supported \\ +element & $n \mod C_{mMsg}$ & pointer links & \\ \hline +speed & fastest & fast & slow \\\hline +mem overhead & large & some & almost none \\\hline +reliability & reliable & reliable & audit-grade\footnote{if configured correctly}\\ +\hline +\end{tabular} + +\subsubsection{Checkmarks} +The following things need to be verified in the actual implementation. + +\paragraph{Queue Full} +Is it possible to set an infinte timeout on queue full condition during enqueue? If not, we must provide it. + +\paragraph{Terminatin the Queue} +If we cancel a worker, we need to start from the physical dequeue pointer and pull everything that is not scheduled for deletion - NOT from the logical dequeue pointer. + \end{document} diff --git a/doc/rsyslog_queue_pointers.jpeg b/doc/rsyslog_queue_pointers.jpeg new file mode 100644 index 00000000..809dd446 Binary files /dev/null and b/doc/rsyslog_queue_pointers.jpeg differ diff --git a/doc/rsyslog_queue_pointers2.jpeg b/doc/rsyslog_queue_pointers2.jpeg new file mode 100644 index 00000000..069301e3 Binary files /dev/null and b/doc/rsyslog_queue_pointers2.jpeg differ diff --git a/doc/src/rsyslog_queue_pointers.dia b/doc/src/rsyslog_queue_pointers.dia new file mode 100644 index 00000000..2ad4cacb Binary files /dev/null and b/doc/src/rsyslog_queue_pointers.dia differ diff --git a/doc/src/rsyslog_queue_pointers2.dia b/doc/src/rsyslog_queue_pointers2.dia new file mode 100644 index 00000000..e630e33e Binary files /dev/null and b/doc/src/rsyslog_queue_pointers2.dia differ -- cgit v1.2.3 From b81311ac70e4de0bd5c0b0286413ff1b527ef906 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 15 May 2009 16:39:27 +0200 Subject: worked more on the design document --- doc/design.tex | 209 +++++++++++++++++++++++++++++++----- doc/queue_msg_state.dot | 25 +++++ doc/queue_msg_state.jpeg | Bin 0 -> 12499 bytes doc/rsyslog_queue_pointers2.jpeg | Bin 20917 -> 20459 bytes doc/src/rsyslog_queue_pointers2.dia | Bin 2911 -> 2899 bytes 5 files changed, 205 insertions(+), 29 deletions(-) create mode 100644 doc/queue_msg_state.dot create mode 100644 doc/queue_msg_state.jpeg diff --git a/doc/design.tex b/doc/design.tex index 2927c517..7c0641b8 100644 --- a/doc/design.tex +++ b/doc/design.tex @@ -44,8 +44,15 @@ There is not much I could add to Mr. Stetter's thought, except, maybe, that the In general, in rsyslog there exists single objects $o$, which are used to build larger sets $O$, which form a superset $\mathcal{O}$ of all those objects that exist at a given time inside a running instance of rsyslog. As seen above, single objects are always described by lower case letters ($o$), larger sets by upper case letters ($O$) and the ``all-sets'' in caligraphic letters ($\mathcal{O}$). Often, objects $O_i, i \in \IN, i \le |\mathcal{O}|$ partition $\mathcal{O}$, but this is not necessarily the case. \subsection{Definitions} +\subsubsection{Sudden Fatal Failure} +As sudden fatal failure is one that occurs at some instant and causes Complete loss of processing capabilities. The two major cases are a sudden power loss or a ``kill -9'' of the process. There are more exotic cases, too, like disasters. + +One may argue that it is possible to protect against many sudden fatal failure cases. For example, using an uninterruptable power supply (UPS) will prevent a sudden power loss. While this is true in most cases, it does not hold if looked very closely: in the case of the UPS, for example, a failure in the UPS itself may cause a sudden power loss, which can not be mitigated. Well, actually there can be several layers of mitigation, but always one more potential failure scenario remains. So it is not possible to totally solve the issue. + +The concept of ``sudden fatal failure'' now covers all these rest risk that result in termiantion of rsyslogd without the ability execute any code before this happens. This is a very important concept in regard to audit-gradeness. + \subsubsection{Audit Grade} -In the context of this document, ``audit grade'' means that a subsystem never loses a message that it has taken responsibility for, not even in cases of sudden power failures. The only limit in this restriction is that a subsystem does not guarantee message survival if the subsytem at large is being destroyed (e.g. during a disaster) or some of its components are not of audit-grade. This draws a fine limitation on the audit-grade of a subsystem. +In the context of this document, ``audit grade'' means that a subsystem never loses a message that it has taken responsibility for, not even in cases of sudden fatal failures. The only limit in this restriction is that a subsystem does not guarantee message survival if the subsytem at large is being destroyed (e.g. during a disaster) or some of its components are not of audit-grade. This draws a fine limitation on the audit-grade of a subsystem. For example, the rsyslog queue subsystem receives messages and acknowledges them to the submitter (e.g. an input), when they have been enqueued in the storage system. If the queue system is configured to provide audit-grade operation\footnote{Audit-grade queue operation is considerably slower than regular operations, as such this mode is not enabled by default. Most installations will never need a completely audit-grade queue}, the queue relies on the storage subsystem to work properly. If, for example, a disk read error occurs, the message may no longer be readable from the disk and as such is lost. The root cause here is that the disk subsystem was not of audit grade, because it otherwise would not have lost the message. So in this case the queue code is of audit grade, but the one of its components, the disk subsytem, was not. So the overall system is not of audit grade. @@ -498,7 +505,7 @@ back to the queue (think: ungetc()) when something goes wrong. Reasonable in unprocessed state outside of the queue. \paragraph{More reliable can actually be less reliable} -On the rsyslog mailing list, we had a discussion about how reliable rsyslog should be. It circles about a small potential window of message loss in the case of sudden power failure. Rsyslog can be configured to put all messages into a disk queue (instead of main memory), so these messages survive such a powerfail condition. However, messages dequeued and scheduled for processing during the power outage may be lost. +On the rsyslog mailing list, we had a discussion about how reliable rsyslog should be. It circles about a small potential window of message loss in the case of sudden fatal failure. Rsyslog can be configured to put all messages into a disk queue (instead of main memory), so these messages survive such a powerfail condition. However, messages dequeued and scheduled for processing during the power outage may be lost. I now consider a case where we have bursty UDP traffic and rsyslog is configured to use a disk-only queue (which obviously is much slower than an in-memory queue). Looking at processing speeds, the max burst rate is limited by using an ultra-reliable queue. To avoid using UDP messages, a second instance could be run that uses an in-memory queue and forwards received messages to the one in ultra-reliable mode (that is with the disk-only queue). So that second instance queues in memory until the (slower) reliable rsyslogd can now accept the message and put it into the reliable queue. Let's say that you have a burst of $r$ messages and that from these burst only $r/2$ can be enqueued (because the ultra reliable queue is so slow). So you lose $r/2$ messages. @@ -535,21 +542,21 @@ Audit-grade queue operations certain perquisites: \item queue is configured to not permit losing any messages\footnote{The queue has several settings that can be used to fine-tune situations in which it may discard messages intentionally. All of these must be turned off. Most importantly, that means the producer is blocked for an infinite time if the queue is full.} \item queue consumer must also be of audit-grade \end{itemize} -Only when these prequisites are met, queue operation can be considered of being audit-grade. Note that when message loss in case of sudden power failure and similar incidents is acceptable, neither disk-only queues nore a checkpoint interval of 1 is necessary. Such a configuration can also be build with rsyslog v3, which is up to that level. +Only when these prequisites are met, queue operation can be considered of being audit-grade. Note that when message loss in case of sudden fatal failure and similar incidents is acceptable, neither disk-only queues nore a checkpoint interval of 1 is necessary. Such a configuration can also be build with rsyslog v3, which is up to that level. Note that in the sections below we describe the implementation in broader terms. Most importantly, we do not restrict ourselves to disk-only queue storage drivers. This is important, because it simplifies design and opens the capability to introduce new, possibly faster-performing, queue storage drivers in the future. But it is important to keep in mind that a concrete queue is only of audit-grade if it matches all the perquisites given here, most importantly with the right configuration. -\subsubsection{Implementation} -Messages are enqueued by the queue producer (either an input module or the main message queue's consumer). The enqueue operation is completed only when the message has been successfully accepted by the queue storage driver. Then and only then the producer is permitted to remove the queue from its own storage system. A rough sketch is given in algorithm \ref{alg_q_enq}. +\subsubsection{Implementation Alternatives} +Messages, or more precisely objects\footnote{While rsyslog deals with messages, the queue is designed to handle any type of thing that is represented as an rsyslog object. This is considered useful as queues may at some time contain other things than just messages, so we keep it generic.}, are enqueued by the queue producer (either an input module or the main message queue's consumer). The enqueue operation is completed only when the message has been successfully accepted by the queue storage driver. Then and only then the producer is permitted to remove the object from its own storage system. A rough sketch is given in algorithm \ref{alg_q_enq}. \begin{algorithm} \caption{enqueueObject($o$)} \begin{algorithmic} \label{alg_q_enq} \STATE lock queue mutex -\WHILE{queue is !ready for enqueue} +\WHILE{queue is not ready for enqueue} \STATE wait on queue to become ready \ENDWHILE \STATE call queue store driver to add $o$ @@ -557,14 +564,18 @@ Messages are enqueued by the queue producer (either an input module or the main \end{algorithmic} \end{algorithm} -The dequeue-operation is more complex. We must ensure that each object stays in the queue until it is finally processed. Hereby, an object is finally processed, when processing of it has been completed. Remeber that to enhance performance objects are dequeued in batches of many. So at any given time, multiple messages may be processed, but not necessarily have finally completed doing so. If another worker thread then tries to obtain a new batch for processing, those ``in-process'' message must not be handed out a second time. Also, if a sudden power loss occurs during processing, queue operation must restart at the point of last commit. This means that all ``in-process'' messages need to be changed back to ``no processed'' state and be restarted again. In those cases the (acceptable) slight message duplication can occur. +The dequeue-operation is more complex. We must ensure that each object stays in the queue until it is finally processed. Hereby, an object is finally processed, when processing of it has been completed. Remember that to enhance performance, objects are dequeued in batches of many. So at any given time, multiple messages may be processed, but not necessarily have finally completed doing so. If another worker thread then tries to obtain a new batch for processing, those ``in-process'' message must not be handed out a second time. Also, if a sudden fatal failure occurs during processing, queue operation must restart at the point of last commit. This means that all ``in-process'' messages need to be changed back to ``no processed'' state and be restarted again. In those cases the (acceptable) slight message duplication can occur. In our design, we differentiate between ``logical'' and ``physical'' dequeuing of batches. If a batch is generated for processing, it is logically dequeued --- in the sense that no other batch generating request will be able to receive another copy of these messages. If no exceptional situation happens, those messages will be processed and thus can be considered consumed under normal circumstances. -However, actual deletion from the physical queue storage happens only after the batch is fully processed. At this point, all objects have been acknowledged by their destinations, which now have the responsibility for the object's survival. Consequently, we can delete them from the queue store. This process we call ``physical dequeue''. A first idea is given in algorithm \ref{alg_pdeq_batch_1} (remember that $O(b)$ contains all objects within the given batch $b$). +However, actual deletion from the physical queue storage happens only after the batch is fully processed. At this point, all objects have been acknowledged by their destinations, which now have the responsibility for the object's survival. Consequently, we can delete them from the queue store. This process is considered the ``physical'' dequeue of the object. + +In order to find some simpler terms, we will call the logical dequeue operation just ``dequeue'' and the physical dequeue operation ``delete''. This is consistent with all previous work on rsyslog and thus probably leads to the least surprise when reading older source code and documentation. + +A first idea for a deletion is given in algorithm \ref{alg_pdeq_batch_1} (remember that $O(b)$ contains all objects within the given batch $b$, this is \emph{not} $O$-notation and should probably in the future be replaced by something else). \begin{algorithm} -\caption{physDequeueBatch($b$), first approach} +\caption{deleteBatch($b$), first approach} \begin{algorithmic} \label{alg_pdeq_batch_1} \STATE lock queue mutex @@ -576,25 +587,25 @@ However, actual deletion from the physical queue storage happens only after the \end{algorithmic} \end{algorithm} -This algorithm is simple, but requires searching the queue store for the object to be dequeued -- a potentially lengthy operation. However, we can improve the searching process if we know more about the inner structure of batch objects. It seems appropriate to logically dequeue objects in queue-sequential order. A drawback of doing so is that we must prevent other worker threads from trying to dequeue concurrently. This is not really a drawback. We need to guard dequeue operations by a mutex in any case, because otherwise internal structures can not be kept consistent. Practical experience and testing have shown that many small dequeue operations cause a lot of locking contention and as such badly affect performance. So it actually is a welcome enhancement to aquire the queue lock only once for the whole batch dequeue operation. As dequeing is a comperatively fast operation, the lock is not held for extended periods of time. +This algorithm is simple, but requires searching the queue store for the object to be deleted -- a potentially lengthy operation. However, we can improve the searching process if we know more about the inner structure of batch objects. It seems appropriate to dequeue objects in queue-sequential order. A drawback of doing so is that we must prevent other worker threads from trying to dequeue concurrently. This is not really a drawback. We need to guard dequeue operations by a mutex in any case, because otherwise internal structures can not be kept consistent. Practical experience and testing have shown that many small dequeue operations cause a lot of locking contention and as such badly affect performance. So it actually is a welcome enhancement to aquire the queue lock only once for the whole batch dequeue operation. As dequeing is a comperatively fast operation, the lock is not held for extended periods of time. -A first approach to this functionality is shown in algorithm \ref{alg_ldeq_batch_1}. Note that $C_mBatch$ is the configured maximum number of elements inside a batch, $i$ is an index to address the objects inside the batch. +A first approach to this functionality is shown in algorithm \ref{alg_ldeq_batch_1}. Note that $C_{mBatch}$ is the configured maximum number of elements inside a batch, $i$ is an index to address the objects inside the batch. \begin{figure}[h] \begin{center} \includegraphics[scale=0.6]{rsyslog_queue_pointers.jpeg} \end{center} -\caption{\textbf{Queue Store Pointers}: boxes represent queue entries, colored boxes entries with objects. Objects in green are unprocessed, in blue are logically but not physicalled dequeued and those in gray are physically dequeued. White indicates not yet used entries. Gray objects may be overwritten at any time. Their entries are actually free, we have used the gray color primarily to indicate there once existed objects. Each queue pointer points to the next entry to process.} +\caption{\textbf{Queue Store Pointers}: boxes represent queue entries, colored boxes entries with objects. Objects in green are unprocessed, in blue are dequeued but not deleted and those in gray have already been deleted. White indicates not yet used entries. Gray objects may be overwritten at any time. Their entries are actually free, we have used the gray color primarily to indicate there once existed objects. Each queue pointer points to the next entry to process.} \label{fig_queue_ptr} \end{figure} \begin{algorithm} -\caption{logicDequeueBatch($b$)} +\caption{dequeueBatch($b$)} \begin{algorithmic} \label{alg_ldeq_batch_1} \STATE lock queue mutex \STATE $0 \to i$ -\WHILE{while queue non-empty and $i < C_mBatch$} +\WHILE{queue non-empty and $i < C_{mBatch}$} \STATE obtain next obj $o$ from queue store \STATE advance logical dequeue position \STATE put $o$ into batch @@ -603,37 +614,37 @@ A first approach to this functionality is shown in algorithm \ref{alg_ldeq_batch \end{algorithmic} \end{algorithm} -A key concept is somewhat hidden in \marginpar{queue pointers} \emph{advance logical dequeue position}. Each queue store is purely sequential, with objects being enqueued at one ``end'' of the store and dequeued at the other. Of course, each queue store has only finite capacity, but we ignore this to explain the overall picture. A queue can be implemented by two pointers: one that points to the tail of the queue, where new messages are enqueued and one that points to the head of it, where new messages are dequeued. The idea is now to duplicate the dequeue pointer and split it into one for logical and one for physical dequeueing. Figure \ref{fig_queue_ptr} shows this three-pointer approach. Now, we can simple advance either the logical or physical dequeue pointer, depending on operation, and do not need to find the first dequeue position inside the queue store. The logical dequeue pointer always points at it. This mode can be implemented with all currently existing queue storage drivers (but the sequential disk driver may need to use a second file handle or stream object instead of two pointers). +A key concept is somewhat hidden in \marginpar{queue pointers} \emph{advance logical dequeue position}. Each queue store is purely sequential, with objects being enqueued at one ``end'' of the store and dequeued at the other. Of course, each queue store has only finite capacity, but we ignore this to explain the overall picture. A queue can be implemented by two pointers: one that points to the tail of the queue, where new messages are enqueued and one that points to the head of it, where new messages are dequeued. The idea is now to duplicate the dequeue pointer and split it into one for (logical) dequeue and one for deletion. Figure \ref{fig_queue_ptr} shows this three-pointer approach. Now, we can simple advance either the dequeue or deletion pointer, depending on operation, and do not need to find the first dequeue position inside the queue store. The dequeue pointer always points at it. This mode can be implemented with all currently existing queue storage drivers (but the sequential disk driver may need to use a second file handle or stream object instead of two pointers). -This makes an efficient implementation of algorithm \ref{alg_ldeq_batch_1} possible: when it logically dequeues, it just needs to advance the logical dequeue pointer. So the algorithm executes in $O(n)$ time where $n$ specifies the number of elements to dequeue with an upper bound of $C_mBatch$. +This makes an efficient implementation of algorithm \ref{alg_ldeq_batch_1} possible: when it logically dequeues, it just needs to advance the dequeue pointer. So the algorithm executes in $O(n)$ time where $n$ specifies the number of elements to dequeue with an upper bound of $C_{mBatch}$. \begin{figure}[h] \begin{center} \includegraphics[scale=0.6]{rsyslog_queue_pointers2.jpeg} \end{center} -\caption{\textbf{Physically Dequeueing Messages}: In this sample, we have two batches. With multiple workers, they may be physically dequeued at any time.} +\caption{\textbf{Physically Dequeueing Messages}: In this sample, we have two batches. With multiple workers, they may be deleted in any order.} \label{fig_queue_ptr_deq} \end{figure} -Furthermore, we can also improve algorithm \ref{alg_pdeq_batch_1}: Consider that each batch is logically dequeued as an atomic operation. That means all batch objects form a sequential subset of the queue. Figure \ref{fig_queue_ptr_deq} shows the situation when two batches have been logically dequeued. So the costly ``find'' operation now needs to be carried out only once at the beginning of the batch. As all other objects are sequential, once we have found the batch begin inside the queue, we can simply physically dequeue the $|b|$ elements in queue-sequential order after it. So the cost of the find operation can be reduced from $O(|b|)$ to $O(1)$. +Furthermore, we can also improve algorithm \ref{alg_pdeq_batch_1}: Consider that each batch is logically dequeued as an atomic operation. That means all batch objects form a sequential subset of the queue. Figure \ref{fig_queue_ptr_deq} shows the situation when two batches have been dequeued. So the costly ``find'' operation now needs to be carried out only once at the beginning of the batch. As all other objects are sequential, once we have found the batch begin inside the queue, we can simply delete the $|b|$ elements in queue-sequential order after it. So the cost of the find operation can be reduced from $O(|b|)$ to $O(1)$. -We can even reduce the remaining cost of the find operation. If the batch to be physically dequeued is right at the queue's head (as as ``B1'' in the figure), the find immediately terminates with the first element and incurs no cost at all. The situation is different if the batch is not at the queue head, ``B2'' is an example for that (assuming that ``B1'' has not yet been dequeued). We would now still need to search over the objects that are not part of the batch and can then finally get to the object at the head of the batch in question. For queue storage drivers that support random access to queue elements, storing a simple pointer to the batches' queue head element further improves the situation and enables $O(1)$ access to the queue element. This is indicated by the dotted lines in figure \ref{fig_queue_ptr_deq}. Once the head of the queue has been found, two things can happen (depending on the capabilities of the queue storage driver): +We can even reduce the remaining cost of the find operation. If the batch to be deleted is right at the queue's head (as is ``B1'' in the figure), the ``find'' immediately terminates with the first element and incurs no cost at all. The situation is different if the batch is not at the queue head, ``B2'' is an example for that (assuming that ``B1'' has not yet been dequeued). We would now still need to search over the objects that are not part of the batch and can then finally get to the object at the head of the batch in question. For queue storage drivers that support random access to queue elements, storing a simple pointer to the batches' queue head element further improves the situation and enables $O(1)$ access to the queue element. This is indicated by the dotted lines in figure \ref{fig_queue_ptr_deq}. Once the head of the queue has been found, two things can happen (depending on the capabilities of the queue storage driver): \begin{enumerate} \item the head element can be flagged as ``this and next $n$ elements are deleted'' \item all elements are actually deleted \end{enumerate} -Note that a mixed form is also possible (and probably useful for our \emph{singly} linked list storage driver: there, some $n'$ elements be actually deleted and the head element is flagged as ``this and next $n - n'$ elements are deleted''. Note that in the linked-list case, all but the first elements can be deleted with ease, so probably just the head would stay inside the queue. Note that removing elements off the queue, where possible, is useful because it frees resources. On a busy system, freeing messages as soon as possible can prevent message loss (in non-audit-grade setup) or system slowdown. So it should be done when possible. +Note that a mixed form is also possible (and probably useful for our \emph{singly} linked list storage driver: there, some $n'$ elements be actually deleted and the head element is flagged as ``this and next $n - n'$ elements are deleted''. Note that in the linked-list case, all but the first elements can be deleted with ease\footnote{It can be considered to change from a singly-linked list to a doubly-linked list, if the benefit outweighs the extra effort required.}, so probably just the head would stay inside the queue. Note that removing elements off the queue, where possible, is useful because it frees resources. On a busy system, freeing messages as soon as possible can prevent message loss (in non-audit-grade setup) or system slowdown. So it should be done when possible. -If we have a purely sequential queue storage driver (currently the sequential disk driver), finding and updating the head element is not an option. Even in this case, we can observe that the batch at the actual physical dequeue pointer will eventually be submitted for dequeuing. So a route to take is to create a list of elements that can be deleted as soon as the physical dequeue pointer reaches any of these elements. We call this the \marginpar{to-delete list}``to-delete list''. To facilitate processing, this list must be ordered in sequence of logical dequeing. This information may not be available from the storage subsystem itself, but it can easily be generated. To do so, a strictly monotonically increasing counter is kept with each logical dequeue operation and stored as part of the batch\footnote{As this must be done via the usual computer-implemented modular arithmetic, we must be careful that we do not see repetion of values because of overflows. Each day has $60 \cdot 60 \cot 24 = 86,400$ seconds (ignoring the subleties of UTC). Now let's assume that we have a moderately-busy system with 1,000 messages per second. We further assume, to be on the save side, that each message is processed inside its own batch. So we have $86,400,000$ batches per day. If we now use a typical $32$-bit integer for generating the batch IDs, we the unique range will be used up after +If we have a purely sequential queue storage driver (currently the sequential disk driver), finding and updating the head element is not an option. Even in this case, we can observe that the batch at the actual deletion pointer will eventually be submitted for deletion. So a route to take is to create a list of elements that can be deleted as soon as the physical dequeue pointer reaches any of these elements. We call this the \marginpar{to-delete list}``to-delete list''. To facilitate processing, this list must be ordered in sequence of dequeing. This information may not be available from the storage subsystem itself, but it can easily be generated. To do so, a strictly monotonically increasing counter is kept with each logical dequeue operation and stored as part of the batch\footnote{As this must be done via the usual computer-implemented modular arithmetic, we must be careful that we do not see repetion of values because of overflows. Each day has $60 \cdot 60 \cot 24 = 86,400$ seconds (ignoring the subleties of UTC). Now let's assume that we have a moderately-busy system with 1,000 messages per second. We further assume, to be on the save side, that each message is processed inside its own batch. So we have $86,400,000$ batches per day. If we now use a typical $32$-bit integer for generating the batch IDs, we the unique range will be used up after $$\frac{2^{32}}{8640000} \approx 497 \text{ days}$$ days of uninterrupted rsyslog operation. While this sounds somewhat save, it goes down to approximately 10 days of messages are submitted at rate of 50,000 messages per second (which is high, but not unheared of). So it is strongly advised to use 64 bits, which we consider to be save, because for our 1,000 messages per second the range would be exhausted only after $$\frac{2^{64}}{8640000} \approx 2.135 \cdot 10^{11} \text{ days}$$ which equals approximately $584,500,000$ \emph{years}. So even at a rate of one million messages per second, the range would be sufficient for over 500,000 years of continuos operations -- that should be far sufficient.} -An example: let us assume that ``B2'' was submitted for physical dequeueing first. Then, the head of ``B2'' is not at the queue's physical dequeue pointer. As such, no action can be carried out immediately. So the batch head pointer is stored into a ``to be deleted'' list. Processing continues. Some time later, batch ``B1'' is submitted for deletion. Now, the head pointer is at the head of the physical dequeue list, as such all batch elements are dequeued. Then, the ``to be deleted'' list is checked, and ``B2'' is found in it. Now, ``B2'' is at the head of the (new) physical dequeue pointer and can also be removed. So, ultimately, all messages are physically dequeued. This is more formally describe in algorithm \ref{alg_phys_deq_seq_store}. In that pseudocode, we made a simplification by always putting the to be deleted batch in the ``to-delete'' list, which then enables us to use somewhat more generic code to carry out the work. +An example: let us assume that ``B2'' was submitted for deletion first. Then, the head of ``B2'' is not at the queue's delete pointer. As such, no action can be carried out immediately. So the batch head pointer is stored into a ``to be deleted'' list. Processing continues. Some time later, batch ``B1'' is submitted for deletion. Now, the head pointer is at the head of the delete list, as such all batch elements are dequeued. Then, the ``to be deleted'' list is checked, and ``B2'' is found in it. Now, ``B2'' is at the head of the (new) deletion pointer and can also be removed. So, ultimately, all messages are physically dequeued. This is more formally describe in algorithm \ref{alg_phys_deq_seq_store}. In that pseudocode, we made a simplification by always putting the to be deleted batch in the ``to-delete'' list, which then enables us to use somewhat more generic code to carry out the work. -Note that there is a price to pay for deletions via the ``to-delete'' list: if a sudden power failure happens during processing, the set of duplicate messages is increased. For example, if power fails after ``B2'' has been fully processed and scheduled for deletion, but \emph{before ``B1'' is also submitted for deletion}, ``B2'' will be reprocessed after recovery. This would not happen if ``B2'' would have been removed from the queue. +Note that there is a price to pay for deletions via the ``to-delete'' list: if a sudden fatal failure happens during processing, the set of duplicate messages is increased. For example, if a fatal failure happens after ``B2'' has been fully processed and scheduled for deletion, but \emph{before ``B1'' is also submitted for deletion}, ``B2'' will be reprocessed after recovery. This would not happen if ``B2'' would have been removed from the queue. \begin{algorithm} \caption{deleteBatch($b$)} @@ -642,10 +653,10 @@ Note that there is a price to pay for deletions via the ``to-delete'' list: if a \REQUIRE queue mutex is locked by caller \STATE enqueue $b.head, |b|$ in ``to-delete'' list $D$ \COMMENT ``to-delete'' list must be in order of logical dequeue -\WHILE{$D.head = Q.pysDeqPtr$} +\WHILE{$D.head = Q.deletePtr$} \FOR{$|b|$ elements} \STATE delete element at queue head - \STATE move $q.pysDeqPtr$ + \STATE move $q.deletePtr$ \ENDFOR \STATE remove head of ``to-delete'' list \ENDWHILE @@ -653,16 +664,65 @@ Note that there is a price to pay for deletions via the ``to-delete'' list: if a \end{algorithm} \paragraph{Warp-Up of Queue Delete Operations} -When evaluating which route to take, the ``to-delete'' list approach looks elegant for all cases. The negative side effect of potentially increased message duplication currently does not even exist: today, the sequential disk queue storage driver permits only a single worker thread and thus there will always only be one thread at a time. Even if we remove that limitation, message duplication could not be avoided, as stated in the algorithm description above. What remains are the other queue storage drivers. however, they operate in-memory, so message duplication will not happen simply because all messages will be lost on sudden fatal failure. The advantage of limited message duplication only exists in the so-far hypothetical case of a random-access, audit-grade disk queue storage driver. Thus, the decision could be postponed unless that happens (if it ever does). +When evaluating which route to take, the ``to-delete'' list approach looks elegant for all cases. The negative side effect of potentially increased message duplication currently does not even exist: today, the sequential disk queue storage driver permits only a single worker thread and thus there always will be only one thread at a time. Even if we remove that limitation, message duplication could not be avoided, as stated in the algorithm description above. What remains are the other queue storage drivers. However, they operate in-memory, so message duplication will not happen simply because all messages will be lost on sudden fatal failure. The advantage of limited message duplication only exists in the so-far hypothetical case of a random-access, audit-grade disk queue storage driver. Thus, the decision could be postponed unless that happens (if it ever does). From a code complexity point of view, the ``to-delete'' list approch is definitely advantagous. Not only because of the reduced number of algorithms required. We also do not need to maintain unique batch IDs and all the logic associated with them. The other aspect to look at is memory consumption. Assuming that we delete the actual objects, just not their containers inside the queue, extra memory consumption is not really that worse. More importantly, currently only the linked-list queue storage driver can benefit at all, because it is the only driver capable of deleting queue entries in mid-queue. All others, including the array memory driver, do not have this capability. -From a performance point of view, the ``to delete'' list approach looks approximately as good as the others, with some mild better performance for some storage drivers for a non-``to delete'' list approach. This can be mitigated, especially if the potentially somewhat-costly maintenance of the ``to-delete'' list is slightly optimized and the algorithm actually checks if the to be deleted batch is right at the queues delete pointer position. The improved code simplicity, together with current CPU's code caching, may even result in an otherwise not expected speedup. +From a performance point of view, the ``to delete'' list approach looks approximately as good as the others, with some mild better performance for some storage drivers for a non-``to delete'' list approach. This can be mitigated, especially if the potentially somewhat-costly maintenance of the ``to-delete'' list is slightly optimized and the algorithm actually checks if the to be deleted batch is right at the queue's delete pointer position. The improved code simplicity, together with current CPU's code caching, may even result in an otherwise not expected speedup. + +In conclusion, we will implement the ``to-delete'' list approach on the queue layer (above the queue storage drivers). However, we will leave the window open to permit overwriting it with queue storage driver specific functionality. How to do this will not be specified now, as there is currently no need and we do not even know if there ever will be. However, we retain the discussion on the various modes as well as the relevant algorithmic discussions and data structurs inside this paper so that it is readily available should need arise. We also think this is important so that everybody later knows that the decision was made based on good argument and not by accident (we consider this useful in another design enhancement attempt). + +\paragraph{Processing Sequence} Looking at the processing sequence, we notice that always objects are dequeued, then processed and then deleted. Then, the whole process starts again. In particular, this meanss that after the previous batch has been deleted, the next batch will be dequeued. Now consider that we need to have exclusive access to the queue for both of these operations. As such it seems natural to combine this into a single step, further reducing potential locking contention. + +Note that a side-effect of this approach is that messages can be deleted only when a new batch is dequeued. With current design, this means that at least one message must reside inside the queue. Otherwise, the last batch will not be deleted. However, this something that can (and must!) be solved on the queue worker layer, in that it deletes a batch when the queue is empty. + +This leads us to the implementation of dequeueBatch() and deleteBatch() shown in algorithms \ref{alg_deq_batch_final} and \ref{alg_del_batch_final}. Note that $l$ is a flag variable that indicates if the queue is already locked. + +\begin{algorithm} +\caption{dequeueBatch($b$): final version} +\begin{algorithmic} +\label{alg_deq_batch_final} +\STATE lock queue mutex +\STATE call deleteBatch(b, 1) +\STATE $0 \to i$ +\WHILE{queue non-empty and $i < C_{mBatch}$} + \STATE obtain next obj $o$ from queue store + \STATE advance dequeue position + \STATE put $o$ into batch +\ENDWHILE +\STATE commit queue changes to storage system (if needed, e.g. fsync()) +\STATE unlock queue mutex +\end{algorithmic} +\end{algorithm} -In conclusion, we will implement the ``to-delte'' list approach on the queue layer (above the queue storage drivers). However, we will leave the window open to permit overwriting it with queue storage driver specific functionality. How to do this will not be specified now, as there is currently no need and we do not even know if there ever will be. However, we retain the discussion on the various modes as well as the relevant algorithmic discussions and data structurs inside this paper so that it is readily available should need arise. We also think this is important so that everybody later knows that the decision was made based on good argument and not by accident (we consider this useful in another design enhancement attempt). +\begin{algorithm} +\caption{deleteBatch($b, l$): final version} +\begin{algorithmic} +\label{alg_del_batch_final} +\IF{queue not yet locked (test via $l$)} + \STATE lock queue mutex +\ENDIF +\FORALL{objects $o$ in $b$} + \STATE destruct $o$ +\ENDFOR +\STATE enqueue $b.head, |b|$ in ``to-delete'' list $D$ +\COMMENT ``to-delete'' list must be in order of logical dequeue +\WHILE{$D.head = Q.deletePtr$} + \FOR{$|b|$ elements} + \STATE delete element at queue head + \STATE move $q.deletePtr$ + \ENDFOR + \STATE remove head of ``to-delete'' list +\ENDWHILE +\STATE commit queue changes to storage system (if needed, e.g. fsync()) +\IF{queue not yet locked (test via $l$)} + \STATE unlock queue mutex +\ENDIF +\end{algorithmic} +\end{algorithm} \subsubsection{Queue Stores} Currently, rsyslog supports three different types of queue store drivers: @@ -694,13 +754,104 @@ reliability & reliable & reliable & audit-grade\footnote{if configured correctl \hline \end{tabular} +\subsubsection{Implementation} +The actual implementation will be based on algorithms \ref{alg_deq_batch_final} and \ref{alg_del_batch_final}. The rsyslog v3 queue storage driver will be extended one additional method, which permits non-destructive dequeueing of elements. As such, the driver now has the $qAdd()$, $qDeq()$, and $qDel()$ entry points (together with the usual construction and destruction entry points). The queue drivers must support the three pointers for enqueue, dequeue and delete. The ``to-delete'' list will be maintained on the upper queue layer (and not the queue driver layer). This functionality will be optimized so that if a batch to delete is right at the queue's delete pointer, it will immediatly be deleted and not be sent to the ``to-delete'' list. This is especially important with the sequential disk driver, as the condition here always is true (and thus the driver can pretend this in the relevant API without even comparing any pointers -- what would otherwise quite complicated in this driver. + +The full list of the queue store driver interface is: + +\paragraph{qConstruct} Initializes the queue store. + +\paragraph{qDestruct} Destructs the queue store, including all messages that may still be present in it. + +\paragraph{qAdd} Enqueue a new object into the queue. Note that this entry point must only be called when the queue is non-full. + +\paragraph{qDeq} Non-destructive dequeue of the object at queue head. Dequeue pointer is advanced. + +\paragraph{qDel} Delete the object at queue head. Delete pointer is advanced. + +\paragraph{qIsAtDelPos} Check if the pointer provided is at the queues current delete position. Returns true, if so, false, otherwise. + +Disk queue store drivers may support additional internal functions. However, they should not be exposed to the rest of the queue subsystem. + +\begin{figure} +\begin{center} +\includegraphics[scale=0.4]{queue_msg_state.jpeg} +\end{center} +\caption{Logical Message States during Queue Processing} +\label{fig_queue_msg_state} +\end{figure} + +Figure \ref{fig_queue_msg_state} shows a logical message state diagram during queue processing. There is no actual state variable, but rather the processing flow demands these state. Note that the state transition from ``dequeued'' to ``queued'' only happens after a fatal failure and a successful system recovery. So this is a rather exceptional case. + +\paragraph{Sequential Disk Queue Store Driver} +The enequeue, deqeueue and delete pointers must be implemented via three stream objects. Most importantly, the dequeue stream must be configured not to delete files when it closes them. A side-effect of this implementation is that data is actually read twice, once to actually obtain it and a second time to delete it. This could only be avoided by an overall redesign on how the disk queue works. + \subsubsection{Checkmarks} The following things need to be verified in the actual implementation. \paragraph{Queue Full} Is it possible to set an infinte timeout on queue full condition during enqueue? If not, we must provide it. -\paragraph{Terminatin the Queue} +\paragraph{Termination the Queue} If we cancel a worker, we need to start from the physical dequeue pointer and pull everything that is not scheduled for deletion - NOT from the logical dequeue pointer. +\paragraph{Failed Messages} +If a message fails on a detached action queue, no backup processing is available (because we detect the failure at a point where the message is already considered processed from the main queue's point of view. We need address this and have two options: + + +I see two approaches at handling this: + +a) we enable an action to configure a backup file that shall receive all +message permanent failures. This is simple (not only to implement but to +configure and understand) + +b) we push the failed message back to the main queue, but with an indication +that it failed in an action. This is harder to implement and most importantly +harder to understand/configure, but more flexible + +\section{Future Development} +This section covers topics that can not currently be developed, but where important thoughts came up in discussions. For obvious reasons, the section has brainstorming character. + +\subsection{Audit-Grade High Performance Queue Storage Driver} +An audit grade driver must ensure that no message is lost, but should also be able to handle large workloads. The sequential disk driver does not support the later. + +An additional disk driver is envisioned with the properties like the linked list driver, but a reliable on-disk store. In particular, random access to queue elements is desired, which requires an addressing capability. + +A potential implementation requires a pre-formatted file. That file is organized in pages of $n$ bytes (e.g. 1K). The page index is used to address a queue item. If an item fits into 1K, it uses one page. If it is larger than 1K, consequtive pages are used to store the element. A page header must be present to indicate how many pages a single element is made up of. + +It may be noted that we could even improve performance by keeping part of the data in-memory. For audit-gradeness, it is required that upon enqueue the message is written to disk and only after final processing it needs to be removed. However, it is not forbidden to keep the same message in main memory. That way, the logical dequeue operation could be done one the in-memory representation. Only the physical dequeue would need to write to disk again. As such, we save one disk read out of three writes and one read otherwise required (so one can roughly say that we save one third of disk operations. + +Note that due to potential multi-pages messages we can not directly address individual elements, but we can reliably and quikly address elements whom's address we know (learned, for example, during logical dequeue). This is similar to the organization of the in-memory linked list. Actally, such a store \emph{is} a linked list implementation, just that memory is allocated on disk instead of in main memory. + +To further improve speed, object representation could be zipped before being written to a page. + +File Layout +Page 0: control structures (most importantyle queue pointers) (can make sense to store in a separate file, which could be moved to a dedicated disk subsystem - can potentially greatly reduce disk seek times). +Page 1 to n: actual object storage + +Algorithms \ref{alg_AuditGradeStoreEnqueue} and \ref{alg_AuditGradeStoreDelete} show how records are enqueued and deleted. Note that the delete part does not even need to read back the record. If we keep at last some records in-memory, the performance cost of ultra-reliable mode can actually comparatively low. Note that we may not even really need to commit data to the storage system in ``AuditGradeStoreDelete()'', because if a fatal failure occurs at this point, at worst message duplication may happen, what we have considered to be acceptable. + +\begin{algorithm} +\caption{AuditGradeStoreEnqueue($o$)} +\begin{algorithmic} +\label{alg_AuditGradeStoreEnqueue} +\REQUIRE queue mutex is locked by caller +\STATE write $o$ to current enqueue location +\STATE update \& write queue structures [page 0] +\STATE sync all files touched +\STATE store $o$ in an in-memory structure (or a cache) +\end{algorithmic} +\end{algorithm} + +\begin{algorithm} +\caption{AuditGradeStoreDelete($o$)} +\begin{algorithmic} +\label{alg_AuditGradeStoreDelete} +\REQUIRE queue mutex is locked by caller +\STATE update queue dequeue pointer \& write queue structures [page 0] +\STATE sync all files touched +\end{algorithmic} +\end{algorithm} + + \end{document} diff --git a/doc/queue_msg_state.dot b/doc/queue_msg_state.dot new file mode 100644 index 00000000..bfef2657 --- /dev/null +++ b/doc/queue_msg_state.dot @@ -0,0 +1,25 @@ +// This file is part of rsyslog. +// +// rsyslog message state in queue processing +// +// see http://www.graphviz.org for how to obtain the graphviz processor +// which is used to build the actual graph. +// +// generate the graph with +// $ dot file.dot -Tpng >file.png + +digraph msgState { + rankdir=LR + + prod [label="producer" style="dotted" shape="box"] + que [label="queued"] + deq [label="dequeued"] + del [label="deleted"] + + prod -> que [label="qEnq()" style="dotted"] + que -> deq [label="qDeq()"] + deq -> del [label="qDel()"] + deq -> que [label="fatal failure\n& restart"] + + //{rank=same; del apf pdn } +} diff --git a/doc/queue_msg_state.jpeg b/doc/queue_msg_state.jpeg new file mode 100644 index 00000000..a215f000 Binary files /dev/null and b/doc/queue_msg_state.jpeg differ diff --git a/doc/rsyslog_queue_pointers2.jpeg b/doc/rsyslog_queue_pointers2.jpeg index 069301e3..2ad60113 100644 Binary files a/doc/rsyslog_queue_pointers2.jpeg and b/doc/rsyslog_queue_pointers2.jpeg differ diff --git a/doc/src/rsyslog_queue_pointers2.dia b/doc/src/rsyslog_queue_pointers2.dia index e630e33e..6a35c664 100644 Binary files a/doc/src/rsyslog_queue_pointers2.dia and b/doc/src/rsyslog_queue_pointers2.dia differ -- cgit v1.2.3 From 93f873277bfe5ebb309ff5e92f5dc7244ebd9f1a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 18 May 2009 17:28:34 +0200 Subject: t-delete list implemented, queue store drivers updated... ... on the way to the ultra-reliable queue modes (redesign doc). This version does not really work, but is a good commit point. Next comes queue size calculation. DA mode does not yet work. --- doc/design.tex | 6 +- runtime/batch.h | 1 + runtime/queue.c | 353 ++++++++++++++++++++++++++++++++++++++++++++++------- runtime/queue.h | 25 +++- runtime/rsyslog.h | 10 +- runtime/stream.c | 5 +- tests/diskqueue.sh | 2 +- 7 files changed, 342 insertions(+), 60 deletions(-) diff --git a/doc/design.tex b/doc/design.tex index 7c0641b8..c03e1fab 100644 --- a/doc/design.tex +++ b/doc/design.tex @@ -769,8 +769,6 @@ The full list of the queue store driver interface is: \paragraph{qDel} Delete the object at queue head. Delete pointer is advanced. -\paragraph{qIsAtDelPos} Check if the pointer provided is at the queues current delete position. Returns true, if so, false, otherwise. - Disk queue store drivers may support additional internal functions. However, they should not be exposed to the rest of the queue subsystem. \begin{figure} @@ -783,6 +781,10 @@ Disk queue store drivers may support additional internal functions. However, the Figure \ref{fig_queue_msg_state} shows a logical message state diagram during queue processing. There is no actual state variable, but rather the processing flow demands these state. Note that the state transition from ``dequeued'' to ``queued'' only happens after a fatal failure and a successful system recovery. So this is a rather exceptional case. +Another subtle issue is that we now need two different queue size counters: one for seeing when the queue is physically full and one for detecting when there are no more messages to be dequeued. + +As a simplification, support for ungetting objects can be removed (as objects never leave the queue), what also means that cancel-processing is probably less complex. + \paragraph{Sequential Disk Queue Store Driver} The enequeue, deqeueue and delete pointers must be implemented via three stream objects. Most importantly, the dequeue stream must be configured not to delete files when it closes them. A side-effect of this implementation is that data is actually read twice, once to actually obtain it and a second time to delete it. This could only be avoided by an overall redesign on how the disk queue works. diff --git a/runtime/batch.h b/runtime/batch.h index fcbbafce..eb266b3f 100644 --- a/runtime/batch.h +++ b/runtime/batch.h @@ -59,6 +59,7 @@ struct batch_obj_s { struct batch_s { int nElem; /* actual number of element in this entry */ int iDoneUpTo; /* all messages below this index have state other than RDY */ + qDeqID deqID; /* ID of dequeue operation that generated this batch */ batch_obj_t *pElem; /* batch elements */ }; diff --git a/runtime/queue.c b/runtime/queue.c index 6bea338a..dc399066 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -79,6 +79,93 @@ static rsRetVal UngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex); #define QUEUE_CHECKPOINT 1 #define QUEUE_NO_CHECKPOINT 0 +/*********************************************************************** + * we need a private data structure, the "to-delete" list. As C does + * not provide any partly private data structures, we implement this + * structure right here inside the module. + * Note that this list must always be kept sorted based on a unique + * dequeue ID (which is monotonically increasing). + * rgerhards, 2009-05-18 + ***********************************************************************/ + +/* generate next uniqueue dequeue ID. Note that uniqueness is only required + * on a per-queue basis and while this instance runs. So a stricly monotonically + * increasing counter is sufficient (if enough bits are used). + */ +static inline qDeqID getNextDeqID(qqueue_t *pQueue) +{ + ISOBJ_TYPE_assert(pQueue, qqueue); + return pQueue->deqIDAdd++; +} + + +/* return the top element of the to-delete list or NULL, if the + * list is empty. + */ +static inline toDeleteLst_t *tdlPeek(qqueue_t *pQueue) +{ + ISOBJ_TYPE_assert(pQueue, qqueue); + return pQueue->toDeleteLst; +} + + +/* remove the top element of the to-delete list. Nothing but the + * element itself is destroyed. Must not be called when the list + * is empty. + */ +static inline rsRetVal tdlPop(qqueue_t *pQueue) +{ + toDeleteLst_t *pRemove; + DEFiRet; + + ISOBJ_TYPE_assert(pQueue, qqueue); + assert(pQueue->toDeleteLst != NULL); + + pRemove = pQueue->toDeleteLst; + pQueue->toDeleteLst = pQueue->toDeleteLst->pNext; + free(pRemove); + + RETiRet; +} + + +/* Add a new to-delete list entry. The function allocates the data + * structure, populates it with the values provided and links the new + * element into the correct place inside the list. + */ +static inline rsRetVal tdlAdd(qqueue_t *pQueue, qDeqID deqID, int nElem) +{ + toDeleteLst_t *pNew; + toDeleteLst_t *pPrev; + DEFiRet; + + ISOBJ_TYPE_assert(pQueue, qqueue); + assert(pQueue->toDeleteLst != NULL); + + CHKmalloc(pNew = malloc(sizeof(toDeleteLst_t))); + pNew->deqID = deqID; + pNew->nElem = nElem; + + /* now find right spot */ + for( pPrev = pQueue->toDeleteLst + ; pPrev != NULL && deqID > pPrev->deqID + ; pPrev = pPrev->pNext) { + /*JUST SEARCH*/; + } + + if(pPrev == NULL) { + pNew->pNext = pQueue->toDeleteLst; + pQueue->toDeleteLst = pNew; + } else { + pNew->pNext = pPrev->pNext; + pPrev->pNext = pNew; + } + +finalize_it: + RETiRet; +} + + /* methods */ @@ -114,7 +201,9 @@ static inline void queueDrain(qqueue_t *pThis) /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ while(pThis->iQueueSize-- > 0) { - pThis->qDel(pThis, &pUsr); + pThis->qDeq(pThis, &pUsr); +// TODO: ULTRA + //pThis->qDel(pThis, &pUsr); if(pUsr != NULL) { objDestruct(pUsr); } @@ -472,6 +561,7 @@ static rsRetVal qConstructFixedArray(qqueue_t *pThis) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } + pThis->tVars.farray.deqhead = 0; pThis->tVars.farray.head = 0; pThis->tVars.farray.tail = 0; @@ -508,13 +598,29 @@ static rsRetVal qAddFixedArray(qqueue_t *pThis, void* in) RETiRet; } -static rsRetVal qDelFixedArray(qqueue_t *pThis, void **out) + +static rsRetVal qDeqFixedArray(qqueue_t *pThis, void **out) { DEFiRet; ASSERT(pThis != NULL); - *out = (void*) pThis->tVars.farray.pBuf[pThis->tVars.farray.head]; + *out = (void*) pThis->tVars.farray.pBuf[pThis->tVars.farray.deqhead]; +//MULTIdbgprintf("ULTRA qDeqFA, deqhead=%d head=%d, tail=%d\n", pThis->tVars.farray.deqhead, pThis->tVars.farray.head, pThis->tVars.farray.tail); + pThis->tVars.farray.deqhead++; + if (pThis->tVars.farray.deqhead == pThis->iMaxQueueSize) + pThis->tVars.farray.deqhead = 0; + + RETiRet; +} + +static rsRetVal qDelFixedArray(qqueue_t *pThis) +{ + DEFiRet; + + ASSERT(pThis != NULL); + +//MULTIdbgprintf("ULTRA qDelFA, deqhead=%d head=%d, tail=%d\n", pThis->tVars.farray.deqhead, pThis->tVars.farray.head, pThis->tVars.farray.tail); pThis->tVars.farray.head++; if (pThis->tVars.farray.head == pThis->iMaxQueueSize) pThis->tVars.farray.head = 0; @@ -529,15 +635,13 @@ static rsRetVal qDelFixedArray(qqueue_t *pThis, void **out) static inline rsRetVal qqueueAddLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, void* pUsr) { - DEFiRet; qLinkedList_t *pEntry; + DEFiRet; ASSERT(ppRoot != NULL); ASSERT(ppLast != NULL); - if((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t)))); pEntry->pNext = NULL; pEntry->pUsr = pUsr; @@ -586,8 +690,9 @@ static rsRetVal qConstructLinkedList(qqueue_t *pThis) ASSERT(pThis != NULL); - pThis->tVars.linklist.pRoot = 0; - pThis->tVars.linklist.pLast = 0; + pThis->tVars.linklist.pDeqRoot = NULL; + pThis->tVars.linklist.pDelRoot = NULL; + pThis->tVars.linklist.pLast = NULL; qqueueChkIsDA(pThis); @@ -610,15 +715,60 @@ static rsRetVal qDestructLinkedList(qqueue_t __attribute__((unused)) *pThis) static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr) { + qLinkedList_t *pEntry; + DEFiRet; + + CHKmalloc((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t)))); + + pEntry->pNext = NULL; + pEntry->pUsr = pUsr; + + if(pThis->tVars.linklist.pDelRoot == NULL) { + pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = pEntry; + } else { + pThis->tVars.linklist.pLast->pNext = pEntry; + pThis->tVars.linklist.pLast = pEntry; + } + + if(pThis->tVars.linklist.pDeqRoot == NULL) { + pThis->tVars.linklist.pDeqRoot = pEntry; + } +RUNLOG_VAR("%p", pThis->tVars.linklist.pDeqRoot); + +finalize_it: + RETiRet; +} + + +static rsRetVal qDeqLinkedList(qqueue_t *pThis, obj_t **ppUsr) +{ + qLinkedList_t *pEntry; DEFiRet; - iRet = qqueueAddLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, pUsr); + +RUNLOG_VAR("%p", pThis->tVars.linklist.pDeqRoot); + pEntry = pThis->tVars.linklist.pDeqRoot; + *ppUsr = pEntry->pUsr; + pThis->tVars.linklist.pDeqRoot = pEntry->pNext; + RETiRet; } -static rsRetVal qDelLinkedList(qqueue_t *pThis, obj_t **ppUsr) + +static rsRetVal qDelLinkedList(qqueue_t *pThis) { + qLinkedList_t *pEntry; DEFiRet; - iRet = qqueueDelLinkedList(&pThis->tVars.linklist.pRoot, &pThis->tVars.linklist.pLast, ppUsr); + + pEntry = pThis->tVars.linklist.pDelRoot; + + if(pThis->tVars.linklist.pDelRoot == pThis->tVars.linklist.pLast) { + pThis->tVars.linklist.pDelRoot = pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pLast = NULL; + } else { + pThis->tVars.linklist.pDelRoot = pEntry->pNext; + } + + free(pEntry); + RETiRet; } @@ -732,11 +882,29 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) /* and now the stream objects (some order as when persisted!) */ CHKiRet(obj.Deserialize(&pThis->tVars.disk.pWrite, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); - CHKiRet(obj.Deserialize(&pThis->tVars.disk.pRead, (uchar*) "strm", psQIF, + CHKiRet(obj.Deserialize(&pThis->tVars.disk.pReadDel, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pWrite)); - CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pRead)); + CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pReadDel)); + + /* we now need to take care of the Deq handle. It is not persisted, so we can create + * a virgin copy based on pReadDel. // TODO duplicat code, same as blow - single function! + */ + + CHKiRet(strmConstruct(&pThis->tVars.disk.pReadDeq)); + CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); + CHKiRet(strmSetDir(pThis->tVars.disk.pReadDeq, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pReadDeq, 10000000)); + CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pReadDeq, STREAMMODE_READ)); + CHKiRet(strmSetsType(pThis->tVars.disk.pReadDeq, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strmConstructFinalize(pThis->tVars.disk.pReadDeq)); + + /* TODO: dirty, need stream methods --> */ + pThis->tVars.disk.pReadDeq->iCurrFNum = pThis->tVars.disk.pReadDel->iCurrFNum; + pThis->tVars.disk.pReadDeq->iCurrOffs = pThis->tVars.disk.pReadDel->iCurrOffs; + /* <-- dirty, need stream methods :TODO */ + CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pReadDeq)); /* OK, we could successfully read the file, so we now can request that it be * deleted when we are done with the persisted information. @@ -787,17 +955,25 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) CHKiRet(strmSetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR)); CHKiRet(strmConstructFinalize(pThis->tVars.disk.pWrite)); - CHKiRet(strmConstruct(&pThis->tVars.disk.pRead)); - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); - CHKiRet(strmSetDir(pThis->tVars.disk.pRead, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pRead, 10000000)); - CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pRead, STREAMMODE_READ)); - CHKiRet(strmSetsType(pThis->tVars.disk.pRead, STREAMTYPE_FILE_CIRCULAR)); - CHKiRet(strmConstructFinalize(pThis->tVars.disk.pRead)); - - - CHKiRet(strmSetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); - CHKiRet(strmSetFName(pThis->tVars.disk.pRead, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strmConstruct(&pThis->tVars.disk.pReadDeq)); + CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); + CHKiRet(strmSetDir(pThis->tVars.disk.pReadDeq, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pReadDeq, 10000000)); + CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pReadDeq, STREAMMODE_READ)); + CHKiRet(strmSetsType(pThis->tVars.disk.pReadDeq, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strmConstructFinalize(pThis->tVars.disk.pReadDeq)); + + CHKiRet(strmConstruct(&pThis->tVars.disk.pReadDel)); + CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1)); + CHKiRet(strmSetDir(pThis->tVars.disk.pReadDel, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pReadDel, 10000000)); + CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pReadDel, STREAMMODE_READ)); + CHKiRet(strmSetsType(pThis->tVars.disk.pReadDel, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strmConstructFinalize(pThis->tVars.disk.pReadDel)); + + CHKiRet(strmSetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strmSetFName(pThis->tVars.disk.pReadDeq, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strmSetFName(pThis->tVars.disk.pReadDel, pThis->pszFilePrefix, pThis->lenFilePrefix)); } /* now we set (and overwrite in case of a persisted restart) some parameters which @@ -806,7 +982,8 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) * ability to read existing queue files. -- rgerhards, 2008-01-12 */ CHKiRet(strmSetiMaxFileSize(pThis->tVars.disk.pWrite, pThis->iMaxFileSize)); - CHKiRet(strmSetiMaxFileSize(pThis->tVars.disk.pRead, pThis->iMaxFileSize)); + CHKiRet(strmSetiMaxFileSize(pThis->tVars.disk.pReadDeq, pThis->iMaxFileSize)); + CHKiRet(strmSetiMaxFileSize(pThis->tVars.disk.pReadDel, pThis->iMaxFileSize)); finalize_it: RETiRet; @@ -820,7 +997,8 @@ static rsRetVal qDestructDisk(qqueue_t *pThis) ASSERT(pThis != NULL); strmDestruct(&pThis->tVars.disk.pWrite); - strmDestruct(&pThis->tVars.disk.pRead); + strmDestruct(&pThis->tVars.disk.pReadDeq); + strmDestruct(&pThis->tVars.disk.pReadDel); RETiRet; } @@ -852,16 +1030,30 @@ finalize_it: RETiRet; } -static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr) + +static rsRetVal qDeqDisk(qqueue_t *pThis, void **ppUsr) { DEFiRet; + CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pReadDeq, NULL, NULL)); + +finalize_it: + RETiRet; +} + + +static rsRetVal qDelDisk(qqueue_t *pThis) +{ + obj_t *pDummyObj; /* we need to deserialize it... */ + DEFiRet; + int64 offsIn; int64 offsOut; - CHKiRet(strmGetCurrOffset(pThis->tVars.disk.pRead, &offsIn)); - CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pRead, NULL, NULL)); - CHKiRet(strmGetCurrOffset(pThis->tVars.disk.pRead, &offsOut)); + CHKiRet(strmGetCurrOffset(pThis->tVars.disk.pReadDel, &offsIn)); + CHKiRet(obj.Deserialize(&pDummyObj, (uchar*) "msg", pThis->tVars.disk.pReadDel, NULL, NULL)); + objDestruct(pDummyObj); + CHKiRet(strmGetCurrOffset(pThis->tVars.disk.pReadDel, &offsOut)); /* This time it is a bit tricky: we free disk space only upon file deletion. So we need * to keep track of what we have read until we get an out-offset that is lower than the @@ -882,6 +1074,7 @@ finalize_it: RETiRet; } + /* -------------------- direct (no queueing) -------------------- */ static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis) { @@ -920,7 +1113,7 @@ static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) } -static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis, __attribute__((unused)) void **out) +static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis) { return RS_RET_OK; } @@ -1023,7 +1216,8 @@ qqueueDel(qqueue_t *pThis, void *pUsr) if(pThis->iUngottenObjs > 0) { iRet = GetUngottenObj(pThis, (obj_t**) pUsr); } else { - iRet = pThis->qDel(pThis, pUsr); + iRet = pThis->qDeq(pThis, pUsr); + // TODO: ULTRA iRet = pThis->qDel(pThis, pUsr); ATOMIC_DEC(pThis->iQueueSize); } @@ -1290,18 +1484,21 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qConstruct = qConstructFixedArray; pThis->qDestruct = qDestructFixedArray; pThis->qAdd = qAddFixedArray; + pThis->qDeq = qDeqFixedArray; pThis->qDel = qDelFixedArray; break; case QUEUETYPE_LINKEDLIST: pThis->qConstruct = qConstructLinkedList; pThis->qDestruct = qDestructLinkedList; pThis->qAdd = qAddLinkedList; - pThis->qDel = (rsRetVal (*)(qqueue_t*,void**)) qDelLinkedList; + pThis->qDeq = (rsRetVal (*)(qqueue_t*,void**)) qDeqLinkedList; + pThis->qDel = (rsRetVal (*)(qqueue_t*)) qDelLinkedList; break; case QUEUETYPE_DISK: pThis->qConstruct = qConstructDisk; pThis->qDestruct = qDestructDisk; pThis->qAdd = qAddDisk; + pThis->qDeq = qDeqDisk; pThis->qDel = qDelDisk; /* special handling */ pThis->iNumWorkerThreads = 1; /* we need exactly one worker */ @@ -1390,8 +1587,68 @@ finalize_it: } +/* Finally remove n elements from the queue store. + */ +static inline rsRetVal +DoDeleteBatchFromQStore(qqueue_t *pThis, int nElem) +{ + int i; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + + /* now send delete request to storage driver */ + for(i = 0 ; i < nElem ; ++i) { + pThis->qDel(pThis); + } + + ++pThis->deqIDDel; /* one more batch dequeued */ + + RETiRet; +} + + +/* remove messages from the physical queue store that are fully processed. This is + * controlled via the to-delete list. We can only delete those elements, that are + * at the current physical tail of the queue. If the batch is from another position, + * we schedule it for deletion, but actual deletion will happen at a later call + * of this function here. We always delete as much as possible, which includes + * picking up things from the to-delete list. + */ +static inline rsRetVal +DeleteBatchFromQStore(qqueue_t *pThis, batch_t *pBatch) +{ + toDeleteLst_t *pTdl; + qDeqID deqIDDel; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pBatch != NULL); + + pTdl = tdlPeek(pThis); + if(pTdl == NULL) { + DoDeleteBatchFromQStore(pThis, pBatch->nElem); + } else if(pBatch->deqID == pThis->deqIDDel) { + deqIDDel = pThis->deqIDDel; + pTdl = tdlPeek(pThis); + while(pTdl != NULL && deqIDDel == pTdl->deqID) { + DoDeleteBatchFromQStore(pThis, pTdl->nElem); + tdlPop(pThis); + ++deqIDDel; + pTdl = tdlPeek(pThis); + } + } else { + /* can not delete, insert into to-delete list */ + CHKiRet(tdlAdd(pThis, pBatch->deqID, pBatch->nElem)); + } + +finalize_it: + RETiRet; +} + + /* Delete a batch of processed user objects from the queue, which includes - * destructing the objects themself. The pointer piRemainingQueu + * destructing the objects themself. * rgerhards, 2009-05-13 */ static inline rsRetVal @@ -1401,30 +1658,31 @@ DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch) void *pUsr; DEFiRet; - /* this is the place to destruct the old messages and pull them off the queue - MULTI-DEQUEUE */ - ISOBJ_TYPE_assert(pThis, qqueue); assert(pBatch != NULL); +// TODO: ULTRA: lock qaueue mutex if instructed to do so - /* if the queue runs in DA mode, the DA worker already deleted the message. But - * in regular mode, we need to do it ourselfs. We differentiate between the two cases, - * because it is actually the easiest way to handle the destruct-Problem in a simple - * and pUsrp-Type agnostic way (else we would need an objAddRef() generic function). + /* if the queue runs in DA mode, the DA worker already deleted the in-memory representation + * of the message. But in regular mode, we need to do it ourselfs. We differentiate between + * the two cases, because it is actually the easiest way to handle the destruct-Problem in + * a simple and pUsrp-Type agnostic way (else we would need an objAddRef() generic function). */ if(!pThis->bRunsDA) { for(i = 0 ; i < pBatch->nElem ; ++i) { - /* TODO: pull msgs off the queue (not yet necessary) */ pUsr = pBatch->pElem[i].pUsrp; objDestruct(pUsr); } } - pBatch->nElem = 0; + + iRet = DeleteBatchFromQStore(pThis, pBatch); + + pBatch->nElem = 0; /* reset batch */ RETiRet; } -/* dequeue as many user points as are available, until we hit the configured +/* dequeue as many user pointers as are available, until we hit the configured * upper limit of pointers. * This must only be called when the queue mutex is LOOKED, otherwise serious * malfunction will happen. @@ -1463,6 +1721,7 @@ dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); qqueueChkPersist(pThis, nDequeued); /* it is sufficient to persist only when the bulk of work is done */ pWti->batch.nElem = nDequeued; + pWti->batch.deqID = getNextDeqID(pThis); *piRemainingQueueSize = iQueueSize; finalize_it: @@ -1957,7 +2216,7 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) pThis->bNeedDelQIF = 0; } /* indicate spool file needs to be deleted */ - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); + CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1)); FINALIZE; /* nothing left to do, so be happy */ } @@ -1992,13 +2251,13 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) /* now persist the stream info */ CHKiRet(strmSerialize(pThis->tVars.disk.pWrite, psQIF)); - CHKiRet(strmSerialize(pThis->tVars.disk.pRead, psQIF)); + CHKiRet(strmSerialize(pThis->tVars.disk.pReadDel, psQIF)); /* tell the input file object that it must not delete the file on close if the queue * is non-empty - but only if we are not during a simple checkpoint */ if(bIsCheckpoint != QUEUE_CHECKPOINT) { - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 0)); + CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pReadDel, 0)); } /* we have persisted the queue object. So whenever it comes to an empty queue, diff --git a/runtime/queue.h b/runtime/queue.h index 4a5f16a1..00cee419 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -30,6 +30,15 @@ #include "batch.h" #include "stream.h" +/* support for the toDelete list */ +typedef struct toDeleteLst_s toDeleteLst_t; +struct toDeleteLst_s { + qDeqID deqID; + int nElem; + struct toDeleteLst_s *pNext; +}; + + /* queue types */ typedef enum { QUEUETYPE_FIXED_ARRAY = 0,/* a simple queue made out of a fixed (initially malloced) array fast but memoryhog */ @@ -85,6 +94,7 @@ typedef struct queue_s { int toQShutdown; /* timeout for regular queue shutdown in ms */ int toActShutdown; /* timeout for long-running action shutdown in ms */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ + toDeleteLst_t *toDeleteLst;/* this queue's to-delete list */ int toEnq; /* enqueue timeout */ int iDeqBatchSize; /* max number of elements that shall be dequeued at once */ /* rate limiting settings (will be expanded) */ @@ -110,7 +120,8 @@ typedef struct queue_s { rsRetVal (*qConstruct)(struct queue_s *pThis); rsRetVal (*qDestruct)(struct queue_s *pThis); rsRetVal (*qAdd)(struct queue_s *pThis, void *pUsr); - rsRetVal (*qDel)(struct queue_s *pThis, void **ppUsr); + rsRetVal (*qDeq)(struct queue_s *pThis, void **ppUsr); + rsRetVal (*qDel)(struct queue_s *pThis); /* end type-specific handler */ /* synchronization variables */ pthread_mutex_t mutThrdMgmt; /* mutex for the queue's thread management */ @@ -134,6 +145,8 @@ typedef struct queue_s { int iNumberFiles; /* how many files make up the queue? */ int64 iMaxFileSize; /* max size for a single queue file */ int64 sizeOnDiskMax; /* maximum size on disk allowed */ + qDeqID deqIDAdd; /* next dequeue ID to use during add to queue store */ + qDeqID deqIDDel; /* queue store delete position */ int bIsDA; /* is this queue disk assisted? */ int bRunsDA; /* is this queue actually *running* disk assisted? */ struct queue_s *pqDA; /* queue for disk-assisted modes */ @@ -148,18 +161,20 @@ typedef struct queue_s { /* now follow queueing mode specific data elements */ union { /* different data elements based on queue type (qType) */ struct { - long head, tail; + long deqhead, head, tail; void** pBuf; /* the queued user data structure */ } farray; struct { - qLinkedList_t *pRoot; + qLinkedList_t *pDeqRoot; + qLinkedList_t *pDelRoot; qLinkedList_t *pLast; } linklist; struct { int64 sizeOnDisk; /* current amount of disk space used */ int64 bytesRead; /* number of bytes read from current (undeleted!) file */ - strm_t *pWrite; /* current file to be written */ - strm_t *pRead; /* current file to be read */ + strm_t *pWrite; /* current file to be written */ + strm_t *pReadDeq; /* current file for dequeueing */ + strm_t *pReadDel; /* current file for deleting */ } disk; } tVars; } qqueue_t; diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 25f9eefe..3dd84ef6 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -67,6 +67,11 @@ /*#define CORE_FEATURE_whatever 2 ... and so on ... */ +/* some universal 64 bit define... */ +typedef long long int64; +typedef long long unsigned uint64; +typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ + /* define some base data types */ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ typedef struct aUsrp_s aUsrp_t; @@ -98,11 +103,8 @@ typedef struct vmstk_s vmstk_t; typedef struct batch_obj_s batch_obj_t; typedef struct batch_s batch_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ +typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ -/* some universal 64 bit define... */ -typedef long long int64; -typedef long long unsigned uint64; -typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ #ifdef __hpux typedef unsigned int u_int32_t; /* TODO: is this correct? */ diff --git a/runtime/stream.c b/runtime/stream.c index f1f69cc8..50d419be 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -99,7 +99,10 @@ static rsRetVal strmOpenFile(strm_t *pThis) pThis->fd = open((char*)pThis->pszCurrFName, iFlags, pThis->tOpenMode); if(pThis->fd == -1) { int ierrnoSave = errno; - dbgoprint((obj_t*) pThis, "open error %d, file '%s'\n", errno, pThis->pszCurrFName); + char errStr[1024]; + dbgoprint((obj_t*) pThis, "open error[%d]: '%s'; file '%s'/%s\n", errno, + rs_strerror_r(errno, errStr, sizeof(errStr)), pThis->pszCurrFName, + (pThis->tOperationsMode == STREAMMODE_READ) ? "READ" : "WRITE"); if(ierrnoSave == ENOENT) ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); else diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index 6384eb64..5ff9ced0 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -17,7 +17,7 @@ if [ "$?" -ne "0" ]; then echo "error during tcpflood! see rsyslog.out.log.save for what was written" cp rsyslog.out.log rsyslog.out.log.save fi -sleep 4 # we need this so that rsyslogd can receive all outstanding messages +sleep 6 # we need this so that rsyslogd can receive all outstanding messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -- cgit v1.2.3 From fe5bea77ac2533faab3b7b73bc253c4dc7d702bc Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 18 May 2009 18:39:52 +0200 Subject: removed queue's UngetObj() call ... which is no longer needed thanks to the new queue design. --- doc/design.tex | 8 ++--- runtime/queue.c | 110 ++++++-------------------------------------------------- runtime/queue.h | 6 ---- 3 files changed, 15 insertions(+), 109 deletions(-) diff --git a/doc/design.tex b/doc/design.tex index c03e1fab..53d25313 100644 --- a/doc/design.tex +++ b/doc/design.tex @@ -433,11 +433,11 @@ message-caused. This is under the assumption that any reasonable responsive admin will hopefully test his configuration at least once before turning it into production. And config SQL errors should manifest immediately, so I expect these to be fixed before a configuration runs in production. So it is -the chore of the output module to interpret the return code it received from -its API and decide whether this is more likely action-caused or +the duty of the output module to interpret the return code it received from +the API call and decide whether the failure is more likely action-caused or message-caused. For database outputs, I would assume that it is always easy -to classify failures that can only be action-caused, especially in the -dominating case of a failed network connection or a failed server. +to classify failures that must be action-caused, especially in the +dominating cases of failed network connections or failed servers. For other outputs it may not be as easy. But, for example, all stream network outputs can detect a broken connection, so this also is a sure fit. diff --git a/runtime/queue.c b/runtime/queue.c index dc399066..8ef3e7db 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -73,7 +73,6 @@ static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal); static int qqueueIsIdleDA(qqueue_t *pThis); static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave); static rsRetVal ConsumerCancelCleanup(void *arg1, void *arg2); -static rsRetVal UngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex); /* some constants for queuePersist () */ #define QUEUE_CHECKPOINT 1 @@ -169,7 +168,7 @@ finalize_it: /* methods */ -/* get the overall queue size, which includes ungotten objects. Must only be called +/* get the overall queue size. Must only be called * while mutex is locked! * rgerhards, 2008-01-29 */ @@ -178,11 +177,11 @@ qqueueGetOverallQueueSize(qqueue_t *pThis) { #if 0 /* leave a bit in for debugging -- rgerhards, 2008-01-30 */ BEGINfunc -dbgoprint((obj_t*) pThis, "queue size: %d (regular %d, ungotten %d)\n", - pThis->iQueueSize + pThis->iUngottenObjs, pThis->iQueueSize, pThis->iUngottenObjs); +dbgoprint((obj_t*) pThis, "queue size: %d (regular %d)\n", + pThis->iQueueSize, pThis->iQueueSize); ENDfunc #endif - return pThis->iQueueSize + pThis->iUngottenObjs; + return pThis->iQueueSize; } @@ -837,8 +836,6 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) uchar pszQIFNam[MAXFNAME]; size_t lenQIFNam; struct stat stat_buf; - int iUngottenObjs; - obj_t *pUsr; ISOBJ_TYPE_assert(pThis, qqueue); @@ -868,18 +865,7 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) /* first, we try to read the property bag for ourselfs */ CHKiRet(obj.DeserializePropBag((obj_t*) pThis, psQIF)); - /* then the ungotten object queue */ - iUngottenObjs = pThis->iUngottenObjs; - pThis->iUngottenObjs = 0; /* will be incremented when we add objects! */ - - while(iUngottenObjs > 0) { - /* fill the queue from disk */ - CHKiRet(obj.Deserialize((void*) &pUsr, (uchar*)"msg", psQIF, NULL, NULL)); - UngetObj(pThis, pUsr, MUTEX_ALREADY_LOCKED); - --iUngottenObjs; /* one less */ - } - - /* and now the stream objects (some order as when persisted!) */ + /* then the stream objects (same order as when persisted!) */ CHKiRet(obj.Deserialize(&pThis->tVars.disk.pWrite, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); CHKiRet(obj.Deserialize(&pThis->tVars.disk.pReadDel, (uchar*) "strm", psQIF, @@ -1122,57 +1108,6 @@ static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis) /* --------------- end type-specific handlers -------------------- */ -/* unget a user pointer that has been dequeued. This functionality is especially important - * for consumer cancel cleanup handlers. To support it, a short list of ungotten user pointers - * is maintened in memory. - * rgerhards, 2008-01-20 - */ -static rsRetVal -UngetObj(qqueue_t *pThis, obj_t *pUsr, int bLockMutex) -{ - DEFiRet; - DEFVARS_mutexProtection; - - ISOBJ_TYPE_assert(pThis, qqueue); - ISOBJ_assert(pUsr); /* TODO: we aborted right at this place at least 3 times -- race? 2008-02-28, -03-10, -03-15 - The second time I noticed it the queue was in destruction with NO worker threads - running. The pUsr ptr was totally off and provided no clue what it may be pointing - at (except that it looked like the static data pool). Both times, the abort happend - inside an action queue */ - - dbgoprint((obj_t*) pThis, "ungetting user object %s\n", obj.GetName(pUsr)); - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex); - iRet = qqueueAddLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, pUsr); - ++pThis->iUngottenObjs; /* indicate one more */ - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - - RETiRet; -} - - -/* dequeues a user pointer from the ungotten queue. Pointers from there should always be - * dequeued first. - * - * This function must only be called when the mutex is locked! - * - * rgerhards, 2008-01-29 - */ -static rsRetVal -GetUngottenObj(qqueue_t *pThis, obj_t **ppUsr) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - ASSERT(ppUsr != NULL); - - iRet = qqueueDelLinkedList(&pThis->pUngetRoot, &pThis->pUngetLast, ppUsr); - --pThis->iUngottenObjs; /* indicate one less */ - dbgoprint((obj_t*) pThis, "dequeued ungotten user object %s\n", obj.GetName(*ppUsr)); - - RETiRet; -} - - /* generic code to add a queue entry * We use some specific code to most efficiently support direct mode * queues. This is justified in spite of the gain and the need to do some @@ -1198,8 +1133,6 @@ finalize_it: /* generic code to remove a queue entry - * rgerhards, 2008-01-29: we must first see if there is any object in the - * ungotten list and, if so, dequeue it first. */ static rsRetVal qqueueDel(qqueue_t *pThis, void *pUsr) @@ -1213,13 +1146,8 @@ qqueueDel(qqueue_t *pThis, void *pUsr) * If we decrement, however, we may lose a message. But that is better than * losing the whole process because it loops... -- rgerhards, 2008-01-03 */ - if(pThis->iUngottenObjs > 0) { - iRet = GetUngottenObj(pThis, (obj_t**) pUsr); - } else { - iRet = pThis->qDeq(pThis, pUsr); - // TODO: ULTRA iRet = pThis->qDel(pThis, pUsr); - ATOMIC_DEC(pThis->iQueueSize); - } + iRet = pThis->qDeq(pThis, pUsr); + ATOMIC_DEC(pThis->iQueueSize); dbgoprint((obj_t*) pThis, "entry deleted, state %d, size now %d entries\n", iRet, pThis->iQueueSize); @@ -1528,6 +1456,8 @@ finalize_it: static rsRetVal ConsumerCancelCleanup(void *arg1, void *arg2) { + //TODO: looks like we no longer need it! + /* DEFiRet; qqueue_t *pThis = (qqueue_t*) arg1; @@ -1535,14 +1465,9 @@ ConsumerCancelCleanup(void *arg1, void *arg2) ISOBJ_TYPE_assert(pThis, qqueue); - if(pUsr != NULL) { - /* make sure the data element is not lost */ - dbgoprint((obj_t*) pThis, "cancelation cleanup handler consumer called, we need to unget one user data element\n"); - CHKiRet(UngetObj(pThis, pUsr, LOCK_MUTEX)); - } - -finalize_it: RETiRet; + */ + return RS_RET_OK; } @@ -2188,7 +2113,6 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) strm_t *psQIF = NULL; /* Queue Info File */ uchar pszQIFNam[MAXFNAME]; size_t lenQIFNam; - obj_t *pUsr; ASSERT(pThis != NULL); @@ -2235,20 +2159,10 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) */ CHKiRet(obj.BeginSerializePropBag(psQIF, (obj_t*) pThis)); objSerializeSCALAR(psQIF, iQueueSize, INT); - objSerializeSCALAR(psQIF, iUngottenObjs, INT); objSerializeSCALAR(psQIF, tVars.disk.sizeOnDisk, INT64); objSerializeSCALAR(psQIF, tVars.disk.bytesRead, INT64); CHKiRet(obj.EndSerialize(psQIF)); - /* now we must persist all objects on the ungotten queue - they can not go to - * to the regular files. -- rgerhards, 2008-01-29 - */ - while(pThis->iUngottenObjs > 0) { - CHKiRet(GetUngottenObj(pThis, &pUsr)); - CHKiRet((objSerialize(pUsr))(pUsr, psQIF)); - objDestruct(pUsr); - } - /* now persist the stream info */ CHKiRet(strmSerialize(pThis->tVars.disk.pWrite, psQIF)); CHKiRet(strmSerialize(pThis->tVars.disk.pReadDel, psQIF)); @@ -2615,8 +2529,6 @@ static rsRetVal qqueueSetProperty(qqueue_t *pThis, var_t *pProp) if(isProp("iQueueSize")) { pThis->iQueueSize = pProp->val.num; - } else if(isProp("iUngottenObjs")) { - pThis->iUngottenObjs = pProp->val.num; } else if(isProp("tVars.disk.sizeOnDisk")) { pThis->tVars.disk.sizeOnDisk = pProp->val.num; } else if(isProp("tVars.disk.bytesRead")) { diff --git a/runtime/queue.h b/runtime/queue.h index 00cee419..e47b8762 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -152,12 +152,6 @@ typedef struct queue_s { struct queue_s *pqDA; /* queue for disk-assisted modes */ struct queue_s *pqParent;/* pointer to the parent (if this is a child queue) */ int bDAEnqOnly; /* EnqOnly setting for DA queue */ - /* some data elements for the queueUngetObj() functionality. This list should always be short - * and is always kept in memory - */ - qLinkedList_t *pUngetRoot; - qLinkedList_t *pUngetLast; - int iUngottenObjs; /* number of objects currently in the "ungotten" list */ /* now follow queueing mode specific data elements */ union { /* different data elements based on queue type (qType) */ struct { -- cgit v1.2.3 From 1bbd2b07f8d7279dd844c7cd52c72af5da5abdb8 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Sun, 17 May 2009 21:53:29 +0200 Subject: Add shave support shave tranforms the verbose autotools output into a pretty Kbuild-like one which makes it easier to spot warnings. See also http://damien.lespiau.name/blog/2009/02/18/shave-making-the-autotools-output-sane/ git clone git://git.lespiau.name/shave By default, shave is disabled and you have to explicitly enable it via ./configure --enable-shave. Signed-off-by: Rainer Gerhards --- .gitignore | 2 + configure.ac | 3 ++ m4/shave.m4 | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++ shave-libtool.in | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ shave.in | 102 +++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 315 insertions(+) create mode 100644 m4/shave.m4 create mode 100644 shave-libtool.in create mode 100644 shave.in diff --git a/.gitignore b/.gitignore index ea044fbe..bc362e7f 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,5 @@ log logfile debug core.* +shave +shave-libtool diff --git a/configure.ac b/configure.ac index 6daf3f5a..db4de163 100644 --- a/configure.ac +++ b/configure.ac @@ -699,6 +699,7 @@ AC_ARG_ENABLE(imtemplate, AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) # end of copy template - be sure to serach for imtemplate to find everything! +SHAVE_INIT # settings for the omprog output module AC_ARG_ENABLE(omprog, @@ -754,6 +755,8 @@ AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) AC_CONFIG_FILES([Makefile \ + shave \ + shave-libtool \ runtime/Makefile \ tools/Makefile \ doc/Makefile \ diff --git a/m4/shave.m4 b/m4/shave.m4 new file mode 100644 index 00000000..e647e579 --- /dev/null +++ b/m4/shave.m4 @@ -0,0 +1,99 @@ +dnl Make automake/libtool output more friendly to humans +dnl +dnl Copyright (c) 2009, Damien Lespiau +dnl +dnl Permission is hereby granted, free of charge, to any person +dnl obtaining a copy of this software and associated documentation +dnl files (the "Software"), to deal in the Software without +dnl restriction, including without limitation the rights to use, +dnl copy, modify, merge, publish, distribute, sublicense, and/or sell +dnl copies of the Software, and to permit persons to whom the +dnl Software is furnished to do so, subject to the following +dnl conditions: +dnl +dnl The above copyright notice and this permission notice shall be +dnl included in all copies or substantial portions of the Software. +dnl +dnl THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +dnl EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +dnl OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +dnl NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +dnl HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +dnl WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +dnl FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +dnl OTHER DEALINGS IN THE SOFTWARE. +dnl +dnl SHAVE_INIT([shavedir],[default_mode]) +dnl +dnl shavedir: the directory where the shave scripts are, it defaults to +dnl $(top_builddir) +dnl default_mode: (enable|disable) default shave mode. This parameter +dnl controls shave's behaviour when no option has been +dnl given to configure. It defaults to disable. +dnl +dnl * SHAVE_INIT should be called late in your configure.(ac|in) file (just +dnl before AC_CONFIG_FILE/AC_OUTPUT is perfect. This macro rewrites CC and +dnl LIBTOOL, you don't want the configure tests to have these variables +dnl re-defined. +dnl * This macro requires GNU make's -s option. + +AC_DEFUN([_SHAVE_ARG_ENABLE], +[ + AC_ARG_ENABLE([shave], + AS_HELP_STRING( + [--enable-shave], + [use shave to make the build pretty [[default=$1]]]),, + [enable_shave=$1] + ) +]) + +AC_DEFUN([SHAVE_INIT], +[ + dnl you can tweak the default value of enable_shave + m4_if([$2], [enable], [_SHAVE_ARG_ENABLE(yes)], [_SHAVE_ARG_ENABLE(no)]) + + if test x"$enable_shave" = xyes; then + dnl where can we find the shave scripts? + m4_if([$1],, + [shavedir="$ac_pwd"], + [shavedir="$ac_pwd/$1"]) + AC_SUBST(shavedir) + + dnl make is now quiet + AC_SUBST([MAKEFLAGS], [-s]) + AC_SUBST([AM_MAKEFLAGS], ['`test -z $V && echo -s`']) + + dnl we need sed + AC_CHECK_PROG(SED,sed,sed,false) + + dnl substitute libtool + SHAVE_SAVED_LIBTOOL=$LIBTOOL + LIBTOOL="${SHELL} ${shavedir}/shave-libtool '${SHAVE_SAVED_LIBTOOL}'" + AC_SUBST(LIBTOOL) + + dnl substitute cc/cxx + SHAVE_SAVED_CC=$CC + SHAVE_SAVED_CXX=$CXX + SHAVE_SAVED_FC=$FC + SHAVE_SAVED_F77=$F77 + SHAVE_SAVED_OBJC=$OBJC + CC="${SHELL} ${shavedir}/shave cc ${SHAVE_SAVED_CC}" + CXX="${SHELL} ${shavedir}/shave cxx ${SHAVE_SAVED_CXX}" + FC="${SHELL} ${shavedir}/shave fc ${SHAVE_SAVED_FC}" + F77="${SHELL} ${shavedir}/shave f77 ${SHAVE_SAVED_F77}" + OBJC="${SHELL} ${shavedir}/shave objc ${SHAVE_SAVED_OBJC}" + AC_SUBST(CC) + AC_SUBST(CXX) + AC_SUBST(FC) + AC_SUBST(F77) + AC_SUBST(OBJC) + + V=@ + else + V=1 + fi + Q='$(V:1=)' + AC_SUBST(V) + AC_SUBST(Q) +]) + diff --git a/shave-libtool.in b/shave-libtool.in new file mode 100644 index 00000000..54ebd690 --- /dev/null +++ b/shave-libtool.in @@ -0,0 +1,109 @@ +#!/bin/sh +# +# Copyright (c) 2009, Damien Lespiau +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +# we need sed +SED=@SED@ +if test -z "$SED" ; then +SED=sed +fi + +lt_unmangle () +{ + last_result=`echo $1 | $SED -e 's#.libs/##' -e 's#[0-9a-zA-Z_\-\.]*_la-##'` +} + +# the real libtool to use +LIBTOOL="$1" +shift + +# if 1, don't print anything, the underlaying wrapper will do it +pass_though=0 + +# scan the arguments, keep the right ones for libtool, and discover the mode +preserved_args= + +# have we seen the --tag option of libtool in the command line ? +tag_seen=0 + +while test "$#" -gt 0; do + opt="$1" + shift + + case $opt in + --mode=*) + mode=`echo $opt | $SED -e 's/[-_a-zA-Z0-9]*=//'` + preserved_args="$preserved_args $opt" + ;; + -o) + lt_output="$1" + preserved_args="$preserved_args $opt" + ;; + --tag=*) + tag_seen=1 + preserved_args="$preserved_args $opt" + ;; + *) + preserved_args="$preserved_args $opt" + ;; + esac +done + +case "$mode" in +compile) + # shave will be called and print the actual CC/CXX/LINK line + preserved_args="$preserved_args --shave-mode=$mode" + pass_though=1 + ;; +link) + preserved_args="$preserved_args --shave-mode=$mode" + Q=" LINK " + ;; +*) + # let's u + # echo "*** libtool: Unimplemented mode: $mode, fill a bug report" + ;; +esac + +lt_unmangle "$lt_output" +output=$last_result + +# automake does not add a --tag switch to its libtool invocation when +# assembling a .s file and rely on libtool to infer the right action based +# on the compiler name. As shave is using CC to hook a wrapper, libtool gets +# confused. Let's detect these cases and add a --tag=CC option. +tag="" +if test $tag_seen -eq 0 -a x"$mode" = xcompile; then + tag="--tag=CC" +fi + +if test -z $V; then + if test $pass_though -eq 0; then + echo "$Q$output" + fi + $LIBTOOL --silent $tag $preserved_args +else + echo $LIBTOOL $tag $preserved_args + $LIBTOOL $tag $preserved_args +fi diff --git a/shave.in b/shave.in new file mode 100644 index 00000000..afed42e1 --- /dev/null +++ b/shave.in @@ -0,0 +1,102 @@ +#!/bin/sh +# +# Copyright (c) 2009, Damien Lespiau +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +# we need sed +SED=@SED@ +if test -z "$SED" ; then +SED=sed +fi + +lt_unmangle () +{ + last_result=`echo $1 | $SED -e 's#.libs/##' -e 's#[0-9a-zA-Z_\-\.]*_la-##'` +} + +# the tool to wrap (cc, cxx, ar, ranlib, ..) +tool="$1" +shift + +# the reel tool (to call) +REEL_TOOL="$1" +shift + +pass_through=0 +preserved_args= +while test "$#" -gt 0; do + opt="$1" + shift + + case $opt in + --shave-mode=*) + mode=`echo $opt | $SED -e 's/[-_a-zA-Z0-9]*=//'` + ;; + -o) + lt_output="$1" + preserved_args="$preserved_args $opt" + ;; + *) + preserved_args="$preserved_args $opt" + ;; + esac +done + +# mode=link is handled in the libtool wrapper +case "$mode,$tool" in +link,*) + pass_through=1 + ;; +*,cxx) + Q=" CXX " + ;; +*,cc) + Q=" CC " + ;; +*,fc) + Q=" FC " + ;; +*,f77) + Q=" F77 " + ;; +*,objc) + Q=" OBJC " + ;; +*,*) + # should not happen + Q=" CC " + ;; +esac + +lt_unmangle "$lt_output" +output=$last_result + +if test -z $V; then + if test $pass_through -eq 0; then + echo "$Q$output" + fi + $REEL_TOOL $preserved_args +else + echo $REEL_TOOL $preserved_args + $REEL_TOOL $preserved_args +fi -- cgit v1.2.3 From 81875845e63bf64922726c3da134e43b56e5d879 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Sun, 17 May 2009 21:55:40 +0200 Subject: Add autogen.sh This file is useful for building rsyslog from git. It is a shortcut for autoreconf and ./configure and enables shave by default. Signed-off-by: Rainer Gerhards --- autogen.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 autogen.sh diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 00000000..daa87a2a --- /dev/null +++ b/autogen.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +(test -f $srcdir/configure.ac) || { + echo -n "**Error**: Directory "\`$srcdir\'" does not look like the" + echo " top-level package directory" + exit 1 +} + +if test -z "$*"; then + echo "**Warning**: I am going to run \`configure' with no arguments." + echo "If you wish to pass any to it, please specify them on the" + echo \`$0\'" command line." + echo +fi + +(cd $srcdir && autoreconf --verbose --force --install) || exit 1 + +conf_flags="--enable-shave --cache-file=config.cache" + +if test x$NOCONFIGURE = x; then + echo Running $srcdir/configure $conf_flags "$@" ... + $srcdir/configure $conf_flags "$@" \ + && echo Now type \`make\' to compile. || exit 1 +else + echo Skipping configure process. +fi + + -- cgit v1.2.3 From 735a02c65622da0bc9c2d6b4ce44d126d4c766aa Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Mon, 18 May 2009 01:13:17 +0200 Subject: Cleanup and typo fixes Signed-off-by: Rainer Gerhards --- configure.ac | 8 ++------ doc/dev_oplugins.html | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/configure.ac b/configure.ac index db4de163..a93a968c 100644 --- a/configure.ac +++ b/configure.ac @@ -697,7 +697,7 @@ AC_ARG_ENABLE(imtemplate, # for samples # AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) -# end of copy template - be sure to serach for imtemplate to find everything! +# end of copy template - be sure to search for imtemplate to find everything! SHAVE_INIT @@ -732,7 +732,7 @@ AC_ARG_ENABLE(omtemplate, # for samples # AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) -# end of copy template - be sure to serach for omtemplate to find everything! +# end of copy template - be sure to search for omtemplate to find everything! # settings for omstdout @@ -747,10 +747,6 @@ AC_ARG_ENABLE(omstdout, esac], [enable_omstdout=no] ) -# -# you may want to do some library checks here - see snmp, mysql, pgsql modules -# for samples -# AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) diff --git a/doc/dev_oplugins.html b/doc/dev_oplugins.html index 5bfc974c..cc2f7f38 100644 --- a/doc/dev_oplugins.html +++ b/doc/dev_oplugins.html @@ -47,7 +47,7 @@ copying omtemplate. Then, the basic steps you need to do are:

    • mv omtemplate.c your-plugin.c
    • cd ../..
    • vi Makefile.am configure.ac -
      serach for omtemplate, copy and modify (follow comments) +
      search for omtemplate, copy and modify (follow comments)

    Basically, this is all you need to do ... Well, except, of course, coding your plugin ;). For testing, you need rsyslog's debugging support. Some useful -- cgit v1.2.3 From 5ab3e8005f62c2016bdb2d3b099d1d8c1a4cf2bc Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Mon, 18 May 2009 21:07:07 +0200 Subject: Fix compiler warnings include for memcpy and strlen. Signed-off-by: Rainer Gerhards --- runtime/vm.c | 1 + tcps_sess.c | 1 + tests/rscript.c | 1 + 3 files changed, 3 insertions(+) diff --git a/runtime/vm.c b/runtime/vm.c index 41d3e483..125b0d21 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -25,6 +25,7 @@ #include "config.h" #include +#include #include #include diff --git a/tcps_sess.c b/tcps_sess.c index 2a3ec0df..ceb6142f 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -29,6 +29,7 @@ */ #include "config.h" #include +#include #include #include #include diff --git a/tests/rscript.c b/tests/rscript.c index 6b232f5f..ce81491c 100644 --- a/tests/rscript.c +++ b/tests/rscript.c @@ -24,6 +24,7 @@ */ #include "config.h" #include +#include #include #include -- cgit v1.2.3 From 460edfe27fcca96e88d6541a3e1ccb49ab4b1a7a Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Mon, 18 May 2009 21:41:31 +0200 Subject: Do not fail "make check" if omstdout is not enabled Only run omod-if-array.sh and parsertest.sh test if omstdout plugin is enabled. Remove the comment and fix the help output for --enable-stdout (default was "no", not "yes") Signed-off-by: Rainer Gerhards --- configure.ac | 4 +--- tests/Makefile.am | 9 ++++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index a93a968c..9b160379 100644 --- a/configure.ac +++ b/configure.ac @@ -736,10 +736,8 @@ AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) # settings for omstdout -# note that "make check" fails, if omstdout is not enabled (thus we enable -# it by default). AC_ARG_ENABLE(omstdout, - [AS_HELP_STRING([--enable-omstdout],[Compiles stdout template module @<:@default=yes@:>@])], + [AS_HELP_STRING([--enable-omstdout],[Compiles stdout template module @<:@default=no@:>@])], [case "${enableval}" in yes) enable_omstdout="yes" ;; no) enable_omstdout="no" ;; diff --git a/tests/Makefile.am b/tests/Makefile.am index 0f4cbce1..ed415a1e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,9 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh parsertest.sh omod-if-array.sh manytcp.sh diskqueue.sh +TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh +if ENABLE_OMSTDOUT +TESTS += omod-if-array.sh parsertest.sh +endif TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid test_files = testbench.h runtime-dummy.c @@ -8,7 +11,7 @@ test_files = testbench.h runtime-dummy.c EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg1.cfgtest \ cfg1.testin \ - cfg2.cfgtest \ + cfg2.cfgtest \ cfg2.testin \ cfg3.cfgtest \ cfg3.testin \ @@ -19,7 +22,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ NoExistFile.cfgtest \ testsuites/parse1.conf \ testsuites/1.parse1 \ - testsuites/rfc3164.parse1 \ + testsuites/rfc3164.parse1 \ testsuites/rfc5424-1.parse1 \ testsuites/rfc5424-2.parse1 \ testsuites/rfc5424-3.parse1 \ -- cgit v1.2.3 From 67974ecd5df103b4d69932410ea0147cba469f9b Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Mon, 18 May 2009 21:47:59 +0200 Subject: Rename want_gssapi_krb5 to enable_gssapi_krb5 Be consistent and rename the configure variable want_gssapi_krb5 to enable_gssapi_krb5. Signed-off-by: Rainer Gerhards --- configure.ac | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index 9b160379..17c527a8 100644 --- a/configure.ac +++ b/configure.ac @@ -197,13 +197,13 @@ fi AC_ARG_ENABLE(gssapi_krb5, [AS_HELP_STRING([--enable-gssapi-krb5],[Enable GSSAPI Kerberos 5 support @<:@default=no@:>@])], [case "${enableval}" in - yes) want_gssapi_krb5="yes" ;; - no) want_gssapi_krb5="no" ;; + yes) enable_gssapi_krb5="yes" ;; + no) enable_gssapi_krb5="no" ;; *) AC_MSG_ERROR(bad value ${enableval} for --enable-gssapi-krb5) ;; esac], - [want_gssapi_krb5=no] + [enable_gssapi_krb5=no] ) -if test $want_gssapi_krb5 = yes; then +if test $enable_gssapi_krb5 = yes; then AC_CHECK_LIB(gssapi_krb5, gss_acquire_cred, [ AC_CHECK_HEADER(gssapi/gssapi.h, [ AC_DEFINE(USE_GSSAPI,, @@ -213,7 +213,7 @@ if test $want_gssapi_krb5 = yes; then ]) ]) fi -AM_CONDITIONAL(ENABLE_GSSAPI, test x$want_gssapi_krb5 = xyes) +AM_CONDITIONAL(ENABLE_GSSAPI, test x$enable_gssapi_krb5 = xyes) # multithreading via pthreads AC_ARG_ENABLE(pthreads, @@ -803,7 +803,7 @@ echo "omstdout module will be compiled: $enable_omstdout" echo "Large file support enabled: $enable_largefile" echo "Networking support enabled: $enable_inet" echo "GnuTLS network stream driver enabled: $enable_gnutls" -echo "Enable GSSAPI Kerberos 5 support: $want_gssapi_krb5" +echo "Enable GSSAPI Kerberos 5 support: $enable_gssapi_krb5" echo "Debug mode enabled: $enable_debug" echo "Runtime Instrumentation enabled: $enable_rtinst" echo "Diagnostic tools enabled: $enable_diagtools" -- cgit v1.2.3 From 541d7ad9401d3a606c9ecc079eb2de875e5d6f0c Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Mon, 18 May 2009 21:50:29 +0200 Subject: Improve ./configure output Improve readability of the ./configure output by grouping relevant entries and indenting them. Signed-off-by: Rainer Gerhards --- configure.ac | 66 ++++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/configure.ac b/configure.ac index 17c527a8..a4f0acfa 100644 --- a/configure.ac +++ b/configure.ac @@ -783,31 +783,41 @@ AC_OUTPUT echo "****************************************************" echo "rsyslog will be compiled with the following settings:" echo -echo "Multithreading support enabled: $enable_pthreads" -echo "Klog functionality enabled: $enable_klog ($os_type)" -echo "Regular expressions support enabled: $enable_regexp" -echo "Zlib compression support enabled: $enable_zlib" -echo "MySql support enabled: $enable_mysql" -echo "libdbi support enabled: $enable_libdbi" -echo "PostgreSQL support enabled: $enable_pgsql" -echo "Oracle (OCI) support enabled: $enable_oracle" -echo "SNMP support enabled: $enable_snmp" -echo "Mail support enabled: $enable_mail" -echo "RELP support enabled: $enable_relp" -echo "imdiag enabled: $enable_imdiag" -echo "file input module enabled: $enable_imfile" -echo "input template module will be compiled: $enable_imtemplate" -echo "output template module will be compiled: $enable_omtemplate" -echo "omprog module will be compiled: $enable_omprog" -echo "omstdout module will be compiled: $enable_omstdout" -echo "Large file support enabled: $enable_largefile" -echo "Networking support enabled: $enable_inet" -echo "GnuTLS network stream driver enabled: $enable_gnutls" -echo "Enable GSSAPI Kerberos 5 support: $enable_gssapi_krb5" -echo "Debug mode enabled: $enable_debug" -echo "Runtime Instrumentation enabled: $enable_rtinst" -echo "Diagnostic tools enabled: $enable_diagtools" -echo "valgrind support settings enabled: $enable_valgrind" -echo "rsyslog runtime will be built: $enable_rsyslogrt" -echo "rsyslogd will be built: $enable_rsyslogd" - +echo " Multithreading support enabled: $enable_pthreads" +echo " Large file support enabled: $enable_largefile" +echo " Networking support enabled: $enable_inet" +echo " Regular expressions support enabled: $enable_regexp" +echo " Zlib compression support enabled: $enable_zlib" +echo " rsyslog runtime will be built: $enable_rsyslogrt" +echo " rsyslogd will be built: $enable_rsyslogd" +echo +echo "---{ input plugins }---" +echo " Klog functionality enabled: $enable_klog ($os_type)" +echo " imdiag enabled: $enable_imdiag" +echo " file input module enabled: $enable_imfile" +echo " input template module will be compiled: $enable_imtemplate" +echo +echo "---{ output plugins }---" +echo " Mail support enabled: $enable_mail" +echo " omprog module will be compiled: $enable_omprog" +echo " omstdout module will be compiled: $enable_omstdout" +echo " output template module will be compiled: $enable_omtemplate" +echo +echo "---{ database support }---" +echo " MySql support enabled: $enable_mysql" +echo " libdbi support enabled: $enable_libdbi" +echo " PostgreSQL support enabled: $enable_pgsql" +echo " Oracle (OCI) support enabled: $enable_oracle" +echo +echo "---{ protocol support }---" +echo " GnuTLS network stream driver enabled: $enable_gnutls" +echo " GSSAPI Kerberos 5 support enabled: $enable_gssapi_krb5" +echo " RELP support enabled: $enable_relp" +echo " SNMP support enabled: $enable_snmp" +echo +echo "---{ debugging support }---" +echo " Debug mode enabled: $enable_debug" +echo " Runtime Instrumentation enabled: $enable_rtinst" +echo " Diagnostic tools enabled: $enable_diagtools" +echo " Valgrind support settings enabled: $enable_valgrind" +echo -- cgit v1.2.3 From 4f691b1e1dd3ee6e7f26e57aef41a730e7168a40 Mon Sep 17 00:00:00 2001 From: Michael Biebl Date: Mon, 18 May 2009 22:00:53 +0200 Subject: Cleanup configure.ac a little for better readability - Use 2 new lines as separator between configure options. - Remove forgotten copy&paste text. - Reorder and move template configure option to the end. Signed-off-by: Rainer Gerhards --- configure.ac | 76 +++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/configure.ac b/configure.ac index a4f0acfa..8c8db8e1 100644 --- a/configure.ac +++ b/configure.ac @@ -144,6 +144,7 @@ AC_ARG_WITH(moddirs, AM_CONDITIONAL(WITH_MODDIRS, test x$moddirs != x) AC_SUBST(moddirs) + # Large file support AC_ARG_ENABLE(largefile, [AS_HELP_STRING([--enable-largefile],[Enable large file support @<:@default=yes@:>@])], @@ -158,6 +159,7 @@ if test "$enable_largefile" = "no"; then AC_DEFINE(NOLARGEFILE, 1, [Defined when large file support is disabled.]) fi + # Regular expressions AC_ARG_ENABLE(regexp, [AS_HELP_STRING([--enable-regexp],[Enable regular expressions support @<:@default=yes@:>@])], @@ -173,6 +175,7 @@ if test "$enable_regexp" = "yes"; then AC_DEFINE(FEATURE_REGEXP, 1, [Regular expressions support enabled.]) fi + # zlib compression AC_ARG_ENABLE(zlib, [AS_HELP_STRING([--enable-zlib],[Enable zlib compression support @<:@default=yes@:>@])], @@ -193,6 +196,7 @@ if test "$enable_zlib" = "yes"; then fi fi + #gssapi AC_ARG_ENABLE(gssapi_krb5, [AS_HELP_STRING([--enable-gssapi-krb5],[Enable GSSAPI Kerberos 5 support @<:@default=no@:>@])], @@ -215,6 +219,7 @@ if test $enable_gssapi_krb5 = yes; then fi AM_CONDITIONAL(ENABLE_GSSAPI, test x$enable_gssapi_krb5 = xyes) + # multithreading via pthreads AC_ARG_ENABLE(pthreads, [AS_HELP_STRING([--enable-pthreads],[Enable multithreading via pthreads @<:@default=yes@:>@])], @@ -255,6 +260,7 @@ if test "x$enable_pthreads" != "xno"; then ) fi + # klog AC_ARG_ENABLE(klog, [AS_HELP_STRING([--enable-klog],[Integrated klog functionality @<:@default=yes@:>@])], @@ -269,6 +275,7 @@ AM_CONDITIONAL(ENABLE_IMKLOG, test x$enable_klog = xyes) AM_CONDITIONAL(ENABLE_IMKLOG_BSD, test x$os_type = xbsd) AM_CONDITIONAL(ENABLE_IMKLOG_LINUX, test x$os_type = xlinux) + # # SYSLOG_UNIXAF # @@ -289,6 +296,7 @@ AC_ARG_ENABLE([unix], AC_DEFINE([SYSLOG_UNIXAF], [1], [Description]) ]) + # inet AC_ARG_ENABLE(inet, [AS_HELP_STRING([--enable-inet],[Enable networking support @<:@default=yes@:>@])], @@ -304,6 +312,7 @@ if test "$enable_inet" = "yes"; then AC_DEFINE(SYSLOG_INET, 1, [network support is integrated.]) fi + # # The following define determines whether the package adheres to the # file system standard. @@ -325,6 +334,7 @@ AC_ARG_ENABLE([fsstnd], AC_DEFINE([FSSTND], [1], [Description]) ]) + # debug AC_ARG_ENABLE(debug, [AS_HELP_STRING([--enable-debug],[Enable debug mode @<:@default=no@:>@])], @@ -342,6 +352,7 @@ if test "$enable_debug" = "no"; then AC_DEFINE(NDEBUG, 1, [Defined if debug mode is disabled.]) fi + # runtime instrumentation AC_ARG_ENABLE(rtinst, [AS_HELP_STRING([--enable-rtinst],[Enable runtime instrumentation mode @<:@default=no@:>@])], @@ -356,6 +367,7 @@ if test "$enable_rtinst" = "yes"; then AC_DEFINE(RTINST, 1, [Defined if runtime instrumentation mode is enabled.]) fi + # valgrind AC_ARG_ENABLE(valgrind, [AS_HELP_STRING([--enable-valgrind],[Enable valgrind support settings @<:@default=no@:>@])], @@ -452,6 +464,7 @@ AM_CONDITIONAL(ENABLE_PGSQL, test x$enable_pgsql = xyes) AC_SUBST(PGSQL_CFLAGS) AC_SUBST(PGSQL_LIBS) + # oracle (OCI) support AC_ARG_ENABLE(oracle, [AS_HELP_STRING([--enable-oracle],[Enable native Oracle database support @<:@default=no@:>@])], @@ -484,7 +497,8 @@ fi AM_CONDITIONAL(ENABLE_ORACLE, test x$enable_oracle = xyes) AC_SUBST(ORACLE_CFLAGS) AC_SUBST(ORACLE_LIBS) - + + # libdbi support AC_ARG_ENABLE(libdbi, [AS_HELP_STRING([--enable-libdbi],[Enable libdbi database support @<:@default=no@:>@])], @@ -645,6 +659,7 @@ AM_CONDITIONAL(ENABLE_RELP, test x$enable_relp = xyes) AC_SUBST(RELP_CFLAGS) AC_SUBST(RELP_LIBS) + # RFC 3195 support AC_ARG_ENABLE(rfc3195, [AS_HELP_STRING([--enable-rfc3195],[Enable RFC3195 support @<:@default=no@:>@])], @@ -663,7 +678,7 @@ AC_SUBST(LIBLOGGING_CFLAGS) AC_SUBST(LIBLOGGING_LIBS) -# settings for the file input module; +# settings for the file input module AC_ARG_ENABLE(imfile, [AS_HELP_STRING([--enable-imfile],[file input module enabled @<:@default=no@:>@])], [case "${enableval}" in @@ -673,13 +688,35 @@ AC_ARG_ENABLE(imfile, esac], [enable_imfile=no] ) -# -# you may want to do some library checks here - see snmp, mysql, pgsql modules -# for samples -# AM_CONDITIONAL(ENABLE_IMFILE, test x$enable_imfile = xyes) +# settings for the omprog output module +AC_ARG_ENABLE(omprog, + [AS_HELP_STRING([--enable-omprog],[Compiles omprog module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omprog="yes" ;; + no) enable_omprog="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omprog) ;; + esac], + [enable_omprog=no] +) +AM_CONDITIONAL(ENABLE_OMPROG, test x$enable_omprog = xyes) + + +# settings for omstdout +AC_ARG_ENABLE(omstdout, + [AS_HELP_STRING([--enable-omstdout],[Compiles stdout module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_omstdout="yes" ;; + no) enable_omstdout="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-omstdout) ;; + esac], + [enable_omstdout=no] +) +AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) + + # settings for the template input module; copy and modify this code # if you intend to add your own module. Be sure to replace imtemplate # by the actual name of your module. @@ -699,21 +736,6 @@ AC_ARG_ENABLE(imtemplate, AM_CONDITIONAL(ENABLE_IMTEMPLATE, test x$enable_imtemplate = xyes) # end of copy template - be sure to search for imtemplate to find everything! -SHAVE_INIT - -# settings for the omprog output module -AC_ARG_ENABLE(omprog, - [AS_HELP_STRING([--enable-omprog],[Compiles omprog template module @<:@default=no@:>@])], - [case "${enableval}" in - yes) enable_omprog="yes" ;; - no) enable_omprog="no" ;; - *) AC_MSG_ERROR(bad value ${enableval} for --enable-omprog) ;; - esac], - [enable_omprog=no] -) -AM_CONDITIONAL(ENABLE_OMPROG, test x$enable_omprog = xyes) -# end of omprog - # settings for the template output module; copy and modify this code # if you intend to add your own module. Be sure to replace omtemplate @@ -735,17 +757,7 @@ AM_CONDITIONAL(ENABLE_OMTEMPLATE, test x$enable_omtemplate = xyes) # end of copy template - be sure to search for omtemplate to find everything! -# settings for omstdout -AC_ARG_ENABLE(omstdout, - [AS_HELP_STRING([--enable-omstdout],[Compiles stdout template module @<:@default=no@:>@])], - [case "${enableval}" in - yes) enable_omstdout="yes" ;; - no) enable_omstdout="no" ;; - *) AC_MSG_ERROR(bad value ${enableval} for --enable-omstdout) ;; - esac], - [enable_omstdout=no] -) -AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) +SHAVE_INIT AC_CONFIG_FILES([Makefile \ -- cgit v1.2.3 From 55714cdb440b4baa32f88d55316755555ef93607 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 19 May 2009 09:17:13 +0200 Subject: updated ChangeLog with Michael Biebl's changes --- ChangeLog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index d37b8e0b..30fb7bff 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,6 @@ --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? -- performance enhancemnt: imtcp calls parser no longer on input thread +- performance enhancement: imtcp calls parser no longer on input thread but rather inside on of the potentially many main msg queue worker threads (an enhancement scheduled for all input plugins where this is possible) @@ -15,6 +15,7 @@ Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? * added tests for queue disk-only mode (checks disk queue logic) - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs +- build system improvements - thanks to Michael Biebl --------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program -- cgit v1.2.3 From a4dad2009992d436ba23c2d0a4a43b483aac40fc Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 19 May 2009 11:03:09 +0200 Subject: queue size calculation now based on logical/physical dequeue ... needed to split the old single counter into two. I wouldn't bet that I made some mistakes while doing so, but at least some ad-hoc tests plus the testbench do no longer indicate errors. --- runtime/queue.c | 114 +++++++++++++++++++++++++++++--------------------- runtime/queue.h | 3 +- tests/da-mainmsg-q.sh | 4 +- tests/diskqueue.sh | 1 + 4 files changed, 71 insertions(+), 51 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index 8ef3e7db..9855dac8 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -168,29 +168,37 @@ finalize_it: /* methods */ -/* get the overall queue size. Must only be called +/* get the physical queue size. Must only be called * while mutex is locked! * rgerhards, 2008-01-29 */ static inline int -qqueueGetOverallQueueSize(qqueue_t *pThis) +getPhysicalQueueSize(qqueue_t *pThis) { -#if 0 /* leave a bit in for debugging -- rgerhards, 2008-01-30 */ -BEGINfunc -dbgoprint((obj_t*) pThis, "queue size: %d (regular %d)\n", - pThis->iQueueSize, pThis->iQueueSize); -ENDfunc -#endif return pThis->iQueueSize; } +/* get the logical queue size (that is store size minus logically dequeued elements). + * Must only be called while mutex is locked! + * rgerhards, 2009-05-19 + */ +static inline int +getLogicalQueueSize(qqueue_t *pThis) +{ + return pThis->iQueueSize - pThis->nLogDeq; +} + + + /* This function drains the queue in cases where this needs to be done. The most probable * reason is a HUP which needs to discard data (because the queue is configured to be lossy). * During a shutdown, this is typically not needed, as the OS frees up ressources and does * this much quicker than when we clean up ourselvs. -- rgerhards, 2008-10-21 * This function returns void, as it makes no sense to communicate an error back, even if * it happens. + * This functions works "around" the regular deque mechanism, because it is only used to + * clean up (in cases where message loss is acceptable). */ static inline void queueDrain(qqueue_t *pThis) { @@ -198,14 +206,14 @@ static inline void queueDrain(qqueue_t *pThis) ASSERT(pThis != NULL); +// TODO: ULTRA it may be a good idea to check validitity once again /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ while(pThis->iQueueSize-- > 0) { pThis->qDeq(pThis, &pUsr); -// TODO: ULTRA - //pThis->qDel(pThis, &pUsr); if(pUsr != NULL) { objDestruct(pUsr); } + pThis->qDel(pThis); } } @@ -229,14 +237,14 @@ static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) /* if we have not yet reached the high water mark, there is no need to start a * worker. -- rgerhards, 2008-01-26 */ - if(qqueueGetOverallQueueSize(pThis) >= pThis->iHighWtrMrk || pThis->bQueueStarted == 0) { + if(getLogicalQueueSize(pThis) >= pThis->iHighWtrMrk || pThis->bQueueStarted == 0) { wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ } } else { if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { iMaxWorkers = 1; } else { - iMaxWorkers = qqueueGetOverallQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; + iMaxWorkers = getLogicalQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; } wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); /* disk queues have always one worker */ } @@ -295,13 +303,13 @@ qqueueTurnOffDAMode(qqueue_t *pThis) */ /* we need to check if the DA queue is empty because the DA worker may simply have - * terminated do to no new messages arriving. That does not, however, mean that the + * terminated due to no new messages arriving. That does not, however, mean that the * DA queue is empty. If there is still data in that queue, we do nothing and leave * that for a later incarnation of this function (it will be called multiple times * during the lifetime of DA-mode, depending on how often the DA worker receives an * inactivity timeout. -- rgerhards, 2008-01-25 */ - if(pThis->pqDA->iQueueSize == 0) { + if(getLogicalQueueSize(pThis->pqDA) == 0) { pThis->bRunsDA = 0; /* tell the world we are back in non-DA mode */ /* we destruct the queue object, which will also shutdown the queue worker. As the queue is empty, * this will be quick. @@ -313,7 +321,7 @@ qqueueTurnOffDAMode(qqueue_t *pThis) * when it is waiting that the high water mark is reached again. If so, we need to start up * a regular worker. -- rgerhards, 2008-01-26 */ - if(qqueueGetOverallQueueSize(pThis) > 0) { + if(getLogicalQueueSize(pThis) > 0) { qqueueAdviseMaxWorkers(pThis); } } @@ -507,7 +515,7 @@ qqueueChkStrtDA(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); /* if we do not hit the high water mark, we have nothing to do */ - if(qqueueGetOverallQueueSize(pThis) != pThis->iHighWtrMrk) + if(getPhysicalQueueSize(pThis) != pThis->iHighWtrMrk) ABORT_FINALIZE(RS_RET_OK); if(pThis->bRunsDA) { @@ -521,14 +529,14 @@ qqueueChkStrtDA(qqueue_t *pThis) * we need at least one). */ dbgoprint((obj_t*) pThis, "%d entries - passed high water mark in DA mode, send notify\n", - qqueueGetOverallQueueSize(pThis)); + getPhysicalQueueSize(pThis)); qqueueAdviseMaxWorkers(pThis); } else { /* this is the case when we are currently not running in DA mode. So it is time * to turn it back on. */ dbgoprint((obj_t*) pThis, "%d entries - passed high water mark for disk-assisted mode, initiating...\n", - qqueueGetOverallQueueSize(pThis)); + getPhysicalQueueSize(pThis)); qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, MUTEX_ALREADY_LOCKED); /* initiate DA mode */ } @@ -1124,7 +1132,8 @@ qqueueAdd(qqueue_t *pThis, void *pUsr) if(pThis->qType != QUEUETYPE_DIRECT) { ATOMIC_INC(pThis->iQueueSize); - dbgoprint((obj_t*) pThis, "entry added, size now %d entries\n", pThis->iQueueSize); + dbgoprint((obj_t*) pThis, "entry added, size now log %d, phys %d entries\n", + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); } finalize_it: @@ -1132,10 +1141,10 @@ finalize_it: } -/* generic code to remove a queue entry +/* generic code to dequeue a queue entry */ static rsRetVal -qqueueDel(qqueue_t *pThis, void *pUsr) +qqueueDeq(qqueue_t *pThis, void *pUsr) { DEFiRet; @@ -1147,10 +1156,10 @@ qqueueDel(qqueue_t *pThis, void *pUsr) * losing the whole process because it loops... -- rgerhards, 2008-01-03 */ iRet = pThis->qDeq(pThis, pUsr); - ATOMIC_DEC(pThis->iQueueSize); + ATOMIC_INC(pThis->nLogDeq); - dbgoprint((obj_t*) pThis, "entry deleted, state %d, size now %d entries\n", - iRet, pThis->iQueueSize); + dbgoprint((obj_t*) pThis, "entry deleted, size now log %d, phys %d entries\n", + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); RETiRet; } @@ -1188,7 +1197,7 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) /* first try to shutdown the queue within the regular shutdown period */ BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(qqueueGetOverallQueueSize(pThis) > 0) { + if(getPhysicalQueueSize(pThis) > 0) { if(pThis->bRunsDA) { /* We may have waited on the low water mark. As it may have changed, we * see if we reactivate the worker. @@ -1259,7 +1268,7 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ /* optimize parameters for shutdown of DA-enabled queues */ - if(pThis->bIsDA && qqueueGetOverallQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { + if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { /* switch to enqueue-only mode so that no more actions happen */ if(pThis->bRunsDA == 0) { qqueueInitDA(pThis, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to DA mode */ @@ -1289,7 +1298,7 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) * they will automatically terminate as there no longer is any message left to process. */ BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(qqueueGetOverallQueueSize(pThis) > 0) { + if(getPhysicalQueueSize(pThis) > 0) { timeoutComp(&tTimeout, pThis->toActShutdown); if(wtpGetCurNumWrkr(pThis->pWtpReg, LOCK_MUTEX) > 0) { END_MTX_PROTECTED_OPERATIONS(pThis->mut); @@ -1358,7 +1367,7 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) * Well, more precisely, they *are in termination*. Some cancel cleanup handlers * may still be running. */ - dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size %d.\n", qqueueGetOverallQueueSize(pThis)); + dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size %d.\n", getPhysicalQueueSize(pThis)); RETiRet; } @@ -1397,6 +1406,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->lenSpoolDir = strlen((char*)pThis->pszSpoolDir); pThis->iMaxFileSize = 1024 * 1024; /* default is 1 MiB */ pThis->iQueueSize = 0; + pThis->nLogDeq = 0; pThis->iMaxQueueSize = iMaxQueueSize; pThis->pConsumer = pConsumer; pThis->iNumWorkerThreads = iWorkerThreads; @@ -1522,11 +1532,16 @@ DoDeleteBatchFromQStore(qqueue_t *pThis, int nElem) ISOBJ_TYPE_assert(pThis, qqueue); +dbgprintf("pre delete batch from store, new sizes: log %d, phys %d, nElem %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis), nElem); /* now send delete request to storage driver */ for(i = 0 ; i < nElem ; ++i) { pThis->qDel(pThis); } + /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ + pThis->iQueueSize -= nElem; + pThis->nLogDeq -= nElem; +dbgprintf("delete batch from store, new sizes: log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); ++pThis->deqIDDel; /* one more batch dequeued */ RETiRet; @@ -1627,11 +1642,13 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz nDequeued = 0; do { dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); - CHKiRet(qqueueDel(pThis, &pUsr)); - iQueueSize = qqueueGetOverallQueueSize(pThis); + CHKiRet(qqueueDeq(pThis, &pUsr)); + iQueueSize = getLogicalQueueSize(pThis); /* check if we should discard this element */ localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr); + //MULTI-DEQUEUE / ULTRA-RELIABLE: we need to handle this case, we need to advance the + // DEQ pointer (or so...) TODO!!! Idea: get a second nElem int in pBatch, nDequeued. Use that when deleting! if(localRet == RS_RET_QUEUE_FULL) continue; else if(localRet != RS_RET_OK) @@ -1880,7 +1897,7 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) && pThis->pqDA->tVars.disk.sizeOnDisk > pThis->pqDA->sizeOnDiskMax) { /* this queue can never grow, so we can give up... */ bStopWrkr = 1; - } else if(qqueueGetOverallQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { + } else if(getPhysicalQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { bStopWrkr = 1; } else { bStopWrkr = 0; @@ -1903,9 +1920,9 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) * the DA queue */ static int -ChkStooWrkrReg(qqueue_t *pThis) +ChkStopWrkrReg(qqueue_t *pThis) { - return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && qqueueGetOverallQueueSize(pThis) == 0); + return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && getPhysicalQueueSize(pThis) == 0); } @@ -1929,7 +1946,7 @@ qqueueIsIdleDA(qqueue_t *pThis) { /* remember: iQueueSize is the DA queue size, not the main queue! */ /* TODO: I think we need just a single function for DA and non-DA mode - but I leave it for now as is */ - return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); + return(getLogicalQueueSize(pThis) == 0 || (pThis->bRunsDA && getLogicalQueueSize(pThis) <= pThis->iLowWtrMrk)); } /* must only be called when the queue mutex is locked, else results * are not stable! Regular queue version @@ -1939,12 +1956,12 @@ IsIdleReg(qqueue_t *pThis) { #if 0 /* enable for performance testing */ int ret; - ret = qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk); + ret = getLogicalQueueSize(pThis) == 0 || (pThis->bRunsDA && getLogicalQueueSize(pThis) <= pThis->iLowWtrMrk); if(ret) fprintf(stderr, "queue is idle\n"); return ret; #else /* regular code! */ - return(qqueueGetOverallQueueSize(pThis) == 0 || (pThis->bRunsDA && qqueueGetOverallQueueSize(pThis) <= pThis->iLowWtrMrk)); + return(getLogicalQueueSize(pThis) == 0 || (pThis->bRunsDA && getLogicalQueueSize(pThis) <= pThis->iLowWtrMrk)); #endif } @@ -2037,11 +2054,12 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ /* call type-specific constructor */ CHKiRet(pThis->qConstruct(pThis)); /* this also sets bIsDA */ - dbgoprint((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, qsize %d, child %d, " + dbgoprint((obj_t*) pThis, "type %d, enq-only %d, disk assisted %d, maxFileSz %lld, lqsize %d, pqsize %d, child %d, " "full delay %d, light delay %d, deq batch size %d starting\n", pThis->qType, pThis->bEnqOnly, pThis->bIsDA, pThis->iMaxFileSize, - qqueueGetOverallQueueSize(pThis), pThis->pqParent == NULL ? 0 : 1, - pThis->iFullDlyMrk, pThis->iLightDlyMrk, pThis->iDeqBatchSize); + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis), + pThis->pqParent == NULL ? 0 : 1, pThis->iFullDlyMrk, pThis->iLightDlyMrk, + pThis->iDeqBatchSize); if(pThis->qType == QUEUETYPE_DIRECT) FINALIZE; /* with direct queues, we are already finished... */ @@ -2053,7 +2071,7 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ CHKiRet(wtpConstruct (&pThis->pWtpReg)); CHKiRet(wtpSetDbgHdr (pThis->pWtpReg, pszBuf, lenBuf)); CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RateLimiter)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) ChkStooWrkrReg)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) ChkStopWrkrReg)); CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) IsIdleReg)); CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerReg)); @@ -2117,7 +2135,7 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) ASSERT(pThis != NULL); if(pThis->qType != QUEUETYPE_DISK) { - if(qqueueGetOverallQueueSize(pThis) > 0) { + if(getPhysicalQueueSize(pThis) > 0) { /* This error code is OK, but we will probably not implement this any time * The reason is that persistence happens via DA queues. But I would like to * leave the code as is, as we so have a hook in case we need one. @@ -2128,13 +2146,13 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) FINALIZE; /* if the queue is empty, we are happy and done... */ } - dbgoprint((obj_t*) pThis, "persisting queue to disk, %d entries...\n", qqueueGetOverallQueueSize(pThis)); + dbgoprint((obj_t*) pThis, "persisting queue to disk, %d entries...\n", getPhysicalQueueSize(pThis)); /* Construct file name */ lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); - if((bIsCheckpoint != QUEUE_CHECKPOINT) && (qqueueGetOverallQueueSize(pThis) == 0)) { + if((bIsCheckpoint != QUEUE_CHECKPOINT) && (getPhysicalQueueSize(pThis) == 0)) { if(pThis->bNeedDelQIF) { unlink((char*)pszQIFNam); pThis->bNeedDelQIF = 0; @@ -2342,13 +2360,13 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) ISOBJ_TYPE_assert(pThis, qqueue); /* first check if we need to discard this message (which will cause CHKiRet() to exit) - * rgerhards, 2008-10-07: It is OK to do this outside of mutex protection. The iQueueSize + * rgerhards, 2008-10-07: It is OK to do this outside of mutex protection. The queue size * and bRunsDA parameters may not reflect the correct settings here, but they are * "good enough" in the sense that they can be used to drive the decision. Valgrind's * threading tools may point this access to be an error, but this is done * intentional. I do not see this causes problems to us. */ - CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); + CHKiRet(qqueueChkDiscardMsg(pThis, getPhysicalQueueSize(pThis), pThis->bRunsDA, pUsr)); /* Please note that this function is not cancel-safe and consequently * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE @@ -2386,12 +2404,12 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) * It's a side effect, but a good one ;) -- rgerhards, 2008-03-14 */ if(flowCtlType == eFLOWCTL_FULL_DELAY) { - while(pThis->iQueueSize >= pThis->iFullDlyMrk) { + while(getPhysicalQueueSize(pThis) >= pThis->iFullDlyMrk) { dbgoprint((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n"); pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut); /* TODO error check? But what do then? */ } } else if(flowCtlType == eFLOWCTL_LIGHT_DELAY) { - if(pThis->iQueueSize >= pThis->iLightDlyMrk) { + if(getPhysicalQueueSize(pThis) >= pThis->iLightDlyMrk) { dbgoprint((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n"); timeoutComp(&t, 1000); /* 1000 millisconds = 1 second TODO: make configurable */ pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); /* TODO error check? But what do then? */ @@ -2403,7 +2421,7 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) * is not the case, basic flow control enters the field, which means we wait for * the queue to become ready or drop the new message. -- rgerhards, 2008-03-14 */ - while( (pThis->iMaxQueueSize > 0 && pThis->iQueueSize >= pThis->iMaxQueueSize) + while( (pThis->iMaxQueueSize > 0 && getPhysicalQueueSize(pThis) >= pThis->iMaxQueueSize) || (pThis->qType == QUEUETYPE_DISK && pThis->sizeOnDiskMax != 0 && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) { dbgoprint((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); diff --git a/runtime/queue.h b/runtime/queue.h index e47b8762..92bf8ae5 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -74,7 +74,8 @@ typedef struct queue_s { int bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ int bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ int bQueueInDestruction;/* 1 if queue is in destruction process, 0 otherwise */ - int iQueueSize; /* Current number of elements in the queue */ + int iQueueSize; /* Current number of elements in queue store (some are already logically dequeued!) */ + int nLogDeq; /* number of elements currently logically dequeued */ int iMaxQueueSize; /* how large can the queue grow? */ int iNumWorkerThreads;/* number of worker threads to use */ int iCurNumWrkThrd;/* current number of active worker threads */ diff --git a/tests/da-mainmsg-q.sh b/tests/da-mainmsg-q.sh index 2ea6278e..91addf68 100755 --- a/tests/da-mainmsg-q.sh +++ b/tests/da-mainmsg-q.sh @@ -24,7 +24,7 @@ if [ "$?" -ne "0" ]; then cp rsyslog.out.log rsyslog.out.log.save fi ls -l test-spool -sleep 1 # we need this so that rsyslogd can receive all outstanding messages +sleep 2 # we need this so that rsyslogd can receive all outstanding messages # # part 2: send bunch of messages. This should trigger DA mode # @@ -35,7 +35,7 @@ if [ "$?" -ne "0" ]; then cp rsyslog.out.log rsyslog.out.log.save fi ls -l test-spool -sleep 5 # we need this so that rsyslogd can receive all outstanding messages +sleep 8 # we need this so that rsyslogd can receive all outstanding messages # # send another handful # diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index 5ff9ced0..efa6728b 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -7,6 +7,7 @@ echo testing queue disk-only mode rm -rf test-spool mkdir test-spool +# enable this, if you need debug output: export RSYSLOG_DEBUG="debug" rm -f work rsyslog.out.log rsyslog.out.log.save # work files ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/diskqueue.conf & sleep 1 -- cgit v1.2.3 From a9c4b26d462dd3c9dbd0575a3a1acc6d8df1c3b3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 19 May 2009 18:47:26 +0200 Subject: some cleanup --- runtime/queue.c | 86 --------------------------------------------------------- runtime/wti.c | 1 - tools/syslogd.c | 3 +- 3 files changed, 1 insertion(+), 89 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index 9855dac8..0f87b235 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -72,7 +72,6 @@ static int qqueueChkStopWrkrDA(qqueue_t *pThis); static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal); static int qqueueIsIdleDA(qqueue_t *pThis); static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave); -static rsRetVal ConsumerCancelCleanup(void *arg1, void *arg2); /* some constants for queuePersist () */ #define QUEUE_CHECKPOINT 1 @@ -206,7 +205,6 @@ static inline void queueDrain(qqueue_t *pThis) ASSERT(pThis != NULL); -// TODO: ULTRA it may be a good idea to check validitity once again /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ while(pThis->iQueueSize-- > 0) { pThis->qDeq(pThis, &pUsr); @@ -473,7 +471,6 @@ qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleDA)); CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerDA)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void*pWti)) ConsumerCancelCleanup)); CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueStartDA)); CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueTurnOffDAMode)); CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); @@ -613,7 +610,6 @@ static rsRetVal qDeqFixedArray(qqueue_t *pThis, void **out) ASSERT(pThis != NULL); *out = (void*) pThis->tVars.farray.pBuf[pThis->tVars.farray.deqhead]; -//MULTIdbgprintf("ULTRA qDeqFA, deqhead=%d head=%d, tail=%d\n", pThis->tVars.farray.deqhead, pThis->tVars.farray.head, pThis->tVars.farray.tail); pThis->tVars.farray.deqhead++; if (pThis->tVars.farray.deqhead == pThis->iMaxQueueSize) pThis->tVars.farray.deqhead = 0; @@ -627,7 +623,6 @@ static rsRetVal qDelFixedArray(qqueue_t *pThis) ASSERT(pThis != NULL); -//MULTIdbgprintf("ULTRA qDelFA, deqhead=%d head=%d, tail=%d\n", pThis->tVars.farray.deqhead, pThis->tVars.farray.head, pThis->tVars.farray.tail); pThis->tVars.farray.head++; if (pThis->tVars.farray.head == pThis->iMaxQueueSize) pThis->tVars.farray.head = 0; @@ -638,58 +633,6 @@ static rsRetVal qDelFixedArray(qqueue_t *pThis) /* -------------------- linked list -------------------- */ -/* first some generic functions which are also used for the unget linked list */ - -static inline rsRetVal qqueueAddLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, void* pUsr) -{ - qLinkedList_t *pEntry; - DEFiRet; - - ASSERT(ppRoot != NULL); - ASSERT(ppLast != NULL); - - CHKmalloc((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t)))); - - pEntry->pNext = NULL; - pEntry->pUsr = pUsr; - - if(*ppRoot == NULL) { - *ppRoot = *ppLast = pEntry; - } else { - (*ppLast)->pNext = pEntry; - *ppLast = pEntry; - } - -finalize_it: - RETiRet; -} - -static inline rsRetVal qqueueDelLinkedList(qLinkedList_t **ppRoot, qLinkedList_t **ppLast, obj_t **ppUsr) -{ - DEFiRet; - qLinkedList_t *pEntry; - - ASSERT(ppRoot != NULL); - ASSERT(ppLast != NULL); - ASSERT(ppUsr != NULL); - ASSERT(*ppRoot != NULL); - - pEntry = *ppRoot; - *ppUsr = pEntry->pUsr; - - if(*ppRoot == *ppLast) { - *ppRoot = NULL; - *ppLast = NULL; - } else { - *ppRoot = pEntry->pNext; - } - free(pEntry); - - RETiRet; -} - -/* end generic functions which are also used for the unget linked list */ - static rsRetVal qConstructLinkedList(qqueue_t *pThis) { @@ -1455,33 +1398,6 @@ finalize_it: } -/* cancellation cleanup handler for queueWorker () - * Updates admin structure and frees ressources. - * Params: - * arg1 - user pointer (in this case a qqueue_t) - * arg2 - user data pointer (in this case a queue data element, any object [queue's pUsr ptr!]) - * Note that arg2 may be NULL, in which case no dequeued but unprocessed pUsr exists! - * rgerhards, 2008-01-16 - */ -static rsRetVal -ConsumerCancelCleanup(void *arg1, void *arg2) -{ - //TODO: looks like we no longer need it! - /* - DEFiRet; - - qqueue_t *pThis = (qqueue_t*) arg1; - obj_t *pUsr = (obj_t*) arg2; - - ISOBJ_TYPE_assert(pThis, qqueue); - - RETiRet; - */ - return RS_RET_OK; -} - - - /* This function checks if the provided message shall be discarded and does so, if needed. * In DA mode, we do not discard any messages as we assume the disk subsystem is fast enough to * provide real-time creation of spool files. @@ -1636,7 +1552,6 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz rsRetVal localRet; DEFiRet; - /* this is the place to destruct the old messages and pull them off the queue - MULTI-DEQUEUE */ DeleteProcessedBatch(pThis, &pWti->batch); nDequeued = 0; @@ -2075,7 +1990,6 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) IsIdleReg)); CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerReg)); - CHKiRet(wtpSetpfOnWorkerCancel (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void*pWti))ConsumerCancelCleanup)); CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RegOnWrkrStartup)); CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RegOnWrkrShutdown)); CHKiRet(wtpSetpmutUsr (pThis->pWtpReg, pThis->mut)); diff --git a/runtime/wti.c b/runtime/wti.c index 7029dcfd..942f7cf1 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -319,7 +319,6 @@ wtiWorkerCancelCleanup(void *arg) DBGPRINTF("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis)); /* call user supplied handler (that one e.g. requeues the element) */ -// MULTIQUEUE: need to change here! pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->batch.pElem[0].pUsrp); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); diff --git a/tools/syslogd.c b/tools/syslogd.c index 88a588e9..2bd43685 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1225,13 +1225,12 @@ msgConsumer(void __attribute__((unused)) *notNeeded, batch_t *pBatch) for(i = 0 ; i < pBatch->nElem ; i++) { pMsg = (msg_t*) pBatch->pElem[i].pUsrp; -dbgprintf("msgConsumer..MULTIQUEUE: i: %d, pMsg: %p\n", i, pMsg); + DBGPRINTF("msgConsumer processes msg %d/%d\n", i, pBatch->nElem); if((pMsg->msgFlags & NEEDS_PARSING) != 0) { parseMsg(pMsg); } processMsg(pMsg); } -dbgprintf("DONE msgConsumer..MULTIQUEUE:\n"); RETiRet; } -- cgit v1.2.3 From 0cf8e88a348dc574244e4f5c2be26f47e8bfff08 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 19 May 2009 18:58:33 +0200 Subject: solved the intended-discard-during-dequeue issue --- runtime/batch.h | 6 ++++++ runtime/queue.c | 22 ++++++++++++---------- runtime/queue.h | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/runtime/batch.h b/runtime/batch.h index eb266b3f..031718a7 100644 --- a/runtime/batch.h +++ b/runtime/batch.h @@ -55,9 +55,15 @@ struct batch_obj_s { * object. We stick to the more generic term because queues may potentially hold * other types of objects, too. * rgerhards, 2009-05-12 + * Note that nElem is not necessarily equal to nElemDeq. This is the case when we + * discard some elements (because of configuration) during dequeue processing. As + * all Elements are only deleted when the batch is processed, we can not immediately + * delete them. So we need to keep their number that we can delete them when the batch + * is completed (else, the whole process does not work correctly). */ struct batch_s { int nElem; /* actual number of element in this entry */ + int nElemDeq; /* actual number of elements dequeued (and thus to be deleted) - see comment above! */ int iDoneUpTo; /* all messages below this index have state other than RDY */ qDeqID deqID; /* ID of dequeue operation that generated this batch */ batch_obj_t *pElem; /* batch elements */ diff --git a/runtime/queue.c b/runtime/queue.c index 0f87b235..672ba9f5 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -131,7 +131,7 @@ static inline rsRetVal tdlPop(qqueue_t *pQueue) * structure, populates it with the values provided and links the new * element into the correct place inside the list. */ -static inline rsRetVal tdlAdd(qqueue_t *pQueue, qDeqID deqID, int nElem) +static inline rsRetVal tdlAdd(qqueue_t *pQueue, qDeqID deqID, int nElemDeq) { toDeleteLst_t *pNew; toDeleteLst_t *pPrev; @@ -142,7 +142,7 @@ static inline rsRetVal tdlAdd(qqueue_t *pQueue, qDeqID deqID, int nElem) CHKmalloc(pNew = malloc(sizeof(toDeleteLst_t))); pNew->deqID = deqID; - pNew->nElem = nElem; + pNew->nElemDeq = nElemDeq; /* now find right spot */ for( pPrev = pQueue->toDeleteLst @@ -1483,19 +1483,19 @@ DeleteBatchFromQStore(qqueue_t *pThis, batch_t *pBatch) pTdl = tdlPeek(pThis); if(pTdl == NULL) { - DoDeleteBatchFromQStore(pThis, pBatch->nElem); + DoDeleteBatchFromQStore(pThis, pBatch->nElemDeq); } else if(pBatch->deqID == pThis->deqIDDel) { deqIDDel = pThis->deqIDDel; pTdl = tdlPeek(pThis); while(pTdl != NULL && deqIDDel == pTdl->deqID) { - DoDeleteBatchFromQStore(pThis, pTdl->nElem); + DoDeleteBatchFromQStore(pThis, pTdl->nElemDeq); tdlPop(pThis); ++deqIDDel; pTdl = tdlPeek(pThis); } } else { /* can not delete, insert into to-delete list */ - CHKiRet(tdlAdd(pThis, pBatch->deqID, pBatch->nElem)); + CHKiRet(tdlAdd(pThis, pBatch->deqID, pBatch->nElemDeq)); } finalize_it: @@ -1547,6 +1547,7 @@ static inline rsRetVal DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSize) { int nDequeued; + int nDiscarded; int iQueueSize; void *pUsr; rsRetVal localRet; @@ -1554,7 +1555,7 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz DeleteProcessedBatch(pThis, &pWti->batch); - nDequeued = 0; + nDequeued = nDiscarded = 0; do { dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); CHKiRet(qqueueDeq(pThis, &pUsr)); @@ -1562,12 +1563,12 @@ dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); /* check if we should discard this element */ localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr); - //MULTI-DEQUEUE / ULTRA-RELIABLE: we need to handle this case, we need to advance the - // DEQ pointer (or so...) TODO!!! Idea: get a second nElem int in pBatch, nDequeued. Use that when deleting! - if(localRet == RS_RET_QUEUE_FULL) + if(localRet == RS_RET_QUEUE_FULL) { + ++nDiscarded; continue; - else if(localRet != RS_RET_OK) + } else if(localRet != RS_RET_OK) { ABORT_FINALIZE(localRet); + } /* all well, use this element */ pWti->batch.pElem[nDequeued].pUsrp = pUsr; @@ -1578,6 +1579,7 @@ dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); qqueueChkPersist(pThis, nDequeued); /* it is sufficient to persist only when the bulk of work is done */ pWti->batch.nElem = nDequeued; + pWti->batch.nElemDeq = nDequeued + nDiscarded; pWti->batch.deqID = getNextDeqID(pThis); *piRemainingQueueSize = iQueueSize; diff --git a/runtime/queue.h b/runtime/queue.h index 92bf8ae5..954a7fd4 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -34,7 +34,7 @@ typedef struct toDeleteLst_s toDeleteLst_t; struct toDeleteLst_s { qDeqID deqID; - int nElem; + int nElemDeq; /* numbe of elements that were dequeued and as such must now be discarded */ struct toDeleteLst_s *pNext; }; -- cgit v1.2.3 From ad7ccabe5ec616a4bf9fda1472d8041eaf1bf815 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 May 2009 11:11:02 +0200 Subject: yield() no longer needed on uniproc thanks to new algorithms --- runtime/queue.c | 7 ------- runtime/wti.c | 27 ++++----------------------- runtime/wtp.c | 16 ++++------------ 3 files changed, 8 insertions(+), 42 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index 672ba9f5..3118bb19 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2361,13 +2361,6 @@ finalize_it: d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n"); - /* the following pthread_yield is experimental, but brought us performance - * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 - * rgerhards, 2008-10-09 - * but this is only true for uniprocessors, so we guard it with an optimize flag -- rgerhards, 2008-10-22 - */ - if(pThis->bOptimizeUniProc) - pthread_yield(); } RETiRet; diff --git a/runtime/wti.c b/runtime/wti.c index 942f7cf1..8a7ee657 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -39,10 +39,10 @@ #include #include -#ifdef OS_SOLARIS -# include -# define pthread_yield() sched_yield() -#endif +/// TODO: check on solaris if this is any longer needed - I don't think so - rgerhards, 2009-09-20 +//#ifdef OS_SOLARIS +//# include +//#endif #include "rsyslog.h" #include "stringbuf.h" @@ -341,20 +341,6 @@ wtiWorkerCancelCleanup(void *arg) * Now make sure we can get canceled - it is not specified if pthread_setcancelstate() is * a cancellation point in itself. As we run most of the time without cancel enabled, I fear * we may never get cancelled if we do not create a cancellation point ourselfs. - * - * On the use of pthread_yield(): - * We yield to give the other threads a chance to obtain the mutex. If we do not - * do that, this thread may very well aquire the mutex again before another thread - * has even a chance to run. The reason is that mutex operations are free to be - * implemented in the quickest possible way (and they typically are!). That is, the - * mutex lock/unlock most probably just does an atomic memory swap and does not necessarily - * schedule other threads waiting on the same mutex. That can lead to the same thread - * aquiring the mutex ever and ever again while all others are starving for it. We - * have exactly seen this behaviour when we deliberately introduced a long-running - * test action which basically did a sleep. I understand that with real actions the - * likelihood of this starvation condition is very low - but it could still happen - * and would be very hard to debug. The yield() is a sure fix, its performance overhead - * should be well accepted given the above facts. -- rgerhards, 2008-01-10 */ #pragma GCC diagnostic ignored "-Wempty-body" rsRetVal @@ -371,7 +357,6 @@ wtiWorker(wti_t *pThis) ISOBJ_TYPE_assert(pWtp, wtp); dbgSetThrdName(pThis->pszDbgHdr); - pThis->batch.nElem = 0; /* flag no elements present */ // MULTIQUEUE: do we really need this any longer (cnacel handeler)? pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); @@ -383,10 +368,6 @@ wtiWorker(wti_t *pThis) /* process any pending thread requests */ wtpProcessThrdChanges(pWtp); pthread_testcancel(); /* see big comment in function header */ -# if !defined(__hpux) /* pthread_yield is missing there! */ - if(pThis->bOptimizeUniProc) - pthread_yield(); /* see big comment in function header */ -# endif /* if we have a rate-limiter set for this worker pool, let's call it. Please * keep in mind that the rate-limiter may hold us for an extended period diff --git a/runtime/wtp.c b/runtime/wtp.c index 9891a55c..dab59562 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -40,10 +40,10 @@ #include #include -#ifdef OS_SOLARIS -# include -# define pthread_yield() sched_yield() -#endif +/// TODO: check on solaris if this is any longer needed - I don't think so - rgerhards, 2009-09-20 +//#ifdef OS_SOLARIS +//# include +//#endif #include "rsyslog.h" #include "stringbuf.h" @@ -510,14 +510,6 @@ wtpStartWrkr(wtp_t *pThis, int bLockMutex) dbgprintf("%s: started with state %d, num workers now %d\n", wtpGetDbgHdr(pThis), iState, pThis->iCurNumWrkThrd); - /* we try to give the starting worker a little boost. It won't help much as we still - * hold the queue's mutex, but at least it has a chance to start on a single-CPU system. - */ -# if !defined(__hpux) /* pthread_yield is missing there! */ - if(pThis->bOptimizeUniProc) - pthread_yield(); -# endif - /* indicate we just started a worker and would like to see it running */ wtpSetInactivityGuard(pThis, 1, MUTEX_ALREADY_LOCKED); -- cgit v1.2.3 From 9f45b80ea9ea86d516c895d97fd8670df37e319e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 May 2009 15:12:49 +0200 Subject: free last processed message in all cases so far, the last processed message was only freed when the next one was processed. This has been changed now. More precisely, a better algorithm has been selected for the queue worker process, which also involves less overhead than the previous one. The fix for "free last processed message" as then more or less a side-effect (easy to do) of the new algorithm. --- runtime/debug.c | 51 ++++++++++++++++++++++--------------- runtime/queue.c | 51 +++++++++++++++++++++++++++++-------- runtime/rsyslog.h | 4 ++- runtime/wti.c | 76 +++++++++++++++++++++++++++++++++++-------------------- runtime/wtp.c | 7 +++-- runtime/wtp.h | 8 +++--- 6 files changed, 129 insertions(+), 68 deletions(-) diff --git a/runtime/debug.c b/runtime/debug.c index 4f45a1e3..248c5ea3 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -828,13 +828,12 @@ sigsegvHdlr(int signum) abort(); } -#if 1 -#pragma GCC diagnostic ignored "-Wempty-body" -/* write the debug message. This is a helper to dbgprintf and dbgoprint which - * contains common code. added 2008-09-26 rgerhards +/* actually write the debug message. This is a separate fuction because the cleanup_push/_pop + * interface otherwise is unsafe to use (generates compiler warnings at least). + * 2009-05-20 rgerhards */ -static void -dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) +static inline void +do_dbgprint(uchar *pszObjName, char *pszMsg, size_t lenMsg) { static pthread_t ptLastThrdID = 0; static int bWasNL = 0; @@ -842,20 +841,6 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) char pszWriteBuf[1024]; size_t lenWriteBuf; struct timespec t; - uchar *pszObjName = NULL; - - /* we must get the object name before we lock the mutex, because the object - * potentially calls back into us. If we locked the mutex, we would deadlock - * ourselfs. On the other hand, the GetName call needs not to be protected, as - * this thread has a valid reference. If such an object is deleted by another - * thread, we are in much more trouble than just for dbgprint(). -- rgerhards, 2008-09-26 - */ - if(pObj != NULL) { - pszObjName = obj.GetName(pObj); - } - - pthread_mutex_lock(&mutdbgprint); - pthread_cleanup_push(dbgMutexCancelCleanupHdlr, &mutdbgprint); /* The bWasNL handler does not really work. It works if no thread * switching occurs during non-NL messages. Else, things are messed @@ -903,11 +888,35 @@ dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) if(altdbg != -1) write(altdbg, pszMsg, lenMsg); bWasNL = (pszMsg[lenMsg - 1] == '\n') ? 1 : 0; +} + +#pragma GCC diagnostic ignored "-Wempty-body" +/* write the debug message. This is a helper to dbgprintf and dbgoprint which + * contains common code. added 2008-09-26 rgerhards + */ +static void +dbgprint(obj_t *pObj, char *pszMsg, size_t lenMsg) +{ + uchar *pszObjName = NULL; + + /* we must get the object name before we lock the mutex, because the object + * potentially calls back into us. If we locked the mutex, we would deadlock + * ourselfs. On the other hand, the GetName call needs not to be protected, as + * this thread has a valid reference. If such an object is deleted by another + * thread, we are in much more trouble than just for dbgprint(). -- rgerhards, 2008-09-26 + */ + if(pObj != NULL) { + pszObjName = obj.GetName(pObj); + } + + pthread_mutex_lock(&mutdbgprint); + pthread_cleanup_push(dbgMutexCancelCleanupHdlr, &mutdbgprint); + + do_dbgprint(pszObjName, pszMsg, lenMsg); pthread_cleanup_pop(1); } #pragma GCC diagnostic warning "-Wempty-body" -#endif /* print some debug output when an object is given * This is mostly a copy of dbgprintf, but I do not know how to combine it diff --git a/runtime/queue.c b/runtime/queue.c index 3118bb19..0019297b 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -469,7 +469,7 @@ qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueIsIdleDA)); + CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, wtp_t*)) qqueueIsIdleDA)); CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerDA)); CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueStartDA)); CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueTurnOffDAMode)); @@ -1257,7 +1257,7 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) /* we need to re-aquire the mutex for the next check in this case! */ BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ } - if(pThis->bIsDA && wtpGetCurNumWrkr(pThis->pWtpDA, LOCK_MUTEX) > 0) { + if(pThis->bRunsDA && wtpGetCurNumWrkr(pThis->pWtpDA, LOCK_MUTEX) > 0) { /* and now the same for the DA queue */ END_MTX_PROTECTED_OPERATIONS(pThis->mut); dbgoprint((obj_t*) pThis, "trying immediate shutdown of DA workers\n"); @@ -1548,18 +1548,19 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz { int nDequeued; int nDiscarded; + int nDeleted; int iQueueSize; void *pUsr; rsRetVal localRet; DEFiRet; + nDeleted = pWti->batch.nElemDeq; DeleteProcessedBatch(pThis, &pWti->batch); nDequeued = nDiscarded = 0; - do { + while((iQueueSize = getLogicalQueueSize(pThis)) > 0 && nDequeued < pThis->iDeqBatchSize) { dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); CHKiRet(qqueueDeq(pThis, &pUsr)); - iQueueSize = getLogicalQueueSize(pThis); /* check if we should discard this element */ localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr); @@ -1574,9 +1575,10 @@ dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); pWti->batch.pElem[nDequeued].pUsrp = pUsr; pWti->batch.pElem[nDequeued].state = BATCH_STATE_RDY; ++nDequeued; - } while(iQueueSize > 0 && nDequeued < pThis->iDeqBatchSize); + } - qqueueChkPersist(pThis, nDequeued); /* it is sufficient to persist only when the bulk of work is done */ + /* it is sufficient to persist only when the bulk of work is done */ + qqueueChkPersist(pThis, nDequeued+nDiscarded+nDeleted); pWti->batch.nElem = nDequeued; pWti->batch.nElemDeq = nDequeued + nDiscarded; @@ -1618,6 +1620,7 @@ DequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) pthread_cond_broadcast(&pThis->belowLightDlyWtrMrk); } + // TODO: MULTI: check physical queue size! pthread_cond_signal(&pThis->notFull); d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); @@ -1727,6 +1730,27 @@ RateLimiter(qqueue_t *pThis) } +/* This dequeues the next batch and checks if the queue is empty. If it is + * empty, return RS_RET_IDLE. That will trigger termination of the function + * and tell the upper layer caller to initiate idle processing. + * rgerhards, 2009-05-20 + */ +static inline rsRetVal +DequeueForConsumer(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ISOBJ_TYPE_assert(pWti, wti); + + CHKiRet(DequeueConsumable(pThis, pWti, iCancelStateSave)); + + if(pWti->batch.nElem == 0) + ABORT_FINALIZE(RS_RET_IDLE); + +finalize_it: + RETiRet; +} /* This is the queue consumer in the regular (non-DA) case. It is * protected by the queue mutex, but MUST release it as soon as possible. @@ -1740,7 +1764,7 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(DequeueConsumable(pThis, pWti, iCancelStateSave)); + CHKiRet(DequeueForConsumer(pThis, pWti, iCancelStateSave)); CHKiRet(pThis->pConsumer(pThis->pUsr, &pWti->batch)); /* we now need to check if we should deliberately delay processing a bit @@ -1754,6 +1778,7 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) } finalize_it: +dbgprintf("XXX: regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); RETiRet; } @@ -1776,7 +1801,7 @@ ConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); - CHKiRet(DequeueConsumable(pThis, pWti, iCancelStateSave)); + CHKiRet(DequeueForConsumer(pThis, pWti, iCancelStateSave)); /* iterate over returned results and enqueue them in DA queue */ for(i = 0 ; i < pWti->batch.nElem ; i++) CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->batch.pElem[i].pUsrp)); @@ -1855,6 +1880,8 @@ GetDeqBatchSize(qqueue_t *pThis, int *pVal) RETiRet; } + +/* common function for the idle functions that deletes the last batch if TODO MULTI */ /* must only be called when the queue mutex is locked, else results * are not stable! DA queue version */ @@ -1990,7 +2017,7 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RateLimiter)); CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) ChkStopWrkrReg)); CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) IsIdleReg)); + CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wtp_t*)) IsIdleReg)); CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerReg)); CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RegOnWrkrStartup)); CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RegOnWrkrShutdown)); @@ -2133,7 +2160,10 @@ static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates) DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); - assert(nUpdates > 0); + assert(nUpdates >= 0); + + if(nUpdates == 0) + FINALIZE; pThis->iUpdsSincePersist += nUpdates; if(pThis->iPersistUpdCnt && pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { @@ -2141,6 +2171,7 @@ static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates) pThis->iUpdsSincePersist = 0; } +finalize_it: RETiRet; } diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 3dd84ef6..01bbbb29 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -102,6 +102,7 @@ typedef struct tcpsrv_s tcpsrv_t; typedef struct vmstk_s vmstk_t; typedef struct batch_obj_s batch_obj_t; typedef struct batch_s batch_t; +typedef struct wtp_s wtp_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ @@ -290,10 +291,11 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ /* some generic error/status codes */ + RS_RET_OK = 0, /**< operation successful */ RS_RET_OK_DELETE_LISTENTRY = 1, /**< operation successful, but callee requested the deletion of an entry (special state) */ RS_RET_TERMINATE_NOW = 2, /**< operation successful, function is requested to terminate (mostly used with threads) */ RS_RET_NO_RUN = 3, /**< operation successful, but function does not like to be executed */ - RS_RET_OK = 0 /**< operation successful */ + RS_RET_IDLE = 4 /**< operation successful, but callee is idle (e.g. because queue is empty) */ }; /* some helpful macros to work with srRetVals. diff --git a/runtime/wti.c b/runtime/wti.c index 8a7ee657..1be008df 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -333,6 +333,32 @@ wtiWorkerCancelCleanup(void *arg) } +/* wait for queue to become non-empty or timeout + * helper to wtiWorker + * IMPORTANT: mutex must be locked when this code is called! + * rgerhards, 2009-05-20 + */ +static inline void +doIdleProcessing(wti_t *pThis, wtp_t *pWtp, int *pbInactivityTOOccured) +{ + struct timespec t; + + DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); + pWtp->pfOnIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED); + + if(pWtp->toWrkShutdown == -1) { + /* never shut down any started worker */ + d_pthread_cond_wait(pWtp->pcondBusy, pWtp->pmutUsr); + } else { + timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */ + if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) { + DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); + *pbInactivityTOOccured = 1; /* indicate we had a timeout */ + } + } +} + + /* generic worker thread framework * * Some special comments below, so that they do not clutter the main function code: @@ -341,16 +367,20 @@ wtiWorkerCancelCleanup(void *arg) * Now make sure we can get canceled - it is not specified if pthread_setcancelstate() is * a cancellation point in itself. As we run most of the time without cancel enabled, I fear * we may never get cancelled if we do not create a cancellation point ourselfs. + * Note on rate-limiters: + * If we have a rate-limiter set for this worker pool, let's call it. Please + * keep in mind that the rate-limiter may hold us for an extended period + * of time. -- rgerhards, 2008-04-02 */ #pragma GCC diagnostic ignored "-Wempty-body" rsRetVal wtiWorker(wti_t *pThis) { - DEFiRet; DEFVARS_mutexProtection; - struct timespec t; wtp_t *pWtp; /* our worker thread pool */ int bInactivityTOOccured = 0; + rsRetVal localRet; + DEFiRet; ISOBJ_TYPE_assert(pThis, wti); pWtp = pThis->pWtp; /* shortcut */ @@ -369,48 +399,38 @@ wtiWorker(wti_t *pThis) wtpProcessThrdChanges(pWtp); pthread_testcancel(); /* see big comment in function header */ - /* if we have a rate-limiter set for this worker pool, let's call it. Please - * keep in mind that the rate-limiter may hold us for an extended period - * of time. -- rgerhards, 2008-04-02 - */ - if(pWtp->pfRateLimiter != NULL) { + if(pWtp->pfRateLimiter != NULL) { /* call rate-limiter, if defined */ pWtp->pfRateLimiter(pWtp->pUsr); } wtpSetInactivityGuard(pThis->pWtp, 0, LOCK_MUTEX); /* must be set before usr mutex is locked! */ BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); - if( (bInactivityTOOccured && pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED)) - || wtpChkStopWrkr(pWtp, LOCK_MUTEX, MUTEX_ALREADY_LOCKED)) { - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); + /* first check if we are in shutdown process */ + if(wtpChkStopWrkr(pWtp, LOCK_MUTEX, MUTEX_ALREADY_LOCKED)) { break; /* end worker thread run */ } - bInactivityTOOccured = 0; /* reset for next run */ - /* if we reach this point, we are still protected by the mutex */ - - if(pWtp->pfIsIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED)) { - DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); - pWtp->pfOnIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED); - - if(pWtp->toWrkShutdown == -1) { - /* never shut down any started worker */ - d_pthread_cond_wait(pWtp->pcondBusy, pWtp->pmutUsr); - } else { - timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */ - if(d_pthread_cond_timedwait(pWtp->pcondBusy, pWtp->pmutUsr, &t) != 0) { - DBGPRINTF("%s: inactivity timeout, worker terminating...\n", wtiGetDbgHdr(pThis)); - bInactivityTOOccured = 1; /* indicate we had a timeout */ - } + /* try to execute and process whatever we have */ + localRet = pWtp->pfDoWork(pWtp->pUsr, pThis, iCancelStateSave); + + if(localRet == RS_RET_IDLE) { + if(bInactivityTOOccured) { + /* we had an inactivity timeout in the last run and are still idle, so it is time to exit... */ + break; /* end worker thread run */ } + doIdleProcessing(pThis, pWtp, &bInactivityTOOccured); END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); continue; /* request next iteration */ } + END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); - /* if we reach this point, we have a non-empty queue (and are still protected by mutex) */ - pWtp->pfDoWork(pWtp->pUsr, pThis, iCancelStateSave); + bInactivityTOOccured = 0; /* reset for next run */ } + /* if we exit the loop, the mutex is locked and must be unlocked */ + END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); + /* indicate termination */ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); d_pthread_mutex_lock(&pThis->mut); diff --git a/runtime/wtp.c b/runtime/wtp.c index dab59562..40a9095b 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -248,7 +248,6 @@ wtpSetState(wtp_t *pThis, wtpState_t iNewState) /* check if the worker shall shutdown (1 = yes, 0 = no) - * TODO: check if we can use atomic operations to enhance performance * Note: there may be two mutexes locked, the bLockUsrMutex is the one in our "user" * (e.g. the queue clas) * rgerhards, 2008-01-21 @@ -263,14 +262,14 @@ wtpChkStopWrkr(wtp_t *pThis, int bLockMutex, int bLockUsrMutex) BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); if( (pThis->wtpState == wtpState_SHUTDOWN_IMMEDIATE) - || ((pThis->wtpState == wtpState_SHUTDOWN) && pThis->pfIsIdle(pThis->pUsr, bLockUsrMutex))) + || ((pThis->wtpState == wtpState_SHUTDOWN) && pThis->pfIsIdle(pThis->pUsr, pThis))) iRet = RS_RET_TERMINATE_NOW; - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); /* try customer handler if one was set and we do not yet have a definite result */ if(iRet == RS_RET_OK && pThis->pfChkStopWrkr != NULL) { iRet = pThis->pfChkStopWrkr(pThis->pUsr, bLockUsrMutex); } + END_MTX_PROTECTED_OPERATIONS(&pThis->mut); RETiRet; } @@ -577,7 +576,7 @@ DEFpropSetMethPTR(wtp, pcondBusy, pthread_cond_t) DEFpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)) DEFpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)) DEFpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)) -DEFpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int)) +DEFpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, wtp_t*)) DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)) DEFpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)) DEFpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*, void*)) diff --git a/runtime/wtp.h b/runtime/wtp.h index 88bd9197..d9d582af 100644 --- a/runtime/wtp.h +++ b/runtime/wtp.h @@ -50,7 +50,7 @@ typedef enum { /* the worker thread pool (wtp) object */ -typedef struct wtp_s { +struct wtp_s { BEGINobjInstance; int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ wtpState_t wtpState; @@ -73,7 +73,7 @@ typedef struct wtp_s { rsRetVal (*pfChkStopWrkr)(void *pUsr, int); rsRetVal (*pfGetDeqBatchSize)(void *pUsr, int*); /* obtains max dequeue count from queue config */ rsRetVal (*pfRateLimiter)(void *pUsr); - rsRetVal (*pfIsIdle)(void *pUsr, int); + rsRetVal (*pfIsIdle)(void *pUsr, wtp_t *pWtp); rsRetVal (*pfDoWork)(void *pUsr, void *pWti, int); rsRetVal (*pfOnIdle)(void *pUsr, int); rsRetVal (*pfOnWorkerCancel)(void *pUsr, void*pWti); @@ -81,7 +81,7 @@ typedef struct wtp_s { rsRetVal (*pfOnWorkerShutdown)(void *pUsr); /* end user objects */ uchar *pszDbgHdr; /* header string for debug messages */ -} wtp_t; +}; /* some symbolic constants for easier reference */ @@ -106,7 +106,7 @@ PROTOTYPEObjClassInit(wtp); PROTOTYPEpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)); PROTOTYPEpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)); PROTOTYPEpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)); -PROTOTYPEpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, int)); +PROTOTYPEpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, wtp_t*)); PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)); PROTOTYPEpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)); PROTOTYPEpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*,void*)); -- cgit v1.2.3 From 8088f9b01a8344c63e7874836721c79d94a01d25 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 May 2009 16:08:23 +0200 Subject: added explicit (base) test for linkedList and FixedArray queue modes --- tests/Makefile.am | 7 ++++++- tests/arrayqueue.sh | 26 ++++++++++++++++++++++++++ tests/linkedlistqueue.sh | 26 ++++++++++++++++++++++++++ tests/testsuites/arrayqueue.conf | 14 ++++++++++++++ tests/testsuites/linkedlistqueue.conf | 14 ++++++++++++++ 5 files changed, 86 insertions(+), 1 deletion(-) create mode 100755 tests/arrayqueue.sh create mode 100755 tests/linkedlistqueue.sh create mode 100644 tests/testsuites/arrayqueue.conf create mode 100644 tests/testsuites/linkedlistqueue.conf diff --git a/tests/Makefile.am b/tests/Makefile.am index 88f6a200..a7736312 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,7 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh fieldtest.sh da-mainmsg-q.sh diskqueue.sh manytcp.sh +TESTS = $(TESTRUNS) cfg.sh fieldtest.sh arrayqueue.sh linkedlistqueue.sh da-mainmsg-q.sh \ + diskqueue.sh manytcp.sh if ENABLE_OMSTDOUT TESTS += omod-if-array.sh parsertest.sh endif @@ -42,6 +43,10 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ fieldtest.sh \ diskqueue.sh \ testsuites/diskqueue.conf \ + arrayqueue.sh \ + testsuites/arrayqueue.conf \ + linkedlistqueue.sh \ + testsuites/linkedlistqueue.conf \ da-mainmsg-q.sh \ testsuites/da-mainmsg-q.conf \ manytcp.sh \ diff --git a/tests/arrayqueue.sh b/tests/arrayqueue.sh new file mode 100755 index 00000000..5b8ebb5f --- /dev/null +++ b/tests/arrayqueue.sh @@ -0,0 +1,26 @@ +# Test for fixedArray queue mode +# added 2009-05-20 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo testing queue fixedArray queue mode +rm -f work rsyslog.out.log +# enable this, if you need debug output: export RSYSLOG_DEBUG="debug" +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/arrayqueue.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +# 40000 messages should be enough +./tcpflood 127.0.0.1 13514 1 40000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 4 # we need this so that rsyslogd can receive all outstanding messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 39999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log diff --git a/tests/linkedlistqueue.sh b/tests/linkedlistqueue.sh new file mode 100755 index 00000000..aac1abb6 --- /dev/null +++ b/tests/linkedlistqueue.sh @@ -0,0 +1,26 @@ +# Test for Linkedlist queue mode +# added 2009-05-20 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo testing queue Linkedlist queue mode +rm -f work rsyslog.out.log +# enable this, if you need debug output: export RSYSLOG_DEBUG="debug" +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/arrayqueue.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +# 40000 messages should be enough +./tcpflood 127.0.0.1 13514 1 40000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 4 # we need this so that rsyslogd can receive all outstanding messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 39999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log diff --git a/tests/testsuites/arrayqueue.conf b/tests/testsuites/arrayqueue.conf new file mode 100644 index 00000000..7fdbc79b --- /dev/null +++ b/tests/testsuites/arrayqueue.conf @@ -0,0 +1,14 @@ +# Test for queue fixedArray mode (see .sh file for details) +# rgerhards, 2009-04-17 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$MainMsgQueueType FixedArray + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/linkedlistqueue.conf b/tests/testsuites/linkedlistqueue.conf new file mode 100644 index 00000000..ecfc7a26 --- /dev/null +++ b/tests/testsuites/linkedlistqueue.conf @@ -0,0 +1,14 @@ +# Test for queue LinkedList mode (see .sh file for details) +# rgerhards, 2009-04-17 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$MainMsgQueueType LinkedList + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt -- cgit v1.2.3 From 737614054caadf8340b638b368d43f55b7bb3741 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 May 2009 17:30:01 +0200 Subject: minor cleanup --- plugins/imuxsock/imuxsock.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 1d88a2b5..9a1e424c 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -324,11 +324,8 @@ CODESTARTafterRun if (funixn[i] && funix[i] != -1) unlink((char*) funixn[i]); /* free no longer needed string */ - if(pLogSockName != NULL) - free(pLogSockName); - if(pLogHostName != NULL) { - free(pLogHostName); - } + free(pLogSockName); + free(pLogHostName); discardFunixn(); nfunix = 1; -- cgit v1.2.3 From 2aca3c2d33dd1a19daf44168e90a5f64bd4095b0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 20 May 2009 17:35:21 +0200 Subject: made imdiag *just* compile & some cleanup imdiag was never finished (not even really begun), but now I need it. I made the few things that are available compile, but more serious work is required. --- plugins/imdiag/imdiag.c | 36 ++++++++++++++++++++---------------- tcpsrv.c | 17 ++++++----------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 3cd2dcf8..90c5d9ee 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -7,7 +7,7 @@ * * File begun on 2008-07-25 by RGerhards * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -48,6 +48,7 @@ #include "module-template.h" #include "net.h" #include "netstrm.h" +#include "netstrms.h" #include "errmsg.h" MODULE_TYPE_INPUT @@ -56,12 +57,13 @@ MODULE_TYPE_INPUT DEF_IMOD_STATIC_DATA DEFobjCurrIf(net) DEFobjCurrIf(netstrm) +DEFobjCurrIf(netstrms) DEFobjCurrIf(errmsg) /* Module static data */ netstrms_t *pNS; /**< pointer to network stream subsystem */ -netstrm_t **ppLstn[10]; /**< our netstream listners */ -int iLstnMax = 0; /**< max nbr of listeners currently supported */ +netstrm_t *arrLstn[10]; /**< our netstream listners */ +int iLstnMax = 0; /**< max nbr of listeners currently supported */ /* config settings */ @@ -71,16 +73,16 @@ int iLstnMax = 0; /**< max nbr of listeners currently supported */ * invoked from the netstrm class. -- rgerhards, 2008-04-23 */ static rsRetVal -addTcpLstn(void *pUsr, netstrm_t *pLstn) +addTcpLstn(netstrm_t *pLstn) { DEFiRet; ISOBJ_TYPE_assert(pLstn, netstrm); - if(iLstnMax >= sizeof(ppLstn)/sizeof(netstrm_t)) + if((unsigned)iLstnMax >= sizeof(arrLstn)/sizeof(netstrm_t*)) ABORT_FINALIZE(RS_RET_MAX_LSTN_REACHED); - ppLstn[pThis->iLstnMax] = pLstn; + arrLstn[iLstnMax] = pLstn; ++iLstnMax; finalize_it: @@ -100,15 +102,15 @@ initNetstrm(void) //CHKiRet(netstrms.SetDrvrAuthMode(pThis->pNS, pThis->pszDrvrAuthMode)); //CHKiRet(netstrms.SetDrvrPermPeers(pThis->pNS, pThis->pPermPeers)); // TODO: set driver! - CHKiRet(netstrms.ConstructFinalize(pThis->pNS)); + CHKiRet(netstrms.ConstructFinalize(&pNS)); /* set up listeners */ - CHKiRet(netstrm.LstnInit(pNS, NULL, addTcpLstn, "127.0.0.1", "44514", 1)); + CHKiRet(netstrm.LstnInit(pNS, NULL, addTcpLstn, "127.0.0.1", (uchar*)"44514", 1)); finalize_it: if(iRet != RS_RET_OK) { - if(pThis->pNS != NULL) - netstrms.Destruct(&pThis->pNS); + if(pNS != NULL) + netstrms.Destruct(&pNS); } RETiRet; } @@ -130,11 +132,12 @@ ENDwillRun BEGINafterRun + int i; CODESTARTafterRun /* do cleanup here */ /* finally close our listen streams */ for(i = 0 ; i < iLstnMax ; ++i) { - netstrm.Destruct(ppLstn + i); + netstrm.Destruct(arrLstn[i]); } /* destruct netstream subsystem */ @@ -146,7 +149,8 @@ BEGINmodExit CODESTARTmodExit /* release objects we used */ objRelease(net, LM_NET_FILENAME); - objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(netstrm, DONT_LOAD_LIB); + objRelease(netstrms, LM_NETSTRMS_FILENAME); objRelease(errmsg, CORE_COMPONENT); ENDmodExit @@ -169,14 +173,14 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr - pOurTcpsrv = NULL; /* request objects we use */ CHKiRet(objUse(net, LM_NET_FILENAME)); - CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); -#if 0 /* register config file handlers */ +#if 0 CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverrun", 0, eCmdHdlrGetWord, addTCPListener, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpmaxsessions", 0, eCmdHdlrInt, @@ -187,9 +191,9 @@ CODEmodInit_QueryRegCFSLineHdlr eCmdHdlrGetWord, NULL, &pszStrmDrvrAuthMode, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdriverpermittedpeer", 0, eCmdHdlrGetWord, setPermittedPeer, NULL, STD_LOADABLE_MODULE_ID)); +#endif CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); -#endif ENDmodInit diff --git a/tcpsrv.c b/tcpsrv.c index b9434398..7af45d3c 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -17,7 +17,7 @@ * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -219,8 +219,7 @@ static void deinit_tcp_listener(tcpsrv_t *pThis) pThis->pSessions = NULL; /* just to make sure... */ } - if(pThis->TCPLstnPort != NULL) - free(pThis->TCPLstnPort); + free(pThis->TCPLstnPort); /* finally close our listen streams */ for(i = 0 ; i < pThis->iLstnMax ; ++i) { @@ -557,12 +556,9 @@ CODESTARTobjDestruct(tcpsrv) if(pThis->pNS != NULL) netstrms.Destruct(&pThis->pNS); - if(pThis->pszDrvrAuthMode != NULL) - free(pThis->pszDrvrAuthMode); - if(pThis->ppLstn != NULL) - free(pThis->ppLstn); - if(pThis->pszInputName != NULL) - free(pThis->pszInputName); + free(pThis->pszDrvrAuthMode); + free(pThis->ppLstn); + free(pThis->pszInputName); ENDobjDestruct(tcpsrv) @@ -683,8 +679,7 @@ SetInputName(tcpsrv_t *pThis, uchar *name) pszName = NULL; else CHKmalloc(pszName = (uchar*)strdup((char*)name)); - if(pThis->pszInputName != NULL) - free(pThis->pszInputName); + free(pThis->pszInputName); pThis->pszInputName = pszName; finalize_it: RETiRet; -- cgit v1.2.3 From aba90e82484118f3568ec51c01de5ba845da589a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 May 2009 17:06:52 +0200 Subject: added capability to run multiple tcp listeners (on different ports) Well, actually this and a lot of related things. I improved the testbench so that the new capabilities are automatically tested and also did some general cleanup. The current multiple tcp listener solution will probably receive some further cleanup, too, but looks quite OK so far. I also reviewed the way tcpsrv et all work, in preparation of using this code for imdiag. I need to document the findings, especially as the code is rather complicated "thanks" to the combination of plain tcp and gssapi transport modes. --- ChangeLog | 1 + doc/imtcp.html | 6 +- plugins/imfile/imfile.c | 5 +- plugins/imgssapi/imgssapi.c | 4 +- plugins/imklog/imklog.c | 7 +- plugins/imtcp/imtcp.c | 35 ++++---- plugins/imudp/imudp.c | 5 +- runtime/Makefile.am | 1 + runtime/datetime.c | 12 +-- runtime/datetime.h | 4 +- runtime/msg.c | 104 +++++++++------------ runtime/msg.h | 8 +- runtime/netstrm.c | 4 +- runtime/netstrm.h | 1 + runtime/parser.c | 2 +- runtime/rsyslog.h | 3 + runtime/unicode-helper.h | 54 +++++++++++ tcps_sess.c | 17 +++- tcps_sess.h | 11 ++- tcpsrv.c | 150 +++++++++++++++++++++---------- tcpsrv.h | 16 +++- tests/Makefile.am | 15 +++- tests/fieldtest.sh | 13 +++ tests/imtcp-multiport.sh | 80 +++++++++++++++++ tests/inputname.sh | 20 +++++ tests/killrsyslog.sh | 7 ++ tests/nettester.c | 101 +++++++++++++++------ tests/omod-if-array.sh | 6 +- tests/parsertest.sh | 6 +- tests/tcpflood.c | 2 - tests/testsuites/1.field1 | 3 + tests/testsuites/1.inputname_imtcp_12514 | 3 + tests/testsuites/1.inputname_imtcp_12515 | 3 + tests/testsuites/1.inputname_imtcp_12516 | 3 + tests/testsuites/field1.conf | 8 ++ tests/testsuites/imtcp-multiport.conf | 13 +++ tests/testsuites/inputname_imtcp.conf | 19 ++++ tools/syslogd.c | 53 +++++------ 38 files changed, 584 insertions(+), 221 deletions(-) create mode 100644 runtime/unicode-helper.h create mode 100755 tests/fieldtest.sh create mode 100755 tests/imtcp-multiport.sh create mode 100755 tests/inputname.sh create mode 100755 tests/killrsyslog.sh create mode 100644 tests/testsuites/1.field1 create mode 100644 tests/testsuites/1.inputname_imtcp_12514 create mode 100644 tests/testsuites/1.inputname_imtcp_12515 create mode 100644 tests/testsuites/1.inputname_imtcp_12516 create mode 100644 tests/testsuites/field1.conf create mode 100644 tests/testsuites/imtcp-multiport.conf create mode 100644 tests/testsuites/inputname_imtcp.conf diff --git a/ChangeLog b/ChangeLog index 30fb7bff..b7a0a67a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +- added capability to run multiple tcp listeners (on different ports) - performance enhancement: imtcp calls parser no longer on input thread but rather inside on of the potentially many main msg queue worker threads (an enhancement scheduled for all input plugins where this is diff --git a/doc/imtcp.html b/doc/imtcp.html index 0ee0f96a..9ea7efa1 100644 --- a/doc/imtcp.html +++ b/doc/imtcp.html @@ -14,9 +14,10 @@ Encryption can be provided by using stunnel (an alternative is the use the imgssapi modul).

    -

    In the future, multiple receivers may be configured by +

    Multiple receivers may be configured by specifying -$InputTCPServerRun multiple times. This is not currently supported. +$InputTCPServerRun multiple times. This is available since version 4.3.1, earlier +versions do NOT support it.

    Configuration Directives:

      @@ -58,7 +59,6 @@ AuthMode and  network stream driver. Permitted Caveats/Known Bugs:
      • module always binds to all interfaces
        • -
      • only a single listener can be bound
      • can not be loaded together with imgssapi (which includes the functionality of imtcp)
      diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index b0211bf6..92fd30c3 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -45,6 +45,7 @@ #include "errmsg.h" #include "glbl.h" #include "datetime.h" +#include "unicode-helper.h" MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ @@ -94,11 +95,11 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); - MsgSetInputName(pMsg, "imfile"); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile")); MsgSetUxTradMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); - MsgSetHOSTNAME(pMsg, (char*)glbl.GetLocalHostName()); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetTAG(pMsg, (char*)pInfo->pszTag); pMsg->iFacility = LOG_FAC(pInfo->iFacility); pMsg->iSeverity = LOG_PRI(pInfo->iSeverity); diff --git a/plugins/imgssapi/imgssapi.c b/plugins/imgssapi/imgssapi.c index b9d7dfe3..d8791880 100644 --- a/plugins/imgssapi/imgssapi.c +++ b/plugins/imgssapi/imgssapi.c @@ -9,7 +9,7 @@ * NOTE: read comments in module-template.h to understand how this file * works! * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -330,7 +330,7 @@ addGSSListener(void __attribute__((unused)) *pVal, uchar *pNewVal) CHKiRet(tcpsrv.SetCBOnSessAccept(pOurTcpsrv, onSessAccept)); CHKiRet(tcpsrv.SetCBOnRegularClose(pOurTcpsrv, onRegularClose)); CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); - tcpsrv.configureTCPListen(pOurTcpsrv, (char *) pNewVal); + tcpsrv.configureTCPListen(pOurTcpsrv, pNewVal); CHKiRet(tcpsrv.ConstructFinalize(pOurTcpsrv)); } diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 20bc34ab..ecb6c100 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -53,6 +53,7 @@ #include "datetime.h" #include "imklog.h" #include "glbl.h" +#include "unicode-helper.h" MODULE_TYPE_INPUT @@ -94,14 +95,14 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); - MsgSetInputName(pMsg, "imklog"); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog")); MsgSetRawMsg(pMsg, (char*)msg); MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); MsgSetMSG(pMsg, (char*)msg); - MsgSetRcvFrom(pMsg, (char*)glbl.GetLocalHostName()); + MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); - MsgSetHOSTNAME(pMsg, (char*)glbl.GetLocalHostName()); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetTAG(pMsg, (char*)pszTag); pMsg->iFacility = LOG_FAC(iFacility); pMsg->iSeverity = LOG_PRI(iSeverity); diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 5a8a62f6..f0efe1ee 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -3,7 +3,7 @@ * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -56,6 +56,7 @@ #include "dirty.h" #include "cfsysline.h" #include "module-template.h" +#include "unicode-helper.h" #include "net.h" #include "netstrm.h" #include "errmsg.h" @@ -91,7 +92,7 @@ static int isPermittedHost(struct sockaddr *addr, char *fromHostFQDN, void __attribute__((unused)) *pUsrSrv, void __attribute__((unused)) *pUsrSess) { - return net.isAllowedSender((uchar*) "TCP", addr, fromHostFQDN); + return net.isAllowedSender(UCHAR_CONSTANT("TCP"), addr, fromHostFQDN); } @@ -169,7 +170,6 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa CHKiRet(tcpsrv.SetCBOnRegularClose(pOurTcpsrv, onRegularClose)); CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); CHKiRet(tcpsrv.SetDrvrMode(pOurTcpsrv, iStrmDrvrMode)); - CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? (uchar*)"imtcp" : pszInputName)); CHKiRet(tcpsrv.SetAddtlFrameDelim(pOurTcpsrv, iAddtlFrameDelim)); /* now set optional params, but only if they were actually configured */ if(pszStrmDrvrAuthMode != NULL) { @@ -178,11 +178,14 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa if(pPermPeersRoot != NULL) { CHKiRet(tcpsrv.SetDrvrPermPeers(pOurTcpsrv, pPermPeersRoot)); } - /* most params set, now start listener */ - tcpsrv.configureTCPListen(pOurTcpsrv, (char *) pNewVal); - CHKiRet(tcpsrv.ConstructFinalize(pOurTcpsrv)); } +dbgprintf("XXX: try add listen port %s\n", pNewVal); + /* initialized, now add socket */ + CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? + UCHAR_CONSTANT("imtcp") : pszInputName)); + tcpsrv.configureTCPListen(pOurTcpsrv, pNewVal); + finalize_it: if(iRet != RS_RET_OK) { errmsg.LogError(0, NO_ERRCODE, "error %d trying to add listener", iRet); @@ -199,7 +202,9 @@ CODESTARTrunInput /* TODO: we must be careful to start the listener here. Currently, tcpsrv.c seems to * do that in ConstructFinalize */ + CHKiRet(tcpsrv.ConstructFinalize(pOurTcpsrv)); iRet = tcpsrv.Run(pOurTcpsrv); +finalize_it: ENDrunInput @@ -217,7 +222,7 @@ ENDwillRun BEGINafterRun CODESTARTafterRun /* do cleanup here */ - net.clearAllowedSenders((uchar*)"TCP"); + net.clearAllowedSenders(UCHAR_CONSTANT("TCP")); ENDafterRun @@ -277,21 +282,21 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* register config file handlers */ - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverrun", 0, eCmdHdlrGetWord, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverrun"), 0, eCmdHdlrGetWord, addTCPListener, NULL, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpmaxsessions", 0, eCmdHdlrInt, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpmaxsessions"), 0, eCmdHdlrInt, NULL, &iTCPSessMax, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdrivermode", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverstreamdrivermode"), 0, eCmdHdlrInt, NULL, &iStrmDrvrMode, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdriverauthmode", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverstreamdriverauthmode"), 0, eCmdHdlrGetWord, NULL, &pszStrmDrvrAuthMode, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdriverpermittedpeer", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverstreamdriverpermittedpeer"), 0, eCmdHdlrGetWord, setPermittedPeer, NULL, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserveraddtlframedelimiter", 0, eCmdHdlrInt, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserveraddtlframedelimiter"), 0, eCmdHdlrInt, NULL, &iAddtlFrameDelim, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverinputname", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverinputname"), 0, eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index c7e8c1d4..f3448095 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -43,6 +43,7 @@ #include "msg.h" #include "parser.h" #include "datetime.h" +#include "unicode-helper.h" MODULE_TYPE_INPUT @@ -219,11 +220,11 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); pMsg->iLenRawMsg = lenRcvBuf; - MsgSetInputName(pMsg, "imudp"); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imudp")); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; - MsgSetRcvFrom(pMsg, (char*)fromHost); + MsgSetRcvFrom(pMsg, fromHost); CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP)); CHKiRet(submitMsg(pMsg)); } diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 2f0a1aa0..8331882f 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -7,6 +7,7 @@ pkglib_LTLIBRARIES = librsyslog_la_SOURCES = \ rsyslog.c \ rsyslog.h \ + unicode_helper.h \ atomic.h \ syslogd-types.h \ module-template.h \ diff --git a/runtime/datetime.c b/runtime/datetime.c index deb66eb4..19e61a0a 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -150,7 +150,7 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) * \retval The number parsed. */ -static int srSLMGParseInt32(char** ppsz) +static int srSLMGParseInt32(uchar** ppsz) { int i; @@ -172,9 +172,9 @@ static int srSLMGParseInt32(char** ppsz) * could be obtained (restriction added 2008-09-16 by rgerhards). */ static rsRetVal -ParseTIMESTAMP3339(struct syslogTime *pTime, char** ppszTS) +ParseTIMESTAMP3339(struct syslogTime *pTime, uchar** ppszTS) { - char *pszTS = *ppszTS; + uchar *pszTS = *ppszTS; /* variables to temporarily hold time information while we parse */ int year; int month; @@ -234,7 +234,7 @@ ParseTIMESTAMP3339(struct syslogTime *pTime, char** ppszTS) /* Now let's see if we have secfrac */ if(*pszTS == '.') { - char *pszStart = ++pszTS; + uchar *pszStart = ++pszTS; secfrac = srSLMGParseInt32(&pszTS); secfracPrecision = (int) (pszTS - pszStart); } else { @@ -307,7 +307,7 @@ finalize_it: * time() call reduction ;). */ static rsRetVal -ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) +ParseTIMESTAMP3164(struct syslogTime *pTime, uchar** ppszTS) { /* variables to temporarily hold time information while we parse */ int month; @@ -317,7 +317,7 @@ ParseTIMESTAMP3164(struct syslogTime *pTime, char** ppszTS) int minute; int second; /* end variables to temporarily hold time information while we parse */ - char *pszTS; + uchar *pszTS; DEFiRet; assert(ppszTS != NULL); diff --git a/runtime/datetime.h b/runtime/datetime.h index 0739588d..efb0a0af 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -36,8 +36,8 @@ typedef struct datetime_s { /* interfaces */ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ void (*getCurrTime)(struct syslogTime *t, time_t *ttSeconds); - rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, char** ppszTS); - rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, char** pszTS); + rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, uchar** ppszTS); + rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, uchar** pszTS); int (*formatTimestampToMySQL)(struct syslogTime *ts, char* pDst, size_t iLenDst); int (*formatTimestampToPgSQL)(struct syslogTime *ts, char *pDst, size_t iLenDst); int (*formatTimestamp3339)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); diff --git a/runtime/msg.c b/runtime/msg.c index 9d5f3838..dbc3c779 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -45,6 +45,7 @@ #include "glbl.h" #include "regexp.h" #include "atomic.h" +#include "unicode-helper.h" /* static data */ DEFobjStaticHelpers @@ -341,52 +342,29 @@ CODESTARTobjDestruct(msg) if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ - if(pThis->pszUxTradMsg != NULL) - free(pThis->pszUxTradMsg); - if(pThis->pszRawMsg != NULL) - free(pThis->pszRawMsg); - if(pThis->pszTAG != NULL) - free(pThis->pszTAG); - if(pThis->pszHOSTNAME != NULL) - free(pThis->pszHOSTNAME); - if(pThis->pszInputName != NULL) - free(pThis->pszInputName); - if(pThis->pszRcvFrom != NULL) - free(pThis->pszRcvFrom); - if(pThis->pszRcvFromIP != NULL) - free(pThis->pszRcvFromIP); - if(pThis->pszMSG != NULL) - free(pThis->pszMSG); - if(pThis->pszFacility != NULL) - free(pThis->pszFacility); - if(pThis->pszFacilityStr != NULL) - free(pThis->pszFacilityStr); - if(pThis->pszSeverity != NULL) - free(pThis->pszSeverity); - if(pThis->pszSeverityStr != NULL) - free(pThis->pszSeverityStr); - if(pThis->pszRcvdAt3164 != NULL) - free(pThis->pszRcvdAt3164); - if(pThis->pszRcvdAt3339 != NULL) - free(pThis->pszRcvdAt3339); - if(pThis->pszRcvdAt_SecFrac != NULL) - free(pThis->pszRcvdAt_SecFrac); - if(pThis->pszRcvdAt_MySQL != NULL) - free(pThis->pszRcvdAt_MySQL); - if(pThis->pszRcvdAt_PgSQL != NULL) - free(pThis->pszRcvdAt_PgSQL); - if(pThis->pszTIMESTAMP3164 != NULL) - free(pThis->pszTIMESTAMP3164); - if(pThis->pszTIMESTAMP3339 != NULL) - free(pThis->pszTIMESTAMP3339); - if(pThis->pszTIMESTAMP_SecFrac != NULL) - free(pThis->pszTIMESTAMP_SecFrac); - if(pThis->pszTIMESTAMP_MySQL != NULL) - free(pThis->pszTIMESTAMP_MySQL); - if(pThis->pszTIMESTAMP_PgSQL != NULL) - free(pThis->pszTIMESTAMP_PgSQL); - if(pThis->pszPRI != NULL) - free(pThis->pszPRI); + free(pThis->pszUxTradMsg); + free(pThis->pszRawMsg); + free(pThis->pszTAG); + free(pThis->pszHOSTNAME); + free(pThis->pszInputName); + free(pThis->pszRcvFrom); + free(pThis->pszRcvFromIP); + free(pThis->pszMSG); + free(pThis->pszFacility); + free(pThis->pszFacilityStr); + free(pThis->pszSeverity); + free(pThis->pszSeverityStr); + free(pThis->pszRcvdAt3164); + free(pThis->pszRcvdAt3339); + free(pThis->pszRcvdAt_SecFrac); + free(pThis->pszRcvdAt_MySQL); + free(pThis->pszRcvdAt_PgSQL); + free(pThis->pszTIMESTAMP3164); + free(pThis->pszTIMESTAMP3339); + free(pThis->pszTIMESTAMP_SecFrac); + free(pThis->pszTIMESTAMP_MySQL); + free(pThis->pszTIMESTAMP_PgSQL); + free(pThis->pszPRI); if(pThis->pCSProgName != NULL) rsCStrDestruct(&pThis->pCSProgName); if(pThis->pCSStrucData != NULL) @@ -1306,15 +1284,15 @@ uchar *getInputName(msg_t *pM) } -char *getRcvFrom(msg_t *pM) +uchar *getRcvFrom(msg_t *pM) { if(pM == NULL) - return ""; + return UCHAR_CONSTANT(""); else if(pM->pszRcvFrom == NULL) - return ""; + return UCHAR_CONSTANT(""); else - return (char*) pM->pszRcvFrom; + return pM->pszRcvFrom; } @@ -1488,13 +1466,13 @@ static int getAPPNAMELen(msg_t *pM) /* rgerhards 2008-09-10: set pszInputName in msg object */ -void MsgSetInputName(msg_t *pMsg, char* pszInputName) +void MsgSetInputName(msg_t *pMsg, uchar* pszInputName) { assert(pMsg != NULL); if(pMsg->pszInputName != NULL) free(pMsg->pszInputName); - pMsg->iLenInputName = strlen(pszInputName); + pMsg->iLenInputName = ustrlen(pszInputName); if((pMsg->pszInputName = malloc(pMsg->iLenInputName + 1)) != NULL) { memcpy(pMsg->pszInputName, pszInputName, pMsg->iLenInputName + 1); } @@ -1502,13 +1480,12 @@ void MsgSetInputName(msg_t *pMsg, char* pszInputName) /* rgerhards 2004-11-16: set pszRcvFrom in msg object */ -void MsgSetRcvFrom(msg_t *pMsg, char* pszRcvFrom) +void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom) { assert(pMsg != NULL); - if(pMsg->pszRcvFrom != NULL) - free(pMsg->pszRcvFrom); + free(pMsg->pszRcvFrom); - pMsg->iLenRcvFrom = strlen(pszRcvFrom); + pMsg->iLenRcvFrom = ustrlen(pszRcvFrom); if((pMsg->pszRcvFrom = malloc(pMsg->iLenRcvFrom + 1)) != NULL) { memcpy(pMsg->pszRcvFrom, pszRcvFrom, pMsg->iLenRcvFrom + 1); } @@ -1559,13 +1536,12 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) * we need it. The rest of the code already knows how to handle an * unset HOSTNAME. */ -void MsgSetHOSTNAME(msg_t *pMsg, char* pszHOSTNAME) +void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME) { assert(pMsg != NULL); - if(pMsg->pszHOSTNAME != NULL) - free(pMsg->pszHOSTNAME); + free(pMsg->pszHOSTNAME); - pMsg->iLenHOSTNAME = strlen(pszHOSTNAME); + pMsg->iLenHOSTNAME = ustrlen(pszHOSTNAME); if((pMsg->pszHOSTNAME = malloc(pMsg->iLenHOSTNAME + 1)) != NULL) memcpy(pMsg->pszHOSTNAME, pszHOSTNAME, pMsg->iLenHOSTNAME + 1); else @@ -1790,7 +1766,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } else if(!strcmp((char*) pName, "inputname")) { pRes = (char*) getInputName(pMsg); } else if(!strcmp((char*) pName, "fromhost")) { - pRes = getRcvFrom(pMsg); + pRes = (char*) getRcvFrom(pMsg); } else if(!strcmp((char*) pName, "fromhost-ip")) { pRes = (char*) getRcvFromIP(pMsg); } else if(!strcmp((char*) pName, "source") || !strcmp((char*) pName, "hostname")) { @@ -2517,13 +2493,13 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("pszTAG")) { MsgSetTAG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszInputName")) { - MsgSetInputName(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszRcvFromIP")) { MsgSetRcvFromIP(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszRcvFrom")) { - MsgSetRcvFrom(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetRcvFrom(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszHOSTNAME")) { - MsgSetHOSTNAME(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetHOSTNAME(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pCSStrucData")) { MsgSetStructuredData(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pCSAPPNAME")) { diff --git a/runtime/msg.h b/runtime/msg.h index c8350626..a14f6b15 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -158,7 +158,7 @@ char *getSeverity(msg_t *pM); char *getSeverityStr(msg_t *pM); char *getFacility(msg_t *pM); char *getFacilityStr(msg_t *pM); -void MsgSetInputName(msg_t *pMsg, char*); +void MsgSetInputName(msg_t *pMsg, uchar*); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); char *getAPPNAME(msg_t *pM); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); @@ -171,15 +171,15 @@ rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); char *getTAG(msg_t *pM); int getHOSTNAMELen(msg_t *pM); char *getHOSTNAME(msg_t *pM); -char *getRcvFrom(msg_t *pM); +uchar *getRcvFrom(msg_t *pM); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); char *getStructuredData(msg_t *pM); int getProgramNameLen(msg_t *pM); char *getProgramName(msg_t *pM); -void MsgSetRcvFrom(msg_t *pMsg, char* pszRcvFrom); +void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); -void MsgSetHOSTNAME(msg_t *pMsg, char* pszHOSTNAME); +void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); void MsgSetMSG(msg_t *pMsg, char* pszMSG); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); diff --git a/runtime/netstrm.c b/runtime/netstrm.c index ffa1c578..459561bc 100644 --- a/runtime/netstrm.c +++ b/runtime/netstrm.c @@ -17,7 +17,7 @@ * Rainer Gerhards and Adiscon GmbH have agreed to permit using the code * under the terms of the GNU Lesser General Public License. * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -114,6 +114,7 @@ AcceptConnReq(netstrm_t *pThis, netstrm_t **ppNew) ISOBJ_TYPE_assert(pThis, netstrm); assert(ppNew != NULL); +RUNLOG_STR("XXX: accept conn reqeust"); /* accept the new connection */ CHKiRet(pThis->Drvr.AcceptConnReq(pThis->pDrvrData, &pNewNsd)); /* construct our object so that we can use it... */ @@ -147,6 +148,7 @@ LstnInit(netstrms_t *pNS, void *pUsr, rsRetVal(*fAddLstn)(void*,netstrm_t*), ISOBJ_TYPE_assert(pNS, netstrms); assert(fAddLstn != NULL); assert(pLstnPort != NULL); +RUNLOG_STR("XXX: Init Lstn"); CHKiRet(pNS->Drvr.LstnInit(pNS, pUsr, fAddLstn, pLstnPort, pLstnIP, iSessMax)); diff --git a/runtime/netstrm.h b/runtime/netstrm.h index 3ab790e8..b00dd223 100644 --- a/runtime/netstrm.h +++ b/runtime/netstrm.h @@ -31,6 +31,7 @@ struct netstrm_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ nsd_t *pDrvrData; /**< the driver's data elements (at most other places, this is called pNsd) */ nsd_if_t Drvr; /**< our stream driver */ + void *pUsr; /**< pointer to user-provided data structure */ netstrms_t *pNS; /**< pointer to our netstream subsystem object */ }; diff --git a/runtime/parser.c b/runtime/parser.c index b4ab0a3e..212d40f3 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -283,7 +283,7 @@ rsRetVal parseMsg(msg_t *pMsg) MsgSetUxTradMsg(pMsg, (char*) msg); if(pMsg->bParseHOSTNAME == 0) - MsgSetHOSTNAME(pMsg, (char*) pMsg->pszRcvFrom); + MsgSetHOSTNAME(pMsg, pMsg->pszRcvFrom); /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have * a traditional syslog message or one formatted according to syslog-protocol. diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index fd5a5371..8e8a9f2a 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -87,6 +87,8 @@ typedef struct tcpsrv_s tcpsrv_t; typedef struct vmstk_s vmstk_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ +typedef struct tcpLstnPortList_s tcpLstnPortList_t; // TODO: rename? + /* some universal 64 bit define... */ typedef long long int64; typedef long long unsigned uint64; @@ -359,6 +361,7 @@ typedef enum rsObjectID rsObjID; /* The following prototype is convenient, even though it may not be the 100% correct place.. -- rgerhards 2008-01-07 */ void dbgprintf(char *, ...) __attribute__((format(printf, 1, 2))); + #include "debug.h" #include "obj.h" diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h new file mode 100644 index 00000000..36d76a78 --- /dev/null +++ b/runtime/unicode-helper.h @@ -0,0 +1,54 @@ +/* This is the header file for unicode support. + * + * Currently, this is a dummy module. + * The following functions are wrappers which hopefully enable us to move + * from 8-bit chars to unicode with relative ease when we finally attack this + * + * Begun 2009-05-21 RGerhards + * + * Copyright (C) 2009 by Rainer Gerhards and Adiscon GmbH + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_UNICODE_HELPER_H +#define INCLUDED_UNICODE_HELPER_H + +#include + +static inline int ustrcmp(uchar *psz1, uchar *psz2) +{ + return strcmp((char*) psz1, (char*) psz2); +} + +static inline int ustrlen(uchar *psz) +{ + return strlen((char*) psz); +} + +static inline uchar* ustrdup(uchar *psz) +{ + return (uchar*) strdup((char*)psz); +} + + +#define UCHAR_CONSTANT(x) ((uchar*) (x)) + +#endif /* multi-include protection */ +/* vim:set ai: + */ diff --git a/tcps_sess.c b/tcps_sess.c index ceb6142f..c564caea 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -185,6 +185,18 @@ SetTcpsrv(tcps_sess_t *pThis, tcpsrv_t *pSrv) } +/* set our parent listener info*/ +static rsRetVal +SetLstnInfo(tcps_sess_t *pThis, tcpLstnPortList_t *pLstnInfo) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, tcps_sess); + assert(pLstnInfo != NULL); + pThis->pLstnInfo = pLstnInfo; + RETiRet; +} + + static rsRetVal SetUsrP(tcps_sess_t *pThis, void *pUsr) { @@ -223,11 +235,11 @@ doSubmitMessage(tcps_sess_t *pThis) CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); pMsg->iLenRawMsg = pThis->iMsg; - MsgSetInputName(pMsg, (char*)pThis->pSrv->pszInputName); + MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; - MsgSetRcvFrom(pMsg, (char*)pThis->fromHost); + MsgSetRcvFrom(pMsg, pThis->fromHost); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); CHKiRet(submitMsg(pMsg)); @@ -457,6 +469,7 @@ CODESTARTobjQueryInterface(tcps_sess) pIf->SetUsrP = SetUsrP; pIf->SetTcpsrv = SetTcpsrv; + pIf->SetLstnInfo = SetLstnInfo; pIf->SetHost = SetHost; pIf->SetHostIP = SetHostIP; pIf->SetStrm = SetStrm; diff --git a/tcps_sess.h b/tcps_sess.h index 576466ff..2ef28264 100644 --- a/tcps_sess.h +++ b/tcps_sess.h @@ -31,7 +31,8 @@ struct tcpsrv_s; /* the tcps_sess object */ typedef struct tcps_sess_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ - struct tcpsrv_s *pSrv; /* pointer back to my server (e.g. for callbacks) */ + tcpsrv_t *pSrv; /* pointer back to my server (e.g. for callbacks) */ + tcpLstnPortList_t *pLstnInfo; /* pointer back to listener info */ netstrm_t *pStrm; int iMsg; /* index of next char to store in msg */ int bAtStrtOfFram; /* are we at the very beginning of a new frame? */ @@ -60,13 +61,19 @@ BEGINinterface(tcps_sess) /* name must also be changed in ENDinterface macro! */ rsRetVal (*DataRcvd)(tcps_sess_t *pThis, char *pData, size_t iLen); /* set methods */ rsRetVal (*SetTcpsrv)(tcps_sess_t *pThis, struct tcpsrv_s *pSrv); + rsRetVal (*SetLstnInfo)(tcps_sess_t *pThis, tcpLstnPortList_t *pLstnInfo); rsRetVal (*SetUsrP)(tcps_sess_t*, void*); rsRetVal (*SetHost)(tcps_sess_t *pThis, uchar*); rsRetVal (*SetHostIP)(tcps_sess_t *pThis, uchar*); rsRetVal (*SetStrm)(tcps_sess_t *pThis, netstrm_t*); rsRetVal (*SetMsgIdx)(tcps_sess_t *pThis, int); ENDinterface(tcps_sess) -#define tcps_sessCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define tcps_sessCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +/* interface changes + * to version v2, rgerhards, 2009-05-22 + * - Data structures changed + * - SetLstnInfo entry point added + */ /* prototypes */ diff --git a/tcpsrv.c b/tcpsrv.c index 7af45d3c..bbd95058 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -66,6 +66,7 @@ #include "netstrm.h" #include "nssel.h" #include "errmsg.h" +#include "unicode-helper.h" MODULE_TYPE_LIB @@ -85,43 +86,61 @@ DEFobjCurrIf(netstrm) DEFobjCurrIf(nssel) -/* configure TCP listener settings. This is called during command - * line parsing. The argument following -t is supplied as an argument. - * The format of this argument is - * ", " - * Typically, there is no whitespace between port and session number. - * (but it may be...). - * NOTE: you can not use dbgprintf() in here - the dbgprintf() system is - * not yet initilized when this function is called. - * rgerhards, 2007-06-21 - * The port in cOptarg is handed over to us - the caller MUST NOT free it! +/* add new listener port to listener port list + * rgerhards, 2009-05-21 + */ +static inline rsRetVal +addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) +{ + tcpLstnPortList_t *pEntry; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, tcpsrv); + +dbgprintf("XXX: tcpsrv.c add port %s, name '%s'\n", pszPort, pThis->pszInputName); + /* create entry */ + CHKmalloc(pEntry = malloc(sizeof(tcpLstnPortList_t))); + pEntry->pszPort = pszPort; + pEntry->pSrv = pThis; + CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); + + /* and add to list */ + pEntry->pNext = pThis->pLstnPorts; + pThis->pLstnPorts = pEntry; + +finalize_it: + RETiRet; +} + + +/* configure TCP listener settings. + * Note: pszPort is handed over to us - the caller MUST NOT free it! * rgerhards, 2008-03-20 */ -static void -configureTCPListen(tcpsrv_t *pThis, char *cOptarg) +static rsRetVal +configureTCPListen(tcpsrv_t *pThis, uchar *pszPort) { - register int i; - register char *pArg = cOptarg; + int i; + uchar *pPort = pszPort; + DEFiRet; - assert(cOptarg != NULL); + assert(pszPort != NULL); ISOBJ_TYPE_assert(pThis, tcpsrv); /* extract port */ i = 0; - while(isdigit((int) *pArg)) { - i = i * 10 + *pArg++ - '0'; + while(isdigit((int) *pPort)) { + i = i * 10 + *pPort++ - '0'; } - if(pThis->TCPLstnPort != NULL) { - free(pThis->TCPLstnPort); - pThis->TCPLstnPort = NULL; - } - - if( i >= 0 && i <= 65535) { - pThis->TCPLstnPort = cOptarg; + if(i >= 0 && i <= 65535) { + CHKiRet(addNewLstnPort(pThis, pszPort)); } else { - errmsg.LogError(0, NO_ERRCODE, "Invalid TCP listen port %s - changed to 514.\n", cOptarg); + errmsg.LogError(0, NO_ERRCODE, "Invalid TCP listen port %s - ignored.\n", pszPort); } + +finalize_it: + RETiRet; } @@ -202,6 +221,8 @@ TCPSessGetNxtSess(tcpsrv_t *pThis, int iCurr) static void deinit_tcp_listener(tcpsrv_t *pThis) { int i; + tcpLstnPortList_t *pEntry; + tcpLstnPortList_t *pDel; ISOBJ_TYPE_assert(pThis, tcpsrv); @@ -219,7 +240,15 @@ static void deinit_tcp_listener(tcpsrv_t *pThis) pThis->pSessions = NULL; /* just to make sure... */ } - free(pThis->TCPLstnPort); + /* free list of tcp listen ports */ + pEntry = pThis->pLstnPorts; + while(pEntry != NULL) { + free(pEntry->pszPort); + free(pEntry->pszInputName); + pDel = pEntry; + pEntry = pEntry->pNext; + free(pDel); + } /* finally close our listen streams */ for(i = 0 ; i < pThis->iLstnMax ; ++i) { @@ -234,9 +263,11 @@ static void deinit_tcp_listener(tcpsrv_t *pThis) static rsRetVal addTcpLstn(void *pUsr, netstrm_t *pLstn) { - tcpsrv_t *pThis = (tcpsrv_t*) pUsr; + tcpLstnPortList_t *pPortList = (tcpLstnPortList_t *) pUsr; + tcpsrv_t *pThis = pPortList->pSrv; DEFiRet; +dbgprintf("XXX: addTcpLst name %s\n", pPortList->pszInputName); ISOBJ_TYPE_assert(pThis, tcpsrv); ISOBJ_TYPE_assert(pLstn, netstrm); @@ -244,6 +275,7 @@ addTcpLstn(void *pUsr, netstrm_t *pLstn) ABORT_FINALIZE(RS_RET_MAX_LSTN_REACHED); pThis->ppLstn[pThis->iLstnMax] = pLstn; + pThis->ppLstnPort[pThis->iLstnMax] = pPortList; ++pThis->iLstnMax; finalize_it: @@ -251,19 +283,20 @@ finalize_it: } -/* Initialize TCP sockets (for listener) and listens on them */ -static rsRetVal -create_tcp_socket(tcpsrv_t *pThis) +/* Initialize TCP listener socket for a single port + * rgerhards, 2009-05-21 + */ +static inline rsRetVal +initTCPListener(tcpsrv_t *pThis, tcpLstnPortList_t *pPortEntry) { DEFiRet; uchar *TCPLstnPort; ISOBJ_TYPE_assert(pThis, tcpsrv); + assert(pPortEntry != NULL); - if(!strcmp((char*)pThis->TCPLstnPort, "0")) - TCPLstnPort = (uchar*)"514"; - // TODO: we need to enable the caller to set a port (based on who is - // using this, 514 may be totally unsuitable... --- rgerhards, 2008-04-22 + if(!ustrcmp(pPortEntry->pszPort, UCHAR_CONSTANT("0"))) + TCPLstnPort = UCHAR_CONSTANT("514"); /* use default - we can not do service db update, because there is * no IANA-assignment for syslog/tcp. In the long term, we might * re-use RFC 3195 port of 601, but that would probably break to @@ -271,10 +304,32 @@ create_tcp_socket(tcpsrv_t *pThis) * rgerhards, 2007-06-28 */ else - TCPLstnPort = (uchar*)pThis->TCPLstnPort; + TCPLstnPort = pPortEntry->pszPort; /* TODO: add capability to specify local listen address! */ - CHKiRet(netstrm.LstnInit(pThis->pNS, (void*)pThis, addTcpLstn, TCPLstnPort, NULL, pThis->iSessMax)); + CHKiRet(netstrm.LstnInit(pThis->pNS, (void*)pPortEntry, addTcpLstn, TCPLstnPort, NULL, pThis->iSessMax)); + +finalize_it: + RETiRet; +} + + +/* Initialize TCP sockets (for listener) and listens on them */ +static rsRetVal +create_tcp_socket(tcpsrv_t *pThis) +{ + tcpLstnPortList_t *pEntry; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, tcpsrv); + + /* init all configured ports */ + pEntry = pThis->pLstnPorts; + while(pEntry != NULL) { +dbgprintf("XXX: tcpsrv.c create_tcp_socket do port %s\n", pEntry->pszPort); + CHKiRet(initTCPListener(pThis, pEntry)); + pEntry = pEntry->pNext; + } /* OK, we had success. Now it is also time to * initialize our connections @@ -296,7 +351,7 @@ finalize_it: /* Accept new TCP connection; make entry in session table. If there * is no more space left in the connection table, the new TCP * connection is immediately dropped. - * ppSess has a pointer to the newly created session, if it succeds. + * ppSess has a pointer to the newly created session, if it succeeds. * If it does not succeed, no session is created and ppSess is * undefined. If the user has provided an OnSessAccept Callback, * this one is executed immediately after creation of the @@ -304,7 +359,7 @@ finalize_it: * rgerhards, 2008-03-02 */ static rsRetVal -SessAccept(tcpsrv_t *pThis, tcps_sess_t **ppSess, netstrm_t *pStrm) +SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, netstrm_t *pStrm) { DEFiRet; tcps_sess_t *pSess = NULL; @@ -315,6 +370,7 @@ SessAccept(tcpsrv_t *pThis, tcps_sess_t **ppSess, netstrm_t *pStrm) uchar *fromHostIP = NULL; ISOBJ_TYPE_assert(pThis, tcpsrv); + assert(pLstnInfo != NULL); CHKiRet(netstrm.AcceptConnReq(pStrm, &pNewStrm)); @@ -328,6 +384,7 @@ SessAccept(tcpsrv_t *pThis, tcps_sess_t **ppSess, netstrm_t *pStrm) /* we found a free spot and can construct our session object */ CHKiRet(tcps_sess.Construct(&pSess)); CHKiRet(tcps_sess.SetTcpsrv(pSess, pThis)); + CHKiRet(tcps_sess.SetLstnInfo(pSess, pLstnInfo)); } /* OK, we have a "good" index... */ @@ -375,12 +432,10 @@ finalize_it: if(iRet != RS_RET_OK) { if(pSess != NULL) tcps_sess.Destruct(&pSess); - if(fromHostFQDN != NULL) - free(fromHostFQDN); - if(fromHostIP != NULL) - free(fromHostIP); if(pNewStrm != NULL) netstrm.Destruct(&pNewStrm); + free(fromHostFQDN); + free(fromHostIP); } RETiRet; @@ -444,7 +499,7 @@ Run(tcpsrv_t *pThis) CHKiRet(nssel.IsReady(pSel, pThis->ppLstn[i], NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { dbgprintf("New connect on NSD %p.\n", pThis->ppLstn[i]); - SessAccept(pThis, &pNewSess, pThis->ppLstn[i]); + SessAccept(pThis, pThis->ppLstnPort[i], &pNewSess, pThis->ppLstn[i]); --nfds; /* indicate we have processed one */ } } @@ -535,6 +590,7 @@ tcpsrvConstructFinalize(tcpsrv_t *pThis) /* set up listeners */ CHKmalloc(pThis->ppLstn = calloc(TCPLSTN_MAX_DEFAULT, sizeof(netstrm_t*))); + CHKmalloc(pThis->ppLstnPort = calloc(TCPLSTN_MAX_DEFAULT, sizeof(tcpLstnPortList_t*))); iRet = pThis->OpenLstnSocks(pThis); finalize_it: @@ -558,6 +614,7 @@ CODESTARTobjDestruct(tcpsrv) netstrms.Destruct(&pThis->pNS); free(pThis->pszDrvrAuthMode); free(pThis->ppLstn); + free(pThis->ppLstnPort); free(pThis->pszInputName); ENDobjDestruct(tcpsrv) @@ -674,11 +731,12 @@ SetInputName(tcpsrv_t *pThis, uchar *name) { uchar *pszName; DEFiRet; +dbgprintf("XXX: SetInputName: %s\n", name); ISOBJ_TYPE_assert(pThis, tcpsrv); if(name == NULL) pszName = NULL; else - CHKmalloc(pszName = (uchar*)strdup((char*)name)); + CHKmalloc(pszName = ustrdup(name)); free(pThis->pszInputName); pThis->pszInputName = pszName; finalize_it: @@ -708,7 +766,7 @@ SetDrvrAuthMode(tcpsrv_t *pThis, uchar *mode) { DEFiRet; ISOBJ_TYPE_assert(pThis, tcpsrv); - CHKmalloc(pThis->pszDrvrAuthMode = (uchar*)strdup((char*)mode)); + CHKmalloc(pThis->pszDrvrAuthMode = ustrdup(mode)); finalize_it: RETiRet; } @@ -763,7 +821,7 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->ConstructFinalize = tcpsrvConstructFinalize; pIf->Destruct = tcpsrvDestruct; - pIf->SessAccept = SessAccept; + //pIf->SessAccept = SessAccept; pIf->configureTCPListen = configureTCPListen; pIf->create_tcp_socket = create_tcp_socket; pIf->Run = Run; diff --git a/tcpsrv.h b/tcpsrv.h index e5ecb865..0f7dd6c6 100644 --- a/tcpsrv.h +++ b/tcpsrv.h @@ -32,6 +32,15 @@ typedef enum ETCPsyslogFramingAnomaly { frame_CiscoIOS = 2 } eTCPsyslogFramingAnomaly; + +/* list of tcp listen ports */ +struct tcpLstnPortList_s { + uchar *pszPort; /**< the ports the listener shall listen on */ + uchar *pszInputName; /**< value to be used as input name */ + tcpsrv_t *pSrv; /**< pointer to higher-level server instance */ + tcpLstnPortList_t *pNext; /**< next port or NULL */ +}; + #define TCPSRV_NO_ADDTL_DELIMITER -1 /* specifies that no additional delimiter is to be used in TCP framing */ /* the tcpsrv object */ @@ -44,8 +53,9 @@ struct tcpsrv_s { permittedPeers_t *pPermPeers;/**< driver's permitted peers */ int iLstnMax; /**< max nbr of listeners currently supported */ netstrm_t **ppLstn; /**< our netstream listners */ + tcpLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ int iSessMax; /**< max number of sessions supported */ - char *TCPLstnPort; /**< the port the listener shall listen on */ + tcpLstnPortList_t *pLstnPorts; /**< head pointer for listen ports */ int addtlFrameDelim; /**< additional frame delimiter for plain TCP syslog framing (e.g. to handle NetScreen) */ tcps_sess_t **pSessions;/**< array of all of our sessions */ void *pUsr; /**< a user-settable pointer (provides extensibility for "derived classes")*/ @@ -70,8 +80,8 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Construct)(tcpsrv_t **ppThis); rsRetVal (*ConstructFinalize)(tcpsrv_t __attribute__((unused)) *pThis); rsRetVal (*Destruct)(tcpsrv_t **ppThis); - void (*configureTCPListen)(tcpsrv_t*, char *cOptarg); - rsRetVal (*SessAccept)(tcpsrv_t *pThis, tcps_sess_t **ppSess, netstrm_t *pStrm); + rsRetVal (*configureTCPListen)(tcpsrv_t*, uchar *pszPort); + //rsRetVal (*SessAccept)(tcpsrv_t *pThis, tcpLstnPortList_t*, tcps_sess_t **ppSess, netstrm_t *pStrm); rsRetVal (*create_tcp_socket)(tcpsrv_t *pThis); rsRetVal (*Run)(tcpsrv_t *pThis); /* set methods */ diff --git a/tests/Makefile.am b/tests/Makefile.am index ed415a1e..bc1849f0 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,8 +1,8 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh +TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh if ENABLE_OMSTDOUT -TESTS += omod-if-array.sh parsertest.sh +TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh endif TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid @@ -29,11 +29,22 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rfc5424-4.parse1 \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ + killrsyslog.sh \ parsertest.sh \ diskqueue.sh \ testsuites/diskqueue.conf \ + imtcp-multiport.sh \ + testsuites/imtcp-multiport.conf \ manytcp.sh \ testsuites/manytcp.conf \ + fieldtest.sh \ + testsuites/field.conf \ + testsuites/1.field1 \ + inputname.sh \ + testsuites/inputname_imtcp.conf \ + testsuites/1.inputname_imtcp_12514 \ + testsuites/1.inputname_imtcp_12515 \ + testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ cfg.sh diff --git a/tests/fieldtest.sh b/tests/fieldtest.sh new file mode 100755 index 00000000..7a646f00 --- /dev/null +++ b/tests/fieldtest.sh @@ -0,0 +1,13 @@ +echo test fieldtest via udp +./killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -tfield1 -iudp +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo test fieldtest via tcp +./nettester -tfield1 -itcp +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/imtcp-multiport.sh b/tests/imtcp-multiport.sh new file mode 100755 index 00000000..aa1f15e8 --- /dev/null +++ b/tests/imtcp-multiport.sh @@ -0,0 +1,80 @@ +# Test for multiple ports in imtcp +# This test checks if multiple tcp listener ports are correctly +# handled by imtcp +# +# NOTE: this test must (and can) be enhanced when we merge in the +# upgraded tcpflood program +# +# added 2009-05-22 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +echo testing imtcp multiple listeners +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +./tcpflood 127.0.0.1 13514 1 10000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 2 # we need this so that rsyslogd can receive all outstanding messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 9999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log +# +# +# ### now complete new cycle with other port ### +# +# +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +./tcpflood 127.0.0.1 13515 1 10000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 2 # we need this so that rsyslogd can receive all outstanding messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 9999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log +# +# +# ### now complete new cycle with other port ### +# +# +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +./tcpflood 127.0.0.1 13516 1 10000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 2 # we need this so that rsyslogd can receive all outstanding messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 9999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log diff --git a/tests/inputname.sh b/tests/inputname.sh new file mode 100755 index 00000000..7d9ea111 --- /dev/null +++ b/tests/inputname.sh @@ -0,0 +1,20 @@ +echo testing $InputTCPServerInputName directive +./killrsyslog.sh # kill rsyslogd if it runs for some reason + +echo port 12514 +./nettester -tinputname_imtcp_12514 -cinputname_imtcp -itcp -p12514 +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo port 12515 +./nettester -tinputname_imtcp_12515 -cinputname_imtcp -itcp -p12515 +if [ "$?" -ne "0" ]; then + exit 1 +fi + +echo port 12516 +./nettester -tinputname_imtcp_12516 -cinputname_imtcp -itcp -p12516 +if [ "$?" -ne "0" ]; then + exit 1 +fi diff --git a/tests/killrsyslog.sh b/tests/killrsyslog.sh new file mode 100755 index 00000000..b1be757b --- /dev/null +++ b/tests/killrsyslog.sh @@ -0,0 +1,7 @@ +#check if rsyslog instance exists and, if so, kill it +if [ -e "rsyslog.pid" ] +then + echo rsyslog.pid exists, trying to shut down rsyslogd process `cat rsyslog.pid`. + kill `cat rsyslog.pid` + sleep 1 +fi diff --git a/tests/nettester.c b/tests/nettester.c index 37183ac9..566f553b 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -45,16 +45,36 @@ #include #include #include +#include #define EXIT_FAILURE 1 #define INVALID_SOCKET -1 /* Name of input file, must match $IncludeConfig in test suite .conf files */ #define NETTEST_INPUT_CONF_FILE "nettest.input.conf" /* name of input file, must match $IncludeConfig in .conf files */ -static enum { inputUDP, inputTCP } inputMode; /* input for which tests are to be run */ +typedef enum { inputUDP, inputTCP } inputMode_t; +inputMode_t inputMode = inputTCP; /* input for which tests are to be run */ static pid_t rsyslogdPid = 0; /* pid of rsyslog instance being tested */ static char *srcdir; /* global $srcdir, set so that we can run outside of "make check" */ -static char *testSuite; /* name of current test suite */ +static char *testSuite = NULL; /* name of current test suite */ +static int iPort = 12514; /* port which shall be used for sending data */ +static char* pszCustomConf = NULL; /* custom config file, use -c conf to specify */ +static int verbose = 0; /* verbose output? -v option */ + + +/* provide user-friednly name of input mode + */ +static char *inputMode2Str(inputMode_t mode) +{ + char *pszMode; + + if(mode == inputUDP) + pszMode = "udp"; + else + pszMode = "tcp"; + + return pszMode; +} void readLine(int fd, char *ln) @@ -94,7 +114,7 @@ tcpSend(char *buf, int lenBuf) memset((char *) &addr, 0, sizeof(addr)); addr.sin_family = AF_INET; - addr.sin_port = htons(13514); + addr.sin_port = htons(iPort); if(inet_aton("127.0.0.1", &addr.sin_addr)==0) { fprintf(stderr, "inet_aton() failed\n"); return(1); @@ -139,7 +159,7 @@ udpSend(char *buf, int lenBuf) memset((char *) &si_other, 0, sizeof(si_other)); si_other.sin_family = AF_INET; - si_other.sin_port = htons(12514); + si_other.sin_port = htons(iPort); if(inet_aton("127.0.0.1", &si_other.sin_addr)==0) { fprintf(stderr, "inet_aton() failed\n"); return(1); @@ -169,9 +189,14 @@ int openPipe(char *configFile, pid_t *pid, int *pfd) "-M../runtime/.libs:../.libs", NULL }; char confFile[1024]; char *newenviron[] = { NULL }; + /* debug aide... + char *newenviron[] = { "RSYSLOG_DEBUG=debug nostdout", + "RSYSLOG_DEBUGLOG=tmp", NULL }; + */ - sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, configFile); + sprintf(confFile, "-f%s/testsuites/%s.conf", srcdir, + (pszCustomConf == NULL) ? configFile : pszCustomConf); newargv[1] = confFile; if (pipe(pipefd) == -1) { @@ -291,12 +316,12 @@ doTests(int fd, char *files) ++iTests; /* all regular files are run through the test logic. Symlinks don't work. */ if(S_ISREG(fileInfo.st_mode)) { /* config file */ - printf("processing test case '%s' ... ", testFile); + if(verbose) printf("processing test case '%s' ... ", testFile); ret = processTestFile(fd, testFile); if(ret == 0) { - printf("successfully completed\n"); + if(verbose) printf("successfully completed\n"); } else { - printf("failed!\n"); + if(verbose) printf("failed!\n"); ++iFailed; } } @@ -334,36 +359,54 @@ void doAtExit(void) int main(int argc, char *argv[]) { int fd; + int opt; int ret = 0; FILE *fp; char buf[4096]; char testcases[4096]; - if(argc != 3) { - printf("Invalid call of nettester\n"); - printf("Usage: nettester testsuite-name input\n"); - printf(" input = udp|tcp\n"); - exit(1); + while((opt = getopt(argc, argv, "c:i:p:t:v")) != EOF) { + switch((char)opt) { + case 'c': + pszCustomConf = optarg; + break; + case 'i': + if(!strcmp(optarg, "udp")) + inputMode = inputUDP; + else if(!strcmp(optarg, "tcp")) + inputMode = inputTCP; + else { + printf("error: unsupported input mode '%s'\n", optarg); + exit(1); + } + break; + case 'p': + iPort = atoi(optarg); + break; + case 't': + testSuite = optarg; + break; + case 'v': + verbose = 1; + break; + default:printf("Invalid call of nettester, invalid option '%c'.\n", opt); + printf("Usage: nettester -ttestsuite-name -iudp|tcp [-pport] [-ccustomConfFile] \n"); + exit(1); + } } - atexit(doAtExit); - - testSuite = argv[1]; - - if(!strcmp(argv[2], "udp")) - inputMode = inputUDP; - else if(!strcmp(argv[2], "tcp")) - inputMode = inputTCP; - else { - printf("error: unsupported input mode '%s'\n", argv[2]); + if(testSuite == NULL) { + printf("error: no testsuite given, need to specify -t testsuite!\n"); exit(1); } + atexit(doAtExit); + if((srcdir = getenv("srcdir")) == NULL) srcdir = "."; - printf("Start of nettester run ($srcdir=%s, testsuite=%s, input=%s)\n", - srcdir, testSuite, argv[2]); + if(verbose) printf("Start of nettester run ($srcdir=%s, testsuite=%s, input=%s/%d)\n", + srcdir, testSuite, inputMode2Str(inputMode), iPort); /* create input config file */ if((fp = fopen(NETTEST_INPUT_CONF_FILE, "w")) == NULL) { @@ -373,15 +416,15 @@ int main(int argc, char *argv[]) } if(inputMode == inputUDP) { fputs("$ModLoad ../plugins/imudp/.libs/imudp\n", fp); - fputs("$UDPServerRun 12514\n", fp); + fprintf(fp, "$UDPServerRun %d\n", iPort); } else { fputs("$ModLoad ../plugins/imtcp/.libs/imtcp\n", fp); - fputs("$InputTCPServerRun 13514\n", fp); + fprintf(fp, "$InputTCPServerRun %d\n", iPort); } fclose(fp); /* start to be tested rsyslogd */ - openPipe(argv[1], &rsyslogdPid, &fd); + openPipe(testSuite, &rsyslogdPid, &fd); readLine(fd, buf); /* generate filename */ @@ -389,6 +432,6 @@ int main(int argc, char *argv[]) if(doTests(fd, testcases) != 0) ret = 1; - printf("End of nettester run (%d).\n", ret); + if(verbose) printf("End of nettester run (%d).\n", ret); exit(ret); } diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh index fd845b4d..7b4b5611 100755 --- a/tests/omod-if-array.sh +++ b/tests/omod-if-array.sh @@ -1,11 +1,13 @@ echo test omod-if-array via udp -./nettester omod-if-array udp +./killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -tomod-if-array -iudp -p4711 if [ "$?" -ne "0" ]; then exit 1 fi echo test omod-if-array via tcp -./nettester omod-if-array tcp +./nettester -tomod-if-array -itcp if [ "$?" -ne "0" ]; then exit 1 fi diff --git a/tests/parsertest.sh b/tests/parsertest.sh index a6b7d45c..152d8b60 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1,11 +1,13 @@ echo test parsertest via udp -./nettester parse1 udp +./killrsyslog.sh # kill rsyslogd if it runs for some reason + +./nettester -tparse1 -iudp if [ "$?" -ne "0" ]; then exit 1 fi echo test parsertest via tcp -./nettester parse1 tcp +./nettester -tparse1 -itcp if [ "$?" -ne "0" ]; then exit 1 fi diff --git a/tests/tcpflood.c b/tests/tcpflood.c index 8dbc201b..c3c9c871 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -154,8 +154,6 @@ int sendMessages(void) int lenBuf; int lenSend; char buf[2048]; - char msgBuf[128]; - size_t lenMsg; srand(time(NULL)); /* seed is good enough for our needs */ diff --git a/tests/testsuites/1.field1 b/tests/testsuites/1.field1 new file mode 100644 index 00000000..54751171 --- /dev/null +++ b/tests/testsuites/1.field1 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: DROP_url_www.sina.com.cn:IN=eth1 OUT=eth0 SRC=192.168.10.78 DST=61.172.201.194 LEN=1182 TOS=0x00 PREC=0x00 TTL=63 ID=14368 DF PROTO=TCP SPT=33343 DPT=80 WINDOW=92 RES=0x00 ACK PSH URGP=0 +DROP_url_www.sina.com.cn:IN=eth1 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/1.inputname_imtcp_12514 b/tests/testsuites/1.inputname_imtcp_12514 new file mode 100644 index 00000000..178b1724 --- /dev/null +++ b/tests/testsuites/1.inputname_imtcp_12514 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: MSG +12514 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/1.inputname_imtcp_12515 b/tests/testsuites/1.inputname_imtcp_12515 new file mode 100644 index 00000000..d616098b --- /dev/null +++ b/tests/testsuites/1.inputname_imtcp_12515 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: MSG +12515 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/1.inputname_imtcp_12516 b/tests/testsuites/1.inputname_imtcp_12516 new file mode 100644 index 00000000..8e6997ce --- /dev/null +++ b/tests/testsuites/1.inputname_imtcp_12516 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: MSG +12516 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/field1.conf b/tests/testsuites/field1.conf new file mode 100644 index 00000000..1ff833dd --- /dev/null +++ b/tests/testsuites/field1.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template fmt,"%msg:F,32:2%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/imtcp-multiport.conf b/tests/testsuites/imtcp-multiport.conf new file mode 100644 index 00000000..00b63cb2 --- /dev/null +++ b/tests/testsuites/imtcp-multiport.conf @@ -0,0 +1,13 @@ +# Test for queue disk mode (see .sh file for details) +# rgerhards, 2009-05-22 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$InputTCPServerRun 13514 +$InputTCPServerRun 13515 +$InputTCPServerRun 13516 + +$ErrorMessagesToStderr off + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/inputname_imtcp.conf b/tests/testsuites/inputname_imtcp.conf new file mode 100644 index 00000000..a25eab37 --- /dev/null +++ b/tests/testsuites/inputname_imtcp.conf @@ -0,0 +1,19 @@ +# This is a special case, thus we define the inputs ourselfs +$ModLoad ../plugins/omstdout/.libs/omstdout + +$ModLoad ../plugins/imtcp/.libs/imtcp + +$InputTCPServerInputname 12514 +$InputTCPServerRun 12514 + +$InputTCPServerInputname 12515 +$InputTCPServerRun 12515 + +$InputTCPServerInputname 12516 +$InputTCPServerRun 12516 + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template fmt,"%inputname%\n" +*.* :omstdout:;fmt diff --git a/tools/syslogd.c b/tools/syslogd.c index 3a751a30..dc5b8fee 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -137,6 +137,7 @@ #include "datetime.h" #include "parser.h" #include "sysvar.h" +#include "unicode-helper.h" /* definitions for objects we access */ DEFobjCurrIf(obj) @@ -620,7 +621,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } if(pszInputName != NULL) - MsgSetInputName(pMsg, (char*) pszInputName); + MsgSetInputName(pMsg, pszInputName); MsgSetFlowControlType(pMsg, flowCtlType); MsgSetRawMsg(pMsg, (char*)msg); @@ -649,8 +650,8 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f * being the local host). rgerhards 2004-11-16 */ if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) - MsgSetHOSTNAME(pMsg, (char*)hname); - MsgSetRcvFrom(pMsg, (char*)hname); + MsgSetHOSTNAME(pMsg, hname); + MsgSetRcvFrom(pMsg, hname); CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP)); /* rgerhards 2004-11-19: well, well... we've now seen that we @@ -928,12 +929,12 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) DEFiRet; CHKiRet(msgConstruct(&pMsg)); - MsgSetInputName(pMsg, "rsyslogd"); + MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd")); MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); - MsgSetHOSTNAME(pMsg, (char*)glbl.GetLocalHostName()); - MsgSetRcvFrom(pMsg, (char*)glbl.GetLocalHostName()); - MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); + MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); + MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1")); /* check if we have an error code associated and, if so, * adjust the tag. -- r5gerhards, 2008-06-27 */ @@ -1233,9 +1234,9 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) * SP-terminated or any other error occurs. * rger, 2005-11-24 */ -static int parseRFCField(char **pp2parse, char *pResult) +static int parseRFCField(uchar **pp2parse, uchar *pResult) { - char *p2parse; + uchar *p2parse; int iRet = 0; assert(pp2parse != NULL); @@ -1271,9 +1272,9 @@ static int parseRFCField(char **pp2parse, char *pResult) * SP-terminated or any other error occurs. * rger, 2005-11-24 */ -static int parseRFCStructuredData(char **pp2parse, char *pResult) +static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult) { - char *p2parse; + uchar *p2parse; int bCont = 1; int iRet = 0; @@ -1340,14 +1341,14 @@ static int parseRFCStructuredData(char **pp2parse, char *pResult) */ int parseRFCSyslogMsg(msg_t *pMsg, int flags) { - char *p2parse; - char *pBuf; + uchar *p2parse; + uchar *pBuf; int bContParse = 1; BEGINfunc assert(pMsg != NULL); assert(pMsg->pszUxTradMsg != NULL); - p2parse = (char*) pMsg->pszUxTradMsg; + p2parse = pMsg->pszUxTradMsg; /* do a sanity check on the version and eat it */ assert(p2parse[0] == '1' && p2parse[1] == ' '); @@ -1358,7 +1359,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) * message, so we can not run into any troubles. I think this is * more wise then to use individual buffers. */ - if((pBuf = malloc(sizeof(char)* strlen(p2parse) + 1)) == NULL) + if((pBuf = malloc(sizeof(uchar) * ustrlen(p2parse) + 1)) == NULL) return 1; /* IMPORTANT NOTE: @@ -1393,29 +1394,29 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) /* APP-NAME */ if(bContParse) { parseRFCField(&p2parse, pBuf); - MsgSetAPPNAME(pMsg, pBuf); + MsgSetAPPNAME(pMsg, (char*)pBuf); } /* PROCID */ if(bContParse) { parseRFCField(&p2parse, pBuf); - MsgSetPROCID(pMsg, pBuf); + MsgSetPROCID(pMsg, (char*)pBuf); } /* MSGID */ if(bContParse) { parseRFCField(&p2parse, pBuf); - MsgSetMSGID(pMsg, pBuf); + MsgSetMSGID(pMsg, (char*)pBuf); } /* STRUCTURED-DATA */ if(bContParse) { parseRFCStructuredData(&p2parse, pBuf); - MsgSetStructuredData(pMsg, pBuf); + MsgSetStructuredData(pMsg, (char*)pBuf); } /* MSG */ - MsgSetMSG(pMsg, p2parse); + MsgSetMSG(pMsg, (char*)p2parse); free(pBuf); ENDfunc @@ -1438,7 +1439,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) */ int parseLegacySyslogMsg(msg_t *pMsg, int flags) { - char *p2parse; + uchar *p2parse; char *pBuf; char *pWork; cstr_t *pStrB; @@ -1448,7 +1449,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) assert(pMsg != NULL); assert(pMsg->pszUxTradMsg != NULL); - p2parse = (char*) pMsg->pszUxTradMsg; + p2parse = pMsg->pszUxTradMsg; /* Check to see if msg contains a timestamp. We start by assuming * that the message timestamp is the time of reciption (which we @@ -1503,7 +1504,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) /* the memory allocated is far too much in most cases. But on the plus side, * it is quite fast... - rgerhards, 2007-09-20 */ - if((pBuf = malloc(sizeof(char)* (strlen(p2parse) +1))) == NULL) + if((pBuf = malloc(sizeof(char)* (ustrlen(p2parse) +1))) == NULL) return 1; pWork = pBuf; /* this is the actual parsing loop */ @@ -1609,7 +1610,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) } /* The rest is the actual MSG */ - MsgSetMSG(pMsg, p2parse); + MsgSetMSG(pMsg, (char*)p2parse); ENDfunc return 0; /* all ok */ @@ -1919,10 +1920,10 @@ static void doDie(int sig) # define MSG1 "DoDie called.\n" # define MSG2 "DoDie called 5 times - unconditional exit\n" static int iRetries = 0; /* debug aid */ - if(Debug || NoFork) + if(Debug) write(1, MSG1, sizeof(MSG1) - 1); if(iRetries++ == 4) { - if(Debug || NoFork) + if(Debug) write(1, MSG2, sizeof(MSG2) - 1); abort(); } -- cgit v1.2.3 From e89ae2e7b18ba935d7eea4c44f3ec15ff6dcfb31 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 May 2009 17:23:28 +0200 Subject: minor cleanup: remove compiler warning --- runtime/obj.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/obj.h b/runtime/obj.h index dc04203b..98bd4854 100644 --- a/runtime/obj.h +++ b/runtime/obj.h @@ -68,7 +68,7 @@ #define objSerializePTR(strm, propName, propType) \ CHKiRet(obj.SerializeProp(strm, (uchar*) #propName, PROPTYPE_##propType, (void*) pThis->propName)); #define DEFobjStaticHelpers \ - static objInfo_t *pObjInfoOBJ = NULL; \ + static __attribute__((unused)) objInfo_t *pObjInfoOBJ = NULL; \ DEFobjCurrIf(obj) -- cgit v1.2.3 From 7adb9877f0c08f929d89f436103dfade03e8ea07 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 22 May 2009 17:40:24 +0200 Subject: fixing some nits with the build system --- runtime/Makefile.am | 2 +- tests/Makefile.am | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 8331882f..bc03c4a7 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -7,7 +7,7 @@ pkglib_LTLIBRARIES = librsyslog_la_SOURCES = \ rsyslog.c \ rsyslog.h \ - unicode_helper.h \ + unicode-helper.h \ atomic.h \ syslogd-types.h \ module-template.h \ diff --git a/tests/Makefile.am b/tests/Makefile.am index bc1849f0..c0e629d3 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -38,7 +38,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ manytcp.sh \ testsuites/manytcp.conf \ fieldtest.sh \ - testsuites/field.conf \ + testsuites/field1.conf \ testsuites/1.field1 \ inputname.sh \ testsuites/inputname_imtcp.conf \ -- cgit v1.2.3 From eb1615068c6a704287eda732d287280df4cc4c44 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 10:47:22 +0200 Subject: added new testing module imdiag which enables to talk to the rsyslog core at runtime. The current implementation is only a beginning, but can be expanded over time --- ChangeLog | 3 + Makefile.am | 2 +- dirty.h | 1 + plugins/imdiag/imdiag.c | 247 +++++++++++++++++++++++++++++++--------- runtime/netstrm.c | 1 - runtime/rsyslog.h | 1 + tcps_sess.c | 27 ++++- tcps_sess.h | 8 +- tcpsrv.c | 26 +++-- tcpsrv.h | 5 +- tests/Makefile.am | 1 + tests/diskqueue.sh | 2 +- tests/fieldtest.sh | 2 +- tests/inputname.sh | 2 +- tests/manytcp.sh | 2 +- tests/omod-if-array.sh | 2 +- tests/parsertest.sh | 2 +- tests/testsuites/diskqueue.conf | 3 + tests/testsuites/manytcp.conf | 3 + tests/waitqueueempty.sh | 4 + tools/syslogd.c | 27 ++++- 21 files changed, 289 insertions(+), 82 deletions(-) create mode 100755 tests/waitqueueempty.sh diff --git a/ChangeLog b/ChangeLog index b7a0a67a..a0357889 100644 --- a/ChangeLog +++ b/ChangeLog @@ -17,6 +17,9 @@ Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs - build system improvements - thanks to Michael Biebl +- added new testing module imdiag, which enables to talk to the + rsyslog core at runtime. The current implementation is only a + beginning, but can be expanded over time --------------------------------------------------------------------------- Version 4.3.0 [DEVEL] (rgerhards), 2009-04-17 - new feature: new output plugin omprog, which permits to start program diff --git a/Makefile.am b/Makefile.am index a5cf879c..8d57700f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -131,5 +131,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog --enable-imdiag ACLOCAL_AMFLAGS = -I m4 diff --git a/dirty.h b/dirty.h index db9bc31b..6d585753 100644 --- a/dirty.h +++ b/dirty.h @@ -32,6 +32,7 @@ rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); int parseRFCSyslogMsg(msg_t *pMsg, int flags); int parseLegacySyslogMsg(msg_t *pMsg, int flags); +rsRetVal diagGetMainMsgQSize(int *piSize); /* for imdiag */ /* TODO: the following 2 need to go in conf obj interface... */ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 90c5d9ee..40f94692 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -26,8 +26,8 @@ * * A copy of the GPL can be found in the file "COPYING" in this distribution. */ - #include "config.h" +#include #include #include #include @@ -43,114 +43,247 @@ #include #endif #include "rsyslog.h" -//#include "dirty.h" +#include "dirty.h" #include "cfsysline.h" #include "module-template.h" +#include "unicode-helper.h" #include "net.h" #include "netstrm.h" -#include "netstrms.h" #include "errmsg.h" +#include "tcpsrv.h" +#include "srUtils.h" +#include "net.h" /* for permittedPeers, may be removed when this is removed */ MODULE_TYPE_INPUT /* static data */ DEF_IMOD_STATIC_DATA +DEFobjCurrIf(tcpsrv) +DEFobjCurrIf(tcps_sess) DEFobjCurrIf(net) DEFobjCurrIf(netstrm) -DEFobjCurrIf(netstrms) DEFobjCurrIf(errmsg) /* Module static data */ -netstrms_t *pNS; /**< pointer to network stream subsystem */ -netstrm_t *arrLstn[10]; /**< our netstream listners */ -int iLstnMax = 0; /**< max nbr of listeners currently supported */ +static tcpsrv_t *pOurTcpsrv = NULL; /* our TCP server(listener) TODO: change for multiple instances */ +static permittedPeers_t *pPermPeersRoot = NULL; /* config settings */ +static int iTCPSessMax = 20; /* max number of sessions */ +static int iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */ +static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ +static uchar *pszInputName = NULL; /* value for inputname property, NULL is OK and handled by core engine */ + + +/* callbacks */ +/* this shall go into a specific ACL module! */ +static int +isPermittedHost(struct sockaddr __attribute__((unused)) *addr, char __attribute__((unused)) *fromHostFQDN, + void __attribute__((unused)) *pUsrSrv, void __attribute__((unused)) *pUsrSess) +{ + return 1; /* TODO: implement ACLs ... or via some other way? */ +} -/* add a listen socket to our listen socket array. This is a callback - * invoked from the netstrm class. -- rgerhards, 2008-04-23 +static rsRetVal +doOpenLstnSocks(tcpsrv_t *pSrv) +{ + ISOBJ_TYPE_assert(pSrv, tcpsrv); + return tcpsrv.create_tcp_socket(pSrv); +} + + +static rsRetVal +doRcvData(tcps_sess_t *pSess, char *buf, size_t lenBuf, ssize_t *piLenRcvd) +{ + DEFiRet; + assert(pSess != NULL); + assert(piLenRcvd != NULL); + + *piLenRcvd = lenBuf; + CHKiRet(netstrm.Rcv(pSess->pStrm, (uchar*) buf, piLenRcvd)); +finalize_it: + RETiRet; +} + +static rsRetVal +onRegularClose(tcps_sess_t *pSess) +{ + DEFiRet; + assert(pSess != NULL); + + /* process any incomplete frames left over */ + tcps_sess.PrepareClose(pSess); + /* Session closed */ + tcps_sess.Close(pSess); + RETiRet; +} + + +static rsRetVal +onErrClose(tcps_sess_t *pSess) +{ + DEFiRet; + assert(pSess != NULL); + + tcps_sess.Close(pSess); + RETiRet; +} + +/* ------------------------------ end callbacks ------------------------------ */ + + +/* This function waits until the main queue is drained (size = 0) */ static rsRetVal -addTcpLstn(netstrm_t *pLstn) +waitMainQEmpty(void) { + int iMsgQueueSize; DEFiRet; - ISOBJ_TYPE_assert(pLstn, netstrm); + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + while(iMsgQueueSize > 0) { + srSleep(0,2); /* wait a little bit */ + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + } + +finalize_it: + RETiRet; +} + - if((unsigned)iLstnMax >= sizeof(arrLstn)/sizeof(netstrm_t*)) - ABORT_FINALIZE(RS_RET_MAX_LSTN_REACHED); +/* Function to handle received messages. This is our core function! + * rgerhards, 2009-05-24 + */ +static rsRetVal +OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) +{ + ssize_t len; + int iMsgQueueSize; + uchar *pszMsg; + uchar buf[1024]; + DEFiRet; - arrLstn[iLstnMax] = pLstn; - ++iLstnMax; + assert(pSess != NULL); + assert(pRcv != NULL); + + /* NOTE: pRcv is NOT a C-String but rather an array of characters + * WITHOUT a termination \0 char. So we need to convert it to one + * before proceeding. + */ + CHKmalloc(pszMsg = malloc(sizeof(uchar) * (iLenMsg + 1))); + memcpy(pszMsg, pRcv, iLenMsg); + pszMsg[iLenMsg] = '\0'; + + if(!ustrcmp(pszMsg, UCHAR_CONSTANT("GetMainMsgQueueSize"))) { + CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); + len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "%d\n", iMsgQueueSize); + CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + } else if(!ustrcmp(pszMsg, UCHAR_CONSTANT("WaitMainQueueEmpty"))) { + CHKiRet(waitMainQEmpty()); + len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "mainqueue empty\n"); + CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + } else { + len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "unkown command '%s'\n", pszMsg); + CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + } finalize_it: RETiRet; } -/* initialize network stream subsystem */ +/* set permitted peer -- rgerhards, 2008-05-19 + */ static rsRetVal -initNetstrm(void) +setPermittedPeer(void __attribute__((unused)) *pVal, uchar *pszID) +{ + DEFiRet; + CHKiRet(net.AddPermittedPeer(&pPermPeersRoot, pszID)); + free(pszID); /* no longer needed, but we need to free as of interface def */ +finalize_it: + RETiRet; +} + + +static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVal) { DEFiRet; - /* prepare network stream subsystem */ - CHKiRet(netstrms.Construct(&pNS)); - CHKiRet(netstrms.SetDrvrMode(pNS, 0)); /* always plain text */ - //CHKiRet(netstrms.SetDrvrAuthMode(pThis->pNS, pThis->pszDrvrAuthMode)); - //CHKiRet(netstrms.SetDrvrPermPeers(pThis->pNS, pThis->pPermPeers)); - // TODO: set driver! - CHKiRet(netstrms.ConstructFinalize(&pNS)); + if(pOurTcpsrv == NULL) { + CHKiRet(tcpsrv.Construct(&pOurTcpsrv)); + CHKiRet(tcpsrv.SetSessMax(pOurTcpsrv, iTCPSessMax)); + CHKiRet(tcpsrv.SetCBIsPermittedHost(pOurTcpsrv, isPermittedHost)); + CHKiRet(tcpsrv.SetCBRcvData(pOurTcpsrv, doRcvData)); + CHKiRet(tcpsrv.SetCBOpenLstnSocks(pOurTcpsrv, doOpenLstnSocks)); + CHKiRet(tcpsrv.SetCBOnRegularClose(pOurTcpsrv, onRegularClose)); + CHKiRet(tcpsrv.SetCBOnErrClose(pOurTcpsrv, onErrClose)); + CHKiRet(tcpsrv.SetDrvrMode(pOurTcpsrv, iStrmDrvrMode)); + CHKiRet(tcpsrv.SetOnMsgReceive(pOurTcpsrv, OnMsgReceived)); + /* now set optional params, but only if they were actually configured */ + if(pszStrmDrvrAuthMode != NULL) { + CHKiRet(tcpsrv.SetDrvrAuthMode(pOurTcpsrv, pszStrmDrvrAuthMode)); + } + if(pPermPeersRoot != NULL) { + CHKiRet(tcpsrv.SetDrvrPermPeers(pOurTcpsrv, pPermPeersRoot)); + } + } - /* set up listeners */ - CHKiRet(netstrm.LstnInit(pNS, NULL, addTcpLstn, "127.0.0.1", (uchar*)"44514", 1)); + /* initialized, now add socket */ + CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? + UCHAR_CONSTANT("imdiag") : pszInputName)); + tcpsrv.configureTCPListen(pOurTcpsrv, pNewVal); finalize_it: if(iRet != RS_RET_OK) { - if(pNS != NULL) - netstrms.Destruct(&pNS); + errmsg.LogError(0, NO_ERRCODE, "error %d trying to add listener", iRet); + if(pOurTcpsrv != NULL) + tcpsrv.Destruct(&pOurTcpsrv); } RETiRet; } - -/* This function is called to gather input. In our case, it is a bit abused - * to drive the listener loop for the diagnostics code. +/* This function is called to gather input. */ BEGINrunInput CODESTARTrunInput + CHKiRet(tcpsrv.ConstructFinalize(pOurTcpsrv)); + iRet = tcpsrv.Run(pOurTcpsrv); +finalize_it: ENDrunInput /* initialize and return if will run or not */ BEGINwillRun CODESTARTwillRun - iRet = initNetstrm(); + /* first apply some config settings */ + if(pOurTcpsrv == NULL) + ABORT_FINALIZE(RS_RET_NO_RUN); +finalize_it: ENDwillRun BEGINafterRun - int i; CODESTARTafterRun /* do cleanup here */ - /* finally close our listen streams */ - for(i = 0 ; i < iLstnMax ; ++i) { - netstrm.Destruct(arrLstn[i]); - } - - /* destruct netstream subsystem */ - netstrms.Destruct(pNS); ENDafterRun BEGINmodExit CODESTARTmodExit + if(pOurTcpsrv != NULL) + iRet = tcpsrv.Destruct(&pOurTcpsrv); + + if(pPermPeersRoot != NULL) { + net.DestructPermittedPeers(&pPermPeersRoot); + } + /* release objects we used */ objRelease(net, LM_NET_FILENAME); - objRelease(netstrm, DONT_LOAD_LIB); - objRelease(netstrms, LM_NETSTRMS_FILENAME); + objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(tcps_sess, LM_TCPSRV_FILENAME); + objRelease(tcpsrv, LM_TCPSRV_FILENAME); objRelease(errmsg, CORE_COMPONENT); ENDmodExit @@ -158,6 +291,14 @@ ENDmodExit static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) { + iTCPSessMax = 200; + iStrmDrvrMode = 0; + free(pszInputName); + pszInputName = NULL; + if(pszStrmDrvrAuthMode != NULL) { + free(pszStrmDrvrAuthMode); + pszStrmDrvrAuthMode = NULL; + } return RS_RET_OK; } @@ -173,26 +314,28 @@ BEGINmodInit() CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + pOurTcpsrv = NULL; /* request objects we use */ CHKiRet(objUse(net, LM_NET_FILENAME)); - CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); - CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); + CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(tcps_sess, LM_TCPSRV_FILENAME)); + CHKiRet(objUse(tcpsrv, LM_TCPSRV_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* register config file handlers */ -#if 0 - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverrun", 0, eCmdHdlrGetWord, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverrun"), 0, eCmdHdlrGetWord, addTCPListener, NULL, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpmaxsessions", 0, eCmdHdlrInt, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagmaxsessions"), 0, eCmdHdlrInt, NULL, &iTCPSessMax, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdrivermode", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverstreamdrivermode"), 0, eCmdHdlrInt, NULL, &iStrmDrvrMode, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdriverauthmode", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverstreamdriverauthmode"), 0, eCmdHdlrGetWord, NULL, &pszStrmDrvrAuthMode, STD_LOADABLE_MODULE_ID)); - CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputtcpserverstreamdriverpermittedpeer", 0, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverstreamdriverpermittedpeer"), 0, eCmdHdlrGetWord, setPermittedPeer, NULL, STD_LOADABLE_MODULE_ID)); -#endif - CHKiRet(omsdRegCFSLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverinputname"), 0, + eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/runtime/netstrm.c b/runtime/netstrm.c index 459561bc..f6a8de7f 100644 --- a/runtime/netstrm.c +++ b/runtime/netstrm.c @@ -148,7 +148,6 @@ LstnInit(netstrms_t *pNS, void *pUsr, rsRetVal(*fAddLstn)(void*,netstrm_t*), ISOBJ_TYPE_assert(pNS, netstrms); assert(fAddLstn != NULL); assert(pLstnPort != NULL); -RUNLOG_STR("XXX: Init Lstn"); CHKiRet(pNS->Drvr.LstnInit(pNS, pUsr, fAddLstn, pLstnPort, pLstnIP, iSessMax)); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 8e8a9f2a..77d845fd 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -84,6 +84,7 @@ typedef rsRetVal (*errLogFunc_t)(uchar*); /* this is a trick to store a function typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct tcpsrv_s tcpsrv_t; +typedef struct tcps_sess_s tcps_sess_t; typedef struct vmstk_s vmstk_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ diff --git a/tcps_sess.c b/tcps_sess.c index c564caea..c4548804 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -58,6 +58,7 @@ static int iMaxLine; /* maximum size of a single message */ /* forward definitions */ static rsRetVal Close(tcps_sess_t *pThis); +static rsRetVal defaultDoSubmitMessage(tcps_sess_t *pThis, uchar*, int); /* Standard-Constructor */ @@ -65,6 +66,7 @@ BEGINobjConstruct(tcps_sess) /* be sure to specify the object type also in END m pThis->iMsg = 0; /* just make sure... */ pThis->bAtStrtOfFram = 1; /* indicate frame header expected */ pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; /* just make sure... */ + pThis->DoSubmitMessage = defaultDoSubmitMessage; /* now allocate the message reception buffer */ CHKmalloc(pThis->pMsg = (uchar*) malloc(sizeof(uchar) * iMaxLine + 1)); finalize_it: @@ -206,6 +208,15 @@ SetUsrP(tcps_sess_t *pThis, void *pUsr) } +static rsRetVal +SetOnMsgReceive(tcps_sess_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)) +{ + DEFiRet; + pThis->DoSubmitMessage = OnMsgReceive; + RETiRet; +} + + /* This is a helper for submitting the message to the rsyslog core. * It does some common processing, including resetting the various * state variables to a "processed" state. @@ -217,8 +228,11 @@ SetUsrP(tcps_sess_t *pThis, void *pUsr) * rgerhards, 2009-04-23 */ static rsRetVal -doSubmitMessage(tcps_sess_t *pThis) +defaultDoSubmitMessage(tcps_sess_t *pThis, uchar *pszMsg, int iLenMsg) { +// TODO: make calling this overridable so that the diag module can ask to be called +// and so it can do its work right in this entry point (but we need to check that +// we have the capability to send a reply at this point). msg_t *pMsg; struct syslogTime stTime; time_t ttGenTime; @@ -233,7 +247,7 @@ doSubmitMessage(tcps_sess_t *pThis) CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); /* first trim the buffer to what we have actually received */ CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); - memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); + memcpy(pMsg->pszRawMsg, pszMsg, iLenMsg); pMsg->iLenRawMsg = pThis->iMsg; MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); @@ -291,7 +305,7 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - doSubmitMessage(pThis); + pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); } finalize_it: @@ -372,7 +386,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); - doSubmitMessage(pThis); + pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -383,7 +397,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(( (c == '\n') || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - doSubmitMessage(pThis); + pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -400,7 +414,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - doSubmitMessage(pThis); + pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); pThis->inputState = eAtStrtFram; } } @@ -474,6 +488,7 @@ CODESTARTobjQueryInterface(tcps_sess) pIf->SetHostIP = SetHostIP; pIf->SetStrm = SetStrm; pIf->SetMsgIdx = SetMsgIdx; + pIf->SetOnMsgReceive = SetOnMsgReceive; finalize_it: ENDobjQueryInterface(tcps_sess) diff --git a/tcps_sess.h b/tcps_sess.h index 2ef28264..5e59aaab 100644 --- a/tcps_sess.h +++ b/tcps_sess.h @@ -29,7 +29,7 @@ struct tcpsrv_s; /* the tcps_sess object */ -typedef struct tcps_sess_s { +struct tcps_sess_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ tcpsrv_t *pSrv; /* pointer back to my server (e.g. for callbacks) */ tcpLstnPortList_t *pLstnInfo; /* pointer back to listener info */ @@ -46,8 +46,9 @@ typedef struct tcps_sess_s { uchar *pMsg; /* message (fragment) received */ uchar *fromHost; uchar *fromHostIP; - void *pUsr; /* a user-pointer */ -} tcps_sess_t; + void *pUsr; /* a user-pointer */ + rsRetVal (*DoSubmitMessage)(tcps_sess_t*, uchar*, int); /* submit message callback */ +}; /* interfaces */ @@ -67,6 +68,7 @@ BEGINinterface(tcps_sess) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetHostIP)(tcps_sess_t *pThis, uchar*); rsRetVal (*SetStrm)(tcps_sess_t *pThis, netstrm_t*); rsRetVal (*SetMsgIdx)(tcps_sess_t *pThis, int); + rsRetVal (*SetOnMsgReceive)(tcps_sess_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)); ENDinterface(tcps_sess) #define tcps_sessCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* interface changes diff --git a/tcpsrv.c b/tcpsrv.c index bbd95058..249eeecf 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -380,14 +380,15 @@ SessAccept(tcpsrv_t *pThis, tcpLstnPortList_t *pLstnInfo, tcps_sess_t **ppSess, errno = 0; errmsg.LogError(0, RS_RET_MAX_SESS_REACHED, "too many tcp sessions - dropping incoming request"); ABORT_FINALIZE(RS_RET_MAX_SESS_REACHED); - } else { - /* we found a free spot and can construct our session object */ - CHKiRet(tcps_sess.Construct(&pSess)); - CHKiRet(tcps_sess.SetTcpsrv(pSess, pThis)); - CHKiRet(tcps_sess.SetLstnInfo(pSess, pLstnInfo)); } - /* OK, we have a "good" index... */ + /* we found a free spot and can construct our session object */ + CHKiRet(tcps_sess.Construct(&pSess)); + CHKiRet(tcps_sess.SetTcpsrv(pSess, pThis)); + CHKiRet(tcps_sess.SetLstnInfo(pSess, pLstnInfo)); + if(pThis->OnMsgReceive != NULL) + CHKiRet(tcps_sess.SetOnMsgReceive(pSess, pThis->OnMsgReceive)); + /* get the host name */ CHKiRet(netstrm.GetRemoteHName(pNewStrm, &fromHostFQDN)); CHKiRet(netstrm.GetRemoteIP(pNewStrm, &fromHostIP)); @@ -568,6 +569,7 @@ finalize_it: /* this is a very special case - this time only we do not exit the BEGINobjConstruct(tcpsrv) /* be sure to specify the object type also in END macro! */ pThis->iSessMax = TCPSESS_MAX_DEFAULT; /* TODO: useful default ;) */ pThis->addtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; + pThis->OnMsgReceive = NULL; ENDobjConstruct(tcpsrv) @@ -713,6 +715,16 @@ SetUsrP(tcpsrv_t *pThis, void *pUsr) RETiRet; } +static rsRetVal +SetOnMsgReceive(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)) +{ + DEFiRet; + assert(OnMsgReceive != NULL); + pThis->OnMsgReceive = OnMsgReceive; + RETiRet; +} + + /* Set additional framing to use (if any) -- rgerhards, 2008-12-10 */ static rsRetVal @@ -731,7 +743,6 @@ SetInputName(tcpsrv_t *pThis, uchar *name) { uchar *pszName; DEFiRet; -dbgprintf("XXX: SetInputName: %s\n", name); ISOBJ_TYPE_assert(pThis, tcpsrv); if(name == NULL) pszName = NULL; @@ -843,6 +854,7 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->SetCBOnDestruct = SetCBOnDestruct; pIf->SetCBOnRegularClose = SetCBOnRegularClose; pIf->SetCBOnErrClose = SetCBOnErrClose; + pIf->SetOnMsgReceive = SetOnMsgReceive; finalize_it: ENDobjQueryInterface(tcpsrv) diff --git a/tcpsrv.h b/tcpsrv.h index 0f7dd6c6..2d174ce0 100644 --- a/tcpsrv.h +++ b/tcpsrv.h @@ -71,6 +71,7 @@ struct tcpsrv_s { rsRetVal (*pOnSessAccept)(tcpsrv_t *, tcps_sess_t*); rsRetVal (*OnSessConstructFinalize)(void*); rsRetVal (*pOnSessDestruct)(void*); + rsRetVal (*OnMsgReceive)(tcps_sess_t *, uchar *pszMsg, int iLenMsg); /* submit message callback */ }; @@ -104,8 +105,10 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetCBOnSessConstructFinalize)(tcpsrv_t*, rsRetVal (*) (void*)); /* added v5 */ rsRetVal (*SetSessMax)(tcpsrv_t *pThis, int iMaxSess); /* 2009-04-09 */ + /* added v6 */ + rsRetVal (*SetOnMsgReceive)(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)); /* 2009-05-24 */ ENDinterface(tcpsrv) -#define tcpsrvCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +#define tcpsrvCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* change for v4: * - SetAddtlFrameDelim() added -- rgerhards, 2008-12-10 * - SetInputName() added -- rgerhards, 2008-12-10 diff --git a/tests/Makefile.am b/tests/Makefile.am index c0e629d3..ed48fce8 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -46,6 +46,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12515 \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ + waitqueueempty.sh \ cfg.sh ourtail_SOURCES = ourtail.c diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index 6384eb64..a91f3414 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -17,7 +17,7 @@ if [ "$?" -ne "0" ]; then echo "error during tcpflood! see rsyslog.out.log.save for what was written" cp rsyslog.out.log rsyslog.out.log.save fi -sleep 4 # we need this so that rsyslogd can receive all outstanding messages +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work diff --git a/tests/fieldtest.sh b/tests/fieldtest.sh index 7a646f00..482fa143 100755 --- a/tests/fieldtest.sh +++ b/tests/fieldtest.sh @@ -1,5 +1,5 @@ echo test fieldtest via udp -./killrsyslog.sh # kill rsyslogd if it runs for some reason +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason ./nettester -tfield1 -iudp if [ "$?" -ne "0" ]; then diff --git a/tests/inputname.sh b/tests/inputname.sh index 7d9ea111..e1a58517 100755 --- a/tests/inputname.sh +++ b/tests/inputname.sh @@ -1,5 +1,5 @@ echo testing $InputTCPServerInputName directive -./killrsyslog.sh # kill rsyslogd if it runs for some reason +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason echo port 12514 ./nettester -tinputname_imtcp_12514 -cinputname_imtcp -itcp -p12514 diff --git a/tests/manytcp.sh b/tests/manytcp.sh index d9b2e9a0..06bd38b6 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -8,7 +8,7 @@ if [ "$?" -ne "0" ]; then echo "error during tcpflood! see rsyslog.out.log.save for what was written" cp rsyslog.out.log rsyslog.out.log.save fi -sleep 5 # we need this so that rsyslogd can receive all outstanding messages +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work diff --git a/tests/omod-if-array.sh b/tests/omod-if-array.sh index 7b4b5611..2c2a8ef3 100755 --- a/tests/omod-if-array.sh +++ b/tests/omod-if-array.sh @@ -1,5 +1,5 @@ echo test omod-if-array via udp -./killrsyslog.sh # kill rsyslogd if it runs for some reason +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason ./nettester -tomod-if-array -iudp -p4711 if [ "$?" -ne "0" ]; then diff --git a/tests/parsertest.sh b/tests/parsertest.sh index 152d8b60..afdb9469 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1,5 +1,5 @@ echo test parsertest via udp -./killrsyslog.sh # kill rsyslogd if it runs for some reason +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason ./nettester -tparse1 -iudp if [ "$?" -ne "0" ]; then diff --git a/tests/testsuites/diskqueue.conf b/tests/testsuites/diskqueue.conf index 8851a459..017ee96d 100644 --- a/tests/testsuites/diskqueue.conf +++ b/tests/testsuites/diskqueue.conf @@ -4,6 +4,9 @@ $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + $ErrorMessagesToStderr off # set spool locations and switch queue to disk-only mode diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf index 8175732e..3867da46 100644 --- a/tests/testsuites/manytcp.conf +++ b/tests/testsuites/manytcp.conf @@ -6,6 +6,9 @@ $MaxOpenFiles 2000 $InputTCPMaxSessions 1100 $InputTCPServerRun 13514 +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + $ErrorMessagesToStderr off $template outfmt,"%msg:F,58:2%\n" diff --git a/tests/waitqueueempty.sh b/tests/waitqueueempty.sh new file mode 100755 index 00000000..2c047588 --- /dev/null +++ b/tests/waitqueueempty.sh @@ -0,0 +1,4 @@ +# wait until main message queue is empty. This is currently done in +# a separate shell script so that we can change the implementation +# at some later point. -- rgerhards, 2009-05-25 +echo WaitMainQueueEmpty | nc 127.0.0.1 13500 diff --git a/tools/syslogd.c b/tools/syslogd.c index dc5b8fee..0b860448 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -350,10 +350,8 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bDebugPrintModuleList = 1; bEscapeCCOnRcv = 1; /* default is to escape control characters */ bReduceRepeatMsgs = 0; - if(pszMainMsgQFName != NULL) { - free(pszMainMsgQFName); - pszMainMsgQFName = NULL; - } + free(pszMainMsgQFName); + pszMainMsgQFName = NULL; iMainMsgQueueSize = 10000; iMainMsgQHighWtrMark = 8000; iMainMsgQLowWtrMark = 2000; @@ -412,6 +410,26 @@ static int usage(void) } +/* ------------------------------ some support functions for imdiag ------------------------------ * + * This is a bit dirty, but the only way to do it, at least with reasonable effort. + * rgerhards, 2009-05-25 + */ + +/* return back the approximate current number of messages in the main message queue + */ +rsRetVal +diagGetMainMsgQSize(int *piSize) +{ + DEFiRet; + assert(piSize != NULL); + *piSize = pMsgQueue->iQueueSize; + RETiRet; +} + + +/* ------------------------------ end support functions for imdiag ------------------------------ */ + + /* function to destruct a selector_t object * rgerhards, 2007-08-01 */ @@ -2658,7 +2676,6 @@ init(void) ABORT_FINALIZE(RS_RET_VALIDATION_RUN); /* switch the message object to threaded operation, if necessary */ -/* TODO:XXX: I think we must do this also if we have action queues! -- rgerhards, 2009-01-26 */ if(MainMsgQueType == QUEUETYPE_DIRECT || iMainMsgQueueNumWorkers > 1) { MsgEnableThreadSafety(); } -- cgit v1.2.3 From aaef9aa018dc030a7b5b2585bad19812ff214fab Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 11:31:53 +0200 Subject: preparing for 4.3.1 --- ChangeLog | 2 +- configure.ac | 2 +- doc/how2help.html | 116 +++++++++++++++++++++++++++--------------------------- doc/manual.html | 2 +- 4 files changed, 60 insertions(+), 62 deletions(-) diff --git a/ChangeLog b/ChangeLog index a0357889..9d57f5e1 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.3.1 [DEVEL] (rgerhards), 2009-04-?? +Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) - performance enhancement: imtcp calls parser no longer on input thread but rather inside on of the potentially many main msg queue worker diff --git a/configure.ac b/configure.ac index 8c8db8e1..66f1cab1 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.3.0],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.3.1],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/how2help.html b/doc/how2help.html index 0caa5a3a..4f0bd57a 100644 --- a/doc/how2help.html +++ b/doc/how2help.html @@ -1,59 +1,57 @@ - - -How you can Help - - -

      How you can Help

      -

      You like rsyslog and would like to lend us a helping hand? This page -tells you how easy it is to help a little bit. You can contribute to the project -even with a single mouse click! If you could pick a single item from the -wish list, that would be awfully helpful!

      -

      This is our wish list:

      -
        -
      • let others know how great rsyslog is -
        • -
      • let us know about rsyslog - we are eager for feedback
          -
        • tell us what you like and what you not like - so that we can include - that into development
          • -
        • tell us what you use rsyslog for - especially if you have high - traffic volume or an otherwise "uncommon" deployment. We are looking for - case studies and experience how rsyslog performs in unusual scenarios.
          • -
        • allow us to post your thoughts and experiences as a "user story" on - the web site (so far, none are there ;))
          • -
        -
        • -
      • if you know how to create packages (rpm, deb, ...)
          -
        • we would very much appreciate your help with package creation. We know - that it is important to have good binary packages for a product to - spread widely. Yet, we do not have the knowledge to do it all ourselves. - Drop Rainer a note if you - could help us out.
          • -
        -
        • -
      • if you have configured a device for sending syslog data, and that device - is not in our - syslog - configuration database, you might want to tell us how to configure it.
        • -
      • if you are a corporate user -
        • -
      -

      We appreciate your help very much. A big thank you for anything you -might do!

      - - - + + +How you can Help + + +

      How you can Help

      +

      You like rsyslog and would like to lend us a helping hand? This page +tells you how easy it is to help a little bit. You can contribute to the project +even with a single mouse click! If you could pick a single item from the +wish list, that would be awfully helpful!

      +

      This is our wish list:

      +
        +
      • let others know how great rsyslog is
          +
        • spread word about rsyslog in forums and newsgroups
          • +
        • place a link to www.rsyslog.com + from your home page
          • +
        +
        • +
      • let us know about rsyslog - we are eager for feedback
          +
        • tell us what you like and what you not like - so that we can include + that into development
          • +
        • tell us what you use rsyslog for - especially if you have high + traffic volume or an otherwise "uncommon" deployment. We are looking for + case studies and experience how rsyslog performs in unusual scenarios.
          • +
        • allow us to post your thoughts and experiences as a "user story" on + the web site (so far, none are there ;))
          • +
        +
        • +
      • if you know how to create packages (rpm, deb, ...)
          +
        • we would very much appreciate your help with package creation. We know + that it is important to have good binary packages for a product to + spread widely. Yet, we do not have the knowledge to do it all ourselves. + Drop Rainer a note if you + could help us out.
          • +
        +
        • +
      • if you have configured a device for sending syslog data, and that device + is not in our + syslog + configuration database, you might want to tell us how to configure it.
        • +
      • if you are a corporate user +
        • +
      +

      We appreciate your help very much. A big thank you for anything you +might do!

      + + + available directly from the source!

      Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

      -

      This documentation is for version 4.3.0 (devel branch) of rsyslog. +

      This documentation is for version 4.3.1 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

      If you like rsyslog, you might -- cgit v1.2.3 From 7a7ec37f99f3dd5120952e6ca6263dd72061abb1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 13:02:06 +0200 Subject: improved testbench / solved imdiag race condition imdiag/imtcp had a modload race condition (as imdiag is a testing aid, this has no implications for production deployments). Also, I replaced netcat by a custom program to talk to imdiag. This, for the first time ever, is now a Java program. I plan to add some GUI troubleshooting tools and thought it is a good idea to start doing things in Java that can simply be done in that language. --- ChangeLog | 3 +++ Makefile.am | 2 +- runtime/modules.c | 21 +++++++++++++++++++++ tcps_sess.c | 23 ++++++++++++----------- tests/DiagTalker.java | 43 +++++++++++++++++++++++++++++++++++++++++++ tests/Makefile.am | 6 +++++- tests/diskqueue.sh | 4 ++++ tests/waitqueueempty.sh | 3 ++- 8 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 tests/DiagTalker.java diff --git a/ChangeLog b/ChangeLog index 9d57f5e1..2fc760c3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,7 @@ --------------------------------------------------------------------------- +Version 4.3.? [DEVEL] (rgerhards), 2009-??-?? +- bugfix: imdiag/imtcp had a race condition +--------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) - performance enhancement: imtcp calls parser no longer on input thread diff --git a/Makefile.am b/Makefile.am index 8d57700f..e991f323 100644 --- a/Makefile.am +++ b/Makefile.am @@ -131,5 +131,5 @@ SUBDIRS += tests # temporarily be removed below. The intent behind forcing everthing to compile # in a make distcheck is so that we detect code that accidently was not updated # when some global update happened. -DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog --enable-imdiag +DISTCHECK_CONFIGURE_FLAGS=--enable-gssapi_krb5 --enable-imfile --enable-snmp --enable-pgsql --enable-libdbi --enable-mysql --enable-omtemplate --enable-imtemplate --enable-relp --enable-rsyslogd --enable-mail --enable-klog --enable-diagtools --enable-gnutls --enable-omstdout --enable-omprog --enable-imdiag --enable-shave ACLOCAL_AMFLAGS = -I m4 diff --git a/runtime/modules.c b/runtime/modules.c index 9fdb48e7..3e8662a3 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -40,6 +40,7 @@ #include #include #include +#include #ifdef OS_BSD # include "libgen.h" #endif @@ -61,6 +62,14 @@ DEFobjStaticHelpers DEFobjCurrIf(errmsg) +/* we must ensure that only one thread at one time tries to load or unload + * modules, otherwise we may see race conditions. This first came up with + * imdiag/imtcp, which both use the same stream drivers. Below is the mutex + * for that handling. + * rgerhards, 2009-05-25 + */ +static pthread_mutex_t mutLoadUnload; + static modInfo_t *pLoadedModules = NULL; /* list of currently-loaded modules */ static modInfo_t *pLoadedModulesLast = NULL; /* tail-pointer */ @@ -479,6 +488,8 @@ modUnlinkAndDestroy(modInfo_t **ppThis) pThis = *ppThis; assert(pThis != NULL); + pthread_mutex_lock(&mutLoadUnload); + /* first check if we are permitted to unload */ if(pThis->eType == eMOD_LIB) { if(pThis->uRefCnt > 0) { @@ -513,6 +524,7 @@ modUnlinkAndDestroy(modInfo_t **ppThis) moduleDestruct(pThis); finalize_it: + pthread_mutex_unlock(&mutLoadUnload); RETiRet; } @@ -587,6 +599,8 @@ Load(uchar *pModName) assert(pModName != NULL); dbgprintf("Requested to load module '%s'\n", pModName); + pthread_mutex_lock(&mutLoadUnload); + iModNameLen = strlen((char *) pModName); if(iModNameLen > 3 && !strcmp((char *) pModName + iModNameLen - 3, ".so")) { iModNameLen -= 3; @@ -696,6 +710,7 @@ Load(uchar *pModName) } finalize_it: + pthread_mutex_unlock(&mutLoadUnload); RETiRet; } @@ -791,6 +806,7 @@ BEGINObjClassExit(module, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MA CODESTARTObjClassExit(module) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); + pthread_mutex_destroy(&mutLoadUnload); # ifdef DEBUG modUsrPrintAll(); /* debug aid - TODO: integrate with debug.c, at least the settings! */ @@ -833,6 +849,7 @@ ENDobjQueryInterface(module) */ BEGINAbstractObjClassInit(module, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE class also in END MACRO! */ uchar *pModPath; + pthread_mutexattr_t mutAttr; /* use any module load path specified in the environment */ if((pModPath = (uchar*) getenv("RSYSLOG_MODDIR")) != NULL) { @@ -850,6 +867,10 @@ BEGINAbstractObjClassInit(module, 1, OBJ_IS_CORE_MODULE) /* class, version - CHA SetModDir(glblModPath); } + pthread_mutexattr_init(&mutAttr); + pthread_mutexattr_settype(&mutAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&mutLoadUnload, &mutAttr); + /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); ENDObjClassInit(module) diff --git a/tcps_sess.c b/tcps_sess.c index c4548804..62d51f66 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -58,7 +58,6 @@ static int iMaxLine; /* maximum size of a single message */ /* forward definitions */ static rsRetVal Close(tcps_sess_t *pThis); -static rsRetVal defaultDoSubmitMessage(tcps_sess_t *pThis, uchar*, int); /* Standard-Constructor */ @@ -66,7 +65,6 @@ BEGINobjConstruct(tcps_sess) /* be sure to specify the object type also in END m pThis->iMsg = 0; /* just make sure... */ pThis->bAtStrtOfFram = 1; /* indicate frame header expected */ pThis->eFraming = TCP_FRAMING_OCTET_STUFFING; /* just make sure... */ - pThis->DoSubmitMessage = defaultDoSubmitMessage; /* now allocate the message reception buffer */ CHKmalloc(pThis->pMsg = (uchar*) malloc(sizeof(uchar) * iMaxLine + 1)); finalize_it: @@ -228,11 +226,8 @@ SetOnMsgReceive(tcps_sess_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar * rgerhards, 2009-04-23 */ static rsRetVal -defaultDoSubmitMessage(tcps_sess_t *pThis, uchar *pszMsg, int iLenMsg) +defaultDoSubmitMessage(tcps_sess_t *pThis) { -// TODO: make calling this overridable so that the diag module can ask to be called -// and so it can do its work right in this entry point (but we need to check that -// we have the capability to send a reply at this point). msg_t *pMsg; struct syslogTime stTime; time_t ttGenTime; @@ -240,6 +235,11 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, uchar *pszMsg, int iLenMsg) ISOBJ_TYPE_assert(pThis, tcps_sess); + if(pThis->DoSubmitMessage != NULL) { + pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); + FINALIZE; + } + //TODO: if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { datetime.getCurrTime(&stTime, &ttGenTime); //} @@ -247,7 +247,7 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, uchar *pszMsg, int iLenMsg) CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); /* first trim the buffer to what we have actually received */ CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); - memcpy(pMsg->pszRawMsg, pszMsg, iLenMsg); + memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); pMsg->iLenRawMsg = pThis->iMsg; MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); @@ -266,6 +266,7 @@ finalize_it: } + /* This should be called before a normal (non forced) close * of a TCP session. This function checks if there is any unprocessed * message left in the TCP stream. Such a message is probably a @@ -305,7 +306,7 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); + defaultDoSubmitMessage(pThis); } finalize_it: @@ -386,7 +387,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); - pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); + defaultDoSubmitMessage(pThis); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -397,7 +398,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(( (c == '\n') || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); + defaultDoSubmitMessage(pThis); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -414,7 +415,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - pThis->DoSubmitMessage(pThis, pThis->pMsg, pThis->iMsg); + defaultDoSubmitMessage(pThis); pThis->inputState = eAtStrtFram; } } diff --git a/tests/DiagTalker.java b/tests/DiagTalker.java new file mode 100644 index 00000000..e33a5867 --- /dev/null +++ b/tests/DiagTalker.java @@ -0,0 +1,43 @@ +//package com.rsyslog.diag; +import java.io.*; +import java.net.*; + +public class DiagTalker { + public static void main(String[] args) throws IOException { + + Socket diagSocket = null; + PrintWriter out = null; + BufferedReader in = null; + final String host = "127.0.0.1"; + final int port = 13500; + + try { + diagSocket = new Socket(host, port); + out = new PrintWriter(diagSocket.getOutputStream(), true); + in = new BufferedReader(new InputStreamReader( + diagSocket.getInputStream())); + } catch (UnknownHostException e) { + System.err.println("can not resolve " + host + "!"); + System.exit(1); + } catch (IOException e) { + System.err.println("Couldn't get I/O for " + + "the connection to: " + host + "."); + System.exit(1); + } + + BufferedReader stdIn = new BufferedReader( + new InputStreamReader(System.in)); + String userInput; + + while ((userInput = stdIn.readLine()) != null) { + out.println(userInput); + System.out.println("imdiag returns: " + in.readLine()); + } + + out.close(); + in.close(); + stdIn.close(); + diagSocket.close(); + } +} + diff --git a/tests/Makefile.am b/tests/Makefile.am index ed48fce8..caa95c51 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -5,9 +5,12 @@ if ENABLE_OMSTDOUT TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh endif TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ -DISTCLEANFILES=rsyslog.pid +DISTCLEANFILES=rsyslog.pid '$(abs_top_builddir)'/DiagTalker.class test_files = testbench.h runtime-dummy.c +check_JAVA = DiagTalker.java +#dist_java_JAVA = DiagTalker.java + EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg1.cfgtest \ cfg1.testin \ @@ -47,6 +50,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ waitqueueempty.sh \ + DiagTalker.java \ cfg.sh ourtail_SOURCES = ourtail.c diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index a91f3414..eabfcf78 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -4,6 +4,10 @@ # memory to disk mode for DA queues. # added 2009-04-17 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +#set -o xtrace +#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUGLOG="tmp" echo testing queue disk-only mode rm -rf test-spool mkdir test-spool diff --git a/tests/waitqueueempty.sh b/tests/waitqueueempty.sh index 2c047588..4825853a 100755 --- a/tests/waitqueueempty.sh +++ b/tests/waitqueueempty.sh @@ -1,4 +1,5 @@ # wait until main message queue is empty. This is currently done in # a separate shell script so that we can change the implementation # at some later point. -- rgerhards, 2009-05-25 -echo WaitMainQueueEmpty | nc 127.0.0.1 13500 +#echo WaitMainQueueEmpty | nc 127.0.0.1 13500 +echo WaitMainQueueEmpty | java -classpath $abs_top_builddir DiagTalker -- cgit v1.2.3 From b9549380270fa68e27e8ee3f049c7d34156a85ff Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 14:16:31 +0200 Subject: solved some issues with testbench & a race condition --- runtime/modules.c | 10 +++++++++- tests/imtcp-multiport.sh | 6 +++--- tests/testsuites/imtcp-multiport.conf | 3 +++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/runtime/modules.c b/runtime/modules.c index 3e8662a3..32ae659f 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -806,7 +806,15 @@ BEGINObjClassExit(module, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MA CODESTARTObjClassExit(module) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); - pthread_mutex_destroy(&mutLoadUnload); + /* We have a problem in our reference counting, which leads to this function + * being called too early. This usually is no problem, but if we destroy + * the mutex object, we get into trouble. So rather than finding the root cause, + * we do not release the mutex right now and have a very, very slight leak. + * We know that otherwise no bad effects happen, so this acceptable for the + * time being. -- rgerhards, 2009-05-25 + * + * TODO: add again: pthread_mutex_destroy(&mutLoadUnload); + */ # ifdef DEBUG modUsrPrintAll(); /* debug aid - TODO: integrate with debug.c, at least the settings! */ diff --git a/tests/imtcp-multiport.sh b/tests/imtcp-multiport.sh index aa1f15e8..17480dae 100755 --- a/tests/imtcp-multiport.sh +++ b/tests/imtcp-multiport.sh @@ -17,7 +17,7 @@ if [ "$?" -ne "0" ]; then echo "error during tcpflood! see rsyslog.out.log.save for what was written" cp rsyslog.out.log rsyslog.out.log.save fi -sleep 2 # we need this so that rsyslogd can receive all outstanding messages +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work @@ -42,7 +42,7 @@ if [ "$?" -ne "0" ]; then echo "error during tcpflood! see rsyslog.out.log.save for what was written" cp rsyslog.out.log rsyslog.out.log.save fi -sleep 2 # we need this so that rsyslogd can receive all outstanding messages +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work @@ -67,7 +67,7 @@ if [ "$?" -ne "0" ]; then echo "error during tcpflood! see rsyslog.out.log.save for what was written" cp rsyslog.out.log rsyslog.out.log.save fi -sleep 2 # we need this so that rsyslogd can receive all outstanding messages +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work diff --git a/tests/testsuites/imtcp-multiport.conf b/tests/testsuites/imtcp-multiport.conf index 00b63cb2..ec059fe4 100644 --- a/tests/testsuites/imtcp-multiport.conf +++ b/tests/testsuites/imtcp-multiport.conf @@ -6,6 +6,9 @@ $InputTCPServerRun 13514 $InputTCPServerRun 13515 $InputTCPServerRun 13516 +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + $ErrorMessagesToStderr off $template outfmt,"%msg:F,58:2%\n" -- cgit v1.2.3 From 598b1d73ca7815dc19906f86c6700402dc869eb4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 14:48:38 +0200 Subject: updated project status --- doc/status.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/status.html b/doc/status.html index c8608f73..4e8f1a5f 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,12 +2,12 @@ rsyslog status page

      rsyslog status page

      -

      This page reflects the status as of 2009-04-21.

      +

      This page reflects the status as of 2009-05-25.

      Current Releases

      -

      development: 4.3.0 [2009-04-17] - -change log - -download +

      development: 4.3.1 [2009-05-25] - +change log - +download
      beta: 3.21.11 [2009-04-03] - change log - -- cgit v1.2.3 From ddf74cc165c828139632170224356175c9c24e96 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 15:39:18 +0200 Subject: added test suite for persistent queue shutdown --- tests/Makefile.am | 5 ++++- tests/memq-persist.sh | 43 +++++++++++++++++++++++++++++++++++++ tests/testsuites/memq-persist1.conf | 25 +++++++++++++++++++++ tests/testsuites/memq-persist2.conf | 20 +++++++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100755 tests/memq-persist.sh create mode 100644 tests/testsuites/memq-persist1.conf create mode 100644 tests/testsuites/memq-persist2.conf diff --git a/tests/Makefile.am b/tests/Makefile.am index caa95c51..92401511 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh +TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh memq-persist.sh if ENABLE_OMSTDOUT TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh endif @@ -50,6 +50,9 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ waitqueueempty.sh \ + memq-persist.sh \ + testsuites/memq-persist1.sh \ + testsuites/memq-persist2.sh \ DiagTalker.java \ cfg.sh diff --git a/tests/memq-persist.sh b/tests/memq-persist.sh new file mode 100755 index 00000000..108cba57 --- /dev/null +++ b/tests/memq-persist.sh @@ -0,0 +1,43 @@ +# Test for memory queue which is persisted at shutdown. The +# plan is to start an instance, emit some data, do a relatively +# fast shutdown and then re-start the engine to process the +# remaining data. +# added 2009-05-25 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +#set -o xtrace +#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUGLOG="log" +echo testing memory queue persisting to disk +$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason +rm -rf test-spool +mkdir test-spool +rm -f work rsyslog.out.log rsyslog.out.log.save # work files +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist1.conf & +sleep 1 +echo "rsyslogd started with pid " `cat rsyslog.pid` +# 20000 messages should be enough +./tcpflood 127.0.0.1 13514 1 10000 +if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save +fi +sleep 3 # we need to wait to ensure everything is received (less 1 second would be better) +kill `cat rsyslog.pid` +sleep 5 # wait for engine to terminate +echo There must exist some files now: +ls -l test-spool +# restart engine and have rest processed +../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist2.conf & +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages +kill `cat rsyslog.pid` +rm -f work +sort < rsyslog.out.log > work +./chkseq work 0 9999 +if [ "$?" -ne "0" ]; then + # rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 +fi +rm -f work rsyslog.out.log +rm -rf test-spool diff --git a/tests/testsuites/memq-persist1.conf b/tests/testsuites/memq-persist1.conf new file mode 100644 index 00000000..5240090f --- /dev/null +++ b/tests/testsuites/memq-persist1.conf @@ -0,0 +1,25 @@ +# Test for persisting messages to disk on shutdown +# rgerhards, 2009-04-17 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 1 +$MainMsgQueueSaveOnShutdown on +$InputTCPServerRun 13514 + +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + +$ModLoad ../plugins/omtesting/.libs/omtesting + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType LinkedList + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt + +# delay execution so that a queue can build up: +*.* :omtesting:sleep 0 1000 diff --git a/tests/testsuites/memq-persist2.conf b/tests/testsuites/memq-persist2.conf new file mode 100644 index 00000000..23e29e2f --- /dev/null +++ b/tests/testsuites/memq-persist2.conf @@ -0,0 +1,20 @@ +# Test for persisting messages to disk on shutdown +# rgerhards, 2009-04-17 +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$MainMsgQueueSaveOnShutdown on +$InputTCPServerRun 13514 + +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$MainMsgQueueType LinkedList + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt -- cgit v1.2.3 From 210f43137d6a077abbd8b77c1f72193dcd81cc99 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 25 May 2009 16:25:53 +0200 Subject: fixing some minor nits --- tests/ourtail.c | 2 +- tools/omfile.c | 6 +++--- tools/syslogd.c | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/ourtail.c b/tests/ourtail.c index f2751c72..6781b5fe 100644 --- a/tests/ourtail.c +++ b/tests/ourtail.c @@ -28,7 +28,7 @@ */ #include -int main(int argc, char *argv[]) +int main(int __attribute__((unused)) argc, char __attribute__((unused)) *argv[]) { int c; diff --git a/tools/omfile.c b/tools/omfile.c index c7283e4d..3e845a73 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -132,7 +132,7 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo if(pData->bDynamicName) { - printf("[dynamic]\n\ttemplate='%s'" + dbgprintf("[dynamic]\n\ttemplate='%s'" "\tfile cache size=%d\n" "\tcreate directories: %s\n" "\tfile owner %d, group %d\n" @@ -146,9 +146,9 @@ CODESTARTdbgPrintInstInfo pData->bFailOnChown ? "yes" : "no" ); } else { /* regular file */ - printf("%s", pData->f_fname); + dbgprintf("%s", pData->f_fname); if (pData->fd == -1) - printf(" (unused)"); + dbgprintf(" (unused)"); } ENDdbgPrintInstInfo diff --git a/tools/syslogd.c b/tools/syslogd.c index e207b5b2..31cc4a4d 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1950,6 +1950,7 @@ static void doDie(int sig) # define MSG1 "DoDie called.\n" # define MSG2 "DoDie called 5 times - unconditional exit\n" static int iRetries = 0; /* debug aid */ + dbgprintf(MSG1); if(Debug) write(1, MSG1, sizeof(MSG1) - 1); if(iRetries++ == 4) { -- cgit v1.2.3 From aa9426f683fa6af9280bc63050ee0187ba4c57e1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 26 May 2009 12:43:43 +0200 Subject: solved design issue with queue termination ... and also improved the test suite. There is a design issue in the v3 queue engine that manifested to some serious problems with the new processing mode. However, in v3 shutdown may take eternally if a queue runs in DA mode, is configured to preserve data AND the action fails and retries immediately. There is no cure available for v3, it would require doing much of the work we have done on the new engine. The window of exposure, as one might guess from the description, is very small. That is probably the reason why we have not seen it in practice. --- runtime/queue.c | 253 +++++++++++++++++++++++++++++------------------ runtime/queue.h | 1 + runtime/rsyslog.h | 3 +- runtime/stream.c | 44 ++++++++- runtime/stream.h | 1 + runtime/wti.c | 13 ++- runtime/wtp.c | 13 ++- tests/DiagTalker.java | 32 +++++- tests/arrayqueue.sh | 2 +- tests/chkseq.c | 63 +++++++++--- tests/da-mainmsg-q.sh | 2 +- tests/diskqueue.sh | 2 +- tests/imtcp-multiport.sh | 6 +- tests/linkedlistqueue.sh | 2 +- tests/manytcp.sh | 2 +- tests/memq-persist.sh | 12 ++- tests/rscript.c | 4 +- 17 files changed, 319 insertions(+), 136 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index 0019297b..539cf4ec 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -202,9 +202,10 @@ getLogicalQueueSize(qqueue_t *pThis) static inline void queueDrain(qqueue_t *pThis) { void *pUsr; - ASSERT(pThis != NULL); + BEGINfunc + dbgoprint((obj_t*) pThis, "queue will lose %d messages, destroying...\n", pThis->iQueueSize); /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ while(pThis->iQueueSize-- > 0) { pThis->qDeq(pThis, &pUsr); @@ -213,6 +214,7 @@ static inline void queueDrain(qqueue_t *pThis) } pThis->qDel(pThis); } + ENDfunc } @@ -617,6 +619,7 @@ static rsRetVal qDeqFixedArray(qqueue_t *pThis, void **out) RETiRet; } + static rsRetVal qDelFixedArray(qqueue_t *pThis) { DEFiRet; @@ -631,6 +634,26 @@ static rsRetVal qDelFixedArray(qqueue_t *pThis) } +/* reset the logical dequeue pointer to the physical dequeue position. + * This is only needed after we cancelled workers (during queue shutdown). + */ +static rsRetVal +qUnDeqAllFixedArray(qqueue_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + + dbgoprint((obj_t*) pThis, "resetting FixedArray deq index to %ld (was %ld), logical dequeue count %d\n", + pThis->tVars.farray.head, pThis->tVars.farray.deqhead, pThis->nLogDeq); + + pThis->tVars.farray.deqhead = pThis->tVars.farray.head; + pThis->nLogDeq = 0; + + RETiRet; +} + + /* -------------------- linked list -------------------- */ @@ -695,7 +718,6 @@ static rsRetVal qDeqLinkedList(qqueue_t *pThis, obj_t **ppUsr) qLinkedList_t *pEntry; DEFiRet; -RUNLOG_VAR("%p", pThis->tVars.linklist.pDeqRoot); pEntry = pThis->tVars.linklist.pDeqRoot; *ppUsr = pEntry->pUsr; pThis->tVars.linklist.pDeqRoot = pEntry->pNext; @@ -723,6 +745,26 @@ static rsRetVal qDelLinkedList(qqueue_t *pThis) } +/* reset the logical dequeue pointer to the physical dequeue position. + * This is only needed after we cancelled workers (during queue shutdown). + */ +static rsRetVal +qUnDeqAllLinkedList(qqueue_t *pThis) +{ + DEFiRet; + + ASSERT(pThis != NULL); + + dbgoprint((obj_t*) pThis, "resetting LinkedList deq ptr to %p (was %p), logical dequeue count %d\n", + pThis->tVars.linklist.pDelRoot, pThis->tVars.linklist.pDeqRoot, pThis->nLogDeq); + + pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pDelRoot; + pThis->nLogDeq = 0; + + RETiRet; +} + + /* -------------------- disk -------------------- */ @@ -822,25 +864,16 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) CHKiRet(obj.Deserialize(&pThis->tVars.disk.pReadDel, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); - CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pWrite)); - CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pReadDel)); - /* we now need to take care of the Deq handle. It is not persisted, so we can create * a virgin copy based on pReadDel. // TODO duplicat code, same as blow - single function! */ - CHKiRet(strmConstruct(&pThis->tVars.disk.pReadDeq)); - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); - CHKiRet(strmSetDir(pThis->tVars.disk.pReadDeq, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pReadDeq, 10000000)); - CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pReadDeq, STREAMMODE_READ)); - CHKiRet(strmSetsType(pThis->tVars.disk.pReadDeq, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strmDup(pThis->tVars.disk.pReadDel, &pThis->tVars.disk.pReadDeq)); + CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pReadDeq, 0)); /* deq must NOT delete the files! */ CHKiRet(strmConstructFinalize(pThis->tVars.disk.pReadDeq)); - /* TODO: dirty, need stream methods --> */ - pThis->tVars.disk.pReadDeq->iCurrFNum = pThis->tVars.disk.pReadDel->iCurrFNum; - pThis->tVars.disk.pReadDeq->iCurrOffs = pThis->tVars.disk.pReadDel->iCurrOffs; - /* <-- dirty, need stream methods :TODO */ + CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pWrite)); + CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pReadDel)); CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pReadDeq)); /* OK, we could successfully read the file, so we now can request that it be @@ -1012,6 +1045,16 @@ finalize_it: } +/* This is a dummy function for disks - we do not need to reset anything + * because everything is already persisted... + */ +static rsRetVal +qUnDeqAllDisk(__attribute__((unused)) qqueue_t *pThis) +{ + return RS_RET_OK; +} + + /* -------------------- direct (no queueing) -------------------- */ static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis) { @@ -1055,6 +1098,12 @@ static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis) return RS_RET_OK; } +static rsRetVal +qUnDeqAllDirect(__attribute__((unused)) qqueue_t *pThis) +{ + return RS_RET_OK; +} + /* --------------- end type-specific handlers -------------------- */ @@ -1109,22 +1158,29 @@ qqueueDeq(qqueue_t *pThis, void *pUsr) /* This function shuts down all worker threads and waits until they - * have terminated. If they timeout, they are cancelled. Parameters have been set - * before this function is called so that DA queues will be fully persisted to - * disk (if configured to do so). + * have terminated. If they timeout, they are cancelled. * rgerhards, 2008-01-24 * Please note that this function shuts down BOTH the parent AND the child queue * in DA case. This is necessary because their timeouts are tightly coupled. Most * importantly, the timeouts would be applied twice (or logic be extremely * complex) if each would have its own shutdown. The function does not self check * this condition - the caller must make sure it is not called with a parent. + * rgerhards, 2009-05-26: we do NO logner persist the queue here if bSaveOnShutdown + * is set. This must be handled by the caller. Not doing that cleans up the queue + * shutdown considerably. Also, older engines had a potential hang condition when + * the DA queue was already started and the DA worker configured for infinite + * retries and the action was during retry processing. This was a design issue, + * which is solved as of now. Note that the shutdown now may take a little bit + * longer, because we no longer can persist the queue in parallel to waiting + * on worker timeouts. */ -static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) +static rsRetVal +ShutdownWorkers(qqueue_t *pThis) { - DEFiRet; DEFVARS_mutexProtection; struct timespec tTimeout; rsRetVal iRetLocal; + DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ @@ -1193,53 +1249,18 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) } /* when we reach this point, both queues are either empty or the regular queue shutdown timeout - * has expired. Now we need to check if we are configured to not loose messages. If so, we need - * to persist the queue to disk (this is only possible if the queue is DA-enabled). We must also - * set the primary queue to SHUTDOWN_IMMEDIATE, as it shall now terminate as soon as its consumer - * is done. This is especially important as we otherwise may interfere with queue order while the - * DA consumer is running. -- rgerhards, 2008-01-27 - * Note: there was a note that we should not wait eternally on the DA worker if we run in - * enqueue-only note. I have reviewed the code and think there is no need for this check. Howerver, - * I'd like to keep this note in here should we happen to run into some related trouble. - * rgerhards, 2008-01-28 + * has expired. We must set the primary queue to SHUTDOWN_IMMEDIATE, as it shall now terminate + * as soon as its consumer is done. In particular, it does no longer need try to empty the queue. */ wtpSetState(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE); /* set primary queue to shutdown only */ /* at this stage, we need to have the DA worker properly initialized and running (if there is one) */ - if(pThis->bRunsDA) + if(pThis->bRunsDA) { qqueueWaitDAModeInitialized(pThis); - - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - /* optimize parameters for shutdown of DA-enabled queues */ - if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { - /* switch to enqueue-only mode so that no more actions happen */ - if(pThis->bRunsDA == 0) { - qqueueInitDA(pThis, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to DA mode */ - } else { - /* TODO: RACE: we may reach this point when the DA worker has been initialized (state 1) - * but is not yet running (state 2). In this case, pThis->pqDA is NULL! rgerhards, 2008-02-27 - */ - qqueueSetEnqOnly(pThis->pqDA, QUEUE_MODE_ENQONLY, MUTEX_ALREADY_LOCKED); /* switch to enqueue-only mode */ - } - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - /* make sure we do not timeout before we are done */ - dbgoprint((obj_t*) pThis, "bSaveOnShutdown configured, eternal timeout set\n"); - timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL); - /* and run the primary queue's DA worker to drain the queue */ - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); - if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying to shut down primary queue in disk save mode, " - "continuing, but results are unpredictable\n", iRetLocal); - } - } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); + wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE); /* also stop DA queue */ } - /* now the primary queue is either empty, persisted to disk - or set to loose messages. So we - * can now request immediate shutdown of any remaining workers. Note that if bSaveOnShutdown was set, - * the queue is now empty. If regular workers are still running, and try to pull the next message, - * they will automatically terminate as there no longer is any message left to process. - */ + /* now give the queue workers a last chance to gracefully shut down (based on action timeout setting) */ BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ if(getPhysicalQueueSize(pThis) > 0) { timeoutComp(&tTimeout, pThis->toActShutdown); @@ -1278,10 +1299,7 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) /* Now queue workers should have terminated. If not, we need to cancel them as we have applied * all timeout setting. If any worker in any queue still executes, its consumer is possibly - * long-running and cancelling is the only way to get rid of it. Note that the - * cancellation handler will probably re-queue a user pointer, so the queue's enqueue - * function is still needed (what is no problem as we do not yet destroy the queue - but I - * thought it's a good idea to mention that fact). -- rgerhards, 2008-01-25 + * long-running and cancelling is the only way to get rid of it. */ dbgoprint((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the primary queue\n"); iRetLocal = wtpCancelAll(pThis->pWtpReg); /* returns immediately if all threads already have terminated */ @@ -1290,12 +1308,6 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) "threads, continuing, but results are unpredictable\n", iRetLocal); } - - /* TODO: think: do we really need to do this here? Can't it happen on DA queue destruction? If we - * disable it, we get an assertion... I think this is OK, as we need to have a certain order and - * canceling the DA workers here ensures that order. But in any instant, we may have a look at this - * code after we have reaced the milestone. -- rgerhards, 2008-01-27 - */ /* ... and now the DA queue, if it exists (should always be after the primary one) */ if(pThis->pqDA != NULL) { dbgoprint((obj_t*) pThis, "checking to see if we need to cancel any worker threads of the DA queue\n"); @@ -1310,7 +1322,8 @@ static rsRetVal qqueueShutdownWorkers(qqueue_t *pThis) * Well, more precisely, they *are in termination*. Some cancel cleanup handlers * may still be running. */ - dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size %d.\n", getPhysicalQueueSize(pThis)); + dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size log %d, phys %d.\n", + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); RETiRet; } @@ -1367,6 +1380,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qAdd = qAddFixedArray; pThis->qDeq = qDeqFixedArray; pThis->qDel = qDelFixedArray; + pThis->qUnDeqAll = qUnDeqAllFixedArray; break; case QUEUETYPE_LINKEDLIST: pThis->qConstruct = qConstructLinkedList; @@ -1374,6 +1388,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qAdd = qAddLinkedList; pThis->qDeq = (rsRetVal (*)(qqueue_t*,void**)) qDeqLinkedList; pThis->qDel = (rsRetVal (*)(qqueue_t*)) qDelLinkedList; + pThis->qUnDeqAll = qUnDeqAllLinkedList; break; case QUEUETYPE_DISK: pThis->qConstruct = qConstructDisk; @@ -1381,6 +1396,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qAdd = qAddDisk; pThis->qDeq = qDeqDisk; pThis->qDel = qDelDisk; + pThis->qUnDeqAll = qUnDeqAllDisk; /* special handling */ pThis->iNumWorkerThreads = 1; /* we need exactly one worker */ break; @@ -1389,6 +1405,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qDestruct = qDestructDirect; pThis->qAdd = qAddDirect; pThis->qDel = qDelDirect; + pThis->qUnDeqAll = qUnDeqAllDirect; break; } @@ -1817,20 +1834,17 @@ finalize_it: * If we are a child, we have done our duty when the queue is empty. In that case, * we can terminate. * Version for the DA worker thread. NOTE: the pThis->bRunsDA is different from - * the DA queue + * the DA queue. + * If our queue is in destruction, we drain to the DA queue and so we shall not terminate + * until we have done so. */ -static int +static rsRetVal qqueueChkStopWrkrDA(qqueue_t *pThis) { - /* if our queue is in destruction, we drain to the DA queue and so we shall not terminate - * until we have done so. - */ - int bStopWrkr; - - BEGINfunc + DEFiRet; if(pThis->bEnqOnly) { - bStopWrkr = 1; + iRet = RS_RET_TERMINATE_NOW; } else { if(pThis->bRunsDA) { ASSERT(pThis->pqDA != NULL); @@ -1838,19 +1852,16 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) && pThis->pqDA->sizeOnDiskMax > 0 && pThis->pqDA->tVars.disk.sizeOnDisk > pThis->pqDA->sizeOnDiskMax) { /* this queue can never grow, so we can give up... */ - bStopWrkr = 1; + iRet = RS_RET_TERMINATE_NOW; } else if(getPhysicalQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { - bStopWrkr = 1; - } else { - bStopWrkr = 0; + iRet = RS_RET_TERMINATE_NOW; } } else { - bStopWrkr = 1; + iRet = RS_RET_TERMINATE_NOW; } } - ENDfunc - return bStopWrkr; + RETiRet; } @@ -1861,10 +1872,20 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) * Version for the regular worker thread. NOTE: the pThis->bRunsDA is different from * the DA queue */ -static int +static rsRetVal ChkStopWrkrReg(qqueue_t *pThis) { + DEFiRet; + /* original condition return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && getPhysicalQueueSize(pThis) == 0); + * TODO: remove when verified! -- rgerhards, 2009-05-26 + */ + if(pThis->bEnqOnly || pThis->bRunsDA) + iRet = RS_RET_TERMINATE_NOW; + else if(pThis->pqParent != NULL) + iRet = RS_RET_TERMINATE_WHEN_IDLE; + + RETiRet; } @@ -1881,7 +1902,6 @@ GetDeqBatchSize(qqueue_t *pThis, int *pVal) } -/* common function for the idle functions that deletes the last batch if TODO MULTI */ /* must only be called when the queue mutex is locked, else results * are not stable! DA queue version */ @@ -1890,7 +1910,7 @@ qqueueIsIdleDA(qqueue_t *pThis) { /* remember: iQueueSize is the DA queue size, not the main queue! */ /* TODO: I think we need just a single function for DA and non-DA mode - but I leave it for now as is */ - return(getLogicalQueueSize(pThis) == 0 || (pThis->bRunsDA && getLogicalQueueSize(pThis) <= pThis->iLowWtrMrk)); + return(getPhysicalQueueSize(pThis) == 0 || (pThis->bRunsDA && getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk)); } /* must only be called when the queue mutex is locked, else results * are not stable! Regular queue version @@ -1905,7 +1925,7 @@ IsIdleReg(qqueue_t *pThis) return ret; #else /* regular code! */ - return(getLogicalQueueSize(pThis) == 0 || (pThis->bRunsDA && getLogicalQueueSize(pThis) <= pThis->iLowWtrMrk)); + return(getPhysicalQueueSize(pThis) == 0 || (pThis->bRunsDA && getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk)); #endif } @@ -2176,19 +2196,62 @@ finalize_it: } +/* persist a queue with all data elements to disk - this is used to handle + * bSaveOnShutdown. We utilize the DA worker to do this. This must only + * be called after all workers have been shut down and if bSaveOnShutdown + * is actually set. Note that this function may potentially run long, + * depending on the queue configuration (e.g. store on remote machine). + * rgerhards, 2009-05-26 + */ +static inline rsRetVal +DoSaveOnShutdown(qqueue_t *pThis) +{ + struct timespec tTimeout; + rsRetVal iRetLocal; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + + qqueueInitDA(pThis, QUEUE_MODE_ENQONLY, LOCK_MUTEX); /* switch to DA mode */ + /* make sure we do not timeout before we are done */ + dbgoprint((obj_t*) pThis, "bSaveOnShutdown configured, eternal timeout set\n"); + timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL); + /* and run the primary queue's DA worker to drain the queue */ + iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); + dbgoprint((obj_t*) pThis, "end queue persistence run, iRet %d, queue size log %d, phys %d\n", + iRetLocal, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); + if(iRetLocal != RS_RET_OK) { + dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying to shut down primary queue in disk save mode, " + "continuing, but results are unpredictable\n", iRetLocal); + } + + RETiRet; +} + + /* destructor for the queue object */ BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(qqueue) pThis->bQueueInDestruction = 1; /* indicate we are in destruction (modifies some behaviour) */ - /* shut down all workers (handles *all* of the persistence logic) - * See function head comment of queueShutdownWorkers () on why we don't call it - * We also do not need to shutdown workers when we are in enqueue-only mode or we are a + /* shut down all workers + * We do not need to shutdown workers when we are in enqueue-only mode or we are a * direct queue - because in both cases we have none... ;) * with a child! -- rgerhards, 2008-01-28 */ if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL) - qqueueShutdownWorkers(pThis); + ShutdownWorkers(pThis); + + /* now all workers are terminated. Messages may exist. Also, some logically dequeued + * messages may never have been processed because their worker was terminated. So + * we need to reset the logical dequeue pointer, persist the queue if configured to do + * so and then destruct everything. -- rgerhards, 2009-05-26 + */ + CHKiRet(pThis->qUnDeqAll(pThis)); + + if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { + CHKiRet(DoSaveOnShutdown(pThis)); + } /* finally destruct our (regular) worker thread pool * Note: currently pWtpReg is never NULL, but if we optimize our logic, this may happen, diff --git a/runtime/queue.h b/runtime/queue.h index 954a7fd4..c1fe597d 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -123,6 +123,7 @@ typedef struct queue_s { rsRetVal (*qAdd)(struct queue_s *pThis, void *pUsr); rsRetVal (*qDeq)(struct queue_s *pThis, void **ppUsr); rsRetVal (*qDel)(struct queue_s *pThis); + rsRetVal (*qUnDeqAll)(struct queue_s *pThis); /* end type-specific handler */ /* synchronization variables */ pthread_mutex_t mutThrdMgmt; /* mutex for the queue's thread management */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 0c671f03..a43c0327 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -298,7 +298,8 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_OK_DELETE_LISTENTRY = 1, /**< operation successful, but callee requested the deletion of an entry (special state) */ RS_RET_TERMINATE_NOW = 2, /**< operation successful, function is requested to terminate (mostly used with threads) */ RS_RET_NO_RUN = 3, /**< operation successful, but function does not like to be executed */ - RS_RET_IDLE = 4 /**< operation successful, but callee is idle (e.g. because queue is empty) */ + RS_RET_IDLE = 4, /**< operation successful, but callee is idle (e.g. because queue is empty) */ + RS_RET_TERMINATE_WHEN_IDLE = 5 /**< operation successful, function is requested to terminate when idle */ }; /* some helpful macros to work with srRetVals. diff --git a/runtime/stream.c b/runtime/stream.c index 50d419be..59e8be3a 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -45,6 +45,7 @@ #include "srUtils.h" #include "obj.h" #include "stream.h" +#include "unicode-helper.h" /* static data */ DEFobjStaticHelpers @@ -80,7 +81,7 @@ static rsRetVal strmOpenFile(strm_t *pThis) pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, pThis->iFileNumDigits)); } else { if(pThis->pszDir == NULL) { - if((pThis->pszCurrFName = (uchar*) strdup((char*) pThis->pszFName)) == NULL) + if((pThis->pszCurrFName = ustrdup(pThis->pszFName)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } else { CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, @@ -814,6 +815,47 @@ finalize_it: } +/* duplicate a stream object excluding dynamic properties. This function is + * primarily meant to provide a duplicate that later on can be used to access + * the data. This is needed, for example, for a restart of the disk queue. + * Note that ConstructFinalize() is NOT called. So our caller may change some + * properties before finalizing things. + * rgerhards, 2009-05-26 + */ +rsRetVal +strmDup(strm_t *pThis, strm_t **ppNew) +{ + strm_t *pNew = NULL; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strm); + assert(ppNew != NULL); + + CHKiRet(strmConstruct(&pNew)); + pNew->sType = pThis->sType; + pNew->iCurrFNum = pThis->iCurrFNum; + CHKmalloc(pNew->pszFName = ustrdup(pThis->pszFName)); + pNew->lenFName = pThis->lenFName; + CHKmalloc(pNew->pszDir = ustrdup(pThis->pszDir)); + pNew->lenDir = pThis->lenDir; + pNew->tOperationsMode = pThis->tOperationsMode; + pNew->tOpenMode = pThis->tOpenMode; + pNew->iAddtlOpenFlags = pThis->iAddtlOpenFlags; + pNew->iMaxFileSize = pThis->iMaxFileSize; + pNew->iMaxFiles = pThis->iMaxFiles; + pNew->iFileNumDigits = pThis->iFileNumDigits; + pNew->bDeleteOnClose = pThis->bDeleteOnClose; + pNew->iCurrOffs = pThis->iCurrOffs; + + *ppNew = pNew; + pNew = NULL; + +finalize_it: + if(pNew != NULL) + strmDestruct(&pNew); + + RETiRet; +} /* set a user write-counter. This counter is initialized to zero and * receives the number of bytes written. It is accurate only after a diff --git a/runtime/stream.h b/runtime/stream.h index 371358ab..e5d05b55 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -119,6 +119,7 @@ rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm); rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal); rsRetVal strmGetCurrOffset(strm_t *pThis, int64 *pOffs); rsRetVal strmSetWCntr(strm_t *pThis, number_t *pWCnt); +rsRetVal strmDup(strm_t *pThis, strm_t **ppNew); PROTOTYPEObjClassInit(strm); PROTOTYPEpropSetMeth(strm, bDeleteOnClose, int); PROTOTYPEpropSetMeth(strm, iMaxFileSize, int); diff --git a/runtime/wti.c b/runtime/wti.c index 1be008df..3b6bf1b9 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -380,6 +380,7 @@ wtiWorker(wti_t *pThis) wtp_t *pWtp; /* our worker thread pool */ int bInactivityTOOccured = 0; rsRetVal localRet; + rsRetVal terminateRet; DEFiRet; ISOBJ_TYPE_assert(pThis, wti); @@ -406,15 +407,21 @@ wtiWorker(wti_t *pThis) wtpSetInactivityGuard(pThis->pWtp, 0, LOCK_MUTEX); /* must be set before usr mutex is locked! */ BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); - /* first check if we are in shutdown process */ - if(wtpChkStopWrkr(pWtp, LOCK_MUTEX, MUTEX_ALREADY_LOCKED)) { - break; /* end worker thread run */ + /* first check if we are in shutdown process (but evaluate a bit later) */ + terminateRet = wtpChkStopWrkr(pWtp, LOCK_MUTEX, MUTEX_ALREADY_LOCKED); + if(terminateRet == RS_RET_TERMINATE_NOW) { + // TODO: we need to free the old batch! -- rgerhards, 2009-05-26 MULTI + break; } /* try to execute and process whatever we have */ localRet = pWtp->pfDoWork(pWtp->pUsr, pThis, iCancelStateSave); if(localRet == RS_RET_IDLE) { + if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE) { + break; /* end of loop */ + } + if(bInactivityTOOccured) { /* we had an inactivity timeout in the last run and are still idle, so it is time to exit... */ break; /* end worker thread run */ diff --git a/runtime/wtp.c b/runtime/wtp.c index 40a9095b..41fcd8d9 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -261,16 +261,19 @@ wtpChkStopWrkr(wtp_t *pThis, int bLockMutex, int bLockUsrMutex) ISOBJ_TYPE_assert(pThis, wtp); BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); - if( (pThis->wtpState == wtpState_SHUTDOWN_IMMEDIATE) - || ((pThis->wtpState == wtpState_SHUTDOWN) && pThis->pfIsIdle(pThis->pUsr, pThis))) - iRet = RS_RET_TERMINATE_NOW; + if(pThis->wtpState == wtpState_SHUTDOWN_IMMEDIATE) { + ABORT_FINALIZE(RS_RET_TERMINATE_NOW); + } else if(pThis->wtpState == wtpState_SHUTDOWN) { + ABORT_FINALIZE(RS_RET_TERMINATE_WHEN_IDLE); + } /* try customer handler if one was set and we do not yet have a definite result */ - if(iRet == RS_RET_OK && pThis->pfChkStopWrkr != NULL) { + if(pThis->pfChkStopWrkr != NULL) { iRet = pThis->pfChkStopWrkr(pThis->pUsr, bLockUsrMutex); } - END_MTX_PROTECTED_OPERATIONS(&pThis->mut); +finalize_it: + END_MTX_PROTECTED_OPERATIONS(&pThis->mut); RETiRet; } diff --git a/tests/DiagTalker.java b/tests/DiagTalker.java index e33a5867..85a6671e 100644 --- a/tests/DiagTalker.java +++ b/tests/DiagTalker.java @@ -1,3 +1,24 @@ +/* A yet very simple tool to talk to imdiag. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ //package com.rsyslog.diag; import java.io.*; import java.net.*; @@ -29,9 +50,14 @@ public class DiagTalker { new InputStreamReader(System.in)); String userInput; - while ((userInput = stdIn.readLine()) != null) { - out.println(userInput); - System.out.println("imdiag returns: " + in.readLine()); + try { + while ((userInput = stdIn.readLine()) != null) { + out.println(userInput); + System.out.println("imdiag returns: " + in.readLine()); + } + } catch (SocketException e) { + System.err.println("We had a socket exception and consider this to be OK: " + + e.getMessage()); } out.close(); diff --git a/tests/arrayqueue.sh b/tests/arrayqueue.sh index 5b8ebb5f..7791ed57 100755 --- a/tests/arrayqueue.sh +++ b/tests/arrayqueue.sh @@ -17,7 +17,7 @@ sleep 4 # we need this so that rsyslogd can receive all outstanding messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -./chkseq work 0 39999 +./chkseq -fwork -e 39999 if [ "$?" -ne "0" ]; then # rm -f work rsyslog.out.log echo "sequence error detected" diff --git a/tests/chkseq.c b/tests/chkseq.c index 3203c250..5ffe855c 100644 --- a/tests/chkseq.c +++ b/tests/chkseq.c @@ -3,9 +3,10 @@ * be set. * * Params - * argv[1] file to check - * argv[2] start number - * argv[3] end number + * -f MUST be given! + * -s -e + * default for s is 0. -e should be given (else it is also 0) + * -d may be specified, in which case duplicate messages are permitted. * * Part of the testbench for rsyslog. * @@ -31,6 +32,7 @@ #include "config.h" #include #include +#include int main(int argc, char *argv[]) { @@ -38,16 +40,36 @@ int main(int argc, char *argv[]) int val; int i; int ret = 0; - int start, end; + int dupsPermitted = 0; + int start = 0, end = 0; + int opt; + int nDups = 0; + char *file = NULL; - if(argc != 4) { - printf("Invalid call of chkseq\n"); - printf("Usage: chkseq file start end\n"); + while((opt = getopt(argc, argv, "e:f:ds:")) != EOF) { + switch((char)opt) { + case 'f': + file = optarg; + break; + case 'd': + dupsPermitted = 1; + break; + case 'e': + end = atoi(optarg); + break; + case 's': + start = atoi(optarg); + break; + default:printf("Invalid call of chkseq\n"); + printf("Usage: chkseq file -sstart -eend -d\n"); + exit(1); + } + } + + if(file == NULL) { + printf("file must be given!\n"); exit(1); } - - start = atoi(argv[2]); - end = atoi(argv[3]); if(start > end) { printf("start must be less than or equal end!\n"); @@ -55,22 +77,35 @@ int main(int argc, char *argv[]) } /* read file */ - fp = fopen(argv[1], "r"); + fp = fopen(file, "r"); if(fp == NULL) { perror(argv[1]); exit(1); } - for(i = start ; i < end ; ++i) { + for(i = start ; i < end+1 ; ++i) { if(fscanf(fp, "%d\n", &val) != 1) { printf("scanf error in index i=%d\n", i); exit(1); } if(val != i) { - printf("read value %d, but expected value %d\n", val, i); - exit(1); + if(val == i - 1 && dupsPermitted) { + --i; + ++nDups; + } else { + printf("read value %d, but expected value %d\n", val, i); + exit(1); + } } } + if(nDups != 0) + printf("info: had %d duplicates (this is no error)\n", nDups); + + if(i - 1 != end) { + printf("only %d records in file, expected %d\n", i - 1, end); + exit(1); + } + exit(ret); } diff --git a/tests/da-mainmsg-q.sh b/tests/da-mainmsg-q.sh index 91addf68..fde9e06e 100755 --- a/tests/da-mainmsg-q.sh +++ b/tests/da-mainmsg-q.sh @@ -52,7 +52,7 @@ sleep 1 # we need this so that rsyslogd can receive all outstanding messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -./chkseq work 0 20099 +./chkseq -fwork -e20099 if [ "$?" -ne "0" ]; then # rm -f work rsyslog.out.log echo "sequence error detected" diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index 20767a90..42018b15 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -26,7 +26,7 @@ $srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -./chkseq work 0 19999 +./chkseq -fwork -e19999 if [ "$?" -ne "0" ]; then # rm -f work rsyslog.out.log echo "sequence error detected" diff --git a/tests/imtcp-multiport.sh b/tests/imtcp-multiport.sh index 17480dae..73ab9558 100755 --- a/tests/imtcp-multiport.sh +++ b/tests/imtcp-multiport.sh @@ -21,7 +21,7 @@ $srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -./chkseq work 0 9999 +./chkseq -fwork -e9999 if [ "$?" -ne "0" ]; then # rm -f work rsyslog.out.log echo "sequence error detected" @@ -46,7 +46,7 @@ $srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -./chkseq work 0 9999 +./chkseq -fwork -e9999 if [ "$?" -ne "0" ]; then # rm -f work rsyslog.out.log echo "sequence error detected" @@ -71,7 +71,7 @@ $srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -./chkseq work 0 9999 +./chkseq -fwork -e9999 if [ "$?" -ne "0" ]; then # rm -f work rsyslog.out.log echo "sequence error detected" diff --git a/tests/linkedlistqueue.sh b/tests/linkedlistqueue.sh index aac1abb6..aa574bd1 100755 --- a/tests/linkedlistqueue.sh +++ b/tests/linkedlistqueue.sh @@ -17,7 +17,7 @@ sleep 4 # we need this so that rsyslogd can receive all outstanding messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -./chkseq work 0 39999 +./chkseq -fwork -e39999 if [ "$?" -ne "0" ]; then # rm -f work rsyslog.out.log echo "sequence error detected" diff --git a/tests/manytcp.sh b/tests/manytcp.sh index 06bd38b6..861f12ff 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -12,7 +12,7 @@ $srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -./chkseq work 0 39999 +./chkseq -fwork -e39999 if [ "$?" -ne "0" ]; then rm -f work rsyslog.out.log echo "sequence error detected" diff --git a/tests/memq-persist.sh b/tests/memq-persist.sh index 108cba57..e935d8db 100755 --- a/tests/memq-persist.sh +++ b/tests/memq-persist.sh @@ -10,9 +10,11 @@ #export RSYSLOG_DEBUGLOG="log" echo testing memory queue persisting to disk $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason +rm -f core.* rm -rf test-spool mkdir test-spool rm -f work rsyslog.out.log rsyslog.out.log.save # work files +#valgrind ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist1.conf & ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist1.conf & sleep 1 echo "rsyslogd started with pid " `cat rsyslog.pid` @@ -22,20 +24,22 @@ if [ "$?" -ne "0" ]; then echo "error during tcpflood! see rsyslog.out.log.save for what was written" cp rsyslog.out.log rsyslog.out.log.save fi -sleep 3 # we need to wait to ensure everything is received (less 1 second would be better) +sleep 4 # we need to wait to ensure everything is received (less 1 second would be better) kill `cat rsyslog.pid` -sleep 5 # wait for engine to terminate +echo wait for shutdown +$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages echo There must exist some files now: ls -l test-spool # restart engine and have rest processed ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist2.conf & +sleep 1 $srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages kill `cat rsyslog.pid` rm -f work sort < rsyslog.out.log > work -./chkseq work 0 9999 +./chkseq -fwork -e9999 -d if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log + rm -f work rsyslog.out.log echo "sequence error detected" exit 1 fi diff --git a/tests/rscript.c b/tests/rscript.c index ce81491c..6361aec4 100644 --- a/tests/rscript.c +++ b/tests/rscript.c @@ -104,8 +104,8 @@ PerformTest(cstr_t *pstrIn, rsRetVal iRetExpected, cstr_t *pstrOut) if(strcmp((char*)rsCStrGetSzStr(pstrPrg), (char*)rsCStrGetSzStr(pstrOut))) { printf("error: compiled program different from expected result!\n"); - printf("generated vmprg (%d bytes):\n%s\n", strlen((char*)rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); - printf("expected (%d bytes):\n%s\n", strlen((char*)rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); + printf("generated vmprg (%d bytes):\n%s\n", (int)strlen((char*)rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); + printf("expected (%d bytes):\n%s\n", (int)strlen((char*)rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); ABORT_FINALIZE(RS_RET_ERR); } -- cgit v1.2.3 From a900a7c34b674573f4b86350af0d68838da6550a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 27 May 2009 11:29:47 +0200 Subject: greatly enhanced testbench The imdiag module now can very effectively inject messages, which also frees us from uncertainties of tcp reception and processing. All shell script based tests have been modularized, what makes it far easier to create new tests. Also, the test bench now executes more reliable and much faster, because we can now rely on actual engine information where we previously did just a dumb sleep. --- ChangeLog | 1 + plugins/imdiag/imdiag.c | 145 +++++++++++++++++++++++++++++++--- plugins/imtcp/imtcp.c | 1 - runtime/netstrm.c | 1 - tcpsrv.c | 3 - tests/Makefile.am | 13 +-- tests/diag.sh | 82 +++++++++++++++++++ tests/diskqueue.sh | 29 ++----- tests/imtcp-multiport.sh | 78 +++++------------- tests/manytcp.sh | 27 ++----- tests/memq-persist.sh | 43 ---------- tests/queue-persist-drvr.sh | 28 +++++++ tests/queue-persist.sh | 11 +++ tests/testsuites/diag-common.conf | 16 ++++ tests/testsuites/diskqueue.conf | 7 +- tests/testsuites/imtcp-multiport.conf | 7 +- tests/testsuites/manytcp.conf | 7 +- tests/testsuites/memq-persist1.conf | 25 ------ tests/testsuites/memq-persist2.conf | 20 ----- tests/testsuites/queue-persist.conf | 23 ++++++ tests/waitqueueempty.sh | 5 -- 21 files changed, 337 insertions(+), 235 deletions(-) create mode 100755 tests/diag.sh delete mode 100755 tests/memq-persist.sh create mode 100755 tests/queue-persist-drvr.sh create mode 100755 tests/queue-persist.sh create mode 100644 tests/testsuites/diag-common.conf delete mode 100644 tests/testsuites/memq-persist1.conf delete mode 100644 tests/testsuites/memq-persist2.conf create mode 100644 tests/testsuites/queue-persist.conf delete mode 100755 tests/waitqueueempty.sh diff --git a/ChangeLog b/ChangeLog index 2fc760c3..d8fae145 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,7 @@ --------------------------------------------------------------------------- Version 4.3.? [DEVEL] (rgerhards), 2009-??-?? - bugfix: imdiag/imtcp had a race condition +- improved testbench (now much better code design and reuse) --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 40f94692..c700cab7 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -52,6 +52,8 @@ #include "errmsg.h" #include "tcpsrv.h" #include "srUtils.h" +#include "msg.h" +#include "datetime.h" #include "net.h" /* for permittedPeers, may be removed when this is removed */ MODULE_TYPE_INPUT @@ -63,6 +65,7 @@ DEFobjCurrIf(tcps_sess) DEFobjCurrIf(net) DEFobjCurrIf(netstrm) DEFobjCurrIf(errmsg) +DEFobjCurrIf(datetime) /* Module static data */ static tcpsrv_t *pOurTcpsrv = NULL; /* our TCP server(listener) TODO: change for multiple instances */ @@ -134,10 +137,123 @@ onErrClose(tcps_sess_t *pSess) /* ------------------------------ end callbacks ------------------------------ */ +/* get the first word delimited by space from a given string. The pointer is + * advanced to after the word. Any leading spaces are discarded. If the + * output buffer is too small, parsing ends on buffer full condition. + * An empty buffer is returned if there is no more data inside the string. + * rgerhards, 2009-05-27 + */ +#define TO_LOWERCASE 1 +#define NO_MODIFY 0 +static void +getFirstWord(uchar **ppszSrc, uchar *pszBuf, size_t lenBuf, int options) +{ + uchar c; + uchar *pszSrc = *ppszSrc; + + while(*pszSrc && *pszSrc == ' ') + ++pszSrc; /* skip to first non-space */ + + while(*pszSrc && *pszSrc != ' ' && lenBuf > 1) { + c = *pszSrc++; + if(options & TO_LOWERCASE) + c = tolower(c); + *pszBuf++ = c; + lenBuf--; + } + + *pszBuf = '\0'; + *ppszSrc = pszSrc; +} + + +/* send a response back to the originator + * rgerhards, 2009-05-27 + */ +static rsRetVal __attribute__((format(printf, 2, 3))) +sendResponse(tcps_sess_t *pSess, char *fmt, ...) +{ + va_list ap; + ssize_t len; + uchar buf[1024]; + DEFiRet; + + va_start(ap, fmt); + len = vsnprintf((char*)buf, sizeof(buf), fmt, ap); + va_end(ap); + CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + +finalize_it: + RETiRet; +} + + +/* actually submit a message to the rsyslog core + */ +static rsRetVal +doInjectMsg(int iNum) +{ + uchar szMsg[1024]; + msg_t *pMsg; + struct syslogTime stTime; + time_t ttGenTime; + DEFiRet; + + snprintf((char*)szMsg, sizeof(szMsg)/sizeof(uchar), + "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", iNum); + + datetime.getCurrTime(&stTime, &ttGenTime); + /* we now create our own message object and submit it to the queue */ + CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); + CHKmalloc(pMsg->pszRawMsg = ustrdup(szMsg)); + pMsg->iLenRawMsg = ustrlen(szMsg); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imdiag")); + MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); + pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; + pMsg->bParseHOSTNAME = 1; + MsgSetRcvFrom(pMsg, UCHAR_CONSTANT("127.0.0.1")); /* TODO: way may use the real sender here... */ + CHKiRet(MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1"))); + CHKiRet(submitMsg(pMsg)); + +finalize_it: + RETiRet; +} + + +/* This function injects messages. Command format: + * injectmsg + * rgerhards, 2009-05-27 + */ +static rsRetVal +injectMsg(uchar *pszCmd, tcps_sess_t *pSess) +{ + uchar wordBuf[1024]; + int iFrom; + int nMsgs; + int i; + DEFiRet; + + /* we do not check errors here! */ + getFirstWord(&pszCmd, wordBuf, sizeof(wordBuf)/sizeof(uchar), TO_LOWERCASE); + iFrom = atoi((char*)wordBuf); + getFirstWord(&pszCmd, wordBuf, sizeof(wordBuf)/sizeof(uchar), TO_LOWERCASE); + nMsgs = atoi((char*)wordBuf); + + for(i = 0 ; i < nMsgs ; ++i) { + doInjectMsg(i + iFrom); + } + + CHKiRet(sendResponse(pSess, "messages injected\n")); + +finalize_it: + RETiRet; +} + + /* This function waits until the main queue is drained (size = 0) */ static rsRetVal -waitMainQEmpty(void) +waitMainQEmpty(tcps_sess_t *pSess) { int iMsgQueueSize; DEFiRet; @@ -148,21 +264,21 @@ waitMainQEmpty(void) CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); } + CHKiRet(sendResponse(pSess, "mainqueue empty\n")); + finalize_it: RETiRet; } - /* Function to handle received messages. This is our core function! * rgerhards, 2009-05-24 */ static rsRetVal OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) { - ssize_t len; int iMsgQueueSize; uchar *pszMsg; - uchar buf[1024]; + uchar cmdBuf[1024]; DEFiRet; assert(pSess != NULL); @@ -176,17 +292,18 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) memcpy(pszMsg, pRcv, iLenMsg); pszMsg[iLenMsg] = '\0'; - if(!ustrcmp(pszMsg, UCHAR_CONSTANT("GetMainMsgQueueSize"))) { + getFirstWord(&pszMsg, cmdBuf, sizeof(cmdBuf)/sizeof(uchar), TO_LOWERCASE); + + if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) { CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); - len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "%d\n", iMsgQueueSize); - CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); - } else if(!ustrcmp(pszMsg, UCHAR_CONSTANT("WaitMainQueueEmpty"))) { - CHKiRet(waitMainQEmpty()); - len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "mainqueue empty\n"); - CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + CHKiRet(sendResponse(pSess, "%d\n", iMsgQueueSize)); + } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("waitmainqueueempty"))) { + CHKiRet(waitMainQEmpty(pSess)); + } else if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("injectmsg"))) { + CHKiRet(injectMsg(pszMsg, pSess)); } else { - len = snprintf((char*)buf, sizeof(buf)/sizeof(uchar), "unkown command '%s'\n", pszMsg); - CHKiRet(netstrm.Send(pSess->pStrm, buf, &len)); + dbgprintf("imdiag unkown command '%s'\n", cmdBuf); + CHKiRet(sendResponse(pSess, "unkown command '%s'\n", cmdBuf)); } finalize_it: @@ -285,6 +402,7 @@ CODESTARTmodExit objRelease(tcps_sess, LM_TCPSRV_FILENAME); objRelease(tcpsrv, LM_TCPSRV_FILENAME); objRelease(errmsg, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); ENDmodExit @@ -321,6 +439,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(tcps_sess, LM_TCPSRV_FILENAME)); CHKiRet(objUse(tcpsrv, LM_TCPSRV_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverrun"), 0, eCmdHdlrGetWord, diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index f0efe1ee..84e660bc 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -180,7 +180,6 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa } } -dbgprintf("XXX: try add listen port %s\n", pNewVal); /* initialized, now add socket */ CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? UCHAR_CONSTANT("imtcp") : pszInputName)); diff --git a/runtime/netstrm.c b/runtime/netstrm.c index f6a8de7f..05bb25c0 100644 --- a/runtime/netstrm.c +++ b/runtime/netstrm.c @@ -114,7 +114,6 @@ AcceptConnReq(netstrm_t *pThis, netstrm_t **ppNew) ISOBJ_TYPE_assert(pThis, netstrm); assert(ppNew != NULL); -RUNLOG_STR("XXX: accept conn reqeust"); /* accept the new connection */ CHKiRet(pThis->Drvr.AcceptConnReq(pThis->pDrvrData, &pNewNsd)); /* construct our object so that we can use it... */ diff --git a/tcpsrv.c b/tcpsrv.c index 249eeecf..02eee88e 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -97,7 +97,6 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) ISOBJ_TYPE_assert(pThis, tcpsrv); -dbgprintf("XXX: tcpsrv.c add port %s, name '%s'\n", pszPort, pThis->pszInputName); /* create entry */ CHKmalloc(pEntry = malloc(sizeof(tcpLstnPortList_t))); pEntry->pszPort = pszPort; @@ -267,7 +266,6 @@ addTcpLstn(void *pUsr, netstrm_t *pLstn) tcpsrv_t *pThis = pPortList->pSrv; DEFiRet; -dbgprintf("XXX: addTcpLst name %s\n", pPortList->pszInputName); ISOBJ_TYPE_assert(pThis, tcpsrv); ISOBJ_TYPE_assert(pLstn, netstrm); @@ -326,7 +324,6 @@ create_tcp_socket(tcpsrv_t *pThis) /* init all configured ports */ pEntry = pThis->pLstnPorts; while(pEntry != NULL) { -dbgprintf("XXX: tcpsrv.c create_tcp_socket do port %s\n", pEntry->pszPort); CHKiRet(initTCPListener(pThis, pEntry)); pEntry = pEntry->pNext; } diff --git a/tests/Makefile.am b/tests/Makefile.am index 92401511..a2e6c34c 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,6 +1,6 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh memq-persist.sh +TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh queue-persist.sh if ENABLE_OMSTDOUT TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh endif @@ -9,7 +9,6 @@ DISTCLEANFILES=rsyslog.pid '$(abs_top_builddir)'/DiagTalker.class test_files = testbench.h runtime-dummy.c check_JAVA = DiagTalker.java -#dist_java_JAVA = DiagTalker.java EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ cfg1.cfgtest \ @@ -34,6 +33,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.omod-if-array \ killrsyslog.sh \ parsertest.sh \ + queue-persist.sh \ + testsuites/queue-persist.conf \ diskqueue.sh \ testsuites/diskqueue.conf \ imtcp-multiport.sh \ @@ -49,10 +50,10 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12515 \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ - waitqueueempty.sh \ - memq-persist.sh \ - testsuites/memq-persist1.sh \ - testsuites/memq-persist2.sh \ + diag.sh \ + queue-persist.sh \ + queue-persist-drvr.sh \ + testsuites/queue-persist.sh \ DiagTalker.java \ cfg.sh diff --git a/tests/diag.sh b/tests/diag.sh new file mode 100755 index 00000000..270b30d6 --- /dev/null +++ b/tests/diag.sh @@ -0,0 +1,82 @@ +# this shell script provides commands to the common diag system. It enables +# test scripts to wait for certain conditions and initiate certain actions. +# needs support in config file. +# NOTE: this file should be included with "source diag.sh", as it otherwise is +# not always able to convey back states to the upper-level test driver +# begun 2009-05-27 by rgerhards +# This file is part of the rsyslog project, released under GPLv3 +#set -o xtrace +#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUGLOG="tmp" +case $1 in + 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + rm -f rsyslogd.started work-*.conf + rm -f work rsyslog.out.log rsyslog.out.log.save # common work files + rm -rf test-spool + mkdir test-spool + ;; + 'exit') rm -f rsyslogd.started work-*.conf + rm -f work rsyslog.out.log rsyslog.out.log.save # common work files + rm -rf test-spool + ;; + 'startup') # start rsyslogd with default params. $2 is the config file name to use + # returns only after successful startup + ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & + $srcdir/diag.sh wait-startup + ;; + 'wait-startup') # wait for rsyslogd startup + while test ! -f rsyslogd.started; do + true + done + echo "rsyslogd started with pid " `cat rsyslog.pid` + ;; + 'wait-shutdown') # actually, we wait for rsyslog.pid to be deleted + while test -f rsyslog.pid; do + true + done + ;; + 'wait-queueempty') # wait for main message queue to be empty + echo WaitMainQueueEmpty | java -classpath $abs_top_builddir DiagTalker + ;; + 'shutdown-when-empty') # shut rsyslogd down when main queue is empty + $srcdir/diag.sh wait-queueempty + kill `cat rsyslog.pid` + # note: we do not wait for the actual termination! + ;; + 'shutdown-immediate') # shut rsyslogd down without emptying the queue + kill `cat rsyslog.pid` + # note: we do not wait for the actual termination! + ;; + 'tcpflood') # do a tcpflood run and check if it worked params are passed to tcpflood + ./tcpflood $2 $3 $4 $5 $6 $7 $8 + if [ "$?" -ne "0" ]; then + echo "error during tcpflood! see rsyslog.out.log.save for what was written" + cp rsyslog.out.log rsyslog.out.log.save + exit 1 + fi + ;; + 'injectmsg') # inject messages via our inject interface (imdiag) + echo injectmsg $2 $3 $4 $5 | java -classpath $abs_top_builddir DiagTalker + # TODO: some return state checking? (does it really make sense here?) + ;; + 'check-mainq-spool') # check if mainqueue spool files exist, if not abort (we just check .qi) + echo There must exist some files now: + ls -l test-spool + if test ! -f test-spool/mainq.qi; then + echo "error: mainq.qi does not exist where expected to do so!" + ls -l test-spool + exit 1 + fi + ;; + 'seq-check') # do the usual sequence check to see if everything was properly received + rm -f work + sort < rsyslog.out.log > work + ./chkseq work $2 $3 + if [ "$?" -ne "0" ]; then + rm -f work rsyslog.out.log + echo "sequence error detected" + exit 1 + fi + ;; + *) echo "invalid argument" $1 +esac diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index eabfcf78..bf1a46fd 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -9,27 +9,10 @@ #export RSYSLOG_DEBUG="debug nostdout" #export RSYSLOG_DEBUGLOG="tmp" echo testing queue disk-only mode -rm -rf test-spool -mkdir test-spool -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/diskqueue.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` +source $srcdir/diag.sh init +source $srcdir/diag.sh startup diskqueue.conf # 20000 messages should be enough - the disk test is slow enough ;) -./tcpflood 127.0.0.1 13514 1 20000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 19999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log -rm -rf test-spool +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 20000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 19999 +source $srcdir/diag.sh exit diff --git a/tests/imtcp-multiport.sh b/tests/imtcp-multiport.sh index 17480dae..702f8834 100755 --- a/tests/imtcp-multiport.sh +++ b/tests/imtcp-multiport.sh @@ -8,73 +8,31 @@ # added 2009-05-22 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 echo testing imtcp multiple listeners -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` -./tcpflood 127.0.0.1 13514 1 10000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 9999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh init +source $srcdir/diag.sh startup imtcp-multiport.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 10000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 9999 +source $srcdir/diag.sh exit # # # ### now complete new cycle with other port ### # # -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` -./tcpflood 127.0.0.1 13515 1 10000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 9999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh init +source $srcdir/diag.sh startup imtcp-multiport.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13515 1 10000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 9999 +source $srcdir/diag.sh exit # # # ### now complete new cycle with other port ### # # -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/imtcp-multiport.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` -./tcpflood 127.0.0.1 13516 1 10000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 9999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh init +source $srcdir/diag.sh startup imtcp-multiport.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13516 1 10000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 9999 +source $srcdir/diag.sh exit diff --git a/tests/manytcp.sh b/tests/manytcp.sh index 06bd38b6..c55eb9c0 100755 --- a/tests/manytcp.sh +++ b/tests/manytcp.sh @@ -1,21 +1,8 @@ -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/manytcp.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` +# test many concurrent tcp connections +source $srcdir/diag.sh init +source $srcdir/diag.sh startup manytcp.conf # the config file specifies exactly 1100 connections -./tcpflood 127.0.0.1 13514 1000 40000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 39999 -if [ "$?" -ne "0" ]; then - rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1000 40000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 39999 +source $srcdir/diag.sh exit diff --git a/tests/memq-persist.sh b/tests/memq-persist.sh deleted file mode 100755 index 108cba57..00000000 --- a/tests/memq-persist.sh +++ /dev/null @@ -1,43 +0,0 @@ -# Test for memory queue which is persisted at shutdown. The -# plan is to start an instance, emit some data, do a relatively -# fast shutdown and then re-start the engine to process the -# remaining data. -# added 2009-05-25 by Rgerhards -# This file is part of the rsyslog project, released under GPLv3 -# uncomment for debugging support: -#set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout" -#export RSYSLOG_DEBUGLOG="log" -echo testing memory queue persisting to disk -$srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason -rm -rf test-spool -mkdir test-spool -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist1.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` -# 20000 messages should be enough -./tcpflood 127.0.0.1 13514 1 10000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -sleep 3 # we need to wait to ensure everything is received (less 1 second would be better) -kill `cat rsyslog.pid` -sleep 5 # wait for engine to terminate -echo There must exist some files now: -ls -l test-spool -# restart engine and have rest processed -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/memq-persist2.conf & -$srcdir/waitqueueempty.sh # wait until rsyslogd is done processing messages -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq work 0 9999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log -rm -rf test-spool diff --git a/tests/queue-persist-drvr.sh b/tests/queue-persist-drvr.sh new file mode 100755 index 00000000..ea5386a7 --- /dev/null +++ b/tests/queue-persist-drvr.sh @@ -0,0 +1,28 @@ +# Test for queue data persisting at shutdown. The +# plan is to start an instance, emit some data, do a relatively +# fast shutdown and then re-start the engine to process the +# remaining data. +# added 2009-05-27 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo testing memory queue persisting to disk, mode $1 +source $srcdir/diag.sh init + +# prepare config +echo \$MainMsgQueueType $1 > work-queuemode.conf +echo "*.* :omtesting:sleep 0 1000" > work-delay.conf + +# inject 5000 msgs, so that we do not hit the high watermark +source $srcdir/diag.sh startup queue-persist.conf +source $srcdir/diag.sh injectmsg 0 5000 +$srcdir/diag.sh shutdown-immediate +$srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh check-mainq-spool + +# restart engine and have rest processed +#remove delay +echo "#" > work-delay.conf +source $srcdir/diag.sh startup queue-persist.conf +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/queue-persist.sh b/tests/queue-persist.sh new file mode 100755 index 00000000..999655b1 --- /dev/null +++ b/tests/queue-persist.sh @@ -0,0 +1,11 @@ +# Test for queue data persisting at shutdown. We use the actual driver +# to carry out multiple tests with different queue modes +# added 2009-05-27 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +source $srcdir/queue-persist-drvr.sh LinkedList +source $srcdir/queue-persist-drvr.sh FixedArray +# the disk test should not fail, however, the config is extreme and using +# it more or less is a config error +source $srcdir/queue-persist-drvr.sh Disk +# we do not test Direct mode because this absolute can not work in direct mode +# (maybe we should do a fail-type of test?) diff --git a/tests/testsuites/diag-common.conf b/tests/testsuites/diag-common.conf new file mode 100644 index 00000000..9e9e28fe --- /dev/null +++ b/tests/testsuites/diag-common.conf @@ -0,0 +1,16 @@ +# This is a config include file. It sets up rsyslog so that the +# diag system can successfully be used. Also, it generates a file +# "rsyslogd.started" after rsyslogd is initialized. This config file +# should be included in all tests that intend to use common code for +# controlling the daemon. +# NOTE: we assume that rsyslogd's current working directory is +# ./tests (or the distcheck equivalent), in particlular that this +# config file resides in the testsuites subdirectory. +# rgerhards, 2009-05-27 +$ModLoad ../plugins/imdiag/.libs/imdiag +$IMDiagServerRun 13500 + +$template startupfile,"rsyslogd.started" # trick to use relative path names! +:syslogtag, contains, "rsyslogd" ?startupfile + +$ErrorMessagesToStderr off diff --git a/tests/testsuites/diskqueue.conf b/tests/testsuites/diskqueue.conf index 017ee96d..d7f323bc 100644 --- a/tests/testsuites/diskqueue.conf +++ b/tests/testsuites/diskqueue.conf @@ -1,14 +1,11 @@ # Test for queue disk mode (see .sh file for details) # rgerhards, 2009-04-17 +$IncludeConfig testsuites/diag-common.conf + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ErrorMessagesToStderr off - # set spool locations and switch queue to disk-only mode $WorkDirectory test-spool $MainMsgQueueFilename mainq diff --git a/tests/testsuites/imtcp-multiport.conf b/tests/testsuites/imtcp-multiport.conf index ec059fe4..9146f6e0 100644 --- a/tests/testsuites/imtcp-multiport.conf +++ b/tests/testsuites/imtcp-multiport.conf @@ -1,16 +1,13 @@ # Test for queue disk mode (see .sh file for details) # rgerhards, 2009-05-22 +$IncludeConfig testsuites/diag-common.conf + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 $InputTCPServerRun 13515 $InputTCPServerRun 13516 -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ErrorMessagesToStderr off - $template outfmt,"%msg:F,58:2%\n" $template dynfile,"rsyslog.out.log" # trick to use relative path names! :msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf index 3867da46..772ec9ce 100644 --- a/tests/testsuites/manytcp.conf +++ b/tests/testsuites/manytcp.conf @@ -1,16 +1,13 @@ # Test for tcp "flood" testing # rgerhards, 2009-04-08 +$IncludeConfig testsuites/diag-common.conf + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $MaxOpenFiles 2000 $InputTCPMaxSessions 1100 $InputTCPServerRun 13514 -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ErrorMessagesToStderr off - $template outfmt,"%msg:F,58:2%\n" $template dynfile,"rsyslog.out.log" # trick to use relative path names! :msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/memq-persist1.conf b/tests/testsuites/memq-persist1.conf deleted file mode 100644 index 5240090f..00000000 --- a/tests/testsuites/memq-persist1.conf +++ /dev/null @@ -1,25 +0,0 @@ -# Test for persisting messages to disk on shutdown -# rgerhards, 2009-04-17 -$ModLoad ../plugins/imtcp/.libs/imtcp -$MainMsgQueueTimeoutShutdown 1 -$MainMsgQueueSaveOnShutdown on -$InputTCPServerRun 13514 - -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ModLoad ../plugins/omtesting/.libs/omtesting - -$ErrorMessagesToStderr off - -# set spool locations and switch queue to disk-only mode -$WorkDirectory test-spool -$MainMsgQueueFilename mainq -$MainMsgQueueType LinkedList - -$template outfmt,"%msg:F,58:2%\n" -$template dynfile,"rsyslog.out.log" # trick to use relative path names! -:msg, contains, "msgnum:" ?dynfile;outfmt - -# delay execution so that a queue can build up: -*.* :omtesting:sleep 0 1000 diff --git a/tests/testsuites/memq-persist2.conf b/tests/testsuites/memq-persist2.conf deleted file mode 100644 index 23e29e2f..00000000 --- a/tests/testsuites/memq-persist2.conf +++ /dev/null @@ -1,20 +0,0 @@ -# Test for persisting messages to disk on shutdown -# rgerhards, 2009-04-17 -$ModLoad ../plugins/imtcp/.libs/imtcp -$MainMsgQueueTimeoutShutdown 10000 -$MainMsgQueueSaveOnShutdown on -$InputTCPServerRun 13514 - -$ModLoad ../plugins/imdiag/.libs/imdiag -$IMDiagServerRun 13500 - -$ErrorMessagesToStderr off - -# set spool locations and switch queue to disk-only mode -$WorkDirectory test-spool -$MainMsgQueueFilename mainq -$MainMsgQueueType LinkedList - -$template outfmt,"%msg:F,58:2%\n" -$template dynfile,"rsyslog.out.log" # trick to use relative path names! -:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/queue-persist.conf b/tests/testsuites/queue-persist.conf new file mode 100644 index 00000000..80f8ba30 --- /dev/null +++ b/tests/testsuites/queue-persist.conf @@ -0,0 +1,23 @@ +# Test for persisting messages on shutdown +# rgerhards, 2009-04-17 +$IncludeConfig testsuites/diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 1 +$MainMsgQueueSaveOnShutdown on +$InputTCPServerRun 13514 + +$ModLoad ../plugins/omtesting/.libs/omtesting + +$ErrorMessagesToStderr off + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueFilename mainq +$IncludeConfig work-queuemode.conf + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt + +$IncludeConfig work-delay.conf diff --git a/tests/waitqueueempty.sh b/tests/waitqueueempty.sh deleted file mode 100755 index 4825853a..00000000 --- a/tests/waitqueueempty.sh +++ /dev/null @@ -1,5 +0,0 @@ -# wait until main message queue is empty. This is currently done in -# a separate shell script so that we can change the implementation -# at some later point. -- rgerhards, 2009-05-25 -#echo WaitMainQueueEmpty | nc 127.0.0.1 13500 -echo WaitMainQueueEmpty | java -classpath $abs_top_builddir DiagTalker -- cgit v1.2.3 From 74ed599ea6ab96cffba35ca14f5654523f89bc61 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 27 May 2009 11:36:54 +0200 Subject: fixed typo --- tests/Makefile.am | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index a2e6c34c..713c721e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -33,8 +33,6 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.omod-if-array \ killrsyslog.sh \ parsertest.sh \ - queue-persist.sh \ - testsuites/queue-persist.conf \ diskqueue.sh \ testsuites/diskqueue.conf \ imtcp-multiport.sh \ @@ -53,7 +51,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ diag.sh \ queue-persist.sh \ queue-persist-drvr.sh \ - testsuites/queue-persist.sh \ + testsuites/queue-persist.conf \ DiagTalker.java \ cfg.sh -- cgit v1.2.3 From affa217cc2d22fa8037a50b8d40d3372694ff855 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 27 May 2009 12:52:28 +0200 Subject: adapted testbench to new capabilities ... and now make check fails again, we have obviously found some new bugs thanks to the additional cases --- plugins/imdiag/imdiag.c | 2 +- tests/Makefile.am | 5 ++- tests/arrayqueue.sh | 28 +++++---------- tests/da-mainmsg-q.sh | 66 ++++++++++------------------------- tests/daqueue-persist-drvr.sh | 28 +++++++++++++++ tests/daqueue-persist.sh | 11 ++++++ tests/diag.sh | 8 ++--- tests/diskqueue.sh | 3 -- tests/linkedlistqueue.sh | 28 +++++---------- tests/testsuites/arrayqueue.conf | 4 +-- tests/testsuites/da-mainmsg-q.conf | 2 +- tests/testsuites/linkedlistqueue.conf | 2 ++ tests/testsuites/queue-persist.conf | 2 -- 13 files changed, 87 insertions(+), 102 deletions(-) create mode 100755 tests/daqueue-persist-drvr.sh create mode 100755 tests/daqueue-persist.sh diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index c700cab7..03396db7 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -243,7 +243,7 @@ injectMsg(uchar *pszCmd, tcps_sess_t *pSess) doInjectMsg(i + iFrom); } - CHKiRet(sendResponse(pSess, "messages injected\n")); + CHKiRet(sendResponse(pSess, "%d messages injected\n", nMsgs)); finalize_it: RETiRet; diff --git a/tests/Makefile.am b/tests/Makefile.am index 279158cf..71daa3bc 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -2,7 +2,8 @@ TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq TESTS = $(TESTRUNS) cfg.sh arrayqueue.sh linkedlistqueue.sh da-mainmsg-q.sh \ diskqueue.sh manytcp.sh \ - queue-persist.sh + queue-persist.sh \ + daqueue-persist.sh if ENABLE_OMSTDOUT TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh @@ -65,6 +66,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ diag.sh \ + daqueue-persist.sh \ + daqueue-persist-drvr.sh \ queue-persist.sh \ queue-persist-drvr.sh \ testsuites/queue-persist.conf \ diff --git a/tests/arrayqueue.sh b/tests/arrayqueue.sh index 7791ed57..01fc133b 100755 --- a/tests/arrayqueue.sh +++ b/tests/arrayqueue.sh @@ -2,25 +2,13 @@ # added 2009-05-20 by rgerhards # This file is part of the rsyslog project, released under GPLv3 echo testing queue fixedArray queue mode -rm -f work rsyslog.out.log -# enable this, if you need debug output: export RSYSLOG_DEBUG="debug" -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/arrayqueue.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` +source $srcdir/diag.sh init +source $srcdir/diag.sh startup arrayqueue.conf + # 40000 messages should be enough -./tcpflood 127.0.0.1 13514 1 40000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -sleep 4 # we need this so that rsyslogd can receive all outstanding messages +source $srcdir/diag.sh injectmsg 0 40000 + +# terminate *now* (don't wait for queue to drain!) kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq -fwork -e 39999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh seq-check 0 39999 +source $srcdir/diag.sh exit diff --git a/tests/da-mainmsg-q.sh b/tests/da-mainmsg-q.sh index fde9e06e..1c947de2 100755 --- a/tests/da-mainmsg-q.sh +++ b/tests/da-mainmsg-q.sh @@ -8,55 +8,25 @@ # added 2009-04-22 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 echo "testing main message queue in DA mode (going to disk)" -rm -f work rsyslog.out.log -rm -rf test-spool -mkdir test-spool -rm -f work rsyslog.out.log rsyslog.out.log.save # work files -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/da-mainmsg-q.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` -# +source $srcdir/diag.sh init +source $srcdir/diag.sh startup da-mainmsg-q.conf + # part1: send first 50 messages (in memory, only) -# -./tcpflood 127.0.0.1 13514 1 50 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -ls -l test-spool -sleep 2 # we need this so that rsyslogd can receive all outstanding messages -# +#source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 50 +source $srcdir/diag.sh injectmsg 0 50 +source $srcdir/diag.sh wait-queueempty # let queue drain for this test case + # part 2: send bunch of messages. This should trigger DA mode -# -# 20000 messages should be enough - the disk test is slow enough ;) -./tcpflood 127.0.0.1 13514 2 20000 50 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -ls -l test-spool -sleep 8 # we need this so that rsyslogd can receive all outstanding messages -# +source $srcdir/diag.sh injectmsg 50 20000 +ls -l test-spool # for manual review + # send another handful -# -ls -l test-spool -./tcpflood 127.0.0.1 13514 1 50 20050 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -sleep 1 # we need this so that rsyslogd can receive all outstanding messages -# +source $srcdir/diag.sh injectmsg 20050 50 +#sleep 1 # we need this so that rsyslogd can receive all outstanding messages + # clean up and check test result -# -kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq -fwork -e20099 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log -rm -rf test-spool +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +### currently, we get a stable abort if we use the former kill logic. With shutdown-when-empty, it hangs (but that still tells us there is a bug ;)) ### +#kill `cat rsyslog.pid` +source $srcdir/diag.sh seq-check 0 20099 +source $srcdir/diag.sh exit diff --git a/tests/daqueue-persist-drvr.sh b/tests/daqueue-persist-drvr.sh new file mode 100755 index 00000000..11058110 --- /dev/null +++ b/tests/daqueue-persist-drvr.sh @@ -0,0 +1,28 @@ +# Test for queue data persisting at shutdown. The +# plan is to start an instance, emit some data, do a relatively +# fast shutdown and then re-start the engine to process the +# remaining data. +# added 2009-05-27 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo testing memory daqueue persisting to disk, mode $1 +source $srcdir/diag.sh init + +# prepare config +echo \$MainMsgQueueType $1 > work-queuemode.conf +echo "*.* :omtesting:sleep 0 1000" > work-delay.conf + +# inject 10000 msgs, so that DO hit the high watermark +source $srcdir/diag.sh startup queue-persist.conf +source $srcdir/diag.sh injectmsg 0 10000 +$srcdir/diag.sh shutdown-immediate +$srcdir/diag.sh wait-shutdown +source $srcdir/diag.sh check-mainq-spool + +# restart engine and have rest processed +#remove delay +echo "#" > work-delay.conf +source $srcdir/diag.sh startup queue-persist.conf +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/daqueue-persist.sh b/tests/daqueue-persist.sh new file mode 100755 index 00000000..157b8f7a --- /dev/null +++ b/tests/daqueue-persist.sh @@ -0,0 +1,11 @@ +# Test for queue data persisting at shutdown. We use the actual driver +# to carry out multiple tests with different queue modes +# added 2009-05-27 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +source $srcdir/daqueue-persist-drvr.sh LinkedList +#source $srcdir/daqueue-persist-drvr.sh FixedArray +# the disk test should not fail, however, the config is extreme and using +# it more or less is a config error +#source $srcdir/daqueue-persist-drvr.sh Disk +# we do not test Direct mode because this absolute can not work in direct mode +# (maybe we should do a fail-type of test?) diff --git a/tests/diag.sh b/tests/diag.sh index a8f9b1b4..01b335ee 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -6,10 +6,11 @@ # begun 2009-05-27 by rgerhards # This file is part of the rsyslog project, released under GPLv3 #set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout" -#export RSYSLOG_DEBUGLOG="log" +export RSYSLOG_DEBUG="debug nostdout" +export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + rm -f core.* vgcore.* # do NOT delete them at exit ;) rm -f rsyslogd.started work-*.conf rm -f work rsyslog.out.log rsyslog.out.log.save # common work files rm -rf test-spool @@ -21,7 +22,7 @@ case $1 in ;; 'startup') # start rsyslogd with default params. $2 is the config file name to use # returns only after successful startup - ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & + valgrind ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & $srcdir/diag.sh wait-startup ;; 'wait-startup') # wait for rsyslogd startup @@ -73,7 +74,6 @@ case $1 in sort < rsyslog.out.log > work ./chkseq -fwork -e$2 $3 if [ "$?" -ne "0" ]; then - rm -f work rsyslog.out.log echo "sequence error detected" exit 1 fi diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index bf1a46fd..2fe31db9 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -5,9 +5,6 @@ # added 2009-04-17 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 # uncomment for debugging support: -#set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout" -#export RSYSLOG_DEBUGLOG="tmp" echo testing queue disk-only mode source $srcdir/diag.sh init source $srcdir/diag.sh startup diskqueue.conf diff --git a/tests/linkedlistqueue.sh b/tests/linkedlistqueue.sh index aa574bd1..9570ed2b 100755 --- a/tests/linkedlistqueue.sh +++ b/tests/linkedlistqueue.sh @@ -2,25 +2,13 @@ # added 2009-05-20 by rgerhards # This file is part of the rsyslog project, released under GPLv3 echo testing queue Linkedlist queue mode -rm -f work rsyslog.out.log -# enable this, if you need debug output: export RSYSLOG_DEBUG="debug" -../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/arrayqueue.conf & -sleep 1 -echo "rsyslogd started with pid " `cat rsyslog.pid` +source $srcdir/diag.sh init +source $srcdir/diag.sh startup linkedlistqueue.conf + # 40000 messages should be enough -./tcpflood 127.0.0.1 13514 1 40000 -if [ "$?" -ne "0" ]; then - echo "error during tcpflood! see rsyslog.out.log.save for what was written" - cp rsyslog.out.log rsyslog.out.log.save -fi -sleep 4 # we need this so that rsyslogd can receive all outstanding messages +source $srcdir/diag.sh injectmsg 0 40000 + +# terminate *now* (don't wait for queue to drain) kill `cat rsyslog.pid` -rm -f work -sort < rsyslog.out.log > work -./chkseq -fwork -e39999 -if [ "$?" -ne "0" ]; then - # rm -f work rsyslog.out.log - echo "sequence error detected" - exit 1 -fi -rm -f work rsyslog.out.log +source $srcdir/diag.sh seq-check 0 39999 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/arrayqueue.conf b/tests/testsuites/arrayqueue.conf index 7fdbc79b..44b9920b 100644 --- a/tests/testsuites/arrayqueue.conf +++ b/tests/testsuites/arrayqueue.conf @@ -1,11 +1,11 @@ # Test for queue fixedArray mode (see .sh file for details) # rgerhards, 2009-04-17 +$IncludeConfig testsuites/diag-common.conf + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 -$ErrorMessagesToStderr off - # set spool locations and switch queue to disk-only mode $MainMsgQueueType FixedArray diff --git a/tests/testsuites/da-mainmsg-q.conf b/tests/testsuites/da-mainmsg-q.conf index fbda27d4..c1e8cb01 100644 --- a/tests/testsuites/da-mainmsg-q.conf +++ b/tests/testsuites/da-mainmsg-q.conf @@ -4,7 +4,7 @@ $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 -$ErrorMessagesToStderr off +$IncludeConfig testsuites/diag-common.conf # set spool locations and switch queue to disk assisted mode $WorkDirectory test-spool diff --git a/tests/testsuites/linkedlistqueue.conf b/tests/testsuites/linkedlistqueue.conf index ecfc7a26..321678a5 100644 --- a/tests/testsuites/linkedlistqueue.conf +++ b/tests/testsuites/linkedlistqueue.conf @@ -1,5 +1,7 @@ # Test for queue LinkedList mode (see .sh file for details) # rgerhards, 2009-04-17 +$IncludeConfig testsuites/diag-common.conf + $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 diff --git a/tests/testsuites/queue-persist.conf b/tests/testsuites/queue-persist.conf index 80f8ba30..4396c7bd 100644 --- a/tests/testsuites/queue-persist.conf +++ b/tests/testsuites/queue-persist.conf @@ -9,8 +9,6 @@ $InputTCPServerRun 13514 $ModLoad ../plugins/omtesting/.libs/omtesting -$ErrorMessagesToStderr off - # set spool locations and switch queue to disk-only mode $WorkDirectory test-spool $MainMsgQueueFilename mainq -- cgit v1.2.3 From d4564f8399f4362c7e79066370049f909cef996c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 27 May 2009 19:43:28 +0200 Subject: interim commit: working on failure cases slightly improved situation, would like to save it before carrying on --- runtime/debug.h | 2 +- runtime/queue.c | 314 ++++++++++++++++++++++++++++-------------- runtime/rsyslog.h | 1 + runtime/srUtils.h | 12 ++ runtime/wti.c | 30 +++- runtime/wti.h | 4 +- runtime/wtp.c | 15 ++ runtime/wtp.h | 2 + tests/daqueue-persist-drvr.sh | 3 + tests/diag.sh | 7 +- 10 files changed, 275 insertions(+), 115 deletions(-) diff --git a/runtime/debug.h b/runtime/debug.h index 1375493d..03702250 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -135,7 +135,7 @@ void dbgPrintAllDebugInfo(void); /* debug aides */ //#ifdef RTINST -#if 0 // temporarily removed for helgrind +#if 1 // temporarily removed for helgrind #define d_pthread_mutex_lock(x) dbgMutexLock(x, pdbgFuncDB, __LINE__, dbgCALLStaCK_POP_POINT ) #define d_pthread_mutex_trylock(x) dbgMutexTryLock(x, pdbgFuncDB, __LINE__, dbgCALLStaCK_POP_POINT ) #define d_pthread_mutex_unlock(x) dbgMutexUnlock(x, pdbgFuncDB, __LINE__, dbgCALLStaCK_POP_POINT ) diff --git a/runtime/queue.c b/runtime/queue.c index 539cf4ec..48d1c6e3 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -66,7 +66,7 @@ DEFobjCurrIf(glbl) /* forward-definitions */ static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates); -static rsRetVal qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); +static rsRetVal SetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); static rsRetVal RateLimiter(qqueue_t *pThis); static int qqueueChkStopWrkrDA(qqueue_t *pThis); static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal); @@ -205,7 +205,7 @@ static inline void queueDrain(qqueue_t *pThis) ASSERT(pThis != NULL); BEGINfunc - dbgoprint((obj_t*) pThis, "queue will lose %d messages, destroying...\n", pThis->iQueueSize); + dbgoprint((obj_t*) pThis, "queue (type %d) will lose %d messages, destroying...\n", pThis->qType, pThis->iQueueSize); /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ while(pThis->iQueueSize-- > 0) { pThis->qDeq(pThis, &pUsr); @@ -366,7 +366,7 @@ qqueueChkIsDA(qqueue_t *pThis) * rgerhards, 2008-01-15 */ static rsRetVal -qqueueStartDA(qqueue_t *pThis) +StartDA(qqueue_t *pThis) { DEFiRet; uchar pszDAQName[128]; @@ -396,7 +396,7 @@ qqueueStartDA(qqueue_t *pThis) CHKiRet(qqueueSetiPersistUpdCnt(pThis->pqDA, pThis->iPersistUpdCnt)); CHKiRet(qqueueSettoActShutdown(pThis->pqDA, pThis->toActShutdown)); CHKiRet(qqueueSettoEnq(pThis->pqDA, pThis->toEnq)); - CHKiRet(qqueueSetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED)); + CHKiRet(SetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED)); CHKiRet(qqueueSetiDeqtWinFromHr(pThis->pqDA, pThis->iDeqtWinFromHr)); CHKiRet(qqueueSetiDeqtWinToHr(pThis->pqDA, pThis->iDeqtWinToHr)); CHKiRet(qqueueSetiHighWtrMrk(pThis->pqDA, 0)); @@ -450,8 +450,8 @@ finalize_it: * If this function fails (should not happen), DA mode is not turned on. * rgerhards, 2008-01-16 */ -static inline rsRetVal -qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) +static rsRetVal +InitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; @@ -464,16 +464,17 @@ qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) * is intentional. We assume that when we need it once, we may also need it on another * occasion. Ressources used are quite minimal when no worker is running. * rgerhards, 2008-01-24 + * NOTE: this is the DA worker *pool*, not the DA queue! */ if(pThis->pWtpDA == NULL) { - lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DA", obj.GetName((obj_t*) pThis)); + lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DAwpool", obj.GetName((obj_t*) pThis)); CHKiRet(wtpConstruct (&pThis->pWtpDA)); CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, wtp_t*)) qqueueIsIdleDA)); CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerDA)); - CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueStartDA)); + CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) StartDA)); CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueTurnOffDAMode)); CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty)); @@ -493,6 +494,7 @@ qqueueInitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) * until the next enqueue request. */ wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* DA queues alsways have just one worker max */ +RUNLOG_VAR("%d", pThis->bRunsDA); finalize_it: END_MTX_PROTECTED_OPERATIONS(pThis->mut); @@ -536,7 +538,7 @@ qqueueChkStrtDA(qqueue_t *pThis) */ dbgoprint((obj_t*) pThis, "%d entries - passed high water mark for disk-assisted mode, initiating...\n", getPhysicalQueueSize(pThis)); - qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, MUTEX_ALREADY_LOCKED); /* initiate DA mode */ + InitDA(pThis, QUEUE_MODE_ENQDEQ, MUTEX_ALREADY_LOCKED); /* initiate DA mode */ } finalize_it: @@ -706,7 +708,6 @@ static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr) if(pThis->tVars.linklist.pDeqRoot == NULL) { pThis->tVars.linklist.pDeqRoot = pEntry; } -RUNLOG_VAR("%p", pThis->tVars.linklist.pDeqRoot); finalize_it: RETiRet; @@ -1157,25 +1158,14 @@ qqueueDeq(qqueue_t *pThis, void *pUsr) } -/* This function shuts down all worker threads and waits until they - * have terminated. If they timeout, they are cancelled. - * rgerhards, 2008-01-24 - * Please note that this function shuts down BOTH the parent AND the child queue - * in DA case. This is necessary because their timeouts are tightly coupled. Most - * importantly, the timeouts would be applied twice (or logic be extremely - * complex) if each would have its own shutdown. The function does not self check - * this condition - the caller must make sure it is not called with a parent. - * rgerhards, 2009-05-26: we do NO logner persist the queue here if bSaveOnShutdown - * is set. This must be handled by the caller. Not doing that cleans up the queue - * shutdown considerably. Also, older engines had a potential hang condition when - * the DA queue was already started and the DA worker configured for infinite - * retries and the action was during retry processing. This was a design issue, - * which is solved as of now. Note that the shutdown now may take a little bit - * longer, because we no longer can persist the queue in parallel to waiting - * on worker timeouts. +/* Try to terminate queue worker threads within the regular shutdown interval. + * Both the regular and DA queue (if it exists) is waited for, but on the same timeout. + * After this function returns, the workers must either be finished or some force + * to finish them must be applied. + * rgerhards, 2009-05-27 */ static rsRetVal -ShutdownWorkers(qqueue_t *pThis) +tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis) { DEFVARS_mutexProtection; struct timespec tTimeout; @@ -1185,16 +1175,6 @@ ShutdownWorkers(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ - dbgoprint((obj_t*) pThis, "initiating worker thread shutdown sequence\n"); - - /* we reduce the low water mark in any case. This is not absolutely necessary, but - * it is useful because we enable DA mode at several spots below and so we do not need - * to think about the low water mark each time. - */ - pThis->iHighWtrMrk = 1; /* if we do not do this, the DA queue will not stop! */ - pThis->iLowWtrMrk = 0; - - /* first try to shutdown the queue within the regular shutdown period */ BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ if(getPhysicalQueueSize(pThis) > 0) { if(pThis->bRunsDA) { @@ -1206,6 +1186,11 @@ ShutdownWorkers(qqueue_t *pThis) } END_MTX_PROTECTED_OPERATIONS(pThis->mut); + /* at this stage, we need to have the DA worker properly initialized and running (if there is one) */ + if(pThis->bRunsDA) { + qqueueWaitDAModeInitialized(pThis); + } + /* Now wait for the queue's workers to shut down. Note that we run into the code even if we just found * out there are no active workers - that doesn't matter: the wtp knows about that and so will * return immediately. @@ -1228,74 +1213,109 @@ ShutdownWorkers(qqueue_t *pThis) if(iRetLocal == RS_RET_TIMED_OUT) { dbgoprint((obj_t*) pThis, "regular shutdown timed out on primary queue (this is OK)\n"); } else { - /* OK, the regular queue is now shut down. So we can now wait for the DA queue (if running DA) */ dbgoprint((obj_t*) pThis, "regular queue workers shut down.\n"); - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(pThis->bRunsDA) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - dbgoprint((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n", - qqueueGetID(pThis->pqDA)); - /* we use the same absolute timeout as above, so we do not use more than the configured - * timeout interval! - */ - dbgoprint((obj_t*) pThis, "trying shutdown of DA workers\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "shutdown timed out on DA queue (this is OK)\n"); - } + } + + /* OK, the worker for the regular queue is processed, on the the DA queue regular worker. */ + BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ + if(pThis->bRunsDA) { + END_MTX_PROTECTED_OPERATIONS(pThis->mut); + dbgoprint((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n", + qqueueGetID(pThis->pqDA)); + /* we use the same absolute timeout as above, so we do not use more than the configured + * timeout interval! + */ + dbgoprint((obj_t*) pThis, "trying shutdown of regular worker of DA queue\n"); + iRetLocal = wtpShutdownAll(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + dbgoprint((obj_t*) pThis, "shutdown timed out on DA queue worker (this is OK)\n"); } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); + dbgoprint((obj_t*) pThis, "DA queue worker shut down.\n"); } + } else { + END_MTX_PROTECTED_OPERATIONS(pThis->mut); } - /* when we reach this point, both queues are either empty or the regular queue shutdown timeout - * has expired. We must set the primary queue to SHUTDOWN_IMMEDIATE, as it shall now terminate - * as soon as its consumer is done. In particular, it does no longer need try to empty the queue. - */ - wtpSetState(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE); /* set primary queue to shutdown only */ + RETiRet; +} - /* at this stage, we need to have the DA worker properly initialized and running (if there is one) */ - if(pThis->bRunsDA) { - qqueueWaitDAModeInitialized(pThis); - wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE); /* also stop DA queue */ + +/* Try to shut down regular and DA queue workers, within the action timeout + * period. Note that the main queue DA worker is still unaffected (and may shuffle + * data to the disk queue while we terminate the other workers). Not finishing + * processing all messages is now OK (but they may be preserved later, depending + * on bSaveOnShutdown setting). + * rgerhards, 2009-05-27 + */ +static rsRetVal +tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis) +{ + DEFVARS_mutexProtection; + struct timespec tTimeout; + rsRetVal iRetLocal; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ + + /* instruct workers to finish ASAP, even if still work exists */ +RUNLOG_STR("setting enqOnly for main queue"); + //TODO:SetEnqOnly(pThis, 1, LOCK_MUTEX); /* start no new workers */ + pThis->bEnqOnly = 1; + wtpSetState(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE); + if(pThis->pqDA != NULL) { +RUNLOG_STR("setting enqOnly for DA queue"); + //TODO:SetEnqOnly(pThis->pqDA, 1, LOCK_MUTEX); + pThis->pqDA->bEnqOnly = 1; + wtpSetState(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE); } /* now give the queue workers a last chance to gracefully shut down (based on action timeout setting) */ + timeoutComp(&tTimeout, pThis->toActShutdown); BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - if(getPhysicalQueueSize(pThis) > 0) { - timeoutComp(&tTimeout, pThis->toActShutdown); - if(wtpGetCurNumWrkr(pThis->pWtpReg, LOCK_MUTEX) > 0) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - dbgoprint((obj_t*) pThis, "trying immediate shutdown of regular workers\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "immediate shutdown timed out on primary queue (this is acceptable and " - "triggers cancellation)\n"); - } else if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the primary queue " - "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); - } - /* we need to re-aquire the mutex for the next check in this case! */ - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ - } - if(pThis->bRunsDA && wtpGetCurNumWrkr(pThis->pWtpDA, LOCK_MUTEX) > 0) { - /* and now the same for the DA queue */ - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - dbgoprint((obj_t*) pThis, "trying immediate shutdown of DA workers\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - dbgoprint((obj_t*) pThis, "immediate shutdown timed out on DA queue (this is acceptable and " - "triggers cancellation)\n"); - } else if(iRetLocal != RS_RET_OK) { - dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the DA queue " - "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); - } - } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); + if(wtpGetCurNumWrkr(pThis->pWtpReg, LOCK_MUTEX) > 0) { + END_MTX_PROTECTED_OPERATIONS(pThis->mut); + dbgoprint((obj_t*) pThis, "trying immediate shutdown of regular workers\n"); + iRetLocal = wtpShutdownAll(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + dbgoprint((obj_t*) pThis, "immediate shutdown timed out on primary queue (this is acceptable and " + "triggers cancellation)\n"); + } else if(iRetLocal != RS_RET_OK) { + dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the primary queue " + "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); } - } else { + /* we need to re-aquire the mutex for the next check in this case! */ + BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); + } + + if(pThis->bRunsDA && wtpGetCurNumWrkr(pThis->pqDA->pWtpReg, LOCK_MUTEX) > 0) { + /* and now the same for the DA queue */ END_MTX_PROTECTED_OPERATIONS(pThis->mut); + dbgoprint((obj_t*) pThis, "trying immediate shutdown of DA queue workers\n"); + iRetLocal = wtpShutdownAll(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + dbgoprint((obj_t*) pThis, "immediate shutdown timed out on DA queue (this is acceptable and " + "triggers cancellation)\n"); + } else if(iRetLocal != RS_RET_OK) { + dbgoprint((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the DA queue " + "in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); + } } + END_MTX_PROTECTED_OPERATIONS(pThis->mut); + + RETiRet; +} + + +/* This function cancels all remenaing regular workers for both the main and the DA + * queue. The main queue's DA worker pool continues to run (if it exists and is active). + * rgerhards, 2009-05-29 + */ +static rsRetVal +cancelWorkers(qqueue_t *pThis) +{ + rsRetVal iRetLocal; + DEFiRet; /* Now queue workers should have terminated. If not, we need to cancel them as we have applied * all timeout setting. If any worker in any queue still executes, its consumer is possibly @@ -1318,13 +1338,60 @@ ShutdownWorkers(qqueue_t *pThis) } } + RETiRet; +} + + +/* This function shuts down all worker threads and waits until they + * have terminated. If they timeout, they are cancelled. + * rgerhards, 2008-01-24 + * Please note that this function shuts down BOTH the parent AND the child queue + * in DA case. This is necessary because their timeouts are tightly coupled. Most + * importantly, the timeouts would be applied twice (or logic be extremely + * complex) if each would have its own shutdown. The function does not self check + * this condition - the caller must make sure it is not called with a parent. + * rgerhards, 2009-05-26: we do NO longer persist the queue here if bSaveOnShutdown + * is set. This must be handled by the caller. Not doing that cleans up the queue + * shutdown considerably. Also, older engines had a potential hang condition when + * the DA queue was already started and the DA worker configured for infinite + * retries and the action was during retry processing. This was a design issue, + * which is solved as of now. Note that the shutdown now may take a little bit + * longer, because we no longer can persist the queue in parallel to waiting + * on worker timeouts. + */ +static rsRetVal +ShutdownWorkers(qqueue_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ + + dbgoprint((obj_t*) pThis, "initiating worker thread shutdown sequence\n"); + + /* we reduce the low water mark in any case. This is not absolutely necessary, but + * it is useful because we enable DA mode at several spots below and so we do not need + * to think about the low water mark each time. + */ + pThis->iHighWtrMrk = 1; /* if we do not do this, the DA queue will not stop! */ + pThis->iLowWtrMrk = 0; + + CHKiRet(tryShutdownWorkersWithinQueueTimeout(pThis)); + + if(getPhysicalQueueSize(pThis) > 0) { + CHKiRet(tryShutdownWorkersWithinActionTimeout(pThis)); + } + + CHKiRet(cancelWorkers(pThis)); + /* ... finally ... all worker threads have terminated :-) * Well, more precisely, they *are in termination*. Some cancel cleanup handlers - * may still be running. + * may still be running. Note that the main queue's DA worker may still be running. */ dbgoprint((obj_t*) pThis, "worker threads terminated, remaining queue size log %d, phys %d.\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); +finalize_it: RETiRet; } @@ -1498,8 +1565,8 @@ DeleteBatchFromQStore(qqueue_t *pThis, batch_t *pBatch) ISOBJ_TYPE_assert(pThis, qqueue); assert(pBatch != NULL); - pTdl = tdlPeek(pThis); - if(pTdl == NULL) { + pTdl = tdlPeek(pThis); /* get current head element */ + if(pTdl == NULL) { /* to-delete list empty */ DoDeleteBatchFromQStore(pThis, pBatch->nElemDeq); } else if(pBatch->deqID == pThis->deqIDDel) { deqIDDel = pThis->deqIDDel; @@ -1512,6 +1579,7 @@ DeleteBatchFromQStore(qqueue_t *pThis, batch_t *pBatch) } } else { /* can not delete, insert into to-delete list */ + dbgprintf("not at head of to-delete list, enqueue %d\n", (int) pBatch->deqID); CHKiRet(tdlAdd(pThis, pBatch->deqID, pBatch->nElemDeq)); } @@ -1769,6 +1837,27 @@ finalize_it: RETiRet; } + +/* This is called when a batch is processed and the worker does not + * ask for another batch (e.g. because it is to be terminated) + * rgerhards, 2009-05-27 + */ +static rsRetVal +batchProcessedReg(qqueue_t *pThis, wti_t *pWti) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + ISOBJ_TYPE_assert(pWti, wti); +dbgprintf("XXX: batchProcessedReg deletes %d records\n", pWti->batch.nElemDeq); + + DeleteProcessedBatch(pThis, &pWti->batch); + qqueueChkPersist(pThis, pWti->batch.nElemDeq); + + RETiRet; +} + + /* This is the queue consumer in the regular (non-DA) case. It is * protected by the queue mutex, but MUST release it as soon as possible. * rgerhards, 2008-01-21 @@ -1844,7 +1933,8 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) DEFiRet; if(pThis->bEnqOnly) { - iRet = RS_RET_TERMINATE_NOW; + iRet = RS_RET_TERMINATE_WHEN_IDLE; +RUNLOG; } else { if(pThis->bRunsDA) { ASSERT(pThis->pqDA != NULL); @@ -1852,11 +1942,14 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) && pThis->pqDA->sizeOnDiskMax > 0 && pThis->pqDA->tVars.disk.sizeOnDisk > pThis->pqDA->sizeOnDiskMax) { /* this queue can never grow, so we can give up... */ +RUNLOG; iRet = RS_RET_TERMINATE_NOW; } else if(getPhysicalQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { +RUNLOG; iRet = RS_RET_TERMINATE_NOW; } } else { +RUNLOG; iRet = RS_RET_TERMINATE_NOW; } } @@ -1880,10 +1973,14 @@ ChkStopWrkrReg(qqueue_t *pThis) return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && getPhysicalQueueSize(pThis) == 0); * TODO: remove when verified! -- rgerhards, 2009-05-26 */ - if(pThis->bEnqOnly || pThis->bRunsDA) +RUNLOG; + if(pThis->bEnqOnly || pThis->bRunsDA) { +RUNLOG; iRet = RS_RET_TERMINATE_NOW; - else if(pThis->pqParent != NULL) + } else if(pThis->pqParent != NULL) { +RUNLOG; iRet = RS_RET_TERMINATE_WHEN_IDLE; + } RETiRet; } @@ -2039,6 +2136,7 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wtp_t*)) IsIdleReg)); CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerReg)); + CHKiRet(wtpSetpfObjProcessed (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessedReg)); CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RegOnWrkrStartup)); CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RegOnWrkrShutdown)); CHKiRet(wtpSetpmutUsr (pThis->pWtpReg, pThis->mut)); @@ -2056,7 +2154,7 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ iRetLocal = qqueueHaveQIF(pThis); if(iRetLocal == RS_RET_OK) { dbgoprint((obj_t*) pThis, "on-disk queue present, needs to be reloaded\n"); - qqueueInitDA(pThis, QUEUE_MODE_ENQDEQ, LOCK_MUTEX); /* initiate DA mode */ + InitDA(pThis, QUEUE_MODE_ENQDEQ, LOCK_MUTEX); /* initiate DA mode */ bInitialized = 1; /* we are done */ } else { /* TODO: use logerror? -- rgerhards, 2008-01-16 */ @@ -2212,9 +2310,16 @@ DoSaveOnShutdown(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); - qqueueInitDA(pThis, QUEUE_MODE_ENQONLY, LOCK_MUTEX); /* switch to DA mode */ +dbgprintf("after InitDA, queue log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); + if(pThis->bRunsDA != 2) { + InitDA(pThis, QUEUE_MODE_ENQONLY, LOCK_MUTEX); /* switch to DA mode */ +dbgprintf("after InitDA, queue log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); +RUNLOG_VAR("%d", pThis->bRunsDA); +RUNLOG_VAR("%d", pThis->pWtpDA->wtpState); + qqueueWaitDAModeInitialized(pThis); /* make sure DA mode is actually started, else we may have a race! */ + } /* make sure we do not timeout before we are done */ - dbgoprint((obj_t*) pThis, "bSaveOnShutdown configured, eternal timeout set\n"); + dbgoprint((obj_t*) pThis, "bSaveOnShutdown configured, infinite timeout set\n"); timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL); /* and run the primary queue's DA worker to drain the queue */ iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); @@ -2247,7 +2352,7 @@ CODESTARTobjDestruct(qqueue) * we need to reset the logical dequeue pointer, persist the queue if configured to do * so and then destruct everything. -- rgerhards, 2009-05-26 */ - CHKiRet(pThis->qUnDeqAll(pThis)); +//!!!! //CHKiRet(pThis->qUnDeqAll(pThis)); if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { CHKiRet(DoSaveOnShutdown(pThis)); @@ -2470,7 +2575,7 @@ finalize_it: * rgerhards, 2008-01-16 */ static rsRetVal -qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex) +SetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; @@ -2495,13 +2600,16 @@ qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex) dbgoprint((obj_t*) pThis, "switching to enqueue-only mode, terminating all worker threads\n"); if(pThis->pWtpReg != NULL) wtpWakeupAllWrkr(pThis->pWtpReg); +RUNLOG; if(pThis->pWtpDA != NULL) wtpWakeupAllWrkr(pThis->pWtpDA); +RUNLOG; } else { /* switch back to regular mode */ ABORT_FINALIZE(RS_RET_NOT_IMPLEMENTED); /* we don't need this so far... */ } } +RUNLOG; pThis->bEnqOnly = bEnqOnly; diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index a43c0327..58346d03 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -89,6 +89,7 @@ typedef struct nsd_gsspi_s nsd_gsspi_t; typedef struct nsd_nss_s nsd_nss_t; typedef struct nsdsel_ptcp_s nsdsel_ptcp_t; typedef struct nsdsel_gtls_s nsdsel_gtls_t; +typedef struct wti_s wti_t; typedef obj_t nsd_t; typedef obj_t nsdsel_t; typedef struct msg msg_t; diff --git a/runtime/srUtils.h b/runtime/srUtils.h index bfce4cbb..288e9dd7 100644 --- a/runtime/srUtils.h +++ b/runtime/srUtils.h @@ -117,11 +117,23 @@ int getSubString(uchar **ppSrc, char *pDst, size_t DstSize, char cSep); if(bMustLock == LOCK_MUTEX) { \ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); \ d_pthread_mutex_lock(mut); \ + assert(bLockedOpIsLocked == 0); \ bLockedOpIsLocked = 1; \ } #define END_MTX_PROTECTED_OPERATIONS(mut) \ if(bLockedOpIsLocked) { \ d_pthread_mutex_unlock(mut); \ + bLockedOpIsLocked = 0; \ pthread_setcancelstate(iCancelStateSave, NULL); \ } +/* The unconditional versions of the macro always lock the mutex. They are preferred in + * complex scenarios, where the simple ones might get mixed up by multiple calls. + */ +#define BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(mut) \ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); \ + d_pthread_mutex_lock(mut); +#define END_MTX_PROTECTED_OPERATIONS_UNCOND(mut) \ + d_pthread_mutex_unlock(mut); \ + pthread_setcancelstate(iCancelStateSave, NULL); + #endif diff --git a/runtime/wti.c b/runtime/wti.c index 3b6bf1b9..75e497b8 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -343,6 +343,7 @@ doIdleProcessing(wti_t *pThis, wtp_t *pWtp, int *pbInactivityTOOccured) { struct timespec t; + BEGINfunc DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); pWtp->pfOnIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED); @@ -356,6 +357,7 @@ doIdleProcessing(wti_t *pThis, wtp_t *pWtp, int *pbInactivityTOOccured) *pbInactivityTOOccured = 1; /* indicate we had a timeout */ } } + ENDfunc } @@ -390,9 +392,12 @@ wtiWorker(wti_t *pThis) dbgSetThrdName(pThis->pszDbgHdr); pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); - BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); +dbgprintf("XXX: worker startup\n"); + RUNLOG_STR("MUTEX lock"); + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); pWtp->pfOnWorkerStartup(pWtp->pUsr); - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); + RUNLOG_STR("MUTEX release"); + END_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); /* now we have our identity, on to real processing */ while(1) { /* loop will be broken below - need to do mutex locks */ @@ -405,17 +410,25 @@ wtiWorker(wti_t *pThis) } wtpSetInactivityGuard(pThis->pWtp, 0, LOCK_MUTEX); /* must be set before usr mutex is locked! */ - BEGIN_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr, LOCK_MUTEX); + RUNLOG_STR("MUTEX lock"); + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); /* first check if we are in shutdown process (but evaluate a bit later) */ +RUNLOG; terminateRet = wtpChkStopWrkr(pWtp, LOCK_MUTEX, MUTEX_ALREADY_LOCKED); +RUNLOG_VAR("%d", terminateRet); if(terminateRet == RS_RET_TERMINATE_NOW) { - // TODO: we need to free the old batch! -- rgerhards, 2009-05-26 MULTI + /* we now need to free the old batch */ + localRet = pWtp->pfObjProcessed(pWtp->pUsr, pThis); + dbgoprint((obj_t*) pThis, "terminating worker because auf TERMINATE_NOW mode, del iRet %d\n", + localRet); break; } +RUNLOG; /* try to execute and process whatever we have */ localRet = pWtp->pfDoWork(pWtp->pUsr, pThis, iCancelStateSave); + /* This function must and does RELEASE the MUTEX! */ if(localRet == RS_RET_IDLE) { if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE) { @@ -426,17 +439,20 @@ wtiWorker(wti_t *pThis) /* we had an inactivity timeout in the last run and are still idle, so it is time to exit... */ break; /* end worker thread run */ } + RUNLOG_STR("MUTEX lock"); + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); doIdleProcessing(pThis, pWtp, &bInactivityTOOccured); - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); + RUNLOG_STR("MUTEX release"); + END_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); continue; /* request next iteration */ } - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); bInactivityTOOccured = 0; /* reset for next run */ } /* if we exit the loop, the mutex is locked and must be unlocked */ - END_MTX_PROTECTED_OPERATIONS(pWtp->pmutUsr); + RUNLOG_STR("MUTEX release"); + END_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); /* indicate termination */ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); diff --git a/runtime/wti.h b/runtime/wti.h index 0990941e..17038ec5 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -31,7 +31,7 @@ /* the worker thread instance class */ -typedef struct wti_s { +struct wti_s { BEGINobjInstance; int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ pthread_t thrdID; /* thread ID */ @@ -42,7 +42,7 @@ typedef struct wti_s { int bShutdownRqtd; /* shutdown for this thread requested? 0 - no , 1 - yes */ batch_t batch; /* pointer to an object array meaningful for current user pointer (e.g. queue pUsr data elemt) */ uchar *pszDbgHdr; /* header string for debug messages */ -} wti_t; +}; /* some symbolic constants for easier reference */ diff --git a/runtime/wtp.c b/runtime/wtp.c index 41fcd8d9..7786a656 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -91,10 +91,12 @@ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! pThis->pfGetDeqBatchSize = NotImplementedDummy; pThis->pfIsIdle = NotImplementedDummy; pThis->pfDoWork = NotImplementedDummy; + pThis->pfObjProcessed = NotImplementedDummy; pThis->pfOnIdle = NotImplementedDummy; pThis->pfOnWorkerCancel = NotImplementedDummy; pThis->pfOnWorkerStartup = NotImplementedDummy; pThis->pfOnWorkerShutdown = NotImplementedDummy; +dbgprintf("XXX: wtpConstruct: %d\n", pThis->wtpState); ENDobjConstruct(wtp) @@ -139,6 +141,7 @@ BEGINobjDestruct(wtp) /* be sure to specify the object type also in END and CODE int i; CODESTARTobjDestruct(wtp) wtpProcessThrdChanges(pThis); /* process thread changes one last time */ +RUNLOG_STR("wtpDestruct"); /* destruct workers */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) @@ -260,17 +263,22 @@ wtpChkStopWrkr(wtp_t *pThis, int bLockMutex, int bLockUsrMutex) ISOBJ_TYPE_assert(pThis, wtp); +RUNLOG; BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); if(pThis->wtpState == wtpState_SHUTDOWN_IMMEDIATE) { +RUNLOG; ABORT_FINALIZE(RS_RET_TERMINATE_NOW); } else if(pThis->wtpState == wtpState_SHUTDOWN) { ABORT_FINALIZE(RS_RET_TERMINATE_WHEN_IDLE); +RUNLOG; } +RUNLOG_VAR("%d", iRet); /* try customer handler if one was set and we do not yet have a definite result */ if(pThis->pfChkStopWrkr != NULL) { iRet = pThis->pfChkStopWrkr(pThis->pUsr, bLockUsrMutex); } +RUNLOG_VAR("%d", iRet); finalize_it: END_MTX_PROTECTED_OPERATIONS(&pThis->mut); @@ -292,13 +300,17 @@ wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout ISOBJ_TYPE_assert(pThis, wtp); +dbgprintf("XXX:10 wtp %p, state %d\n", pThis, pThis->wtpState); wtpSetState(pThis, tShutdownCmd); +dbgprintf("XXX:20 wtp %p, state %d\n", pThis, pThis->wtpState); wtpWakeupAllWrkr(pThis); +dbgprintf("XXX:30 wtp %p, state %d\n", pThis, pThis->wtpState); /* see if we need to harvest (join) any terminated threads (even in timeout case, * some may have terminated... */ wtpProcessThrdChanges(pThis); +dbgprintf("XXX:40 wtp %p, state %d\n", pThis, pThis->wtpState); /* and wait for their termination */ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); @@ -306,7 +318,9 @@ wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout pthread_cleanup_push(mutexCancelCleanup, &pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); bTimedOut = 0; +dbgprintf("XXX:50 wtp %p, state %d\n", pThis, pThis->wtpState); while(pThis->iCurNumWrkThrd > 0 && !bTimedOut) { +dbgprintf("XXX:60 wtp %p, state %d\n", pThis, pThis->wtpState); dbgprintf("%s: waiting %ldms on worker thread termination, %d still running\n", wtpGetDbgHdr(pThis), timeoutVal(ptTimeout), pThis->iCurNumWrkThrd); @@ -581,6 +595,7 @@ DEFpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)) DEFpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)) DEFpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, wtp_t*)) DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)) +DEFpropSetMethFP(wtp, pfObjProcessed, rsRetVal(*pVal)(void*, wti_t*)) DEFpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)) DEFpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*, void*)) DEFpropSetMethFP(wtp, pfOnWorkerStartup, rsRetVal(*pVal)(void*)) diff --git a/runtime/wtp.h b/runtime/wtp.h index d9d582af..e1180177 100644 --- a/runtime/wtp.h +++ b/runtime/wtp.h @@ -72,6 +72,7 @@ struct wtp_s { pthread_cond_t *pcondBusy; /* condition the user will signal "busy again, keep runing" on (awakes worker) */ rsRetVal (*pfChkStopWrkr)(void *pUsr, int); rsRetVal (*pfGetDeqBatchSize)(void *pUsr, int*); /* obtains max dequeue count from queue config */ + rsRetVal (*pfObjProcessed)(void *pUsr, wti_t *pWti); /* indicate user object is processed */ rsRetVal (*pfRateLimiter)(void *pUsr); rsRetVal (*pfIsIdle)(void *pUsr, wtp_t *pWtp); rsRetVal (*pfDoWork)(void *pUsr, void *pWti, int); @@ -108,6 +109,7 @@ PROTOTYPEpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)); PROTOTYPEpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)); PROTOTYPEpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, wtp_t*)); PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*, int)); +PROTOTYPEpropSetMethFP(wtp, pfObjProcessed, rsRetVal(*pVal)(void*, wti_t*)); PROTOTYPEpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)); PROTOTYPEpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*,void*)); PROTOTYPEpropSetMethFP(wtp, pfOnWorkerStartup, rsRetVal(*pVal)(void*)); diff --git a/tests/daqueue-persist-drvr.sh b/tests/daqueue-persist-drvr.sh index 11058110..0ec76b47 100755 --- a/tests/daqueue-persist-drvr.sh +++ b/tests/daqueue-persist-drvr.sh @@ -19,6 +19,9 @@ $srcdir/diag.sh shutdown-immediate $srcdir/diag.sh wait-shutdown source $srcdir/diag.sh check-mainq-spool +echo DEBUG EXIT! +#exit + # restart engine and have rest processed #remove delay echo "#" > work-delay.conf diff --git a/tests/diag.sh b/tests/diag.sh index 01b335ee..5c6a0ab8 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -5,8 +5,11 @@ # not always able to convey back states to the upper-level test driver # begun 2009-05-27 by rgerhards # This file is part of the rsyslog project, released under GPLv3 +#valgrind="valgrind --log-fd=1" +#valgrind="valgrind --tool=drd --log-fd=1" +#valgrind="valgrind --tool=helgrind --log-fd=1" #set -o xtrace -export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUG="debug Xnostdout printmutexaction" export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason @@ -22,7 +25,7 @@ case $1 in ;; 'startup') # start rsyslogd with default params. $2 is the config file name to use # returns only after successful startup - valgrind ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & + $valgrind ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & $srcdir/diag.sh wait-startup ;; 'wait-startup') # wait for rsyslogd startup -- cgit v1.2.3 From 9517e19b6427c295e206ece9562ce70f4a6d7044 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 28 May 2009 09:59:45 +0200 Subject: preserving current changes ... in preparation for some larger changes - I need to apply some serious design changes, as the current system does not play well at all with ultra-reliable queues. Will do that in a totally new version. --- plugins/imdiag/imdiag.c | 4 ++++ runtime/queue.c | 35 +++++++++++++++++++++++------------ runtime/wti.c | 13 ++----------- runtime/wtp.c | 6 ------ tests/diag.sh | 2 +- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 03396db7..cb1c1686 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -256,10 +256,13 @@ static rsRetVal waitMainQEmpty(tcps_sess_t *pSess) { int iMsgQueueSize; + int iPrint = 0; DEFiRet; CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); while(iMsgQueueSize > 0) { + if(iPrint++ % 500 == 0) + dbgprintf("imdiag sleeping, wait mainq drain, curr size %d\n", iMsgQueueSize); srSleep(0,2); /* wait a little bit */ CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); } @@ -294,6 +297,7 @@ OnMsgReceived(tcps_sess_t *pSess, uchar *pRcv, int iLenMsg) getFirstWord(&pszMsg, cmdBuf, sizeof(cmdBuf)/sizeof(uchar), TO_LOWERCASE); + dbgprintf("imdiag received command '%s'\n", cmdBuf); if(!ustrcmp(cmdBuf, UCHAR_CONSTANT("getmainmsgqueuesize"))) { CHKiRet(diagGetMainMsgQSize(&iMsgQueueSize)); CHKiRet(sendResponse(pSess, "%d\n", iMsgQueueSize)); diff --git a/runtime/queue.c b/runtime/queue.c index 48d1c6e3..4405dd39 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -255,7 +255,11 @@ static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) /* wait until we have a fully initialized DA queue. Sometimes, we need to - * sync with it, as we expect it for some function. + * sync with it, as we expect it for some function. Note that in extreme + * cases, the DA queue may already have started up AND terminated when we + * call this function. As such,it may validly be that DA is already shut down. + * So we just check if we are in init phase and then wait for full startup. + * If in non-DA mode, we silently return. * rgerhards, 2008-02-27 */ static rsRetVal @@ -264,9 +268,8 @@ qqueueWaitDAModeInitialized(qqueue_t *pThis) DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); - ASSERT(pThis->bRunsDA); - while(pThis->bRunsDA != 2) { + while(pThis->bRunsDA == 1) { d_pthread_cond_wait(&pThis->condDAReady, pThis->mut); } @@ -289,13 +292,16 @@ qqueueTurnOffDAMode(qqueue_t *pThis) { DEFiRet; +RUNLOG_STR("XXX: TurnOffDAMode\n"); ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->bRunsDA); /* at this point, we need a fully initialized DA queue. So if it isn't, we finally need * to wait for its startup... -- rgerhards, 2008-01-25 */ +RUNLOG; qqueueWaitDAModeInitialized(pThis); +RUNLOG; /* if we need to pull any data that we still need from the (child) disk queue, * now would be the time to do so. At present, we do not need this, but I'd like to @@ -309,6 +315,7 @@ qqueueTurnOffDAMode(qqueue_t *pThis) * during the lifetime of DA-mode, depending on how often the DA worker receives an * inactivity timeout. -- rgerhards, 2008-01-25 */ +dbgprintf("XXX: getLogicalQueueSize(pThis->pqDA): %d\n", getLogicalQueueSize(pThis->pqDA)); if(getLogicalQueueSize(pThis->pqDA) == 0) { pThis->bRunsDA = 0; /* tell the world we are back in non-DA mode */ /* we destruct the queue object, which will also shutdown the queue worker. As the queue is empty, @@ -1151,8 +1158,8 @@ qqueueDeq(qqueue_t *pThis, void *pUsr) iRet = pThis->qDeq(pThis, pUsr); ATOMIC_INC(pThis->nLogDeq); - dbgoprint((obj_t*) pThis, "entry deleted, size now log %d, phys %d entries\n", - getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); +// dbgoprint((obj_t*) pThis, "entry deleted, size now log %d, phys %d entries\n", +// getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); RETiRet; } @@ -1259,13 +1266,12 @@ tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis) ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ /* instruct workers to finish ASAP, even if still work exists */ -RUNLOG_STR("setting enqOnly for main queue"); - //TODO:SetEnqOnly(pThis, 1, LOCK_MUTEX); /* start no new workers */ + /* note that we modify bEnqOnly direclty, because going through the method would + * startup some workers again. So this is OK here. -- rgerhards, 2009-05-28 + */ pThis->bEnqOnly = 1; wtpSetState(pThis->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE); if(pThis->pqDA != NULL) { -RUNLOG_STR("setting enqOnly for DA queue"); - //TODO:SetEnqOnly(pThis->pqDA, 1, LOCK_MUTEX); pThis->pqDA->bEnqOnly = 1; wtpSetState(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE); } @@ -1644,7 +1650,7 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz nDequeued = nDiscarded = 0; while((iQueueSize = getLogicalQueueSize(pThis)) > 0 && nDequeued < pThis->iDeqBatchSize) { -dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); +//dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); CHKiRet(qqueueDeq(pThis, &pUsr)); /* check if we should discard this element */ @@ -1945,7 +1951,7 @@ RUNLOG; RUNLOG; iRet = RS_RET_TERMINATE_NOW; } else if(getPhysicalQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { -RUNLOG; +dbgprintf("XXX: queue size %d, high water mark %d\n", getPhysicalQueueSize(pThis), pThis->iHighWtrMrk); iRet = RS_RET_TERMINATE_NOW; } } else { @@ -1973,7 +1979,6 @@ ChkStopWrkrReg(qqueue_t *pThis) return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && getPhysicalQueueSize(pThis) == 0); * TODO: remove when verified! -- rgerhards, 2009-05-26 */ -RUNLOG; if(pThis->bEnqOnly || pThis->bRunsDA) { RUNLOG; iRet = RS_RET_TERMINATE_NOW; @@ -2314,6 +2319,7 @@ dbgprintf("after InitDA, queue log %d, phys %d\n", getLogicalQueueSize(pThis), g if(pThis->bRunsDA != 2) { InitDA(pThis, QUEUE_MODE_ENQONLY, LOCK_MUTEX); /* switch to DA mode */ dbgprintf("after InitDA, queue log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); +//!!! TODO !!!das passiert wohl, wenn die queue empty wird! (aber es vorher noch nciht war) RUNLOG_VAR("%d", pThis->bRunsDA); RUNLOG_VAR("%d", pThis->pWtpDA->wtpState); qqueueWaitDAModeInitialized(pThis); /* make sure DA mode is actually started, else we may have a race! */ @@ -2322,6 +2328,7 @@ RUNLOG_VAR("%d", pThis->pWtpDA->wtpState); dbgoprint((obj_t*) pThis, "bSaveOnShutdown configured, infinite timeout set\n"); timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL); /* and run the primary queue's DA worker to drain the queue */ +RUNLOG; iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); dbgoprint((obj_t*) pThis, "end queue persistence run, iRet %d, queue size log %d, phys %d\n", iRetLocal, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); @@ -2339,6 +2346,7 @@ BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and C CODESTARTobjDestruct(qqueue) pThis->bQueueInDestruction = 1; /* indicate we are in destruction (modifies some behaviour) */ +RUNLOG_STR("XXX: queue destruct\n"); /* shut down all workers * We do not need to shutdown workers when we are in enqueue-only mode or we are a * direct queue - because in both cases we have none... ;) @@ -2353,6 +2361,9 @@ CODESTARTobjDestruct(qqueue) * so and then destruct everything. -- rgerhards, 2009-05-26 */ //!!!! //CHKiRet(pThis->qUnDeqAll(pThis)); +dbgprintf("XXX: pre unDeq disk log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); + CHKiRet(pThis->qUnDeqAll(pThis)); +dbgprintf("XXX: post unDeq disk log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { CHKiRet(DoSaveOnShutdown(pThis)); diff --git a/runtime/wti.c b/runtime/wti.c index 75e497b8..2fb5eea2 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -392,11 +392,8 @@ wtiWorker(wti_t *pThis) dbgSetThrdName(pThis->pszDbgHdr); pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); -dbgprintf("XXX: worker startup\n"); - RUNLOG_STR("MUTEX lock"); BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); pWtp->pfOnWorkerStartup(pWtp->pUsr); - RUNLOG_STR("MUTEX release"); END_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); /* now we have our identity, on to real processing */ @@ -410,21 +407,17 @@ dbgprintf("XXX: worker startup\n"); } wtpSetInactivityGuard(pThis->pWtp, 0, LOCK_MUTEX); /* must be set before usr mutex is locked! */ - RUNLOG_STR("MUTEX lock"); BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); /* first check if we are in shutdown process (but evaluate a bit later) */ -RUNLOG; terminateRet = wtpChkStopWrkr(pWtp, LOCK_MUTEX, MUTEX_ALREADY_LOCKED); -RUNLOG_VAR("%d", terminateRet); if(terminateRet == RS_RET_TERMINATE_NOW) { /* we now need to free the old batch */ localRet = pWtp->pfObjProcessed(pWtp->pUsr, pThis); - dbgoprint((obj_t*) pThis, "terminating worker because auf TERMINATE_NOW mode, del iRet %d\n", + dbgoprint((obj_t*) pThis, "terminating worker because of TERMINATE_NOW mode, del iRet %d\n", localRet); break; } -RUNLOG; /* try to execute and process whatever we have */ localRet = pWtp->pfDoWork(pWtp->pUsr, pThis, iCancelStateSave); @@ -439,10 +432,8 @@ RUNLOG; /* we had an inactivity timeout in the last run and are still idle, so it is time to exit... */ break; /* end worker thread run */ } - RUNLOG_STR("MUTEX lock"); BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); doIdleProcessing(pThis, pWtp, &bInactivityTOOccured); - RUNLOG_STR("MUTEX release"); END_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); continue; /* request next iteration */ } @@ -451,7 +442,6 @@ RUNLOG; } /* if we exit the loop, the mutex is locked and must be unlocked */ - RUNLOG_STR("MUTEX release"); END_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); /* indicate termination */ @@ -459,6 +449,7 @@ RUNLOG; d_pthread_mutex_lock(&pThis->mut); pthread_cleanup_pop(0); /* remove cleanup handler */ +RUNLOG_STR("XXX: Worker shutdown"); pWtp->pfOnWorkerShutdown(pWtp->pUsr); wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); diff --git a/runtime/wtp.c b/runtime/wtp.c index 7786a656..a5836da3 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -141,7 +141,6 @@ BEGINobjDestruct(wtp) /* be sure to specify the object type also in END and CODE int i; CODESTARTobjDestruct(wtp) wtpProcessThrdChanges(pThis); /* process thread changes one last time */ -RUNLOG_STR("wtpDestruct"); /* destruct workers */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) @@ -263,22 +262,17 @@ wtpChkStopWrkr(wtp_t *pThis, int bLockMutex, int bLockUsrMutex) ISOBJ_TYPE_assert(pThis, wtp); -RUNLOG; BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); if(pThis->wtpState == wtpState_SHUTDOWN_IMMEDIATE) { -RUNLOG; ABORT_FINALIZE(RS_RET_TERMINATE_NOW); } else if(pThis->wtpState == wtpState_SHUTDOWN) { ABORT_FINALIZE(RS_RET_TERMINATE_WHEN_IDLE); -RUNLOG; } -RUNLOG_VAR("%d", iRet); /* try customer handler if one was set and we do not yet have a definite result */ if(pThis->pfChkStopWrkr != NULL) { iRet = pThis->pfChkStopWrkr(pThis->pUsr, bLockUsrMutex); } -RUNLOG_VAR("%d", iRet); finalize_it: END_MTX_PROTECTED_OPERATIONS(&pThis->mut); diff --git a/tests/diag.sh b/tests/diag.sh index 5c6a0ab8..8bf6b129 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -9,7 +9,7 @@ #valgrind="valgrind --tool=drd --log-fd=1" #valgrind="valgrind --tool=helgrind --log-fd=1" #set -o xtrace -#export RSYSLOG_DEBUG="debug Xnostdout printmutexaction" +export RSYSLOG_DEBUG="debug nostdout printmutexaction" export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason -- cgit v1.2.3 From 7405a3057003bab0361a111b0f9d013b881b6db0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 28 May 2009 10:03:14 +0200 Subject: begun rsyslog v5 branch (for ultra-reliable queue) --- ChangeLog | 2 ++ configure.ac | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 1c5742f4..877a987e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +--------------------------------------------------------------------------- +Version 5.1.1 [DEVEL] (rgerhards), 2009-??-?? - added $MainMsgQueueDequeueBatchSize and $ActionQueueDequeueBatchSize configuration directives - implemented a new transactional output module interface which provides diff --git a/configure.ac b/configure.ac index 66f1cab1..fc8cdc36 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.3.1],[rsyslog@lists.adiscon.com]) +rC_INIT([rsyslog],[5.1.1],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) -- cgit v1.2.3 From fc3e56941ca6dbf401bee2f9dc0f9e4c5cd87f40 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 28 May 2009 11:57:30 +0200 Subject: fixing an issue during DA mode queue shutdown also changed DA queue mode in that the regular workers now run concurrently. --- configure.ac | 2 +- runtime/queue.c | 108 +++++++++++++++++++++----------------------------- runtime/wti.c | 1 + tests/chkseq.c | 3 +- tests/da-mainmsg-q.sh | 8 ++-- tests/diag.sh | 4 +- 6 files changed, 57 insertions(+), 69 deletions(-) diff --git a/configure.ac b/configure.ac index fc8cdc36..d8d17a96 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -rC_INIT([rsyslog],[5.1.1],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.1.1],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/runtime/queue.c b/runtime/queue.c index 4405dd39..698495ef 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -72,6 +72,7 @@ static int qqueueChkStopWrkrDA(qqueue_t *pThis); static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal); static int qqueueIsIdleDA(qqueue_t *pThis); static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave); +static rsRetVal batchProcessed(qqueue_t *pThis, wti_t *pWti); /* some constants for queuePersist () */ #define QUEUE_CHECKPOINT 1 @@ -240,14 +241,15 @@ static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) if(getLogicalQueueSize(pThis) >= pThis->iHighWtrMrk || pThis->bQueueStarted == 0) { wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ } + } + /* regular workers always run */ + if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { + iMaxWorkers = 1; } else { - if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { - iMaxWorkers = 1; - } else { - iMaxWorkers = getLogicalQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; - } - wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); /* disk queues have always one worker */ + iMaxWorkers = getLogicalQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; } +dbgprintf("YYY: wtp advise max reg workers %d\n", iMaxWorkers); + wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); /* disk queues have always one worker */ } RETiRet; @@ -288,7 +290,7 @@ qqueueWaitDAModeInitialized(qqueue_t *pThis) * rgerhards, 2008-01-15 */ static rsRetVal -qqueueTurnOffDAMode(qqueue_t *pThis) +TurnOffDAMode(qqueue_t *pThis) { DEFiRet; @@ -299,9 +301,7 @@ RUNLOG_STR("XXX: TurnOffDAMode\n"); /* at this point, we need a fully initialized DA queue. So if it isn't, we finally need * to wait for its startup... -- rgerhards, 2008-01-25 */ -RUNLOG; - qqueueWaitDAModeInitialized(pThis); -RUNLOG; + //TODO: MULTI del, can not happen (but verify first) qqueueWaitDAModeInitialized(pThis); /* if we need to pull any data that we still need from the (child) disk queue, * now would be the time to do so. At present, we do not need this, but I'd like to @@ -321,16 +321,13 @@ dbgprintf("XXX: getLogicalQueueSize(pThis->pqDA): %d\n", getLogicalQueueSize(pTh /* we destruct the queue object, which will also shutdown the queue worker. As the queue is empty, * this will be quick. */ - qqueueDestruct(&pThis->pqDA); /* and now we are ready to destruct the DA queue */ +//XXX: TODO qqueueDestruct(&pThis->pqDA); /* and now we are ready to destruct the DA queue */ dbgoprint((obj_t*) pThis, "disk-assistance has been turned off, disk queue was empty (iRet %d)\n", iRet); - /* now we need to check if the regular queue has some messages. This may be the case - * when it is waiting that the high water mark is reached again. If so, we need to start up - * a regular worker. -- rgerhards, 2008-01-26 - */ - if(getLogicalQueueSize(pThis) > 0) { - qqueueAdviseMaxWorkers(pThis); - } + } else { + /* the queue has data again! */ + dbgprintf("DA queue has data during shutdown, restarting...\n"); + qqueueAdviseMaxWorkers(pThis->pqDA); } RETiRet; @@ -408,6 +405,10 @@ StartDA(qqueue_t *pThis) CHKiRet(qqueueSetiDeqtWinToHr(pThis->pqDA, pThis->iDeqtWinToHr)); CHKiRet(qqueueSetiHighWtrMrk(pThis->pqDA, 0)); CHKiRet(qqueueSetiDiscardMrk(pThis->pqDA, 0)); + + // experimental: XXX + CHKiRet(qqueueSettoWrkShutdown(pThis->pqDA, 0)); + if(pThis->toQShutdown == 0) { CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 0)); /* if the user really wants... */ } else { @@ -423,14 +424,6 @@ StartDA(qqueue_t *pThis) if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) FINALIZE; /* something is wrong */ - /* as we are right now starting DA mode because we are so busy, it is - * extremely unlikely that any regular worker is sleeping on empty queue. HOWEVER, - * we want to be on the safe side, and so we awake anyone that is waiting - * on one. So even if the scheduler plays badly with us, things should be - * quite well. -- rgerhards, 2008-01-15 - */ - wtpWakeupWrkr(pThis->pWtpReg); /* awake all workers, but not ourselves ;) */ - pThis->bRunsDA = 2; /* we are now in DA mode, but not fully initialized */ pThis->bChildIsDone = 0;/* set to 1 when child's worker detect queue is finished */ pthread_cond_broadcast(&pThis->condDAReady); /* signal we are now initialized and ready to go ;) */ @@ -481,8 +474,9 @@ InitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, wtp_t*)) qqueueIsIdleDA)); CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerDA)); + CHKiRet(wtpSetpfObjProcessed (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed)); CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) StartDA)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) qqueueTurnOffDAMode)); + CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) TurnOffDAMode)); CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty)); CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1)); @@ -500,8 +494,7 @@ InitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) * that will also start one up. If we forgot that step, everything would be stalled * until the next enqueue request. */ - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* DA queues alsways have just one worker max */ -RUNLOG_VAR("%d", pThis->bRunsDA); + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* DA queues always have just one worker max */ finalize_it: END_MTX_PROTECTED_OPERATIONS(pThis->mut); @@ -516,7 +509,7 @@ finalize_it: * rgerhards, 2008-01-14 */ static inline rsRetVal -qqueueChkStrtDA(qqueue_t *pThis) +ChkStrtDA(qqueue_t *pThis) { DEFiRet; @@ -1538,7 +1531,7 @@ DoDeleteBatchFromQStore(qqueue_t *pThis, int nElem) ISOBJ_TYPE_assert(pThis, qqueue); -dbgprintf("pre delete batch from store, new sizes: log %d, phys %d, nElem %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis), nElem); +//dbgprintf("pre delete batch from store, new sizes: log %d, phys %d, nElem %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis), nElem); /* now send delete request to storage driver */ for(i = 0 ; i < nElem ; ++i) { pThis->qDel(pThis); @@ -1849,13 +1842,13 @@ finalize_it: * rgerhards, 2009-05-27 */ static rsRetVal -batchProcessedReg(qqueue_t *pThis, wti_t *pWti) +batchProcessed(qqueue_t *pThis, wti_t *pWti) { DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); -dbgprintf("XXX: batchProcessedReg deletes %d records\n", pWti->batch.nElemDeq); +dbgprintf("XXX: batchProcessed deletes %d records\n", pWti->batch.nElemDeq); DeleteProcessedBatch(pThis, &pWti->batch); qqueueChkPersist(pThis, pWti->batch.nElemDeq); @@ -1940,7 +1933,6 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) if(pThis->bEnqOnly) { iRet = RS_RET_TERMINATE_WHEN_IDLE; -RUNLOG; } else { if(pThis->bRunsDA) { ASSERT(pThis->pqDA != NULL); @@ -1948,15 +1940,17 @@ RUNLOG; && pThis->pqDA->sizeOnDiskMax > 0 && pThis->pqDA->tVars.disk.sizeOnDisk > pThis->pqDA->sizeOnDiskMax) { /* this queue can never grow, so we can give up... */ -RUNLOG; iRet = RS_RET_TERMINATE_NOW; } else if(getPhysicalQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { -dbgprintf("XXX: queue size %d, high water mark %d\n", getPhysicalQueueSize(pThis), pThis->iHighWtrMrk); +dbgprintf("XXX: terminate_NOW DA worker: queue size %d, high water mark %d\n", getPhysicalQueueSize(pThis), pThis->iHighWtrMrk); iRet = RS_RET_TERMINATE_NOW; +RUNLOG_STR("XXX: re-start reg worker"); +qqueueAdviseMaxWorkers(pThis); +RUNLOG_STR("XXX: done re-start reg worker"); } } else { -RUNLOG; - iRet = RS_RET_TERMINATE_NOW; + // experimental iRet = RS_RET_TERMINATE_NOW; + ; } } @@ -1979,11 +1973,11 @@ ChkStopWrkrReg(qqueue_t *pThis) return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && getPhysicalQueueSize(pThis) == 0); * TODO: remove when verified! -- rgerhards, 2009-05-26 */ - if(pThis->bEnqOnly || pThis->bRunsDA) { -RUNLOG; + // TODO: DEL - we now keep the workers running! if(pThis->bEnqOnly || pThis->bRunsDA) { + if(pThis->bEnqOnly) { +dbgprintf("XXX: terminate_NOW queue:Reg worker: enqOnly! queue size %d\n", getPhysicalQueueSize(pThis)); iRet = RS_RET_TERMINATE_NOW; } else if(pThis->pqParent != NULL) { -RUNLOG; iRet = RS_RET_TERMINATE_WHEN_IDLE; } @@ -2000,35 +1994,27 @@ GetDeqBatchSize(qqueue_t *pThis, int *pVal) DEFiRet; assert(pVal != NULL); *pVal = pThis->iDeqBatchSize; +if(pThis->pqParent != NULL) + *pVal = 16; RETiRet; } /* must only be called when the queue mutex is locked, else results - * are not stable! DA queue version + * are not stable! DA worker version (pThis *is* the *main* queue, not DA!) */ static int qqueueIsIdleDA(qqueue_t *pThis) { - /* remember: iQueueSize is the DA queue size, not the main queue! */ - /* TODO: I think we need just a single function for DA and non-DA mode - but I leave it for now as is */ - return(getPhysicalQueueSize(pThis) == 0 || (pThis->bRunsDA && getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk)); + return(getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk); } /* must only be called when the queue mutex is locked, else results - * are not stable! Regular queue version + * are not stable! Regular worker version. */ static int IsIdleReg(qqueue_t *pThis) { -#if 0 /* enable for performance testing */ - int ret; - ret = getLogicalQueueSize(pThis) == 0 || (pThis->bRunsDA && getLogicalQueueSize(pThis) <= pThis->iLowWtrMrk); - if(ret) fprintf(stderr, "queue is idle\n"); - return ret; -#else - /* regular code! */ - return(getPhysicalQueueSize(pThis) == 0 || (pThis->bRunsDA && getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk)); -#endif + return(getPhysicalQueueSize(pThis) == 0); } @@ -2084,7 +2070,8 @@ RegOnWrkrStartup(qqueue_t *pThis) /* start up the queue - it must have been constructed and parameters defined * before. */ -rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ +rsRetVal +qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; rsRetVal iRetLocal; @@ -2141,7 +2128,7 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wtp_t*)) IsIdleReg)); CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti, int)) ConsumerReg)); - CHKiRet(wtpSetpfObjProcessed (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessedReg)); + CHKiRet(wtpSetpfObjProcessed (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed)); CHKiRet(wtpSetpfOnWorkerStartup (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RegOnWrkrStartup)); CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RegOnWrkrShutdown)); CHKiRet(wtpSetpmutUsr (pThis->pWtpReg, pThis->mut)); @@ -2168,7 +2155,7 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ } } - if(!bInitialized) { + if(Debug && !bInitialized) { dbgoprint((obj_t*) pThis, "queue starts up without (loading) any DA disk state (this is normal for the DA " "queue itself!)\n"); } @@ -2507,7 +2494,7 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* then check if we need to add an assistance disk queue */ if(pThis->bIsDA) - CHKiRet(qqueueChkStrtDA(pThis)); + CHKiRet(ChkStrtDA(pThis)); /* handle flow control * There are two different flow control mechanisms: basic and advanced flow control. @@ -2611,16 +2598,13 @@ SetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex) dbgoprint((obj_t*) pThis, "switching to enqueue-only mode, terminating all worker threads\n"); if(pThis->pWtpReg != NULL) wtpWakeupAllWrkr(pThis->pWtpReg); -RUNLOG; if(pThis->pWtpDA != NULL) wtpWakeupAllWrkr(pThis->pWtpDA); -RUNLOG; } else { /* switch back to regular mode */ ABORT_FINALIZE(RS_RET_NOT_IMPLEMENTED); /* we don't need this so far... */ } } -RUNLOG; pThis->bEnqOnly = bEnqOnly; diff --git a/runtime/wti.c b/runtime/wti.c index 2fb5eea2..9428cd47 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -392,6 +392,7 @@ wtiWorker(wti_t *pThis) dbgSetThrdName(pThis->pszDbgHdr); pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); + pThis->batch.nElemDeq = 0; /* re-init dequeue count */ BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); pWtp->pfOnWorkerStartup(pWtp->pUsr); END_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); diff --git a/tests/chkseq.c b/tests/chkseq.c index 5ffe855c..8c5fc61a 100644 --- a/tests/chkseq.c +++ b/tests/chkseq.c @@ -79,7 +79,8 @@ int main(int argc, char *argv[]) /* read file */ fp = fopen(file, "r"); if(fp == NULL) { - perror(argv[1]); + printf("error opening file '%s'\n", file); + perror(file); exit(1); } diff --git a/tests/da-mainmsg-q.sh b/tests/da-mainmsg-q.sh index 1c947de2..25e7fd95 100755 --- a/tests/da-mainmsg-q.sh +++ b/tests/da-mainmsg-q.sh @@ -17,16 +17,18 @@ source $srcdir/diag.sh injectmsg 0 50 source $srcdir/diag.sh wait-queueempty # let queue drain for this test case # part 2: send bunch of messages. This should trigger DA mode -source $srcdir/diag.sh injectmsg 50 20000 +#source $srcdir/diag.sh injectmsg 50 20000 +source $srcdir/diag.sh injectmsg 50 2000 ls -l test-spool # for manual review # send another handful -source $srcdir/diag.sh injectmsg 20050 50 +source $srcdir/diag.sh injectmsg 2050 50 #sleep 1 # we need this so that rsyslogd can receive all outstanding messages # clean up and check test result source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages ### currently, we get a stable abort if we use the former kill logic. With shutdown-when-empty, it hangs (but that still tells us there is a bug ;)) ### #kill `cat rsyslog.pid` -source $srcdir/diag.sh seq-check 0 20099 +echo seqchk? +source $srcdir/diag.sh seq-check 2099 source $srcdir/diag.sh exit diff --git a/tests/diag.sh b/tests/diag.sh index 8bf6b129..a34f3ede 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -9,8 +9,8 @@ #valgrind="valgrind --tool=drd --log-fd=1" #valgrind="valgrind --tool=helgrind --log-fd=1" #set -o xtrace -export RSYSLOG_DEBUG="debug nostdout printmutexaction" -export RSYSLOG_DEBUGLOG="log" +#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" +#export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason rm -f core.* vgcore.* # do NOT delete them at exit ;) -- cgit v1.2.3 From 13d4a23e92996e24d6a833ca75d06428c5387aa4 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 28 May 2009 14:24:37 +0200 Subject: some more fixes for queue engine The enhanced testbench now runs without failures, again --- runtime/datetime.c | 2 ++ runtime/queue.c | 67 ++++++++++++++++++++++++------------------- runtime/wti.c | 2 +- runtime/wtp.c | 6 ---- tests/DiagTalker.java | 1 + tests/da-mainmsg-q.sh | 3 -- tests/daqueue-persist-drvr.sh | 1 - tests/diag.sh | 2 +- 8 files changed, 42 insertions(+), 42 deletions(-) diff --git a/runtime/datetime.c b/runtime/datetime.c index 19e61a0a..2a0df91a 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -121,6 +121,8 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) t->OffsetMode = '+'; t->OffsetHour = lBias / 3600; t->OffsetMinute = lBias % 3600; + + t->timeType = 0; /* this is new and may cause format errors -- rgerhards, 2009-05-28 */ } diff --git a/runtime/queue.c b/runtime/queue.c index 698495ef..57385056 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -54,6 +54,7 @@ #include "wtp.h" #include "wti.h" #include "atomic.h" +#include "msg.h" /* TODO: remove one we removed MsgAddRef() call */ #ifdef OS_SOLARIS # include @@ -248,7 +249,6 @@ static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) } else { iMaxWorkers = getLogicalQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; } -dbgprintf("YYY: wtp advise max reg workers %d\n", iMaxWorkers); wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); /* disk queues have always one worker */ } @@ -294,7 +294,6 @@ TurnOffDAMode(qqueue_t *pThis) { DEFiRet; -RUNLOG_STR("XXX: TurnOffDAMode\n"); ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->bRunsDA); @@ -720,6 +719,7 @@ static rsRetVal qDeqLinkedList(qqueue_t *pThis, obj_t **ppUsr) DEFiRet; pEntry = pThis->tVars.linklist.pDeqRoot; + ISOBJ_TYPE_assert(pEntry->pUsr, msg); *ppUsr = pEntry->pUsr; pThis->tVars.linklist.pDeqRoot = pEntry->pNext; @@ -1137,7 +1137,7 @@ finalize_it: /* generic code to dequeue a queue entry */ static rsRetVal -qqueueDeq(qqueue_t *pThis, void *pUsr) +qqueueDeq(qqueue_t *pThis, void **ppUsr) { DEFiRet; @@ -1148,7 +1148,7 @@ qqueueDeq(qqueue_t *pThis, void *pUsr) * If we decrement, however, we may lose a message. But that is better than * losing the whole process because it loops... -- rgerhards, 2008-01-03 */ - iRet = pThis->qDeq(pThis, pUsr); + iRet = pThis->qDeq(pThis, ppUsr); ATOMIC_INC(pThis->nLogDeq); // dbgoprint((obj_t*) pThis, "entry deleted, size now log %d, phys %d entries\n", @@ -1162,6 +1162,8 @@ qqueueDeq(qqueue_t *pThis, void *pUsr) * Both the regular and DA queue (if it exists) is waited for, but on the same timeout. * After this function returns, the workers must either be finished or some force * to finish them must be applied. + * This function also instructs the DA worker pool (if it exists) to terminate. This is done + * in preparation of final queue shutdown. * rgerhards, 2009-05-27 */ static rsRetVal @@ -1175,7 +1177,7 @@ tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(pThis->mut); /* some workers may be running in parallel! */ if(getPhysicalQueueSize(pThis) > 0) { if(pThis->bRunsDA) { /* We may have waited on the low water mark. As it may have changed, we @@ -1184,7 +1186,7 @@ tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis) wtpAdviseMaxWorkers(pThis->pWtpDA, 1); } } - END_MTX_PROTECTED_OPERATIONS(pThis->mut); + END_MTX_PROTECTED_OPERATIONS_UNCOND(pThis->mut); /* at this stage, we need to have the DA worker properly initialized and running (if there is one) */ if(pThis->bRunsDA) { @@ -1217,9 +1219,7 @@ tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis) } /* OK, the worker for the regular queue is processed, on the the DA queue regular worker. */ - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, LOCK_MUTEX); /* some workers may be running in parallel! */ if(pThis->bRunsDA) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); dbgoprint((obj_t*) pThis, "we have a DA queue (0x%lx), requesting its shutdown.\n", qqueueGetID(pThis->pqDA)); /* we use the same absolute timeout as above, so we do not use more than the configured @@ -1232,8 +1232,16 @@ tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis) } else { dbgoprint((obj_t*) pThis, "DA queue worker shut down.\n"); } - } else { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); + /* we also instruct the DA worker pool to shutdown ASAP. If we need it for persisting + * the queue, it is restarted at a later stage. We don't care here if a timeout happens. + */ + dbgoprint((obj_t*) pThis, "trying shutdown of regular worker of DA queue\n"); + iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); + if(iRetLocal == RS_RET_TIMED_OUT) { + dbgoprint((obj_t*) pThis, "shutdown timed out on main queue DA worker pool (this is OK)\n"); + } else { + dbgoprint((obj_t*) pThis, "main queue DA worker pool shut down.\n"); + } } RETiRet; @@ -1335,6 +1343,14 @@ cancelWorkers(qqueue_t *pThis) dbgoprint((obj_t*) pThis, "unexpected iRet state %d trying to cancel DA queue worker " "threads, continuing, but results are unpredictable\n", iRetLocal); } + + /* finally, we cancel the main queue's DA worker pool, if it still is running. It may be + * restarted later to persist the queue. But we stop it, because otherwise we get into + * big trouble when resetting the logical dequeue pointer. This operation can only be + * done when *no* worker is running. So time for a shutdown... -- rgerhards, 2009-05-28 + */ + dbgoprint((obj_t*) pThis, "checking to see if we need to cancel the main queue's DA worker pool\n"); + iRetLocal = wtpCancelAll(pThis->pWtpDA); /* returns immediately if all threads already have terminated */ } RETiRet; @@ -1600,23 +1616,15 @@ DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch) ISOBJ_TYPE_assert(pThis, qqueue); assert(pBatch != NULL); -// TODO: ULTRA: lock qaueue mutex if instructed to do so - /* if the queue runs in DA mode, the DA worker already deleted the in-memory representation - * of the message. But in regular mode, we need to do it ourselfs. We differentiate between - * the two cases, because it is actually the easiest way to handle the destruct-Problem in - * a simple and pUsrp-Type agnostic way (else we would need an objAddRef() generic function). - */ - if(!pThis->bRunsDA) { - for(i = 0 ; i < pBatch->nElem ; ++i) { - pUsr = pBatch->pElem[i].pUsrp; - objDestruct(pUsr); - } + for(i = 0 ; i < pBatch->nElem ; ++i) { + pUsr = pBatch->pElem[i].pUsrp; + objDestruct(pUsr); } iRet = DeleteBatchFromQStore(pThis, pBatch); - pBatch->nElem = 0; /* reset batch */ + pBatch->nElem = pBatch->nElemDeq = 0; /* reset batch */ RETiRet; } @@ -1908,8 +1916,13 @@ ConsumerDA(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) CHKiRet(DequeueForConsumer(pThis, pWti, iCancelStateSave)); /* iterate over returned results and enqueue them in DA queue */ - for(i = 0 ; i < pWti->batch.nElem ; i++) - CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, pWti->batch.pElem[i].pUsrp)); + for(i = 0 ; i < pWti->batch.nElem ; i++) { + /* TODO: we must add a generic "addRef" mechanism, because the disk queue enqueue destructs + * the message. So far, we simply assume we always have msg_t, what currently is always the case. + * rgerhards, 2009-05-28 + */ + CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, (obj_t*)MsgAddRef((msg_t*)(pWti->batch.pElem[i].pUsrp)))); + } finalize_it: dbgoprint((obj_t*) pThis, "DAConsumer returns with iRet %d\n", iRet); @@ -2306,16 +2319,12 @@ dbgprintf("after InitDA, queue log %d, phys %d\n", getLogicalQueueSize(pThis), g if(pThis->bRunsDA != 2) { InitDA(pThis, QUEUE_MODE_ENQONLY, LOCK_MUTEX); /* switch to DA mode */ dbgprintf("after InitDA, queue log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); -//!!! TODO !!!das passiert wohl, wenn die queue empty wird! (aber es vorher noch nciht war) -RUNLOG_VAR("%d", pThis->bRunsDA); -RUNLOG_VAR("%d", pThis->pWtpDA->wtpState); qqueueWaitDAModeInitialized(pThis); /* make sure DA mode is actually started, else we may have a race! */ } /* make sure we do not timeout before we are done */ dbgoprint((obj_t*) pThis, "bSaveOnShutdown configured, infinite timeout set\n"); timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL); /* and run the primary queue's DA worker to drain the queue */ -RUNLOG; iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); dbgoprint((obj_t*) pThis, "end queue persistence run, iRet %d, queue size log %d, phys %d\n", iRetLocal, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); @@ -2333,7 +2342,6 @@ BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and C CODESTARTobjDestruct(qqueue) pThis->bQueueInDestruction = 1; /* indicate we are in destruction (modifies some behaviour) */ -RUNLOG_STR("XXX: queue destruct\n"); /* shut down all workers * We do not need to shutdown workers when we are in enqueue-only mode or we are a * direct queue - because in both cases we have none... ;) @@ -2347,7 +2355,6 @@ RUNLOG_STR("XXX: queue destruct\n"); * we need to reset the logical dequeue pointer, persist the queue if configured to do * so and then destruct everything. -- rgerhards, 2009-05-26 */ -//!!!! //CHKiRet(pThis->qUnDeqAll(pThis)); dbgprintf("XXX: pre unDeq disk log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); CHKiRet(pThis->qUnDeqAll(pThis)); dbgprintf("XXX: post unDeq disk log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); diff --git a/runtime/wti.c b/runtime/wti.c index 9428cd47..465dc3e1 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -392,7 +392,7 @@ wtiWorker(wti_t *pThis) dbgSetThrdName(pThis->pszDbgHdr); pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); - pThis->batch.nElemDeq = 0; /* re-init dequeue count */ + // TODO: if we have a problem, enable again! pThis->batch.nElemDeq = 0; /* re-init dequeue count */ BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); pWtp->pfOnWorkerStartup(pWtp->pUsr); END_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); diff --git a/runtime/wtp.c b/runtime/wtp.c index a5836da3..6b39793b 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -294,17 +294,13 @@ wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout ISOBJ_TYPE_assert(pThis, wtp); -dbgprintf("XXX:10 wtp %p, state %d\n", pThis, pThis->wtpState); wtpSetState(pThis, tShutdownCmd); -dbgprintf("XXX:20 wtp %p, state %d\n", pThis, pThis->wtpState); wtpWakeupAllWrkr(pThis); -dbgprintf("XXX:30 wtp %p, state %d\n", pThis, pThis->wtpState); /* see if we need to harvest (join) any terminated threads (even in timeout case, * some may have terminated... */ wtpProcessThrdChanges(pThis); -dbgprintf("XXX:40 wtp %p, state %d\n", pThis, pThis->wtpState); /* and wait for their termination */ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); @@ -312,9 +308,7 @@ dbgprintf("XXX:40 wtp %p, state %d\n", pThis, pThis->wtpState); pthread_cleanup_push(mutexCancelCleanup, &pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); bTimedOut = 0; -dbgprintf("XXX:50 wtp %p, state %d\n", pThis, pThis->wtpState); while(pThis->iCurNumWrkThrd > 0 && !bTimedOut) { -dbgprintf("XXX:60 wtp %p, state %d\n", pThis, pThis->wtpState); dbgprintf("%s: waiting %ldms on worker thread termination, %d still running\n", wtpGetDbgHdr(pThis), timeoutVal(ptTimeout), pThis->iCurNumWrkThrd); diff --git a/tests/DiagTalker.java b/tests/DiagTalker.java index 85a6671e..04e12327 100644 --- a/tests/DiagTalker.java +++ b/tests/DiagTalker.java @@ -34,6 +34,7 @@ public class DiagTalker { try { diagSocket = new Socket(host, port); + diagSocket.setSoTimeout(0); /* wait for lenghty operations */ out = new PrintWriter(diagSocket.getOutputStream(), true); in = new BufferedReader(new InputStreamReader( diagSocket.getInputStream())); diff --git a/tests/da-mainmsg-q.sh b/tests/da-mainmsg-q.sh index 25e7fd95..6ec2f3a9 100755 --- a/tests/da-mainmsg-q.sh +++ b/tests/da-mainmsg-q.sh @@ -27,8 +27,5 @@ source $srcdir/diag.sh injectmsg 2050 50 # clean up and check test result source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages -### currently, we get a stable abort if we use the former kill logic. With shutdown-when-empty, it hangs (but that still tells us there is a bug ;)) ### -#kill `cat rsyslog.pid` -echo seqchk? source $srcdir/diag.sh seq-check 2099 source $srcdir/diag.sh exit diff --git a/tests/daqueue-persist-drvr.sh b/tests/daqueue-persist-drvr.sh index 0ec76b47..7b6ec6dd 100755 --- a/tests/daqueue-persist-drvr.sh +++ b/tests/daqueue-persist-drvr.sh @@ -19,7 +19,6 @@ $srcdir/diag.sh shutdown-immediate $srcdir/diag.sh wait-shutdown source $srcdir/diag.sh check-mainq-spool -echo DEBUG EXIT! #exit # restart engine and have rest processed diff --git a/tests/diag.sh b/tests/diag.sh index a34f3ede..ce323143 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -9,7 +9,7 @@ #valgrind="valgrind --tool=drd --log-fd=1" #valgrind="valgrind --tool=helgrind --log-fd=1" #set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" +#export RSYSLOG_DEBUG="debug nostdout printmutexaction" #export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason -- cgit v1.2.3 From 5514acb7eeaca41f1fd196402ef08efc55bce11c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 28 May 2009 15:19:03 +0200 Subject: fixed make distcheck --- tests/Makefile.am | 1 + tests/diag.sh | 5 +++-- tests/testsuites/diskqueue.conf | 2 +- tests/testsuites/imtcp-multiport.conf | 2 +- tests/testsuites/manytcp.conf | 2 +- tests/testsuites/queue-persist.conf | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 713c721e..7a7b8147 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -49,6 +49,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ diag.sh \ + testsuites/diag-common.conf \ queue-persist.sh \ queue-persist-drvr.sh \ testsuites/queue-persist.conf \ diff --git a/tests/diag.sh b/tests/diag.sh index 270b30d6..ec2c1190 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -7,15 +7,16 @@ # This file is part of the rsyslog project, released under GPLv3 #set -o xtrace #export RSYSLOG_DEBUG="debug nostdout" -#export RSYSLOG_DEBUGLOG="tmp" +#export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason + cp $srcdir/testsuites/diag-common.conf diag-common.conf rm -f rsyslogd.started work-*.conf rm -f work rsyslog.out.log rsyslog.out.log.save # common work files rm -rf test-spool mkdir test-spool ;; - 'exit') rm -f rsyslogd.started work-*.conf + 'exit') rm -f rsyslogd.started work-*.conf diag-common.conf rm -f work rsyslog.out.log rsyslog.out.log.save # common work files rm -rf test-spool ;; diff --git a/tests/testsuites/diskqueue.conf b/tests/testsuites/diskqueue.conf index d7f323bc..a992c5a5 100644 --- a/tests/testsuites/diskqueue.conf +++ b/tests/testsuites/diskqueue.conf @@ -1,6 +1,6 @@ # Test for queue disk mode (see .sh file for details) # rgerhards, 2009-04-17 -$IncludeConfig testsuites/diag-common.conf +$IncludeConfig diag-common.conf $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 diff --git a/tests/testsuites/imtcp-multiport.conf b/tests/testsuites/imtcp-multiport.conf index 9146f6e0..ccdc15fb 100644 --- a/tests/testsuites/imtcp-multiport.conf +++ b/tests/testsuites/imtcp-multiport.conf @@ -1,6 +1,6 @@ # Test for queue disk mode (see .sh file for details) # rgerhards, 2009-05-22 -$IncludeConfig testsuites/diag-common.conf +$IncludeConfig diag-common.conf $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 diff --git a/tests/testsuites/manytcp.conf b/tests/testsuites/manytcp.conf index 772ec9ce..eb9db257 100644 --- a/tests/testsuites/manytcp.conf +++ b/tests/testsuites/manytcp.conf @@ -1,6 +1,6 @@ # Test for tcp "flood" testing # rgerhards, 2009-04-08 -$IncludeConfig testsuites/diag-common.conf +$IncludeConfig diag-common.conf $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 diff --git a/tests/testsuites/queue-persist.conf b/tests/testsuites/queue-persist.conf index 80f8ba30..81ee1be5 100644 --- a/tests/testsuites/queue-persist.conf +++ b/tests/testsuites/queue-persist.conf @@ -1,6 +1,6 @@ # Test for persisting messages on shutdown # rgerhards, 2009-04-17 -$IncludeConfig testsuites/diag-common.conf +$IncludeConfig diag-common.conf $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 1 -- cgit v1.2.3 From cbbf1586a204a897a050b6cb294b120e3ff5f23c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 28 May 2009 15:57:39 +0200 Subject: some cleanup & fix make distcheck --- runtime/debug.h | 4 ++-- runtime/queue.c | 17 +++-------------- tests/Makefile.am | 2 +- tests/testsuites/arrayqueue.conf | 2 +- tests/testsuites/da-mainmsg-q.conf | 2 +- tests/testsuites/linkedlistqueue.conf | 2 +- 6 files changed, 9 insertions(+), 20 deletions(-) diff --git a/runtime/debug.h b/runtime/debug.h index 03702250..8b66d784 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -134,8 +134,8 @@ void dbgPrintAllDebugInfo(void); /* debug aides */ -//#ifdef RTINST -#if 1 // temporarily removed for helgrind +#ifdef RTINST +//#if 0 // temporarily removed for helgrind #define d_pthread_mutex_lock(x) dbgMutexLock(x, pdbgFuncDB, __LINE__, dbgCALLStaCK_POP_POINT ) #define d_pthread_mutex_trylock(x) dbgMutexTryLock(x, pdbgFuncDB, __LINE__, dbgCALLStaCK_POP_POINT ) #define d_pthread_mutex_unlock(x) dbgMutexUnlock(x, pdbgFuncDB, __LINE__, dbgCALLStaCK_POP_POINT ) diff --git a/runtime/queue.c b/runtime/queue.c index 57385056..23d60ddc 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -54,7 +54,7 @@ #include "wtp.h" #include "wti.h" #include "atomic.h" -#include "msg.h" /* TODO: remove one we removed MsgAddRef() call */ +#include "msg.h" /* TODO: remove once we remove MsgAddRef() call */ #ifdef OS_SOLARIS # include @@ -297,11 +297,6 @@ TurnOffDAMode(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->bRunsDA); - /* at this point, we need a fully initialized DA queue. So if it isn't, we finally need - * to wait for its startup... -- rgerhards, 2008-01-25 - */ - //TODO: MULTI del, can not happen (but verify first) qqueueWaitDAModeInitialized(pThis); - /* if we need to pull any data that we still need from the (child) disk queue, * now would be the time to do so. At present, we do not need this, but I'd like to * keep that comment if future need arises. @@ -865,8 +860,7 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) CHKiRet(obj.Deserialize(&pThis->tVars.disk.pReadDel, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); - /* we now need to take care of the Deq handle. It is not persisted, so we can create - * a virgin copy based on pReadDel. // TODO duplicat code, same as blow - single function! + /* create a duplicate for the read "pointer". */ CHKiRet(strmDup(pThis->tVars.disk.pReadDel, &pThis->tVars.disk.pReadDeq)); @@ -1712,7 +1706,7 @@ DequeueConsumable(qqueue_t *pThis, wti_t *pWti, int iCancelStateSave) pthread_cond_broadcast(&pThis->belowLightDlyWtrMrk); } - // TODO: MULTI: check physical queue size! + // TODO: MULTI: check physical queue size? pthread_cond_signal(&pThis->notFull); d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); @@ -1982,11 +1976,6 @@ static rsRetVal ChkStopWrkrReg(qqueue_t *pThis) { DEFiRet; - /* original condition - return pThis->bEnqOnly || pThis->bRunsDA || (pThis->pqParent != NULL && getPhysicalQueueSize(pThis) == 0); - * TODO: remove when verified! -- rgerhards, 2009-05-26 - */ - // TODO: DEL - we now keep the workers running! if(pThis->bEnqOnly || pThis->bRunsDA) { if(pThis->bEnqOnly) { dbgprintf("XXX: terminate_NOW queue:Reg worker: enqOnly! queue size %d\n", getPhysicalQueueSize(pThis)); iRet = RS_RET_TERMINATE_NOW; diff --git a/tests/Makefile.am b/tests/Makefile.am index 14d15247..a0723c9d 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -43,7 +43,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/rfc5424-4.parse1 \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ - testsuites/1.field1-if-array \ + testsuites/1.field1 \ killrsyslog.sh \ parsertest.sh \ fieldtest.sh \ diff --git a/tests/testsuites/arrayqueue.conf b/tests/testsuites/arrayqueue.conf index 44b9920b..c5874a83 100644 --- a/tests/testsuites/arrayqueue.conf +++ b/tests/testsuites/arrayqueue.conf @@ -1,6 +1,6 @@ # Test for queue fixedArray mode (see .sh file for details) # rgerhards, 2009-04-17 -$IncludeConfig testsuites/diag-common.conf +$IncludeConfig diag-common.conf $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 diff --git a/tests/testsuites/da-mainmsg-q.conf b/tests/testsuites/da-mainmsg-q.conf index c1e8cb01..843a3e4f 100644 --- a/tests/testsuites/da-mainmsg-q.conf +++ b/tests/testsuites/da-mainmsg-q.conf @@ -4,7 +4,7 @@ $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 $InputTCPServerRun 13514 -$IncludeConfig testsuites/diag-common.conf +$IncludeConfig diag-common.conf # set spool locations and switch queue to disk assisted mode $WorkDirectory test-spool diff --git a/tests/testsuites/linkedlistqueue.conf b/tests/testsuites/linkedlistqueue.conf index 321678a5..92a9649c 100644 --- a/tests/testsuites/linkedlistqueue.conf +++ b/tests/testsuites/linkedlistqueue.conf @@ -1,6 +1,6 @@ # Test for queue LinkedList mode (see .sh file for details) # rgerhards, 2009-04-17 -$IncludeConfig testsuites/diag-common.conf +$IncludeConfig diag-common.conf $ModLoad ../plugins/imtcp/.libs/imtcp $MainMsgQueueTimeoutShutdown 10000 -- cgit v1.2.3 From c6a2dafb2fc5d186ef146035f270a4819075eab1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 28 May 2009 16:30:57 +0200 Subject: doc: added info on when $OptimizeForUniprocessor was made available --- doc/rsyslog_conf_global.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 43eacc43..fe76912d 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -196,7 +196,7 @@ large enough for the whole message. (Introduced with 4.1.5). Once set, it affect

    • $ResetConfigVariables
    • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The -default may change as uniprocessor systems become less common.
      • +default may change as uniprocessor systems become less common. [available since 4.1.0]
    • $PreserveFQDN [on/off) - if set to off (legacy default to remain compatible to sysklogd), the domain part from a name that is within the same domain as the receiving system is stripped. If set to on, full names are always used.
      • -- cgit v1.2.3 From cd6f19ab7ab3356dd545d0bfc7e338d5f5f9c9d8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 29 May 2009 11:40:07 +0200 Subject: preparing for 5.1.0 release --- ChangeLog | 27 ++++++++++++++++++++++++++- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 834f9391..43293400 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,30 @@ --------------------------------------------------------------------------- -Version 5.1.1 [DEVEL] (rgerhards), 2009-??-?? +Version 5.1.0 [DEVEL] (rgerhards), 2009-05-29 + +*********************************** NOTE ********************************** +The v5 versions of rsyslog feature a greatly redesigned queue engine. The +major theme for the v5 release is twofold: + +a) greatly improved performance +b) enable audit-grade processing + +Here, audit-grade processing means that rsyslog, if used together with +audit-grade transports and configured correctly, will never lose messages +that already have been acknowledged, not even in fatal failure cases like +sudden loss of power. + +Note that large parts of rsyslog's important core components have been +restructured to support these design goals. As such, early versions of +the engine will probably be less stable than the v3/v4 engine. + +Also note that the initial versions do not cover all and everything. As +usual, the code will evolve toward the final goal as version numbers +increase. +*********************************** NOTE ********************************** + +- redesigned queue engine so that it supports ultra-reliable operations + This resulted in a rewrite of large parts. The new capability can be + used to build audit-grade systems on the basis of rsyslog. - added $MainMsgQueueDequeueBatchSize and $ActionQueueDequeueBatchSize configuration directives - implemented a new transactional output module interface which provides diff --git a/configure.ac b/configure.ac index 13999395..ffa1f230 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[5.1.1],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[5.1.0],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/manual.html b/doc/manual.html index adf89587..81180f39 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

      Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

      -

      This documentation is for version 4.3.1 (devel branch) of rsyslog. +

      This documentation is for version 5.1.0 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

      If you like rsyslog, you might -- cgit v1.2.3 From 8e180e152ab1cd332a212648d6a0f2e2583a2bca Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 29 May 2009 12:06:59 +0200 Subject: added forgotten file --- doc/rsyslog-vers.png | Bin 0 -> 162637 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 doc/rsyslog-vers.png diff --git a/doc/rsyslog-vers.png b/doc/rsyslog-vers.png new file mode 100644 index 00000000..e8ec8b81 Binary files /dev/null and b/doc/rsyslog-vers.png differ -- cgit v1.2.3 From eafafc63597a600fceda5ac9ada90acd6e2f037e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 29 May 2009 14:31:50 +0200 Subject: fix: import from beta accidently disable imdiag --- plugins/imdiag/imdiag.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 129241fa..c700cab7 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -1,5 +1,3 @@ -#warning "imdiag is NOT supported in this version of rsyslog" -#if 0 /* imdiag.c * This is a diagnostics module, primarily meant for troubleshooting * and information about the runtime state of rsyslog. It is implemented @@ -459,7 +457,6 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit -#endif /* vim:set ai: -- cgit v1.2.3 From 36fe8d92f8bad4ddc1f24ede14405129ec46c0d8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 29 May 2009 17:25:16 +0200 Subject: added ability for the UDP output action to rebind its send socket after sending n messages New config directive $ActionSendUDPRebindInterval added for the purpose. By default, rebinding is disabled. This is considered useful for load balancers. --- ChangeLog | 6 +++- doc/rsyslog_conf_global.html | 3 ++ runtime/conf.c | 3 +- tools/omfwd.c | 69 ++++++++++++++++++++++++++++++++------------ 4 files changed, 60 insertions(+), 21 deletions(-) diff --git a/ChangeLog b/ChangeLog index 931f21f8..832d7a6c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ --------------------------------------------------------------------------- -Version 4.3.? [DEVEL] (rgerhards), 2009-??-?? +Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +- added ability for the UDP output action to rebind its send socket after + sending n messages. New config directive $ActionSendUDPRebindInterval + added for the purpose. By default, rebinding is disabled. This is + considered useful for load balancers. - bugfix: imdiag/imtcp had a race condition - improved testbench (now much better code design and reuse) --------------------------------------------------------------------------- diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index fe76912d..778e18f8 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -96,6 +96,9 @@ default 60000 (1 minute)] (driver-specific)

    • $ActionSendStreamDriverAuthMode <mode>,  authentication mode to use with the stream driver (driver-specific)
    • $ActionSendStreamDriverPermittedPeer <ID>,  accepted fingerprint (SHA1) or name of remote peer (driver-specific) - directive may go away!
      • +
    • $ActionSendUDPRebindInterval nbr- [available since 4.3.2] - instructs the UDP send +action to rebind the send socket every nbr of messages sent. Zero, the default, means +that no rebind is done. This directive is useful for use with load-balancers.
    • $AllowedSender
    • $ControlCharacterEscapePrefix
    • $DebugPrintCFSyslineHandlerList
      • diff --git a/runtime/conf.c b/runtime/conf.c index 602a5eb1..bb1fa282 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -484,7 +484,8 @@ finalize_it: } if(bHadAnError && (iRet == RS_RET_OK)) { /* a bit dirty, enhance in future releases */ - iRet = RS_RET_ERR; +// TODO: fix regression! + // iRet = RS_RET_ERR; } RETiRet; } diff --git a/tools/omfwd.c b/tools/omfwd.c index 88a382e0..c8fedfc9 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -82,12 +82,14 @@ typedef struct _instanceData { permittedPeers_t *pPermPeers; int iStrmDrvrMode; char *f_hname; - int *pSockArray; /* sockets to use for UDP */ + int *pSockArray; /* sockets to use for UDP */ int bIsConnected; /* are we connected to remote host? 0 - no, 1 - yes, UDP means addr resolved */ struct addrinfo *f_addr; - int compressionLevel; /* 0 - no compression, else level for zlib */ + int compressionLevel; /* 0 - no compression, else level for zlib */ char *port; int protocol; + int iUDPRebindInterval; /* rebind interval */ + int nXmit; /* number of transmissions since last (re-)bind */ # define FORW_UDP 0 # define FORW_TCP 1 /* following fields for TCP-based delivery */ @@ -100,9 +102,31 @@ static uchar *pszStrmDrvr = NULL; /* name of the stream driver to use */ static short iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mostly means plain tcp) */ static short bResendLastOnRecon = 0; /* should the last message be re-sent on a successful reconnect? */ static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ +static int iUDPRebindInterval = 0; /* support for automatic re-binding (load balancers!). 0 - no rebind */ static permittedPeers_t *pPermPeers = NULL; +static rsRetVal doTryResume(instanceData *pData); + +/* Close the UDP sockets. + * rgerhards, 2009-05-29 + */ +static rsRetVal +closeUDPSockets(instanceData *pData) +{ + DEFiRet; + assert(pData != NULL); + if(pData->pSockArray != NULL) { + net.closeUDPListenSockets(pData->pSockArray); + pData->pSockArray = NULL; + freeaddrinfo(pData->f_addr); + pData->f_addr = NULL; + } +pData->bIsConnected = 0; // TODO: remove this variable altogether + RETiRet; +} + + /* get the syslog forward port from selector_t. The passed in * struct must be one that is setup for forwarding. * rgerhards, 2007-06-28 @@ -148,30 +172,19 @@ ENDisCompatibleWithFeature BEGINfreeInstance CODESTARTfreeInstance - if(pData->f_addr != NULL) { /* TODO: is the check ok? */ - freeaddrinfo(pData->f_addr); - pData->f_addr = NULL; - } - if(pData->port != NULL) - free(pData->port); - /* final cleanup */ DestructTCPInstanceData(pData); - if(pData->pSockArray != NULL) - net.closeUDPListenSockets(pData->pSockArray); + closeUDPSockets(pData); if(pData->protocol == FORW_TCP) { tcpclt.Destruct(&pData->pTCPClt); } - if(pData->f_hname != NULL) - free(pData->f_hname); - if(pData->pszStrmDrvr != NULL) - free(pData->pszStrmDrvr); - if(pData->pszStrmDrvrAuthMode != NULL) - free(pData->pszStrmDrvrAuthMode); - if(pData->pPermPeers != NULL) - net.DestructPermittedPeers(&pData->pPermPeers); + free(pData->port); + free(pData->f_hname); + free(pData->pszStrmDrvr); + free(pData->pszStrmDrvrAuthMode); + net.DestructPermittedPeers(&pData->pPermPeers); ENDfreeInstance @@ -192,6 +205,18 @@ static rsRetVal UDPSend(instanceData *pData, char *msg, size_t len) unsigned lsent = 0; int bSendSuccess; +dbgprintf("rebind logic: interval %d, curr %d, mod %d, if %d\n", pData->iUDPRebindInterval, pData->nXmit, + (pData->nXmit % pData->iUDPRebindInterval), ((pData->nXmit % pData->iUDPRebindInterval) == 0)); + if(pData->iUDPRebindInterval && (pData->nXmit++ % pData->iUDPRebindInterval == 0)) { + dbgprintf("omfwd dropping UDP 'connection' (as configured)\n"); + pData->nXmit = 1; /* else we have an addtl wrap at 2^31-1 */ + CHKiRet(closeUDPSockets(pData)); + } + + if(pData->pSockArray == NULL) { + CHKiRet(doTryResume(pData)); + } + if(pData->pSockArray != NULL) { /* we need to track if we have success sending to the remote * peer. Success is indicated by at least one sendto() call @@ -224,6 +249,7 @@ static rsRetVal UDPSend(instanceData *pData, char *msg, size_t len) } } +finalize_it: RETiRet; } @@ -616,6 +642,9 @@ CODE_STD_STRING_REQUESTparseSelectorAct(1) CHKmalloc(pData->f_hname = strdup((char*) q)); } + /* copy over config data as needed */ + pData->iUDPRebindInterval = iUDPRebindInterval; + /* process template */ CHKiRet(cflineParseTemplateName(&p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (pszTplName == NULL) ? (uchar*)"RSYSLOG_TraditionalForwardFormat" : pszTplName)); @@ -699,6 +728,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a /* we now must reset all non-string values */ iStrmDrvrMode = 0; bResendLastOnRecon = 0; + iUDPRebindInterval = 0; return RS_RET_OK; } @@ -713,6 +743,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(net,LM_NET_FILENAME)); CHKiRet(regCfSysLineHdlr((uchar *)"actionforwarddefaulttemplate", 0, eCmdHdlrGetWord, NULL, &pszTplName, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionsendudprebindinterval", 0, eCmdHdlrInt, NULL, &iUDPRebindInterval, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionsendstreamdriver", 0, eCmdHdlrGetWord, NULL, &pszStrmDrvr, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionsendstreamdrivermode", 0, eCmdHdlrInt, NULL, &iStrmDrvrMode, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionsendstreamdriverauthmode", 0, eCmdHdlrGetWord, NULL, &pszStrmDrvrAuthMode, NULL)); -- cgit v1.2.3 From 25daa04902f2754a487973c4de707869d66b84c1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 2 Jun 2009 07:44:07 +0200 Subject: added support for custom modules ... this provides some basic support to integrate extensions that are not direct parts of rsyslog to be built during its build process. --- Makefile.am | 4 ++++ configure.ac | 16 ++++++++++++++++ plugins/cust1/Makefile.am | 6 ++++++ 3 files changed, 26 insertions(+) create mode 100644 plugins/cust1/Makefile.am diff --git a/Makefile.am b/Makefile.am index e991f323..8a130655 100644 --- a/Makefile.am +++ b/Makefile.am @@ -84,6 +84,10 @@ if ENABLE_SNMP SUBDIRS += plugins/omsnmp endif +if ENABLE_CUST1 +SUBDIRS += plugins/cust1 +endif + if ENABLE_IMTEMPLATE SUBDIRS += plugins/imtemplate endif diff --git a/configure.ac b/configure.ac index 7150d211..9fc59f76 100644 --- a/configure.ac +++ b/configure.ac @@ -716,6 +716,20 @@ AC_ARG_ENABLE(omstdout, ) AM_CONDITIONAL(ENABLE_OMSTDOUT, test x$enable_omstdout = xyes) +# This provides a vehicle to integrate custom modules, that are not +# part of rsyslog, into the build process. It is named cust1, so that +# additional such modules can easily be added. +AC_ARG_ENABLE(cust1, + [AS_HELP_STRING([--enable-cust1],[Compiles stdout module @<:@default=no@:>@])], + [case "${enableval}" in + yes) enable_cust1="yes" ;; + no) enable_cust1="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-cust1) ;; + esac], + [enable_cust1=no] +) +AM_CONDITIONAL(ENABLE_CUST1, test x$enable_cust1 = xyes) + # settings for the template input module; copy and modify this code # if you intend to add your own module. Be sure to replace imtemplate @@ -789,6 +803,7 @@ AC_CONFIG_FILES([Makefile \ plugins/ommail/Makefile \ plugins/omsnmp/Makefile \ plugins/omoracle/Makefile \ + plugins/cust1/Makefile \ tests/Makefile]) AC_OUTPUT @@ -802,6 +817,7 @@ echo " Regular expressions support enabled: $enable_regexp" echo " Zlib compression support enabled: $enable_zlib" echo " rsyslog runtime will be built: $enable_rsyslogrt" echo " rsyslogd will be built: $enable_rsyslogd" +echo " custom module 1 will be built: $enable_cust1" echo echo "---{ input plugins }---" echo " Klog functionality enabled: $enable_klog ($os_type)" diff --git a/plugins/cust1/Makefile.am b/plugins/cust1/Makefile.am new file mode 100644 index 00000000..d2e075f9 --- /dev/null +++ b/plugins/cust1/Makefile.am @@ -0,0 +1,6 @@ +pkglib_LTLIBRARIES = cust1.la + +cust1_la_SOURCES = cust1.c +cust1_la_CPPFLAGS = -I$(top_srcdir) $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +cust1_la_LDFLAGS = -module -avoid-version +cust1_la_LIBADD = -- cgit v1.2.3 From 14d1209640e3554c4284eab136f9932cfc441126 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 2 Jun 2009 10:31:42 +0200 Subject: added a generic network stream server (in addition to rather specific syslog tcp server) --- ChangeLog | 2 + runtime/Makefile.am | 7 + runtime/rsyslog.h | 3 + runtime/strms_sess.c | 292 ++++++++++++++++ runtime/strms_sess.h | 74 ++++ runtime/strmsrv.c | 952 +++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/strmsrv.h | 110 ++++++ tcpsrv.c | 5 +- 8 files changed, 1444 insertions(+), 1 deletion(-) create mode 100644 runtime/strms_sess.c create mode 100644 runtime/strms_sess.h create mode 100644 runtime/strmsrv.c create mode 100644 runtime/strmsrv.h diff --git a/ChangeLog b/ChangeLog index 832d7a6c..ec971916 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +- added a generic network stream server (in addition to rather specific + syslog tcp server) - added ability for the UDP output action to rebind its send socket after sending n messages. New config directive $ActionSendUDPRebindInterval added for the purpose. By default, rebinding is disabled. This is diff --git a/runtime/Makefile.am b/runtime/Makefile.am index bc03c4a7..eeb656d6 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -121,6 +121,13 @@ lmnetstrms_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) lmnetstrms_la_LDFLAGS = -module -avoid-version lmnetstrms_la_LIBADD = +# generic stream server framework +pkglib_LTLIBRARIES += lmstrmsrv.la +lmstrmsrv_la_SOURCES = strmsrv.c strmsrv.h strms_sess.c strms_sess.h +lmstrmsrv_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +lmstrmsrv_la_LDFLAGS = -module -avoid-version +lmstrmsrv_la_LIBADD = + # netstream drivers # plain tcp driver - main driver diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 2f15f926..293e559b 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -85,10 +85,13 @@ typedef struct permittedPeers_s permittedPeers_t; /* this should go away in the typedef struct permittedPeerWildcard_s permittedPeerWildcard_t; /* this should go away in the long term -- rgerhards, 2008-05-19 */ typedef struct tcpsrv_s tcpsrv_t; typedef struct tcps_sess_s tcps_sess_t; +typedef struct strmsrv_s strmsrv_t; +typedef struct strms_sess_s strms_sess_t; typedef struct vmstk_s vmstk_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ typedef struct tcpLstnPortList_s tcpLstnPortList_t; // TODO: rename? +typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename? /* some universal 64 bit define... */ typedef long long int64; diff --git a/runtime/strms_sess.c b/runtime/strms_sess.c new file mode 100644 index 00000000..20920430 --- /dev/null +++ b/runtime/strms_sess.c @@ -0,0 +1,292 @@ +/* strms_sess.c + * + * This implements a session of the strmsrv object. For general + * comments, see header of strmsrv.c. + * + * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#include "config.h" +#include +#include +#include +#include +#include + +#include "rsyslog.h" +#include "dirty.h" +#include "module-template.h" +#include "net.h" +#include "strmsrv.h" +#include "strms_sess.h" +#include "obj.h" +#include "errmsg.h" +#include "netstrm.h" +#include "msg.h" +#include "datetime.h" + + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(glbl) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(netstrm) +DEFobjCurrIf(datetime) + +static int iMaxLine; /* maximum size of a single message */ + +/* forward definitions */ +static rsRetVal Close(strms_sess_t *pThis); + + +/* Standard-Constructor */ +BEGINobjConstruct(strms_sess) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(strms_sess) + + +/* ConstructionFinalizer + */ +static rsRetVal +strms_sessConstructFinalize(strms_sess_t __attribute__((unused)) *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + if(pThis->pSrv->OnSessConstructFinalize != NULL) { + CHKiRet(pThis->pSrv->OnSessConstructFinalize(&pThis->pUsr)); + } + +finalize_it: + RETiRet; +} + + +/* destructor for the strms_sess object */ +BEGINobjDestruct(strms_sess) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(strms_sess) + if(pThis->pStrm != NULL) + netstrm.Destruct(&pThis->pStrm); + + if(pThis->pSrv->pOnSessDestruct != NULL) { + pThis->pSrv->pOnSessDestruct(&pThis->pUsr); + } + /* now destruct our own properties */ + free(pThis->fromHost); + free(pThis->fromHostIP); +ENDobjDestruct(strms_sess) + + +/* debugprint for the strms_sess object */ +BEGINobjDebugPrint(strms_sess) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(strms_sess) +ENDobjDebugPrint(strms_sess) + + +/* set property functions */ +/* set the hostname. Note that the caller *hands over* the string. That is, + * the caller no longer controls it once SetHost() has received it. Most importantly, + * the caller must not free it. -- rgerhards, 2008-04-24 + */ +static rsRetVal +SetHost(strms_sess_t *pThis, uchar *pszHost) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + free(pThis->fromHost); + pThis->fromHost = pszHost; + RETiRet; +} + +/* set the remote host's IP. Note that the caller *hands over* the string. That is, + * the caller no longer controls it once SetHostIP() has received it. Most importantly, + * the caller must not free it. -- rgerhards, 2008-05-16 + */ +static rsRetVal +SetHostIP(strms_sess_t *pThis, uchar *pszHostIP) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + free(pThis->fromHostIP); + pThis->fromHostIP = pszHostIP; + RETiRet; +} + +static rsRetVal +SetStrm(strms_sess_t *pThis, netstrm_t *pStrm) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + pThis->pStrm = pStrm; + RETiRet; +} + + +/* set our parent, the strmsrv object */ +static rsRetVal +SetStrmsrv(strms_sess_t *pThis, strmsrv_t *pSrv) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + ISOBJ_TYPE_assert(pSrv, strmsrv); + pThis->pSrv = pSrv; + RETiRet; +} + + +/* set our parent listener info*/ +static rsRetVal +SetLstnInfo(strms_sess_t *pThis, strmLstnPortList_t *pLstnInfo) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strms_sess); + assert(pLstnInfo != NULL); + pThis->pLstnInfo = pLstnInfo; + RETiRet; +} + + +static rsRetVal +SetUsrP(strms_sess_t *pThis, void *pUsr) +{ + DEFiRet; + pThis->pUsr = pUsr; + RETiRet; +} + + +/* Closes a STRM session + * No attention is paid to the return code + * of close, so potential-double closes are not detected. + */ +static rsRetVal +Close(strms_sess_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strms_sess); + netstrm.Destruct(&pThis->pStrm); + free(pThis->fromHost); + pThis->fromHost = NULL; /* not really needed, but... */ + free(pThis->fromHostIP); + pThis->fromHostIP = NULL; /* not really needed, but... */ + + RETiRet; +} + + + +/* Processes the data received via a STRM session. If there + * is no other way to handle it, data is discarded. + * Input parameter data is the data received, iLen is its + * len as returned from recv(). iLen must be 1 or more (that + * is errors must be handled by caller!). iSTRMSess must be + * the index of the STRM session that received the data. + * rgerhards 2005-07-04 + * And another change while generalizing. We now return either + * RS_RET_OK, which means the session should be kept open + * or anything else, which means it must be closed. + * rgerhards, 2008-03-01 + */ +static rsRetVal +DataRcvd(strms_sess_t *pThis, char *pData, size_t iLen) +{ + DEFiRet; + char *pEnd; + + ISOBJ_TYPE_assert(pThis, strms_sess); + assert(pData != NULL); + assert(iLen > 0); + + /* We now copy the message to the session buffer. */ + pEnd = pData + iLen; /* this is one off, which is intensional */ + + while(pData < pEnd) { + CHKiRet(pThis->pSrv->OnCharRcvd(pThis, (uchar)*pData++)); + } + +finalize_it: + RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-02-29 + */ +BEGINobjQueryInterface(strms_sess) +CODESTARTobjQueryInterface(strms_sess) + if(pIf->ifVersion != strms_sessCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->DebugPrint = strms_sessDebugPrint; + pIf->Construct = strms_sessConstruct; + pIf->ConstructFinalize = strms_sessConstructFinalize; + pIf->Destruct = strms_sessDestruct; + + pIf->Close = Close; + pIf->DataRcvd = DataRcvd; + + pIf->SetUsrP = SetUsrP; + pIf->SetStrmsrv = SetStrmsrv; + pIf->SetLstnInfo = SetLstnInfo; + pIf->SetHost = SetHost; + pIf->SetHostIP = SetHostIP; + pIf->SetStrm = SetStrm; +finalize_it: +ENDobjQueryInterface(strms_sess) + + +/* exit our class + * rgerhards, 2008-03-10 + */ +BEGINObjClassExit(strms_sess, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(strms_sess) + /* release objects we no longer need */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(datetime, CORE_COMPONENT); +ENDObjClassExit(strms_sess) + + +/* Initialize our class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-29 + */ +BEGINObjClassInit(strms_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE class also in END MACRO! */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + + CHKiRet(objUse(glbl, CORE_COMPONENT)); + iMaxLine = glbl.GetMaxLine(); /* get maximum size we currently support */ + objRelease(glbl, CORE_COMPONENT); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, strms_sessDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strms_sessConstructFinalize); +ENDObjClassInit(strms_sess) + +/* vim:set ai: + */ diff --git a/runtime/strms_sess.h b/runtime/strms_sess.h new file mode 100644 index 00000000..c8cb5349 --- /dev/null +++ b/runtime/strms_sess.h @@ -0,0 +1,74 @@ +/* Definitions for strms_sess class. This implements a session of the + * generic stream server. + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_STRMS_SESS_H +#define INCLUDED_STRMS_SESS_H + +#include "obj.h" + +/* a forward-definition, we are somewhat cyclic */ +struct strmsrv_s; + +/* the strms_sess object */ +struct strms_sess_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + strmsrv_t *pSrv; /* pointer back to my server (e.g. for callbacks) */ + strmLstnPortList_t *pLstnInfo; /* pointer back to listener info */ + netstrm_t *pStrm; +// uchar *pMsg; /* message (fragment) received */ + uchar *fromHost; + uchar *fromHostIP; + void *pUsr; /* a user-pointer */ +}; + + +/* interfaces */ +BEGINinterface(strms_sess) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(strms_sess); + rsRetVal (*Construct)(strms_sess_t **ppThis); + rsRetVal (*ConstructFinalize)(strms_sess_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(strms_sess_t **ppThis); + rsRetVal (*Close)(strms_sess_t *pThis); + rsRetVal (*DataRcvd)(strms_sess_t *pThis, char *pData, size_t iLen); + /* set methods */ + rsRetVal (*SetStrmsrv)(strms_sess_t *pThis, struct strmsrv_s *pSrv); + rsRetVal (*SetLstnInfo)(strms_sess_t *pThis, strmLstnPortList_t *pLstnInfo); + rsRetVal (*SetUsrP)(strms_sess_t*, void*); + rsRetVal (*SetHost)(strms_sess_t *pThis, uchar*); + rsRetVal (*SetHostIP)(strms_sess_t *pThis, uchar*); + rsRetVal (*SetStrm)(strms_sess_t *pThis, netstrm_t*); + rsRetVal (*SetOnMsgReceive)(strms_sess_t *pThis, rsRetVal (*OnMsgReceive)(strms_sess_t*, uchar*, int)); +ENDinterface(strms_sess) +#define strms_sessCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* interface changes + * to version v2, rgerhards, 2009-05-22 + * - Data structures changed + * - SetLstnInfo entry point added + */ + + +/* prototypes */ +PROTOTYPEObj(strms_sess); + + +#endif /* #ifndef INCLUDED_STRMS_SESS_H */ diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c new file mode 100644 index 00000000..6c803ee2 --- /dev/null +++ b/runtime/strmsrv.c @@ -0,0 +1,952 @@ +/* strmsrv.c + * + * This builds a basic stream server. It handles connection creation but + * not any protocol. Instead, it calls a "data received" entry point of the + * caller with any data received, in which case the caller must react accordingly. + * This module works together with the netstream drivers. + * + * There are actually two classes within the stream server code: one is + * the strmsrv itself, the other one is its sessions. This is a helper + * class to strmsrv. + * + * File begun on 2009-06-01 by RGerhards based on strmsrv.c. Note that strmsrv is + * placed under LGPL, which is possible because I carefully evaluated and + * eliminated all those parts of strmsrv which were not written by me. + * + * TODO: I would consider it useful to migrate tcpsrv.c/tcps_sess.c to this stream + * class here. The requires a little bit redesign, but should not be too hard. The + * core idea, already begun here, is that we still support lots of callbacks, but + * provide "canned" implementations for standard cases. That way, most upper-layer + * modules can be kept rather simple and without any extra overhead. Note that + * to support this, tcps_sess.c would need to extract the message reception state + * machine to a separate module which then is called via the DoCharRcvd() interface + * of this class here. -- rgerhards, 2009-06-01 + * + * Copyright 2007, 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if HAVE_FCNTL_H +#include +#endif +#include "rsyslog.h" +#include "dirty.h" +#include "cfsysline.h" +#include "module-template.h" +#include "net.h" +#include "srUtils.h" +#include "conf.h" +#include "strmsrv.h" +#include "obj.h" +#include "glbl.h" +#include "netstrms.h" +#include "netstrm.h" +#include "nssel.h" +#include "errmsg.h" +#include "unicode-helper.h" + +MODULE_TYPE_LIB + +/* defines */ +#define STRMSESS_MAX_DEFAULT 200 /* default for nbr of strm sessions if no number is given */ +#define STRMLSTN_MAX_DEFAULT 20 /* default for nbr of listeners */ + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(conf) +DEFobjCurrIf(glbl) +DEFobjCurrIf(strms_sess) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(net) +DEFobjCurrIf(netstrms) +DEFobjCurrIf(netstrm) +DEFobjCurrIf(nssel) + +/* forward definitions */ +static rsRetVal create_strm_socket(strmsrv_t *pThis); + +/* standard callbacks, if the caller did not provide us with them (this helps keep us + * flexible while at the same time permits very simple upper-layer modules) + */ +/* this shall go into a specific ACL module! */ +static int +isPermittedHost(struct sockaddr __attribute__((unused)) *addr, char __attribute__((unused)) *fromHostFQDN, + void __attribute__((unused)) *pUsrSrv, void __attribute__((unused)) *pUsrSess) +{ + return 1; +} + + +static rsRetVal +doOpenLstnSocks(strmsrv_t *pSrv) +{ + ISOBJ_TYPE_assert(pSrv, strmsrv); + return create_strm_socket(pSrv); +} + + +static rsRetVal +doRcvData(strms_sess_t *pSess, char *buf, size_t lenBuf, ssize_t *piLenRcvd) +{ + DEFiRet; + assert(pSess != NULL); + assert(piLenRcvd != NULL); + + *piLenRcvd = lenBuf; + CHKiRet(netstrm.Rcv(pSess->pStrm, (uchar*) buf, piLenRcvd)); +finalize_it: + RETiRet; +} + +static rsRetVal +onRegularClose(strms_sess_t *pSess) +{ + DEFiRet; + assert(pSess != NULL); + + /* process any incomplete frames left over */ + //strms_sess.PrepareClose(pSess); + /* Session closed */ + strms_sess.Close(pSess); + RETiRet; +} + + +static rsRetVal +onErrClose(strms_sess_t *pSess) +{ + DEFiRet; + assert(pSess != NULL); + + strms_sess.Close(pSess); + RETiRet; +} + +/* ------------------------------ end callbacks ------------------------------ */ + +/* add new listener port to listener port list + * rgerhards, 2009-05-21 + */ +static inline rsRetVal +addNewLstnPort(strmsrv_t *pThis, uchar *pszPort) +{ + strmLstnPortList_t *pEntry; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* create entry */ + CHKmalloc(pEntry = malloc(sizeof(strmLstnPortList_t))); + pEntry->pszPort = pszPort; + pEntry->pSrv = pThis; + CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); + + /* and add to list */ + pEntry->pNext = pThis->pLstnPorts; + pThis->pLstnPorts = pEntry; + +finalize_it: + RETiRet; +} + + +/* configure STRM listener settings. + * Note: pszPort is handed over to us - the caller MUST NOT free it! + * rgerhards, 2008-03-20 + */ +static rsRetVal +configureSTRMListen(strmsrv_t *pThis, uchar *pszPort) +{ + int i; + uchar *pPort = pszPort; + DEFiRet; + + assert(pszPort != NULL); + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* extract port */ + i = 0; + while(isdigit((int) *pPort)) { + i = i * 10 + *pPort++ - '0'; + } + + if(i >= 0 && i <= 65535) { + CHKiRet(addNewLstnPort(pThis, pszPort)); + } else { + errmsg.LogError(0, NO_ERRCODE, "Invalid STRM listen port %s - ignored.\n", pszPort); + } + +finalize_it: + RETiRet; +} + + +/* Initialize the session table + * returns 0 if OK, somewhat else otherwise + */ +static rsRetVal +STRMSessTblInit(strmsrv_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pThis->pSessions == NULL); + + dbgprintf("Allocating buffer for %d STRM sessions.\n", pThis->iSessMax); + if((pThis->pSessions = (strms_sess_t **) calloc(pThis->iSessMax, sizeof(strms_sess_t *))) == NULL) { + dbgprintf("Error: STRMSessInit() could not alloc memory for STRM session table.\n"); + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } + +finalize_it: + RETiRet; +} + + +/* find a free spot in the session table. If the table + * is full, -1 is returned, else the index of the free + * entry (0 or higher). + */ +static int +STRMSessTblFindFreeSpot(strmsrv_t *pThis) +{ + register int i; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + for(i = 0 ; i < pThis->iSessMax ; ++i) { + if(pThis->pSessions[i] == NULL) + break; + } + + return((i < pThis->iSessMax) ? i : -1); +} + + +/* Get the next session index. Free session tables entries are + * skipped. This function is provided the index of the last + * session entry, or -1 if no previous entry was obtained. It + * returns the index of the next session or -1, if there is no + * further entry in the table. Please note that the initial call + * might as well return -1, if there is no session at all in the + * session table. + */ +static int +STRMSessGetNxtSess(strmsrv_t *pThis, int iCurr) +{ + register int i; + + BEGINfunc + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pThis->pSessions != NULL); + for(i = iCurr + 1 ; i < pThis->iSessMax ; ++i) { + if(pThis->pSessions[i] != NULL) + break; + } + + ENDfunc + return((i < pThis->iSessMax) ? i : -1); +} + + +/* De-Initialize STRM listner sockets. + * This function deinitializes everything, including freeing the + * session table. No STRM listen receive operations are permitted + * unless the subsystem is reinitialized. + * rgerhards, 2007-06-21 + */ +static void deinit_strm_listener(strmsrv_t *pThis) +{ + int i; + strmLstnPortList_t *pEntry; + strmLstnPortList_t *pDel; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + if(pThis->pSessions != NULL) { + /* close all STRM connections! */ + i = STRMSessGetNxtSess(pThis, -1); + while(i != -1) { + strms_sess.Destruct(&pThis->pSessions[i]); + /* now get next... */ + i = STRMSessGetNxtSess(pThis, i); + } + + /* we are done with the session table - so get rid of it... */ + free(pThis->pSessions); + pThis->pSessions = NULL; /* just to make sure... */ + } + + /* free list of strm listen ports */ + pEntry = pThis->pLstnPorts; + while(pEntry != NULL) { + free(pEntry->pszPort); + free(pEntry->pszInputName); + pDel = pEntry; + pEntry = pEntry->pNext; + free(pDel); + } + + /* finally close our listen streams */ + for(i = 0 ; i < pThis->iLstnMax ; ++i) { + netstrm.Destruct(pThis->ppLstn + i); + } +} + + +/* add a listen socket to our listen socket array. This is a callback + * invoked from the netstrm class. -- rgerhards, 2008-04-23 + */ +static rsRetVal +addStrmLstn(void *pUsr, netstrm_t *pLstn) +{ + strmLstnPortList_t *pPortList = (strmLstnPortList_t *) pUsr; + strmsrv_t *pThis = pPortList->pSrv; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + ISOBJ_TYPE_assert(pLstn, netstrm); + + if(pThis->iLstnMax >= STRMLSTN_MAX_DEFAULT) + ABORT_FINALIZE(RS_RET_MAX_LSTN_REACHED); + + pThis->ppLstn[pThis->iLstnMax] = pLstn; + pThis->ppLstnPort[pThis->iLstnMax] = pPortList; + ++pThis->iLstnMax; + +finalize_it: + RETiRet; +} + + +/* Initialize STRM listener socket for a single port + * rgerhards, 2009-05-21 + */ +static inline rsRetVal +initSTRMListener(strmsrv_t *pThis, strmLstnPortList_t *pPortEntry) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pPortEntry != NULL); + + /* TODO: add capability to specify local listen address! */ + CHKiRet(netstrm.LstnInit(pThis->pNS, (void*)pPortEntry, addStrmLstn, pPortEntry->pszPort, NULL, pThis->iSessMax)); + +finalize_it: + RETiRet; +} + + +/* Initialize STRM sockets (for listener) and listens on them */ +static rsRetVal +create_strm_socket(strmsrv_t *pThis) +{ + strmLstnPortList_t *pEntry; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* init all configured ports */ + pEntry = pThis->pLstnPorts; + while(pEntry != NULL) { + CHKiRet(initSTRMListener(pThis, pEntry)); + pEntry = pEntry->pNext; + } + + /* OK, we had success. Now it is also time to + * initialize our connections + */ + if(STRMSessTblInit(pThis) != 0) { + /* OK, we are in some trouble - we could not initialize the + * session table, so we can not continue. We need to free all + * we have assigned so far, because we can not really use it... + */ + errmsg.LogError(0, RS_RET_ERR, "Could not initialize STRM session table, suspending STRM message reception."); + ABORT_FINALIZE(RS_RET_ERR); + } + +finalize_it: + RETiRet; +} + + +/* Accept new STRM connection; make entry in session table. If there + * is no more space left in the connection table, the new STRM + * connection is immediately dropped. + * ppSess has a pointer to the newly created session, if it succeeds. + * If it does not succeed, no session is created and ppSess is + * undefined. If the user has provided an OnSessAccept Callback, + * this one is executed immediately after creation of the + * session object, so that it can do its own initialization. + * rgerhards, 2008-03-02 + */ +static rsRetVal +SessAccept(strmsrv_t *pThis, strmLstnPortList_t *pLstnInfo, strms_sess_t **ppSess, netstrm_t *pStrm) +{ + DEFiRet; + strms_sess_t *pSess = NULL; + netstrm_t *pNewStrm = NULL; + int iSess = -1; + struct sockaddr_storage *addr; + uchar *fromHostFQDN = NULL; + uchar *fromHostIP = NULL; + + ISOBJ_TYPE_assert(pThis, strmsrv); + assert(pLstnInfo != NULL); + + CHKiRet(netstrm.AcceptConnReq(pStrm, &pNewStrm)); + + /* Add to session list */ + iSess = STRMSessTblFindFreeSpot(pThis); + if(iSess == -1) { + errno = 0; + errmsg.LogError(0, RS_RET_MAX_SESS_REACHED, "too many strm sessions - dropping incoming request"); + ABORT_FINALIZE(RS_RET_MAX_SESS_REACHED); + } + + /* we found a free spot and can construct our session object */ + CHKiRet(strms_sess.Construct(&pSess)); + CHKiRet(strms_sess.SetStrmsrv(pSess, pThis)); + CHKiRet(strms_sess.SetLstnInfo(pSess, pLstnInfo)); + + /* get the host name */ + CHKiRet(netstrm.GetRemoteHName(pNewStrm, &fromHostFQDN)); + CHKiRet(netstrm.GetRemoteIP(pNewStrm, &fromHostIP)); + CHKiRet(netstrm.GetRemAddr(pNewStrm, &addr)); + /* TODO: check if we need to strip the domain name here -- rgerhards, 2008-04-24 */ + + /* Here we check if a host is permitted to send us messages. If it isn't, we do not further + * process the message but log a warning (if we are configured to do this). + * rgerhards, 2005-09-26 + */ + if(pThis->pIsPermittedHost != NULL + && !pThis->pIsPermittedHost((struct sockaddr*) addr, (char*) fromHostFQDN, pThis->pUsr, pSess->pUsr)) { + dbgprintf("%s is not an allowed sender\n", fromHostFQDN); + if(glbl.GetOption_DisallowWarning()) { + errno = 0; + errmsg.LogError(0, RS_RET_HOST_NOT_PERMITTED, "STRM message from disallowed sender %s discarded", fromHostFQDN); + } + ABORT_FINALIZE(RS_RET_HOST_NOT_PERMITTED); + } + + /* OK, we have an allowed sender, so let's continue, what + * means we can finally fill in the session object. + */ + CHKiRet(strms_sess.SetHost(pSess, fromHostFQDN)); + fromHostFQDN = NULL; /* we handed this string over */ + CHKiRet(strms_sess.SetHostIP(pSess, fromHostIP)); + fromHostIP = NULL; /* we handed this string over */ + CHKiRet(strms_sess.SetStrm(pSess, pNewStrm)); + pNewStrm = NULL; /* prevent it from being freed in error handler, now done in strms_sess! */ + CHKiRet(strms_sess.ConstructFinalize(pSess)); + + /* check if we need to call our callback */ + if(pThis->pOnSessAccept != NULL) { + CHKiRet(pThis->pOnSessAccept(pThis, pSess)); + } + + *ppSess = pSess; + pThis->pSessions[iSess] = pSess; + pSess = NULL; /* this is now also handed over */ + +finalize_it: + if(iRet != RS_RET_OK) { + if(pSess != NULL) + strms_sess.Destruct(&pSess); + if(pNewStrm != NULL) + netstrm.Destruct(&pNewStrm); + free(fromHostFQDN); + free(fromHostIP); + } + + RETiRet; +} + + +static void +RunCancelCleanup(void *arg) +{ + nssel_t **ppSel = (nssel_t**) arg; + + if(*ppSel != NULL) + nssel.Destruct(ppSel); +} + + +/* This function is called to gather input. */ +#pragma GCC diagnostic ignored "-Wempty-body" +static rsRetVal +Run(strmsrv_t *pThis) +{ + DEFiRet; + int nfds; + int i; + int iSTRMSess; + int bIsReady; + strms_sess_t *pNewSess; + nssel_t *pSel; + ssize_t iRcvd; + + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* this is an endless loop - it is terminated by the framework canelling + * this thread. Thus, we also need to instantiate a cancel cleanup handler + * to prevent us from leaking anything. -- rgerharsd, 20080-04-24 + */ + pthread_cleanup_push(RunCancelCleanup, (void*) &pSel); + while(1) { + CHKiRet(nssel.Construct(&pSel)); + // TODO: set driver + CHKiRet(nssel.ConstructFinalize(pSel)); + + /* Add the STRM listen sockets to the list of read descriptors. */ + for(i = 0 ; i < pThis->iLstnMax ; ++i) { + CHKiRet(nssel.Add(pSel, pThis->ppLstn[i], NSDSEL_RD)); + } + + /* do the sessions */ + iSTRMSess = STRMSessGetNxtSess(pThis, -1); + while(iSTRMSess != -1) { + /* TODO: access to pNsd is NOT really CLEAN, use method... */ + CHKiRet(nssel.Add(pSel, pThis->pSessions[iSTRMSess]->pStrm, NSDSEL_RD)); + /* now get next... */ + iSTRMSess = STRMSessGetNxtSess(pThis, iSTRMSess); + } + + /* wait for io to become ready */ + CHKiRet(nssel.Wait(pSel, &nfds)); + + for(i = 0 ; i < pThis->iLstnMax ; ++i) { + CHKiRet(nssel.IsReady(pSel, pThis->ppLstn[i], NSDSEL_RD, &bIsReady, &nfds)); + if(bIsReady) { + dbgprintf("New connect on NSD %p.\n", pThis->ppLstn[i]); + SessAccept(pThis, pThis->ppLstnPort[i], &pNewSess, pThis->ppLstn[i]); + --nfds; /* indicate we have processed one */ + } + } + + /* now check the sessions */ + iSTRMSess = STRMSessGetNxtSess(pThis, -1); + while(nfds && iSTRMSess != -1) { + CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iSTRMSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); + if(bIsReady) { + char buf[8*1024]; /* reception buffer - may hold a partial or multiple messages */ + dbgprintf("netstream %p with new data\n", pThis->pSessions[iSTRMSess]->pStrm); + + /* Receive message */ + iRet = pThis->pRcvData(pThis->pSessions[iSTRMSess], buf, sizeof(buf), &iRcvd); + switch(iRet) { + case RS_RET_CLOSED: + pThis->pOnRegularClose(pThis->pSessions[iSTRMSess]); + strms_sess.Destruct(&pThis->pSessions[iSTRMSess]); + break; + case RS_RET_RETRY: + /* we simply ignore retry - this is not an error, but we also have not received anything */ + break; + case RS_RET_OK: + /* valid data received, process it! */ + if(strms_sess.DataRcvd(pThis->pSessions[iSTRMSess], buf, iRcvd) != RS_RET_OK) { + /* in this case, something went awfully wrong. + * We are instructed to terminate the session. + */ + errmsg.LogError(0, NO_ERRCODE, "Tearing down STRM Session %d - see " + "previous messages for reason(s)\n", iSTRMSess); + pThis->pOnErrClose(pThis->pSessions[iSTRMSess]); + strms_sess.Destruct(&pThis->pSessions[iSTRMSess]); + } + break; + default: + errno = 0; + errmsg.LogError(0, iRet, "netstream session %p will be closed due to error\n", + pThis->pSessions[iSTRMSess]->pStrm); + pThis->pOnErrClose(pThis->pSessions[iSTRMSess]); + strms_sess.Destruct(&pThis->pSessions[iSTRMSess]); + break; + } + --nfds; /* indicate we have processed one */ + } + iSTRMSess = STRMSessGetNxtSess(pThis, iSTRMSess); + } + CHKiRet(nssel.Destruct(&pSel)); +finalize_it: /* this is a very special case - this time only we do not exit the function, + * because that would not help us either. So we simply retry it. Let's see + * if that actually is a better idea. Exiting the loop wasn't we always + * crashed, which made sense (the rest of the engine was not prepared for + * that) -- rgerhards, 2008-05-19 + */ + /*EMPTY*/; + } + + /* note that this point is usually not reached */ + pthread_cleanup_pop(0); /* remove cleanup handler */ + + RETiRet; +} +#pragma GCC diagnostic warning "-Wempty-body" + + +/* Standard-Constructor */ +BEGINobjConstruct(strmsrv) /* be sure to specify the object type also in END macro! */ + pThis->iSessMax = STRMSESS_MAX_DEFAULT; /* TODO: useful default ;) */ + /* set default callbacks (used if caller does not overwrite them) */ + pThis->pIsPermittedHost = isPermittedHost; + pThis->OpenLstnSocks = doOpenLstnSocks; + pThis->pRcvData = doRcvData; + pThis->pOnRegularClose = onRegularClose; + pThis->pOnErrClose = onErrClose; + /* session specific callbacks */ + //pThis->OnSessConstructFinalize = + //pThis->pOnSessDestruct = +ENDobjConstruct(strmsrv) + + +/* ConstructionFinalizer */ +static rsRetVal +strmsrvConstructFinalize(strmsrv_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + + /* prepare network stream subsystem */ + CHKiRet(netstrms.Construct(&pThis->pNS)); + CHKiRet(netstrms.SetDrvrMode(pThis->pNS, pThis->iDrvrMode)); + if(pThis->pszDrvrAuthMode != NULL) + CHKiRet(netstrms.SetDrvrAuthMode(pThis->pNS, pThis->pszDrvrAuthMode)); + if(pThis->pPermPeers != NULL) + CHKiRet(netstrms.SetDrvrPermPeers(pThis->pNS, pThis->pPermPeers)); + // TODO: set driver! + CHKiRet(netstrms.ConstructFinalize(pThis->pNS)); + + /* set up listeners */ + CHKmalloc(pThis->ppLstn = calloc(STRMLSTN_MAX_DEFAULT, sizeof(netstrm_t*))); + CHKmalloc(pThis->ppLstnPort = calloc(STRMLSTN_MAX_DEFAULT, sizeof(strmLstnPortList_t*))); + iRet = pThis->OpenLstnSocks(pThis); + +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis->pNS != NULL) + netstrms.Destruct(&pThis->pNS); + } + RETiRet; +} + + +/* destructor for the strmsrv object */ +BEGINobjDestruct(strmsrv) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(strmsrv) + if(pThis->OnDestruct != NULL) + pThis->OnDestruct(pThis->pUsr); + + deinit_strm_listener(pThis); + + if(pThis->pNS != NULL) + netstrms.Destruct(&pThis->pNS); + free(pThis->pszDrvrAuthMode); + free(pThis->ppLstn); + free(pThis->ppLstnPort); + free(pThis->pszInputName); +ENDobjDestruct(strmsrv) + + +/* debugprint for the strmsrv object */ +BEGINobjDebugPrint(strmsrv) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(strmsrv) +ENDobjDebugPrint(strmsrv) + +/* set functions */ +static rsRetVal +SetCBIsPermittedHost(strmsrv_t *pThis, int (*pCB)(struct sockaddr *addr, char *fromHostFQDN, void*, void*)) +{ + DEFiRet; + pThis->pIsPermittedHost = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnSessAccept(strmsrv_t *pThis, rsRetVal (*pCB)(strmsrv_t*, strms_sess_t*)) +{ + DEFiRet; + pThis->pOnSessAccept = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnDestruct(strmsrv_t *pThis, rsRetVal (*pCB)(void*)) +{ + DEFiRet; + pThis->OnDestruct = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnSessConstructFinalize(strmsrv_t *pThis, rsRetVal (*pCB)(void*)) +{ + DEFiRet; + pThis->OnSessConstructFinalize = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnSessDestruct(strmsrv_t *pThis, rsRetVal (*pCB)(void*)) +{ + DEFiRet; + pThis->pOnSessDestruct = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnRegularClose(strmsrv_t *pThis, rsRetVal (*pCB)(strms_sess_t*)) +{ + DEFiRet; + pThis->pOnRegularClose = pCB; + RETiRet; +} + +static rsRetVal +SetCBOnErrClose(strmsrv_t *pThis, rsRetVal (*pCB)(strms_sess_t*)) +{ + DEFiRet; + pThis->pOnErrClose = pCB; + RETiRet; +} + +static rsRetVal +SetCBOpenLstnSocks(strmsrv_t *pThis, rsRetVal (*pCB)(strmsrv_t*)) +{ + DEFiRet; + pThis->OpenLstnSocks = pCB; + RETiRet; +} + +static rsRetVal +SetUsrP(strmsrv_t *pThis, void *pUsr) +{ + DEFiRet; + pThis->pUsr = pUsr; + RETiRet; +} + +static rsRetVal +SetOnCharRcvd(strmsrv_t *pThis, rsRetVal (*OnCharRcvd)(strms_sess_t*, uchar)) +{ + DEFiRet; + assert(OnCharRcvd != NULL); + pThis->OnCharRcvd = OnCharRcvd; + RETiRet; +} + +/* Set the input name to use -- rgerhards, 2008-12-10 */ +static rsRetVal +SetInputName(strmsrv_t *pThis, uchar *name) +{ + uchar *pszName; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + if(name == NULL) + pszName = NULL; + else + CHKmalloc(pszName = ustrdup(name)); + free(pThis->pszInputName); + pThis->pszInputName = pszName; +finalize_it: + RETiRet; +} + + +/* here follows a number of methods that shuffle authentication settings down + * to the drivers. Drivers not supporting these settings may return an error + * state. + * -------------------------------------------------------------------------- */ + +/* set the driver mode -- rgerhards, 2008-04-30 */ +static rsRetVal +SetDrvrMode(strmsrv_t *pThis, int iMode) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + pThis->iDrvrMode = iMode; + RETiRet; +} + + +/* set the driver authentication mode -- rgerhards, 2008-05-19 */ +static rsRetVal +SetDrvrAuthMode(strmsrv_t *pThis, uchar *mode) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + CHKmalloc(pThis->pszDrvrAuthMode = ustrdup(mode)); +finalize_it: + RETiRet; +} + + +/* set the driver's permitted peers -- rgerhards, 2008-05-19 */ +static rsRetVal +SetDrvrPermPeers(strmsrv_t *pThis, permittedPeers_t *pPermPeers) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + pThis->pPermPeers = pPermPeers; + RETiRet; +} + + +/* End of methods to shuffle autentication settings to the driver.; + + * -------------------------------------------------------------------------- */ + + +/* set max number of sessions + * this must be called before ConstructFinalize, or it will have no effect! + * rgerhards, 2009-04-09 + */ +static rsRetVal +SetSessMax(strmsrv_t *pThis, int iMax) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strmsrv); + pThis->iSessMax = iMax; + RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-02-29 + */ +BEGINobjQueryInterface(strmsrv) +CODESTARTobjQueryInterface(strmsrv) + if(pIf->ifVersion != strmsrvCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->DebugPrint = strmsrvDebugPrint; + pIf->Construct = strmsrvConstruct; + pIf->ConstructFinalize = strmsrvConstructFinalize; + pIf->Destruct = strmsrvDestruct; + + pIf->configureSTRMListen = configureSTRMListen; + pIf->create_strm_socket = create_strm_socket; + pIf->Run = Run; + + pIf->SetUsrP = SetUsrP; + pIf->SetInputName = SetInputName; + pIf->SetSessMax = SetSessMax; + pIf->SetDrvrMode = SetDrvrMode; + pIf->SetDrvrAuthMode = SetDrvrAuthMode; + pIf->SetDrvrPermPeers = SetDrvrPermPeers; + pIf->SetCBIsPermittedHost = SetCBIsPermittedHost; + pIf->SetCBOpenLstnSocks = SetCBOpenLstnSocks; + pIf->SetCBOnSessAccept = SetCBOnSessAccept; + pIf->SetCBOnSessConstructFinalize = SetCBOnSessConstructFinalize; + pIf->SetCBOnSessDestruct = SetCBOnSessDestruct; + pIf->SetCBOnDestruct = SetCBOnDestruct; + pIf->SetCBOnRegularClose = SetCBOnRegularClose; + pIf->SetCBOnErrClose = SetCBOnErrClose; + pIf->SetOnCharRcvd = SetOnCharRcvd; + +finalize_it: +ENDobjQueryInterface(strmsrv) + + +/* exit our class + * rgerhards, 2008-03-10 + */ +BEGINObjClassExit(strmsrv, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MACRO! */ +CODESTARTObjClassExit(strmsrv) + /* release objects we no longer need */ + objRelease(strms_sess, DONT_LOAD_LIB); + objRelease(conf, CORE_COMPONENT); + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(netstrms, DONT_LOAD_LIB); + objRelease(nssel, DONT_LOAD_LIB); + objRelease(netstrm, LM_NETSTRMS_FILENAME); + objRelease(net, LM_NET_FILENAME); +ENDObjClassExit(strmsrv) + + +/* Initialize our class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-29 + */ +BEGINObjClassInit(strmsrv, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE class also in END MACRO! */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(net, LM_NET_FILENAME)); + CHKiRet(objUse(netstrms, LM_NETSTRMS_FILENAME)); + CHKiRet(objUse(netstrm, DONT_LOAD_LIB)); + CHKiRet(objUse(nssel, DONT_LOAD_LIB)); + CHKiRet(objUse(strms_sess, DONT_LOAD_LIB)); + CHKiRet(objUse(conf, CORE_COMPONENT)); + CHKiRet(objUse(glbl, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, strmsrvDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strmsrvConstructFinalize); +ENDObjClassInit(strmsrv) + + +/* --------------- here now comes the plumbing that makes as a library module --------------- */ + + +BEGINmodExit +CODESTARTmodExit + /* de-init in reverse order! */ + strmsrvClassExit(); + strms_sessClassExit(); +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_LIB_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ + + /* Initialize all classes that are in our module - this includes ourselfs */ + CHKiRet(strms_sessClassInit(pModInfo)); + CHKiRet(strmsrvClassInit(pModInfo)); /* must be done after strms_sess, as we use it */ +ENDmodInit + +/* vim:set ai: + */ diff --git a/runtime/strmsrv.h b/runtime/strmsrv.h new file mode 100644 index 00000000..e63d3a47 --- /dev/null +++ b/runtime/strmsrv.h @@ -0,0 +1,110 @@ +/* Definitions for strmsrv class. + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_STRMSRV_H +#define INCLUDED_STRMSRV_H + +#include "obj.h" +#include "strms_sess.h" + +/* list of strm listen ports */ +struct strmLstnPortList_s { + uchar *pszPort; /**< the ports the listener shall listen on */ + uchar *pszInputName; /**< value to be used as input name */ + strmsrv_t *pSrv; /**< pointer to higher-level server instance */ + strmLstnPortList_t *pNext; /**< next port or NULL */ +}; + + +/* the strmsrv object */ +struct strmsrv_s { + BEGINobjInstance; /**< Data to implement generic object - MUST be the first data element! */ + netstrms_t *pNS; /**< pointer to network stream subsystem */ + int iDrvrMode; /**< mode of the stream driver to use */ + uchar *pszDrvrAuthMode; /**< auth mode of the stream driver to use */ + uchar *pszInputName; /**< value to be used as input name */ + permittedPeers_t *pPermPeers;/**< driver's permitted peers */ + int iLstnMax; /**< max nbr of listeners currently supported */ + netstrm_t **ppLstn; /**< our netstream listners */ + strmLstnPortList_t **ppLstnPort; /**< pointer to relevant listen port description */ + int iSessMax; /**< max number of sessions supported */ + strmLstnPortList_t *pLstnPorts; /**< head pointer for listen ports */ + int addtlFrameDelim; /**< additional frame delimiter for plain STRM syslog framing (e.g. to handle NetScreen) */ + strms_sess_t **pSessions;/**< array of all of our sessions */ + void *pUsr; /**< a user-settable pointer (provides extensibility for "derived classes")*/ + /* callbacks */ + int (*pIsPermittedHost)(struct sockaddr *addr, char *fromHostFQDN, void*pUsrSrv, void*pUsrSess); + rsRetVal (*pRcvData)(strms_sess_t*, char*, size_t, ssize_t *); + rsRetVal (*OpenLstnSocks)(struct strmsrv_s*); + rsRetVal (*pOnListenDeinit)(void*); + rsRetVal (*OnDestruct)(void*); + rsRetVal (*pOnRegularClose)(strms_sess_t *pSess); + rsRetVal (*pOnErrClose)(strms_sess_t *pSess); + /* session specific callbacks */ + rsRetVal (*pOnSessAccept)(strmsrv_t *, strms_sess_t*); + rsRetVal (*OnSessConstructFinalize)(void*); + rsRetVal (*pOnSessDestruct)(void*); + rsRetVal (*OnCharRcvd)(strms_sess_t*, uchar); +}; + + +/* interfaces */ +BEGINinterface(strmsrv) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(strmsrv); + rsRetVal (*Construct)(strmsrv_t **ppThis); + rsRetVal (*ConstructFinalize)(strmsrv_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(strmsrv_t **ppThis); + rsRetVal (*configureSTRMListen)(strmsrv_t*, uchar *pszPort); + //rsRetVal (*SessAccept)(strmsrv_t *pThis, strmLstnPortList_t*, strms_sess_t **ppSess, netstrm_t *pStrm); + rsRetVal (*create_strm_socket)(strmsrv_t *pThis); + rsRetVal (*Run)(strmsrv_t *pThis); + /* set methods */ + rsRetVal (*SetAddtlFrameDelim)(strmsrv_t*, int); + rsRetVal (*SetInputName)(strmsrv_t*, uchar*); + rsRetVal (*SetUsrP)(strmsrv_t*, void*); + rsRetVal (*SetCBIsPermittedHost)(strmsrv_t*, int (*) (struct sockaddr *addr, char*, void*, void*)); + rsRetVal (*SetCBOpenLstnSocks)(strmsrv_t *, rsRetVal (*)(strmsrv_t*)); + rsRetVal (*SetCBOnDestruct)(strmsrv_t*, rsRetVal (*) (void*)); + rsRetVal (*SetCBOnRegularClose)(strmsrv_t*, rsRetVal (*) (strms_sess_t*)); + rsRetVal (*SetCBOnErrClose)(strmsrv_t*, rsRetVal (*) (strms_sess_t*)); + rsRetVal (*SetDrvrMode)(strmsrv_t *pThis, int iMode); + rsRetVal (*SetDrvrAuthMode)(strmsrv_t *pThis, uchar *pszMode); + rsRetVal (*SetDrvrPermPeers)(strmsrv_t *pThis, permittedPeers_t*); + /* session specifics */ + rsRetVal (*SetCBOnSessAccept)(strmsrv_t*, rsRetVal (*) (strmsrv_t*, strms_sess_t*)); + rsRetVal (*SetCBOnSessDestruct)(strmsrv_t*, rsRetVal (*) (void*)); + rsRetVal (*SetCBOnSessConstructFinalize)(strmsrv_t*, rsRetVal (*) (void*)); + rsRetVal (*SetSessMax)(strmsrv_t *pThis, int iMaxSess); + rsRetVal (*SetOnCharRcvd)(strmsrv_t *pThis, rsRetVal (*OnMsgCharRcvd)(strms_sess_t*, uchar)); +ENDinterface(strmsrv) +#define strmsrvCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +/* change for v?: + */ + + +/* prototypes */ +PROTOTYPEObj(strmsrv); + +/* the name of our library binary */ +#define LM_STRMSRV_FILENAME "lmstrmsrv" + +#endif /* #ifndef INCLUDED_STRMSRV_H */ diff --git a/tcpsrv.c b/tcpsrv.c index 02eee88e..3516b2e3 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -1,8 +1,11 @@ /* tcpsrv.c * - * Common code for plain TCP based servers. This is currently being + * Common code for plain TCP syslog based servers. This is currently being * utilized by imtcp and imgssapi. * + * NOTE: this is *not* a generic TCP server, but one for syslog servers. For + * generic stream servers, please use ./runtime/strmsrv.c! + * * There are actually two classes within the tcpserver code: one is * the tcpsrv itself, the other one is its sessions. This is a helper * class to tcpsrv. -- cgit v1.2.3 From 05ba5fc29fb13857b6fbb7c8f3c22489da3e73cf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 2 Jun 2009 12:22:00 +0200 Subject: slightly extended strms_sess interface --- runtime/strms_sess.c | 10 +++++++++- runtime/strms_sess.h | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/runtime/strms_sess.c b/runtime/strms_sess.c index 20920430..0aeebe03 100644 --- a/runtime/strms_sess.c +++ b/runtime/strms_sess.c @@ -64,7 +64,7 @@ ENDobjConstruct(strms_sess) /* ConstructionFinalizer */ static rsRetVal -strms_sessConstructFinalize(strms_sess_t __attribute__((unused)) *pThis) +strms_sessConstructFinalize(strms_sess_t *pThis) { DEFiRet; ISOBJ_TYPE_assert(pThis, strms_sess); @@ -170,6 +170,13 @@ SetUsrP(strms_sess_t *pThis, void *pUsr) } +static void * +GetUsrP(strms_sess_t *pThis) +{ + return pThis->pUsr; +} + + /* Closes a STRM session * No attention is paid to the return code * of close, so potential-double closes are not detected. @@ -248,6 +255,7 @@ CODESTARTobjQueryInterface(strms_sess) pIf->DataRcvd = DataRcvd; pIf->SetUsrP = SetUsrP; + pIf->GetUsrP = GetUsrP; pIf->SetStrmsrv = SetStrmsrv; pIf->SetLstnInfo = SetLstnInfo; pIf->SetHost = SetHost; diff --git a/runtime/strms_sess.h b/runtime/strms_sess.h index c8cb5349..6483c0c3 100644 --- a/runtime/strms_sess.h +++ b/runtime/strms_sess.h @@ -54,6 +54,7 @@ BEGINinterface(strms_sess) /* name must also be changed in ENDinterface macro! * rsRetVal (*SetStrmsrv)(strms_sess_t *pThis, struct strmsrv_s *pSrv); rsRetVal (*SetLstnInfo)(strms_sess_t *pThis, strmLstnPortList_t *pLstnInfo); rsRetVal (*SetUsrP)(strms_sess_t*, void*); + void* (*GetUsrP)(strms_sess_t*); rsRetVal (*SetHost)(strms_sess_t *pThis, uchar*); rsRetVal (*SetHostIP)(strms_sess_t *pThis, uchar*); rsRetVal (*SetStrm)(strms_sess_t *pThis, netstrm_t*); -- cgit v1.2.3 From b61e5fffc42c98b49a18a95c297653fb0ba06b72 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 2 Jun 2009 13:11:18 +0200 Subject: strmsrv now supports KEEPALIVE socket option --- runtime/netstrm.c | 14 ++++++++++++++ runtime/netstrm.h | 8 ++++++-- runtime/nsd.h | 8 ++++++-- runtime/nsd_gtls.c | 11 +++++++++++ runtime/nsd_ptcp.c | 29 +++++++++++++++++++++++++++++ runtime/strmsrv.c | 14 ++++++++++++++ runtime/strmsrv.h | 2 ++ 7 files changed, 82 insertions(+), 4 deletions(-) diff --git a/runtime/netstrm.c b/runtime/netstrm.c index 05bb25c0..3658006f 100644 --- a/runtime/netstrm.c +++ b/runtime/netstrm.c @@ -233,6 +233,19 @@ Send(netstrm_t *pThis, uchar *pBuf, ssize_t *pLenBuf) RETiRet; } +/* Enable Keep-Alive handling for those drivers that support it. + * rgerhards, 2009-06-02 + */ +static rsRetVal +EnableKeepAlive(netstrm_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, netstrm); + iRet = pThis->Drvr.EnableKeepAlive(pThis->pDrvrData); + RETiRet; +} + + /* check connection - slim wrapper for NSD driver function */ static void @@ -337,6 +350,7 @@ CODESTARTobjQueryInterface(netstrm) pIf->SetDrvrPermPeers = SetDrvrPermPeers; pIf->CheckConnection = CheckConnection; pIf->GetSock = GetSock; + pIf->EnableKeepAlive = EnableKeepAlive; finalize_it: ENDobjQueryInterface(netstrm) diff --git a/runtime/netstrm.h b/runtime/netstrm.h index b00dd223..f6931104 100644 --- a/runtime/netstrm.h +++ b/runtime/netstrm.h @@ -69,9 +69,13 @@ BEGINinterface(netstrm) /* name must also be changed in ENDinterface macro! */ * sockets - not really desirable, but not the end of the world... TODO: should be * reconsidered when a new ACL system is build. -- rgerhards, 2008-12-01 */ + /* v4 */ + rsRetVal (*EnableKeepAlive)(netstrm_t *pThis); ENDinterface(netstrm) -#define netstrmCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ -/* interface version 3 added GetRemAddr() */ +#define netstrmCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ +/* interface version 3 added GetRemAddr() + * interface version 4 added EnableKeepAlive() -- rgerhards, 2009-06-02 + * */ /* prototypes */ PROTOTYPEObj(netstrm); diff --git a/runtime/nsd.h b/runtime/nsd.h index f0c9b9b6..8668c934 100644 --- a/runtime/nsd.h +++ b/runtime/nsd.h @@ -69,9 +69,13 @@ BEGINinterface(nsd) /* name must also be changed in ENDinterface macro! */ * sockets - not really desirable, but not the end of the world... TODO: should be * reconsidered when a new ACL system is build. -- rgerhards, 2008-12-01 */ + /* v5 */ + rsRetVal (*EnableKeepAlive)(nsd_t *pThis); ENDinterface(nsd) -#define nsdCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ -/* interface version 4 added GetRemAddr() */ +#define nsdCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ +/* interface version 4 added GetRemAddr() + * interface version 5 added EnableKeepAlive() -- rgerhards, 2009-06-02 + */ /* interface for the select call */ BEGINinterface(nsdsel) /* name must also be changed in ENDinterface macro! */ diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 5786e191..1a50e2f8 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -1560,6 +1560,16 @@ finalize_it: RETiRet; } +/* Enable KEEPALIVE handling on the socket. + * rgerhards, 2009-06-02 + */ +static rsRetVal +EnableKeepAlive(nsd_t *pNsd) +{ + return nsd_ptcp.EnableKeepAlive(pNsd); +} + + /* open a connection to a remote host (server). With GnuTLS, we always * open a plain tcp socket and then, if in TLS mode, do a handshake on it. @@ -1669,6 +1679,7 @@ CODESTARTobjQueryInterface(nsd_gtls) pIf->GetRemoteHName = GetRemoteHName; pIf->GetRemoteIP = GetRemoteIP; pIf->GetRemAddr = GetRemAddr; + pIf->EnableKeepAlive = EnableKeepAlive; finalize_it: ENDobjQueryInterface(nsd_gtls) diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index cc531ca0..54ee0666 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -614,6 +614,34 @@ finalize_it: } +/* Enable KEEPALIVE handling on the socket. + * rgerhards, 2009-06-02 + */ +static rsRetVal +EnableKeepAlive(nsd_t *pNsd) +{ + nsd_ptcp_t *pThis = (nsd_ptcp_t*) pNsd; + int ret; + int optval; + socklen_t optlen; + DEFiRet; + ISOBJ_TYPE_assert(pThis, nsd_ptcp); + + optval = 1; + optlen = sizeof(optval); + ret = setsockopt(pThis->sock, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen); + if(ret < 0) { + dbgprintf("EnableKeepAlive socket call returns error %d\n", ret); + ABORT_FINALIZE(RS_RET_ERR); + } + + dbgprintf("KEEPALIVE enabled for nsd %p\n", pThis); + +finalize_it: + RETiRet; +} + + /* open a connection to a remote host (server). * rgerhards, 2008-03-19 */ @@ -754,6 +782,7 @@ CODESTARTobjQueryInterface(nsd_ptcp) pIf->GetRemoteHName = GetRemoteHName; pIf->GetRemoteIP = GetRemoteIP; pIf->CheckConnection = CheckConnection; + pIf->EnableKeepAlive = EnableKeepAlive; finalize_it: ENDobjQueryInterface(nsd_ptcp) diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c index 6c803ee2..3dc53a97 100644 --- a/runtime/strmsrv.c +++ b/runtime/strmsrv.c @@ -434,6 +434,10 @@ SessAccept(strmsrv_t *pThis, strmLstnPortList_t *pLstnInfo, strms_sess_t **ppSes ABORT_FINALIZE(RS_RET_MAX_SESS_REACHED); } + if(pThis->bUseKeepAlive) { + CHKiRet(netstrm.EnableKeepAlive(pNewStrm)); + } + /* we found a free spot and can construct our session object */ CHKiRet(strms_sess.Construct(&pSess)); CHKiRet(strms_sess.SetStrmsrv(pSess, pThis)); @@ -756,6 +760,15 @@ SetUsrP(strmsrv_t *pThis, void *pUsr) RETiRet; } +static rsRetVal +SetKeepAlive(strmsrv_t *pThis, int iVal) +{ + DEFiRet; + dbgprintf("keep-alive set to %d\n", iVal); + pThis->bUseKeepAlive = iVal; + RETiRet; +} + static rsRetVal SetOnCharRcvd(strmsrv_t *pThis, rsRetVal (*OnCharRcvd)(strms_sess_t*, uchar)) { @@ -864,6 +877,7 @@ CODESTARTobjQueryInterface(strmsrv) pIf->create_strm_socket = create_strm_socket; pIf->Run = Run; + pIf->SetKeepAlive = SetKeepAlive; pIf->SetUsrP = SetUsrP; pIf->SetInputName = SetInputName; pIf->SetSessMax = SetSessMax; diff --git a/runtime/strmsrv.h b/runtime/strmsrv.h index e63d3a47..86e529c3 100644 --- a/runtime/strmsrv.h +++ b/runtime/strmsrv.h @@ -38,6 +38,7 @@ struct strmLstnPortList_s { /* the strmsrv object */ struct strmsrv_s { BEGINobjInstance; /**< Data to implement generic object - MUST be the first data element! */ + int bUseKeepAlive; /**< use socket layer KEEPALIVE handling? */ netstrms_t *pNS; /**< pointer to network stream subsystem */ int iDrvrMode; /**< mode of the stream driver to use */ uchar *pszDrvrAuthMode; /**< auth mode of the stream driver to use */ @@ -80,6 +81,7 @@ BEGINinterface(strmsrv) /* name must also be changed in ENDinterface macro! */ /* set methods */ rsRetVal (*SetAddtlFrameDelim)(strmsrv_t*, int); rsRetVal (*SetInputName)(strmsrv_t*, uchar*); + rsRetVal (*SetKeepAlive)(strmsrv_t*, int); rsRetVal (*SetUsrP)(strmsrv_t*, void*); rsRetVal (*SetCBIsPermittedHost)(strmsrv_t*, int (*) (struct sockaddr *addr, char*, void*, void*)); rsRetVal (*SetCBOpenLstnSocks)(strmsrv_t *, rsRetVal (*)(strmsrv_t*)); -- cgit v1.2.3 From 4f742a8e32c43dc9b514ceaf80f4d17e697dfdf6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 3 Jun 2009 07:51:47 +0200 Subject: added config switch --enable-testbench=no to turn off testbench --- ChangeLog | 1 + configure.ac | 15 +++++++++++++++ tests/Makefile.am | 3 +++ 3 files changed, 19 insertions(+) diff --git a/ChangeLog b/ChangeLog index ec971916..f9e3e32b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -8,6 +8,7 @@ Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? considered useful for load balancers. - bugfix: imdiag/imtcp had a race condition - improved testbench (now much better code design and reuse) +- added config switch --enable-testbench=no to turn off testbench --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) diff --git a/configure.ac b/configure.ac index 9fc59f76..70eea038 100644 --- a/configure.ac +++ b/configure.ac @@ -678,6 +678,20 @@ AC_SUBST(LIBLOGGING_CFLAGS) AC_SUBST(LIBLOGGING_LIBS) +# enable/disable the testbench (e.g. because some important parts +# are missing) +AC_ARG_ENABLE(testbench, + [AS_HELP_STRING([--enable-testbench],[file input module enabled @<:@default=yes@:>@])], + [case "${enableval}" in + yes) enable_testbench="yes" ;; + no) enable_testbench="no" ;; + *) AC_MSG_ERROR(bad value ${enableval} for --enable-testbench) ;; + esac], + [enable_testbench=yes] +) +AM_CONDITIONAL(ENABLE_TESTBENCH, test x$enable_testbench = xyes) + + # settings for the file input module AC_ARG_ENABLE(imfile, [AS_HELP_STRING([--enable-imfile],[file input module enabled @<:@default=no@:>@])], @@ -844,6 +858,7 @@ echo " RELP support enabled: $enable_relp" echo " SNMP support enabled: $enable_snmp" echo echo "---{ debugging support }---" +echo " Testbench enabled: $enable_testbench" echo " Debug mode enabled: $enable_debug" echo " Runtime Instrumentation enabled: $enable_rtinst" echo " Diagnostic tools enabled: $enable_diagtools" diff --git a/tests/Makefile.am b/tests/Makefile.am index e93aba10..402c1d83 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,3 +1,4 @@ +if ENABLE_TESTBENCH TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh queue-persist.sh \ @@ -6,6 +7,8 @@ if ENABLE_OMSTDOUT TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh endif +endif # if ENABLE_TESTBENCH + TESTS_ENVIRONMENT = RSYSLOG_MODDIR='$(abs_top_builddir)'/runtime/.libs/ DISTCLEANFILES=rsyslog.pid '$(abs_top_builddir)'/DiagTalker.class test_files = testbench.h runtime-dummy.c -- cgit v1.2.3 From f54e72cec06f21f4af939c70541e8a339b7e56ff Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 3 Jun 2009 14:45:09 +0200 Subject: first proof of concept on zipped file writer This DOES NOT work sufficiently well, I just wanted to verify that zip writing is possible and files are readable. Will be refined soon. --- configure.ac | 1 + runtime/Makefile.am | 11 ++ runtime/rsyslog.h | 1 + runtime/unicode-helper.h | 1 + runtime/zlibw.c | 119 ++++++++++++++++++++++ runtime/zlibw.h | 45 +++++++++ tools/Makefile.am | 4 +- tools/omfile.c | 234 ++++++++++++++++++++++++++++++++++++------- tools/zpipe.c | 254 +++++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 631 insertions(+), 39 deletions(-) create mode 100644 runtime/zlibw.c create mode 100644 runtime/zlibw.h create mode 100644 tools/zpipe.c diff --git a/configure.ac b/configure.ac index 70eea038..4a040e6a 100644 --- a/configure.ac +++ b/configure.ac @@ -186,6 +186,7 @@ AC_ARG_ENABLE(zlib, esac], [enable_zlib=yes] ) +AM_CONDITIONAL(ENABLE_ZLIB, test x$enable_zlib = xyes) if test "$enable_zlib" = "yes"; then AC_CHECK_HEADER(zlib.h, [zlib_header="yes"], [zlib_header="no" enable_zlib="false"]) if test "$zlib_header" = "yes"; then diff --git a/runtime/Makefile.am b/runtime/Makefile.am index eeb656d6..81422d64 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -105,6 +105,17 @@ lmregexp_la_LDFLAGS = -module -avoid-version lmregexp_la_LIBADD = endif +# +# zlib support +# +if ENABLE_ZLIB +pkglib_LTLIBRARIES += lmzlibw.la +lmzlibw_la_SOURCES = zlibw.c zlibw.h +lmzlibw_la_CPPFLAGS = $(PTHREADS_CFLAGS) $(RSRT_CFLAGS) +lmzlibw_la_LDFLAGS = -module -avoid-version +lmzlibw_la_LIBADD = +endif + if ENABLE_INET pkglib_LTLIBRARIES += lmnet.la lmnetstrms.la # diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 293e559b..57c5dcdf 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -275,6 +275,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ RS_RET_NONFATAL_CONFIG_ERR = -2123, /**< non-fatal error during config processing */ RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ + RS_RET_ZLIB_ERR = -2141, /**< error during zlib call */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h index 36d76a78..fe4cc1f8 100644 --- a/runtime/unicode-helper.h +++ b/runtime/unicode-helper.h @@ -48,6 +48,7 @@ static inline uchar* ustrdup(uchar *psz) #define UCHAR_CONSTANT(x) ((uchar*) (x)) +#define CHAR_CONVERT(x) ((char*) (x)) #endif /* multi-include protection */ /* vim:set ai: diff --git a/runtime/zlibw.c b/runtime/zlibw.c new file mode 100644 index 00000000..95ac9138 --- /dev/null +++ b/runtime/zlibw.c @@ -0,0 +1,119 @@ +/* The zlibwrap object. + * + * This is an rsyslog object wrapper around zlib. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include + +#include "rsyslog.h" +#include "module-template.h" +#include "obj.h" +#include "zlibw.h" + +MODULE_TYPE_LIB + +/* static data */ +DEFobjStaticHelpers + + +/* ------------------------------ methods ------------------------------ */ + +/* zlib make strong use of macros for its interface functions, so we can not simply + * pass function pointers to them. Instead, we create very small wrappers which call + * the relevant entry points. + */ + +static int myDeflateInit(z_streamp strm, int level) +{ + return deflateInit(strm, level); +} + +static int myDeflate(z_streamp strm, int flush) +{ + return deflate(strm, flush); +} + +static int myDeflateEnd(z_streamp strm) +{ + return deflateEnd(strm); +} + + +/* queryInterface function + * rgerhards, 2008-03-05 + */ +BEGINobjQueryInterface(zlibw) +CODESTARTobjQueryInterface(zlibw) + if(pIf->ifVersion != zlibwCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->DeflateInit = myDeflateInit; + pIf->Deflate = myDeflate; + pIf->DeflateEnd = myDeflateEnd; +finalize_it: +ENDobjQueryInterface(zlibw) + + +/* Initialize the zlibw class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINAbstractObjClassInit(zlibw, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */ + /* request objects we use */ + + /* set our own handlers */ +ENDObjClassInit(zlibw) + + +/* --------------- here now comes the plumbing that makes as a library module --------------- */ + + +BEGINmodExit +CODESTARTmodExit +ENDmodExit + + +BEGINqueryEtryPt +CODESTARTqueryEtryPt +CODEqueryEtryPt_STD_LIB_QUERIES +ENDqueryEtryPt + + +BEGINmodInit() +CODESTARTmodInit + *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ + + CHKiRet(zlibwClassInit(pModInfo)); /* must be done after tcps_sess, as we use it */ + /* Initialize all classes that are in our module - this includes ourselfs */ +ENDmodInit +/* vi:set ai: + */ diff --git a/runtime/zlibw.h b/runtime/zlibw.h new file mode 100644 index 00000000..01e0b054 --- /dev/null +++ b/runtime/zlibw.h @@ -0,0 +1,45 @@ +/* The zlibw object. It encapsulates the zlib functionality. The primary + * purpose of this wrapper class is to enable rsyslogd core to be build without + * zlib libraries. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_ZLIBW_H +#define INCLUDED_ZLIBW_H + +#include + +/* interfaces */ +BEGINinterface(zlibw) /* name must also be changed in ENDinterface macro! */ + int (*DeflateInit)(z_streamp strm, int); + int (*Deflate)(z_streamp strm, int); + int (*DeflateEnd)(z_streamp strm); +ENDinterface(zlibw) +#define zlibwCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(zlibw); + +/* the name of our library binary */ +#define LM_ZLIBW_FILENAME "lmzlibw" + +#endif /* #ifndef INCLUDED_ZLIBW_H */ diff --git a/tools/Makefile.am b/tools/Makefile.am index e523b854..f0f9afab 100644 --- a/tools/Makefile.am +++ b/tools/Makefile.am @@ -26,8 +26,10 @@ rsyslogd_LDADD = $(ZLIB_LIBS) $(PTHREADS_LIBS) $(RSRT_LIBS) $(SOL_LIBS) rsyslogd_LDFLAGS = -export-dynamic if ENABLE_DIAGTOOLS -sbin_PROGRAMS += rsyslog_diag_hostname msggen +sbin_PROGRAMS += rsyslog_diag_hostname msggen zpipe rsyslog_diag_hostname_SOURCES = gethostn.c +zpipe_SOURCES = zpipe.c +zpipe_LDADD = -lz msggen_SOURCES = msggen.c endif diff --git a/tools/omfile.c b/tools/omfile.c index c7283e4d..62a044d8 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -12,7 +12,7 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -57,6 +57,8 @@ #include "cfsysline.h" #include "module-template.h" #include "errmsg.h" +#include "unicode-helper.h" +#include "zlibw.h" MODULE_TYPE_OUTPUT @@ -64,16 +66,30 @@ MODULE_TYPE_OUTPUT */ DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) +DEFobjCurrIf(zlibw) /* The following structure is a dynafile name cache entry. */ struct s_dynaFileCacheEntry { - uchar *pName; /* name currently open, if dynamic name */ + uchar *pName; /* name currently open, if dynamic name */ short fd; /* name associated with file name in cache */ time_t lastUsed; /* for LRU - last access */ }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; +/* the output buffer structure */ +// TODO: later make this part of the dynafile cache! +#define OUTBUF_LEN 128 // TODO: make dynamic! +typedef struct { + uchar pszBuf[OUTBUF_LEN]; /* output buffer for buffered writing */ + size_t lenBuf; /* max size of buffer */ + size_t iBuf; /* current buffer index */ + int fd; /* which file descriptor is this buf for? */ + /* elements for zip writing */ + z_stream zStrm; + char zipBuf[OUTBUF_LEN]; +} outbuf_t; + /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ @@ -92,7 +108,8 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ - short fd; /* file descriptor for (current) file */ + short fd; /* file descriptor for (current) file */ + outbuf_t *poBuf; /* output buffer */ enum { eTypeFILE, eTypeTTY, @@ -132,7 +149,7 @@ ENDisCompatibleWithFeature BEGINdbgPrintInstInfo CODESTARTdbgPrintInstInfo if(pData->bDynamicName) { - printf("[dynamic]\n\ttemplate='%s'" + dbgprintf("[dynamic]\n\ttemplate='%s'" "\tfile cache size=%d\n" "\tcreate directories: %s\n" "\tfile owner %d, group %d\n" @@ -146,9 +163,9 @@ CODESTARTdbgPrintInstInfo pData->bFailOnChown ? "yes" : "no" ); } else { /* regular file */ - printf("%s", pData->f_fname); + dbgprintf("%s", pData->f_fname); if (pData->fd == -1) - printf(" (unused)"); + dbgprintf(" (unused)"); } ENDdbgPrintInstInfo @@ -321,7 +338,8 @@ int resolveFileSizeLimit(instanceData *pData) * function immediately returns. Parameter bFreeEntry is 1 * if the entry should be d_free()ed and 0 if not. */ -static void dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) +static void +dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) { ASSERT(pCache != NULL); @@ -356,7 +374,8 @@ finalize_it: * relevant files. Part of Shutdown and HUP processing. * rgerhards, 2008-10-23 */ -static inline void dynaFileFreeCacheEntries(instanceData *pData) +static inline void +dynaFileFreeCacheEntries(instanceData *pData) { register int i; ASSERT(pData != NULL); @@ -562,48 +581,32 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg } -/* rgerhards 2004-11-11: write to a file output. This - * will be called for all outputs using file semantics, - * for example also for pipes. +/* physically write the file */ -static rsRetVal writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +static rsRetVal +doPhysWrite(instanceData *pData, int fd, char *pszBuf, size_t lenBuf) { off_t actualFileSize; int iLenWritten; DEFiRet; - ASSERT(pData != NULL); - /* first check if we have a dynamic file name and, if so, - * check if it still is ok or a new file needs to be created - */ - if(pData->bDynamicName) { - if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - if(pData->fd == -1) { - rsRetVal iRetLocal; - iRetLocal = prepareFile(pData, pData->f_fname); - if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - /* create the message based on format specified */ +dbgprintf("doPhysWrite, fd %d, iBuf %d\n", fd, (int) lenBuf); again: /* check if we have a file size limit and, if so, * obey to it. */ if(pData->f_sizeLimit != 0) { - actualFileSize = lseek(pData->fd, 0, SEEK_END); + actualFileSize = lseek(fd, 0, SEEK_END); if(actualFileSize >= pData->f_sizeLimit) { char errMsg[256]; /* for now, we simply disable a file once it is * beyond the maximum size. This is better than having * us aborted by the OS... rgerhards 2005-06-21 */ - (void) close(pData->fd); + (void) close(fd); /* try to resolve the situation */ + // TODO: *doesn't work, will need to use new fd ! if(resolveFileSizeLimit(pData) != 0) { /* didn't work out, so disable... */ snprintf(errMsg, sizeof(errMsg), @@ -622,13 +625,12 @@ again: } } - iLenWritten = write(pData->fd, ppString[0], strlen((char*)ppString[0])); -//dbgprintf("lenwritten: %d\n", iLenWritten); + iLenWritten = write(fd, pszBuf, lenBuf); if(iLenWritten < 0) { int e = errno; char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); - DBGPRINTF("log file (%d) write error %d: %s\n", pData->fd, e, errStr); + DBGPRINTF("log file (%d) write error %d: %s\n", fd, e, errStr); /* If a named pipe is full, we suspend this action for a while */ if(pData->fileType == eTypePIPE && e == EAGAIN) @@ -667,9 +669,159 @@ again: errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); } } else if (pData->bSyncFile) { - fsync(pData->fd); + fsync(fd); + } + + pData->poBuf->iBuf = 0; + +finalize_it: + RETiRet; +} + + +/* write the output buffer in zip mode + * This means we compress it first and then do a physical write. + */ +static rsRetVal +doZipWrite(instanceData *pData) +{ + outbuf_t *poBuf; + z_stream strm; + int zRet; /* zlib return state */ + DEFiRet; + assert(pData != NULL); + + poBuf = pData->poBuf; /* use as a shortcut */ + strm = poBuf->zStrm; /* another shortcut */ + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + zRet = zlibw.DeflateInit(&strm, 9); + if(zRet != Z_OK) { + dbgprintf("error %d returned from zlib/deflateInit()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } +RUNLOG_STR("deflateInit() done successfully\n"); + + /* now doing the compression */ + strm.avail_in = poBuf->iBuf; + strm.next_in = (Bytef*) poBuf->pszBuf; + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", strm.avail_in, strm.total_in); + strm.avail_out = OUTBUF_LEN; + strm.next_out = (Bytef*) poBuf->zipBuf; + zRet = zlibw.Deflate(&strm, Z_FINISH); /* no bad return value */ + dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, strm.avail_out); + assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ + CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - strm.avail_out)); + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + +RUNLOG_STR("deflate() should be done successfully\n"); + + zRet = zlibw.DeflateEnd(&strm); + if(zRet != Z_OK) { + dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } +RUNLOG_STR("deflateEnd() done successfully\n"); + +finalize_it: + RETiRet; +} + + +/* flush the output buffer + */ +static rsRetVal +doFlush(instanceData *pData) +{ + DEFiRet; + assert(pData != NULL); + + if(pData->poBuf->iBuf == 0) + FINALIZE; /* nothing to write, but make this a valid case */ + + if(1) { // zlib enabled! + CHKiRet(doZipWrite(pData)); + } else { + CHKiRet(doPhysWrite(pData, pData->poBuf->fd, (char*)pData->poBuf->pszBuf, pData->poBuf->iBuf)); + } + +finalize_it: + RETiRet; +} + + +/* do the actual write process. This function is to be called once we are ready for writing. + * It will do buffered writes and persist data only when the buffer is full. Note that we must + * be careful to detect when the file handle changed. + * rgerhards, 2009-06-03 + */ +static rsRetVal +doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) +{ + int i; + outbuf_t *poBuf; + DEFiRet; + ASSERT(pData != NULL); + ASSERT(pszBuf != NULL); + + poBuf = pData->poBuf; /* use as a shortcut */ +dbgprintf("doWrite, pData->fd %d, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", +pData->fd, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); + + if(pData->fd != poBuf->fd) { + // TODO: more efficient use for dynafiles + CHKiRet(doFlush(pData)); + poBuf->fd = pData->fd; + } + + for(i = 0 ; i < lenBuf ; ++i) { + poBuf->pszBuf[poBuf->iBuf++] = pszBuf[i]; + if(poBuf->iBuf == poBuf->lenBuf) { + CHKiRet(doFlush(pData)); + } + } + +finalize_it: + RETiRet; +} + + +/* rgerhards 2004-11-11: write to a file output. This + * will be called for all outputs using file semantics, + * for example also for pipes. + */ +static rsRetVal +writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) +{ + DEFiRet; + + ASSERT(pData != NULL); + + /* first check if we have a dynamic file name and, if so, + * check if it still is ok or a new file needs to be created + */ + if(pData->bDynamicName) { + if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ + } + + if(pData->fd == -1) { + rsRetVal iRetLocal; + iRetLocal = prepareFile(pData, pData->f_fname); + if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) + ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ } + /* create the message based on format specified */ + CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0])))); + finalize_it: RETiRet; } @@ -678,15 +830,20 @@ finalize_it: BEGINcreateInstance CODESTARTcreateInstance pData->fd = -1; + CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); + pData->poBuf->lenBuf = OUTBUF_LEN; +finalize_it: ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance + doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->bDynamicName) { dynaFileFreeCache(pData); } else if(pData->fd != -1) close(pData->fd); + free(pData->poBuf); ENDfreeInstance @@ -696,7 +853,7 @@ ENDtryResume BEGINdoAction CODESTARTdoAction - DBGPRINTF(" (%s)\n", pData->f_fname); + DBGPRINTF("file to log to: %s\n", pData->f_fname); iRet = writeFile(ppString, iMsgOpts, pData); ENDdoAction @@ -875,8 +1032,8 @@ ENDdoHUP BEGINmodExit CODESTARTmodExit - if(pszTplName != NULL) - free(pszTplName); + objRelease(zlibw, LM_ZLIBW_FILENAME); + free(pszTplName); ENDmodExit @@ -891,6 +1048,7 @@ BEGINmodInit(File) CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr + CHKiRet(objUse(zlibw, LM_ZLIBW_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); diff --git a/tools/zpipe.c b/tools/zpipe.c new file mode 100644 index 00000000..bde6c5c1 --- /dev/null +++ b/tools/zpipe.c @@ -0,0 +1,254 @@ +/* zpipe.c: example of proper use of zlib's inflate() and deflate() + Not copyrighted -- provided to the public domain + Version 1.5 11 December 2005 Mark Adler + Version 2.0 03 June 2009 Rainer Gerhards */ + +/* RSYSLOG NOTE: + * This file is beeing distributed as part of rsyslog, but is just an + * add-on. Most importantly, rsyslog's copyright does not apply but + * rather the (non-) copyright stated above. + */ + +/* Version history: + 1.0 30 Oct 2004 First version + 1.1 8 Nov 2004 Add void casting for unused return values + Use switch statement for inflate() return values + 1.2 9 Nov 2004 Add assertions to document zlib guarantees + 1.3 6 Apr 2005 Remove incorrect assertion in inf() + 1.4 11 Dec 2005 Add hack to avoid MSDOS end-of-line conversions + Avoid some compiler warnings for input and output buffers + 2.0 03 Jun 2009 Add hack to support multiple deflate records inside a single + file on inflate. This is needed in order to support reading + files created by rsyslog's zip output writer. + */ + +#include +#include +#include +#include "zlib.h" + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) +# include +# include +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#define CHUNK 16384 + +/* Compress from file source to file dest until EOF on source. + def() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_STREAM_ERROR if an invalid compression + level is supplied, Z_VERSION_ERROR if the version of zlib.h and the + version of the library linked do not match, or Z_ERRNO if there is + an error reading or writing the files. */ +int def(FILE *source, FILE *dest, int level) +{ + int ret, flush; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + + /* allocate deflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + ret = deflateInit(&strm, level); + if (ret != Z_OK) + return ret; + + /* compress until end of file */ + do { + strm.avail_in = fread(in, 1, CHUNK, source); + if (ferror(source)) { + (void)deflateEnd(&strm); + return Z_ERRNO; + } + flush = feof(source) ? Z_FINISH : Z_NO_FLUSH; + strm.next_in = in; + + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = deflate(&strm, flush); /* no bad return value */ + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)deflateEnd(&strm); + return Z_ERRNO; + } + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + + /* done when last data in file processed */ + } while (flush != Z_FINISH); + assert(ret == Z_STREAM_END); /* stream will be complete */ + + /* clean up and return */ + (void)deflateEnd(&strm); + return Z_OK; +} + + +/* initialize stream for deflating (we need this in case of + * multiple records. + * rgerhards, 2009-06-03 + */ +int doInflateInit(z_stream *strm) +{ + int ret; + + /* allocate inflate state */ + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + strm->avail_in = 0; + strm->next_in = Z_NULL; + ret = inflateInit(strm); + return ret; +} + + +/* Decompress from file source to file dest until stream ends or EOF. + inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_DATA_ERROR if the deflate data is + invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and + the version of the library linked do not match, or Z_ERRNO if there + is an error reading or writing the files. */ +int inf(FILE *source, FILE *dest) +{ + int ret; + unsigned have; + z_stream strm; + unsigned char in[CHUNK]; + int len; + unsigned char *next_in_save; + unsigned char out[CHUNK]; + + ret = doInflateInit(&strm); + if (ret != Z_OK) + return ret; + + /* decompress until deflate stream ends or end of file */ + do { + len = fread(in, 1, CHUNK, source); + if (ferror(source)) { + (void)inflateEnd(&strm); + return Z_ERRNO; + } + if (len == 0) { + break; + } + strm.avail_in = len; + strm.next_in = in; + + /* run inflate() on input until output buffer not full */ + strm.avail_out = CHUNK; + strm.next_out = out; + do { + /* fprintf(stderr, "---inner LOOP---, avail_in %d, avail_out %d Byte 0: %x, 1: %x\n", strm.avail_in, strm.avail_out, *strm.next_in, *(strm.next_in+1));*/ + do { + ret = inflate(&strm, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR); /* state not clobbered */ + switch (ret) { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return ret; + } + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)inflateEnd(&strm); + return Z_ERRNO; + } + } while (strm.avail_out == 0); + /* handle the case that more than one deflate record is contained + * in a single file. -- rgerhards, 2009-06-03 + */ + if(ret == Z_STREAM_END) { + len -= strm.total_in; + if(len > 0) { + next_in_save = strm.next_in; + (void)inflateEnd(&strm); + ret = doInflateInit(&strm); + if (ret != Z_OK) + return ret; + strm.avail_in = len; + strm.next_in = next_in_save; + strm.avail_out = CHUNK; + strm.next_out = out; + ret = Z_OK; /* continue outer loop */ + } + } + } while (strm.avail_in > 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR; +} + +/* report a zlib or i/o error */ +void zerr(int ret) +{ + fputs("zpipe: ", stdout); + switch (ret) { + case Z_ERRNO: + if (ferror(stdin)) + fputs("error reading stdin\n", stdout); + if (ferror(stdout)) + fputs("error writing stdout\n", stdout); + break; + case Z_STREAM_ERROR: + fputs("invalid compression level\n", stdout); + break; + case Z_DATA_ERROR: + fputs("invalid or incomplete deflate data\n", stdout); + break; + case Z_MEM_ERROR: + fputs("out of memory\n", stdout); + break; + case Z_VERSION_ERROR: + fputs("zlib version mismatch!\n", stdout); + } +} + +/* compress or decompress from stdin to stdout */ +int main(int argc, char **argv) +{ + int ret; + + /* avoid end-of-line conversions */ + SET_BINARY_MODE(stdin); + SET_BINARY_MODE(stdout); + + /* do compression if no arguments */ + if (argc == 1) { + ret = def(stdin, stdout, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK) + zerr(ret); + return ret; + } + + /* do decompression if -d specified */ + else if (argc == 2 && strcmp(argv[1], "-d") == 0) { + ret = inf(stdin, stdout); + if (ret != Z_OK) + zerr(ret); + return ret; + } + + /* otherwise, report usage */ + else { + fputs("zpipe usage: zpipe [-d] < source > dest\n", stdout); + return 1; + } +} -- cgit v1.2.3 From 69eac5341d49c5c7278e6d86a77dc9d6b4a4edde Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 3 Jun 2009 15:32:39 +0200 Subject: added gzip header to output files so they can now be processed with the "regular" gzip tools --- runtime/zlibw.c | 10 ++++++++-- runtime/zlibw.h | 1 + tools/omfile.c | 30 +++++++++++++++++++++++++++--- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/runtime/zlibw.c b/runtime/zlibw.c index 95ac9138..2b386213 100644 --- a/runtime/zlibw.c +++ b/runtime/zlibw.c @@ -51,9 +51,9 @@ static int myDeflateInit(z_streamp strm, int level) return deflateInit(strm, level); } -static int myDeflate(z_streamp strm, int flush) +static int myDeflateInit2(z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy) { - return deflate(strm, flush); + return deflateInit2(strm, level, method, windowBits, memLevel, strategy); } static int myDeflateEnd(z_streamp strm) @@ -61,6 +61,11 @@ static int myDeflateEnd(z_streamp strm) return deflateEnd(strm); } +static int myDeflate(z_streamp strm, int flush) +{ + return deflate(strm, flush); +} + /* queryInterface function * rgerhards, 2008-03-05 @@ -77,6 +82,7 @@ CODESTARTobjQueryInterface(zlibw) * of course, also affects the "if" above). */ pIf->DeflateInit = myDeflateInit; + pIf->DeflateInit2 = myDeflateInit2; pIf->Deflate = myDeflate; pIf->DeflateEnd = myDeflateEnd; finalize_it: diff --git a/runtime/zlibw.h b/runtime/zlibw.h index 01e0b054..63d8f386 100644 --- a/runtime/zlibw.h +++ b/runtime/zlibw.h @@ -30,6 +30,7 @@ /* interfaces */ BEGINinterface(zlibw) /* name must also be changed in ENDinterface macro! */ int (*DeflateInit)(z_streamp strm, int); + int (*DeflateInit2)(z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy); int (*Deflate)(z_streamp strm, int); int (*DeflateEnd)(z_streamp strm); ENDinterface(zlibw) diff --git a/tools/omfile.c b/tools/omfile.c index 62a044d8..08c33a7f 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -14,6 +14,29 @@ * * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * + * An important note on writing gzip format via zlib (kept anonymous + * by request): + * + * -------------------------------------------------------------------------- + * We'd like to make sure the output file is in full gzip format + * (compatible with gzip -d/zcat etc). There is a flag in how the output + * is initialized within zlib to properly add the gzip wrappers to the + * output. (gzip is effectively a small metadata wrapper around raw + * zstream output.) + * + * I had written an old bit of code to do this - the documentation on + * deflatInit2() was pretty tricky to nail down on this specific feature: + * + * int deflateInit2 (z_streamp strm, int level, int method, int windowBits, + * int memLevel, int strategy); + * + * I believe "31" would be the value for the "windowBits" field that you'd + * want to try: + * + * deflateInit2(zstrmptr, 6, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + * -------------------------------------------------------------------------- + * + * * This file is part of rsyslog. * * Rsyslog is free software: you can redistribute it and/or modify @@ -698,12 +721,13 @@ doZipWrite(instanceData *pData) strm.zalloc = Z_NULL; strm.zfree = Z_NULL; strm.opaque = Z_NULL; - zRet = zlibw.DeflateInit(&strm, 9); + /* see note in file header for the params we use with deflateInit2() */ + zRet = zlibw.DeflateInit2(&strm, 9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); if(zRet != Z_OK) { - dbgprintf("error %d returned from zlib/deflateInit()\n", zRet); + dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); } -RUNLOG_STR("deflateInit() done successfully\n"); +RUNLOG_STR("deflateInit2() done successfully\n"); /* now doing the compression */ strm.avail_in = poBuf->iBuf; -- cgit v1.2.3 From 58e707b441aea88cd318762e6968e1db1211f949 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 09:57:45 +0200 Subject: backported some of the v5 testbench this permits us to keep a persistent test environment between v4 and v5, most importantly using the same tools. As far as the actual tests are concerned, some had issues. I had no time to check if that was an issue with the test or an actual issue with the v3/4 engine. Will do that at some later stage. --- runtime/rsyslog.h | 5 ++- tests/DiagTalker.java | 33 +++++++++++++++++-- tests/Makefile.am | 26 +++++++++++---- tests/chkseq.c | 66 ++++++++++++++++++++++++++++--------- tests/diag.sh | 10 +++--- tests/diskqueue.sh | 3 -- tests/ourtail.c | 2 +- tests/rscript.c | 4 +-- tests/tcpflood.c | 13 +++++--- tests/testsuites/queue-persist.conf | 2 -- 10 files changed, 123 insertions(+), 41 deletions(-) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 293e559b..ea303a51 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -273,7 +273,10 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_ERR_FORK = -2118, /**< error during fork() */ RS_RET_ERR_WRITE_PIPE = -2119, /**< error writing to pipe */ RS_RET_RSCORE_TOO_OLD = -2120, /**< rsyslog core is too old for ... (eg this plugin) */ - RS_RET_NONFATAL_CONFIG_ERR = -2123, /**< non-fatal error during config processing */ + RS_RET_DEFER_COMMIT = -2121, /**< output plugin status: not yet committed (an OK state!) */ + RS_RET_PREVIOUS_COMMITTED = -2122, /**< output plugin status: previous record was committed (an OK state!) */ + RS_RET_ACTION_FAILED = -2123, /**< action failed and is now suspended (consider this permanent for the time being) */ + RS_RET_NONFATAL_CONFIG_ERR = -2124, /**< non-fatal error during config processing */ RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ /* RainerScript error messages (range 1000.. 1999) */ diff --git a/tests/DiagTalker.java b/tests/DiagTalker.java index e33a5867..04e12327 100644 --- a/tests/DiagTalker.java +++ b/tests/DiagTalker.java @@ -1,3 +1,24 @@ +/* A yet very simple tool to talk to imdiag. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of rsyslog. + * + * Rsyslog is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Rsyslog is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Rsyslog. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + */ //package com.rsyslog.diag; import java.io.*; import java.net.*; @@ -13,6 +34,7 @@ public class DiagTalker { try { diagSocket = new Socket(host, port); + diagSocket.setSoTimeout(0); /* wait for lenghty operations */ out = new PrintWriter(diagSocket.getOutputStream(), true); in = new BufferedReader(new InputStreamReader( diagSocket.getInputStream())); @@ -29,9 +51,14 @@ public class DiagTalker { new InputStreamReader(System.in)); String userInput; - while ((userInput = stdIn.readLine()) != null) { - out.println(userInput); - System.out.println("imdiag returns: " + in.readLine()); + try { + while ((userInput = stdIn.readLine()) != null) { + out.println(userInput); + System.out.println("imdiag returns: " + in.readLine()); + } + } catch (SocketException e) { + System.err.println("We had a socket exception and consider this to be OK: " + + e.getMessage()); } out.close(); diff --git a/tests/Makefile.am b/tests/Makefile.am index 402c1d83..a95139f2 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,8 +1,13 @@ if ENABLE_TESTBENCH TESTRUNS = rt_init rscript check_PROGRAMS = $(TESTRUNS) ourtail nettester tcpflood chkseq -TESTS = $(TESTRUNS) cfg.sh manytcp.sh diskqueue.sh imtcp-multiport.sh queue-persist.sh \ - validation-run.sh +TESTS = $(TESTRUNS) cfg.sh \ + validation-run.sh \ + imtcp-multiport.sh \ + diskqueue.sh \ + manytcp.sh \ + queue-persist.sh + if ENABLE_OMSTDOUT TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh endif @@ -32,25 +37,32 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ err1.rstest \ NoExistFile.cfgtest \ testsuites/parse1.conf \ + testsuites/field1.conf \ testsuites/1.parse1 \ - testsuites/rfc3164.parse1 \ + testsuites/2.parse1 \ + testsuites/3.parse1 \ + testsuites/date1.parse1 \ + testsuites/date2.parse1 \ + testsuites/date3.parse1 \ + testsuites/date4.parse1 \ + testsuites/date5.parse1 \ + testsuites/rfc3164.parse1 \ testsuites/rfc5424-1.parse1 \ testsuites/rfc5424-2.parse1 \ testsuites/rfc5424-3.parse1 \ testsuites/rfc5424-4.parse1 \ testsuites/omod-if-array.conf \ testsuites/1.omod-if-array \ + testsuites/1.field1 \ killrsyslog.sh \ parsertest.sh \ + fieldtest.sh \ diskqueue.sh \ testsuites/diskqueue.conf \ imtcp-multiport.sh \ testsuites/imtcp-multiport.conf \ manytcp.sh \ testsuites/manytcp.conf \ - fieldtest.sh \ - testsuites/field1.conf \ - testsuites/1.field1 \ inputname.sh \ testsuites/inputname_imtcp.conf \ testsuites/1.inputname_imtcp_12514 \ @@ -58,6 +70,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ diag.sh \ + daqueue-persist.sh \ + daqueue-persist-drvr.sh \ testsuites/diag-common.conf \ queue-persist.sh \ queue-persist-drvr.sh \ diff --git a/tests/chkseq.c b/tests/chkseq.c index 3203c250..8c5fc61a 100644 --- a/tests/chkseq.c +++ b/tests/chkseq.c @@ -3,9 +3,10 @@ * be set. * * Params - * argv[1] file to check - * argv[2] start number - * argv[3] end number + * -f MUST be given! + * -s -e + * default for s is 0. -e should be given (else it is also 0) + * -d may be specified, in which case duplicate messages are permitted. * * Part of the testbench for rsyslog. * @@ -31,6 +32,7 @@ #include "config.h" #include #include +#include int main(int argc, char *argv[]) { @@ -38,16 +40,36 @@ int main(int argc, char *argv[]) int val; int i; int ret = 0; - int start, end; + int dupsPermitted = 0; + int start = 0, end = 0; + int opt; + int nDups = 0; + char *file = NULL; - if(argc != 4) { - printf("Invalid call of chkseq\n"); - printf("Usage: chkseq file start end\n"); + while((opt = getopt(argc, argv, "e:f:ds:")) != EOF) { + switch((char)opt) { + case 'f': + file = optarg; + break; + case 'd': + dupsPermitted = 1; + break; + case 'e': + end = atoi(optarg); + break; + case 's': + start = atoi(optarg); + break; + default:printf("Invalid call of chkseq\n"); + printf("Usage: chkseq file -sstart -eend -d\n"); + exit(1); + } + } + + if(file == NULL) { + printf("file must be given!\n"); exit(1); } - - start = atoi(argv[2]); - end = atoi(argv[3]); if(start > end) { printf("start must be less than or equal end!\n"); @@ -55,22 +77,36 @@ int main(int argc, char *argv[]) } /* read file */ - fp = fopen(argv[1], "r"); + fp = fopen(file, "r"); if(fp == NULL) { - perror(argv[1]); + printf("error opening file '%s'\n", file); + perror(file); exit(1); } - for(i = start ; i < end ; ++i) { + for(i = start ; i < end+1 ; ++i) { if(fscanf(fp, "%d\n", &val) != 1) { printf("scanf error in index i=%d\n", i); exit(1); } if(val != i) { - printf("read value %d, but expected value %d\n", val, i); - exit(1); + if(val == i - 1 && dupsPermitted) { + --i; + ++nDups; + } else { + printf("read value %d, but expected value %d\n", val, i); + exit(1); + } } } + if(nDups != 0) + printf("info: had %d duplicates (this is no error)\n", nDups); + + if(i - 1 != end) { + printf("only %d records in file, expected %d\n", i - 1, end); + exit(1); + } + exit(ret); } diff --git a/tests/diag.sh b/tests/diag.sh index ec2c1190..1ceca75b 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -5,8 +5,11 @@ # not always able to convey back states to the upper-level test driver # begun 2009-05-27 by rgerhards # This file is part of the rsyslog project, released under GPLv3 +#valgrind="valgrind --log-fd=1" +#valgrind="valgrind --tool=drd --log-fd=1" +#valgrind="valgrind --tool=helgrind --log-fd=1" #set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout" +#export RSYSLOG_DEBUG="debug nostdout printmutexaction" #export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason @@ -22,7 +25,7 @@ case $1 in ;; 'startup') # start rsyslogd with default params. $2 is the config file name to use # returns only after successful startup - ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & + $valgrind ../tools/rsyslogd -c4 -u2 -n -irsyslog.pid -M../runtime/.libs:../.libs -f$srcdir/testsuites/$2 & $srcdir/diag.sh wait-startup ;; 'wait-startup') # wait for rsyslogd startup @@ -72,9 +75,8 @@ case $1 in 'seq-check') # do the usual sequence check to see if everything was properly received rm -f work sort < rsyslog.out.log > work - ./chkseq work $2 $3 + ./chkseq -fwork -e$2 $3 if [ "$?" -ne "0" ]; then - rm -f work rsyslog.out.log echo "sequence error detected" exit 1 fi diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index bf1a46fd..2fe31db9 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -5,9 +5,6 @@ # added 2009-04-17 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 # uncomment for debugging support: -#set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout" -#export RSYSLOG_DEBUGLOG="tmp" echo testing queue disk-only mode source $srcdir/diag.sh init source $srcdir/diag.sh startup diskqueue.conf diff --git a/tests/ourtail.c b/tests/ourtail.c index f2751c72..6781b5fe 100644 --- a/tests/ourtail.c +++ b/tests/ourtail.c @@ -28,7 +28,7 @@ */ #include -int main(int argc, char *argv[]) +int main(int __attribute__((unused)) argc, char __attribute__((unused)) *argv[]) { int c; diff --git a/tests/rscript.c b/tests/rscript.c index ce81491c..6361aec4 100644 --- a/tests/rscript.c +++ b/tests/rscript.c @@ -104,8 +104,8 @@ PerformTest(cstr_t *pstrIn, rsRetVal iRetExpected, cstr_t *pstrOut) if(strcmp((char*)rsCStrGetSzStr(pstrPrg), (char*)rsCStrGetSzStr(pstrOut))) { printf("error: compiled program different from expected result!\n"); - printf("generated vmprg (%d bytes):\n%s\n", strlen((char*)rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); - printf("expected (%d bytes):\n%s\n", strlen((char*)rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); + printf("generated vmprg (%d bytes):\n%s\n", (int)strlen((char*)rsCStrGetSzStr(pstrPrg)), rsCStrGetSzStr(pstrPrg)); + printf("expected (%d bytes):\n%s\n", (int)strlen((char*)rsCStrGetSzStr(pstrOut)), rsCStrGetSzStr(pstrOut)); ABORT_FINALIZE(RS_RET_ERR); } diff --git a/tests/tcpflood.c b/tests/tcpflood.c index c3c9c871..2ca796ca 100644 --- a/tests/tcpflood.c +++ b/tests/tcpflood.c @@ -6,6 +6,7 @@ * argv[2] target port * argv[3] number of connections * argv[4] number of messages to send (connection is random) + * argv[5] initial message number (optional) * * Part of the testbench for rsyslog. * @@ -51,6 +52,7 @@ static int targetPort; static int numMsgsToSend; /* number of messages to send */ static int numConnections; /* number of connections to create */ static int *sockArray; /* array of sockets to use */ +static int msgNum = 0; /* initial message number to start with */ /* open a single tcp connection @@ -166,19 +168,20 @@ int sendMessages(void) socknum = i - (numMsgsToSend - numConnections); else socknum = rand() % numConnections; - lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", i); + lenBuf = sprintf(buf, "<167>Mar 1 01:00:00 172.20.245.8 tag msgnum:%8.8d:\n", msgNum); lenSend = send(sockArray[socknum], buf, lenBuf, 0); if(lenSend != lenBuf) { printf("\r%5.5d\n", i); fflush(stdout); perror("send test data"); - printf("send() failed at socket %d, index %d\n", socknum, i); + printf("send() failed at socket %d, index %d, msgNum %d\n", socknum, i, msgNum); fflush(stderr); return(1); } if(i % 100 == 0) { printf("\r%5.5d", i); } + ++msgNum; } printf("\r%5.5d messages sent\n", i); @@ -260,9 +263,9 @@ int main(int argc, char *argv[]) setvbuf(stdout, buf, _IONBF, 48); - if(argc != 5) { + if(argc != 5 && argc != 6) { printf("Invalid call of tcpflood\n"); - printf("Usage: tcpflood target-host target-port num-connections num-messages\n"); + printf("Usage: tcpflood target-host target-port num-connections num-messages [initial msgnum]\n"); exit(1); } @@ -270,6 +273,8 @@ int main(int argc, char *argv[]) targetPort = atoi(argv[2]); numConnections = atoi(argv[3]); numMsgsToSend = atoi(argv[4]); + if(argc == 6) + msgNum = atoi(argv[5]); if(openConnections() != 0) { printf("error opening connections\n"); diff --git a/tests/testsuites/queue-persist.conf b/tests/testsuites/queue-persist.conf index 81ee1be5..8903042d 100644 --- a/tests/testsuites/queue-persist.conf +++ b/tests/testsuites/queue-persist.conf @@ -9,8 +9,6 @@ $InputTCPServerRun 13514 $ModLoad ../plugins/omtesting/.libs/omtesting -$ErrorMessagesToStderr off - # set spool locations and switch queue to disk-only mode $WorkDirectory test-spool $MainMsgQueueFilename mainq -- cgit v1.2.3 From e893a5916473c45d6b90dc2401637fe713133291 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 10:32:12 +0200 Subject: backported macros necessary to create output modules with transaction support this does not mean the v4 engine will support batches and output transactions, but I can now write plugins that will work equally well with v4 AND v5. I consider this useful during the rewrite of omfile. --- runtime/module-template.h | 72 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/runtime/module-template.h b/runtime/module-template.h index 6f7d877c..3e963199 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -39,7 +39,8 @@ #define DEF_OMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA \ - DEFobjCurrIf(obj) + DEFobjCurrIf(obj) \ + static __attribute__((unused)) int bCoreSupportsBatching; #define DEF_IMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA \ DEFobjCurrIf(obj) @@ -160,6 +161,37 @@ static rsRetVal isCompatibleWithFeature(syslogFeature __attribute__((unused)) eF RETiRet;\ } + +/* beginTransaction() + * introduced in v4.3.3 -- rgerhards, 2009-04-27 + */ +#define BEGINbeginTransaction \ +static rsRetVal beginTransaction(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTbeginTransaction /* currently empty, but may be extended */ + +#define ENDbeginTransaction \ + RETiRet;\ +} + + +/* endTransaction() + * introduced in v4.3.3 -- rgerhards, 2009-04-27 + */ +#define BEGINendTransaction \ +static rsRetVal endTransaction(instanceData __attribute__((unused)) *pData)\ +{\ + DEFiRet; + +#define CODESTARTendTransaction /* currently empty, but may be extended */ + +#define ENDendTransaction \ + RETiRet;\ +} + + /* doAction() */ #define BEGINdoAction \ @@ -324,6 +356,18 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ *pEtryPoint = tryResume;\ } + +/* the following definition is queryEtryPt block that must be added + * if an output module supports the transactional interface. + * rgerhards, 2009-04-27 + */ +#define CODEqueryEtryPt_TXIF_OMOD_QUERIES \ + else if(!strcmp((char*) name, "beginTransaction")) {\ + *pEtryPoint = beginTransaction;\ + } else if(!strcmp((char*) name, "endTransaction")) {\ + *pEtryPoint = endTransaction;\ + } + /* the following definition is the standard block for queryEtryPt for INPUT * modules. This can be used if no specific handling (e.g. to cover version * differences) is needed. @@ -393,6 +437,32 @@ finalize_it:\ } +/* now come some check functions, which enable a standard way of obtaining feature + * information from the core. feat is the to-be-tested feature and featVar is a + * variable that receives the result (0-not support, 1-supported). + * This must be a macro, so that it is put into the output's code. Otherwise, we + * would need to rely on a library entry point, which is what we intend to avoid ;) + * rgerhards, 2009-04-27 + */ +#define INITChkCoreFeature(featVar, feat) \ +{ \ + rsRetVal MACRO_Ret; \ + rsRetVal (*pQueryCoreFeatureSupport)(int*, unsigned); \ + int bSupportsIt; \ + featVar = 0; \ + MACRO_Ret = pHostQueryEtryPt((uchar*)"queryCoreFeatureSupport", &pQueryCoreFeatureSupport); \ + if(MACRO_Ret == RS_RET_OK) { \ + /* found entry point, so let's see if core supports it */ \ + CHKiRet((*pQueryCoreFeatureSupport)(&bSupportsIt, feat)); \ + if(bSupportsIt) \ + featVar = 1; \ + } else if(MACRO_Ret != RS_RET_ENTRY_POINT_NOT_FOUND) { \ + ABORT_FINALIZE(MACRO_Ret); /* Something else went wrong, what is not acceptable */ \ + } \ +} + + + /* definitions for host API queries */ #define CODEmodInit_QueryRegCFSLineHdlr \ CHKiRet(pHostQueryEtryPt((uchar*)"regCfSysLineHdlr", &omsdRegCFSLineHdlr)); -- cgit v1.2.3 From 768836ab79268e1091fc3af8bb920c03287db91a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 11:05:41 +0200 Subject: reduced max value for $DynaFileCacheSize to 1,000 The former maximum of 10,000 really made no sense, even 1,000 is very high, but we like to keep the user in control ;)). Also, some general cleanup was done. --- ChangeLog | 3 +++ tools/omfile.c | 66 +++++++++++++++++++++++++--------------------------------- 2 files changed, 31 insertions(+), 38 deletions(-) diff --git a/ChangeLog b/ChangeLog index f9e3e32b..9d398f09 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,6 +9,9 @@ Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? - bugfix: imdiag/imtcp had a race condition - improved testbench (now much better code design and reuse) - added config switch --enable-testbench=no to turn off testbench +- reduced max value for $DynaFileCacheSize to 1,000 (the former maximum + of 10,000 really made no sense, even 1,000 is very high, but we like + to keep the user in control ;)). --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) diff --git a/tools/omfile.c b/tools/omfile.c index 08c33a7f..d1274f1b 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -12,6 +12,12 @@ * of the "old" message code without any modifications. However, it * helps to have things at the right place one we go to the meat of it. * + * A large re-write of this file was done in June, 2009. The focus was + * to introduce many more features (like zipped writing), clean up the code + * and make it more reliable. In short, that rewrite tries to provide a new + * solid basis for the next three to five years to come. During it, bugs + * may have been introduced ;) -- rgerhards, 2009-06-04 + * * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * An important note on writing gzip format via zlib (kept anonymous @@ -158,7 +164,7 @@ typedef struct _instanceData { */ dynaFileCacheEntry **dynCache; off_t f_sizeLimit; /* file size limit, 0 = no limit */ - char *f_sizeLimitCmd; /* command to carry out when size limit is reached */ + uchar *f_sizeLimitCmd; /* command to carry out when size limit is reached */ } instanceData; @@ -208,13 +214,13 @@ rsRetVal setDynaFileCacheSize(void __attribute__((unused)) *pVal, int iNewVal) errmsg.LogError(0, RS_RET_VAL_OUT_OF_RANGE, "%s", errMsg); iRet = RS_RET_VAL_OUT_OF_RANGE; iNewVal = 1; - } else if(iNewVal > 10000) { + } else if(iNewVal > 1000) { snprintf((char*) errMsg, sizeof(errMsg)/sizeof(uchar), - "DynaFileCacheSize maximum is 10,000 (%d given), changed to 10,000.", iNewVal); + "DynaFileCacheSize maximum is 1,000 (%d given), changed to 1,000.", iNewVal); errno = 0; errmsg.LogError(0, RS_RET_VAL_OUT_OF_RANGE, "%s", errMsg); iRet = RS_RET_VAL_OUT_OF_RANGE; - iNewVal = 10000; + iNewVal = 1000; } iDynaFileCacheSize = iNewVal; @@ -286,7 +292,7 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR /* WARNING: It is dangerous "just" to pass the pointer. As we * never rebuild the output channel description, this is acceptable here. */ - pData->f_sizeLimitCmd = (char*) pOch->cmdOnSizeLimit; + pData->f_sizeLimitCmd = pOch->cmdOnSizeLimit; iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName); @@ -322,7 +328,7 @@ int resolveFileSizeLimit(instanceData *pData) * when we have a space in the program name. If we find it, everything after * the space is treated as a single argument. */ - if((pCmd = (uchar*)strdup((char*)pData->f_sizeLimitCmd)) == NULL) { + if((pCmd = ustrdup(pData->f_sizeLimitCmd)) == NULL) { /* there is not much we can do - we make syslogd close the file in this case */ return 1; } @@ -372,7 +378,7 @@ dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) FINALIZE; DBGPRINTF("Removed entry %d for file '%s' from dynaCache.\n", iEntry, - pCache[iEntry]->pName == NULL ? "[OPEN FAILED]" : (char*)pCache[iEntry]->pName); + pCache[iEntry]->pName == NULL ? UCHAR_CONSTANT("[OPEN FAILED]") : pCache[iEntry]->pName); /* if the name is NULL, this is an improperly initilized entry which * needs to be discarded. In this case, neither the file is to be closed * not the name to be freed. @@ -452,7 +458,7 @@ prepareFile(instanceData *pData, uchar *newFileName) * We do not report any errors here ourselfs but let the code * fall through to error handler below. */ - if(makeFileParentDirs(newFileName, strlen((char*)newFileName), + if(makeFileParentDirs(newFileName, ustrlen(newFileName), pData->fDirCreateMode, pData->dirUID, pData->dirGID, pData->bFailOnChown) != 0) { ABORT_FINALIZE(RS_RET_ERR); /* we give up */ @@ -523,7 +529,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg * I *hope* this will be a performance enhancement. */ if( (pData->iCurrElt != -1) - && !strcmp((char*) newFileName, (char*) pCache[pData->iCurrElt]->pName)) { + && !ustrcmp(newFileName, pCache[pData->iCurrElt]->pName)) { /* great, we are all set */ pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ return 0; @@ -540,7 +546,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg if(iFirstFree == -1) iFirstFree = i; } else { /* got an element, let's see if it matches */ - if(!strcmp((char*) newFileName, (char*) pCache[i]->pName)) { + if(!ustrcmp(newFileName, pCache[i]->pName)) { /* we found our element! */ pData->fd = pCache[i]->fd; pData->iCurrElt = i; @@ -585,7 +591,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg if(iMsgOpts & INTERNAL_MSG) { DBGPRINTF("Could not open dynaFile, discarding message\n"); } else { - errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", (char*)newFileName); + errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", newFileName); } dynaFileDelCacheEntry(pCache, iFirstFree, 1); pData->iCurrElt = -1; @@ -593,7 +599,7 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg } pCache[iFirstFree]->fd = pData->fd; - pCache[iFirstFree]->pName = (uchar*)strdup((char*)newFileName); /* TODO: check for NULL (very unlikely) */ + pCache[iFirstFree]->pName = ustrdup(newFileName); /* TODO: check for NULL (very unlikely) */ pCache[iFirstFree]->lastUsed = time(NULL); pData->iCurrElt = iFirstFree; DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); @@ -884,21 +890,10 @@ ENDdoAction BEGINparseSelectorAct CODESTARTparseSelectorAct - /* yes, the if below is redundant, but I need it now. Will go away as - * the code further changes. -- rgerhards, 2007-07-25 - */ - if(*p == '$' || *p == '?' || *p == '|' || *p == '/' || *p == '-') { - if((iRet = createInstance(&pData)) != RS_RET_OK) { - ENDfunc - return iRet; /* this can not use RET_iRet! */ - } - } else { - /* this is not clean, but we need it for the time being - * TODO: remove when cleaning up modularization - */ - ENDfunc - return RS_RET_CONFLINE_UNPROCESSED; - } + if(!(*p == '$' || *p == '?' || *p == '|' || *p == '/' || *p == '-')) + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + + CHKiRet(createInstance(&pData)); if(*p == '-') { pData->bSyncFile = 0; @@ -933,15 +928,12 @@ CODESTARTparseSelectorAct */ CODE_STD_STRING_REQUESTparseSelectorAct(2) ++p; /* eat '?' */ - if((iRet = cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, - (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)) - != RS_RET_OK) - break; + CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)); /* "filename" is actually a template name, we need this as string 1. So let's add it * to the pOMSR. -- rgerhards, 2007-07-27 */ - if((iRet = OMSRsetEntry(*ppOMSR, 1, (uchar*)strdup((char*) pData->f_fname), OMSR_NO_RQD_TPL_OPTS)) != RS_RET_OK) - break; + CHKiRet(OMSRsetEntry(*ppOMSR, 1, ustrdup(pData->f_fname), OMSR_NO_RQD_TPL_OPTS)); pData->bDynamicName = 1; pData->iCurrElt = -1; /* no current element */ @@ -979,10 +971,8 @@ CODESTARTparseSelectorAct * to use is specified. So we need to scan for the first coma first * and then look at the rest of the line. */ - if((iRet = cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, - (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)) - != RS_RET_OK) - break; + CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, + (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)); pData->bDynamicName = 0; pData->fCreateMode = fCreateMode; /* preserve current setting */ @@ -1005,7 +995,7 @@ CODESTARTparseSelectorAct errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); break; } - if(strcmp((char*) p, _PATH_CONSOLE) == 0) + if(ustrcmp(p, UCHAR_CONSTANT(_PATH_CONSOLE)) == 0) pData->fileType = eTypeCONSOLE; break; default: -- cgit v1.2.3 From 9e434f19a9baa4a6f411808b5cb6bc22d6a32781 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 12:15:59 +0200 Subject: cleaned up stream class ... ... and also made it callable via an rsyslog interface rather then relying on the OS loader (important if we go for using it inside loadbale modules, which we soon possible will) --- plugins/imfile/imfile.c | 51 ++++++++++++------------ runtime/obj.c | 64 ++++++++++++++++--------------- runtime/queue.c | 90 ++++++++++++++++++++++--------------------- runtime/rsyslog.c | 2 - runtime/stream.c | 100 ++++++++++++++++++++++++++++++------------------ runtime/stream.h | 56 +++++++++++++-------------- runtime/sysvar.c | 2 - tools/omfile.c | 2 +- tools/syslogd.c | 21 ---------- 9 files changed, 198 insertions(+), 190 deletions(-) diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 92fd30c3..ff32c857 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -56,6 +56,7 @@ DEF_IMOD_STATIC_DATA /* must be present, starts static data */ DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(datetime) +DEFobjCurrIf(strm) typedef struct fileInfo_s { uchar *pszFileName; @@ -139,16 +140,16 @@ openFile(fileInfo_t *pThis) /* If we reach this point, we have a .si file */ - CHKiRet(strmConstruct(&psSF)); - CHKiRet(strmSettOperationsMode(psSF, STREAMMODE_READ)); - CHKiRet(strmSetsType(psSF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psSF, pszSFNam, lenSFNam)); - CHKiRet(strmConstructFinalize(psSF)); + CHKiRet(strm.Construct(&psSF)); + CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_READ)); + CHKiRet(strm.SetsType(psSF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psSF, pszSFNam, lenSFNam)); + CHKiRet(strm.ConstructFinalize(psSF)); /* read back in the object */ CHKiRet(obj.Deserialize(&pThis->pStrm, (uchar*) "strm", psSF, NULL, pThis)); - CHKiRet(strmSeekCurrOffs(pThis->pStrm)); + CHKiRet(strm.SeekCurrOffs(pThis->pStrm)); /* OK, we could successfully read the file, so we now can request that it be deleted. * If we need it again, it will be written on the next shutdown. @@ -157,14 +158,14 @@ openFile(fileInfo_t *pThis) finalize_it: if(psSF != NULL) - strmDestruct(&psSF); + strm.Destruct(&psSF); if(iRet != RS_RET_OK) { - CHKiRet(strmConstruct(&pThis->pStrm)); - CHKiRet(strmSettOperationsMode(pThis->pStrm, STREAMMODE_READ)); - CHKiRet(strmSetsType(pThis->pStrm, STREAMTYPE_FILE_MONITOR)); - CHKiRet(strmSetFName(pThis->pStrm, pThis->pszFileName, strlen((char*) pThis->pszFileName))); - CHKiRet(strmConstructFinalize(pThis->pStrm)); + CHKiRet(strm.Construct(&pThis->pStrm)); + CHKiRet(strm.SettOperationsMode(pThis->pStrm, STREAMMODE_READ)); + CHKiRet(strm.SetsType(pThis->pStrm, STREAMTYPE_FILE_MONITOR)); + CHKiRet(strm.SetFName(pThis->pStrm, pThis->pszFileName, strlen((char*) pThis->pszFileName))); + CHKiRet(strm.ConstructFinalize(pThis->pStrm)); } RETiRet; @@ -203,7 +204,7 @@ static rsRetVal pollFile(fileInfo_t *pThis, int *pbHadFileData) /* loop below will be exited when strmReadLine() returns EOF */ while(1) { - CHKiRet(strmReadLine(pThis->pStrm, &pCStr)); + CHKiRet(strm.ReadLine(pThis->pStrm, &pCStr)); *pbHadFileData = 1; /* this is just a flag, so set it and forget it */ CHKiRet(enqLine(pThis, pCStr)); /* process line */ rsCStrDestruct(&pCStr); /* discard string (must be done by us!) */ @@ -354,21 +355,21 @@ persistStrmState(fileInfo_t *pInfo) ASSERT(pInfo != NULL); /* TODO: create a function persistObj in obj.c? */ - CHKiRet(strmConstruct(&psSF)); - CHKiRet(strmSetDir(psSF, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSettOperationsMode(psSF, STREAMMODE_WRITE)); - CHKiRet(strmSetiAddtlOpenFlags(psSF, O_TRUNC)); - CHKiRet(strmSetsType(psSF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psSF, pInfo->pszStateFile, strlen((char*) pInfo->pszStateFile))); - CHKiRet(strmConstructFinalize(psSF)); + CHKiRet(strm.Construct(&psSF)); + CHKiRet(strm.SetDir(psSF, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_WRITE)); + CHKiRet(strm.SetiAddtlOpenFlags(psSF, O_TRUNC)); + CHKiRet(strm.SetsType(psSF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psSF, pInfo->pszStateFile, strlen((char*) pInfo->pszStateFile))); + CHKiRet(strm.ConstructFinalize(psSF)); - CHKiRet(strmSerialize(pInfo->pStrm, psSF)); + CHKiRet(strm.Serialize(pInfo->pStrm, psSF)); - CHKiRet(strmDestruct(&psSF)); + CHKiRet(strm.Destruct(&psSF)); finalize_it: if(psSF != NULL) - strmDestruct(&psSF); + strm.Destruct(&psSF); RETiRet; } @@ -388,7 +389,7 @@ CODESTARTafterRun for(i = 0 ; i < iFilPtr ; ++i) { if(files[i].pStrm != NULL) { /* stream open? */ persistStrmState(&files[i]); - strmDestruct(&(files[i].pStrm)); + strm.Destruct(&(files[i].pStrm)); } } ENDafterRun @@ -401,6 +402,7 @@ ENDafterRun BEGINmodExit CODESTARTmodExit /* release objects we used */ + objRelease(strm, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); @@ -512,6 +514,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilename", 0, eCmdHdlrGetWord, NULL, &pszFileName, STD_LOADABLE_MODULE_ID)); diff --git a/runtime/obj.c b/runtime/obj.c index 2a9df9ed..f5ec66b2 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -93,6 +93,7 @@ DEFobjCurrIf(obj) /* we define our own interface, as this is expected by some ma DEFobjCurrIf(var) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) +DEFobjCurrIf(strm) static objInfo_t *arrObjInfo[OBJ_NUM_IDS]; /* array with object information pointers */ @@ -228,20 +229,20 @@ static rsRetVal objSerializeHeader(strm_t *pStrm, obj_t *pObj, uchar *pszRecType assert(!strcmp((char*) pszRecType, "Obj") || !strcmp((char*) pszRecType, "OPB")); /* object cookie and serializer version (so far always 1) */ - CHKiRet(strmWriteChar(pStrm, COOKIE_OBJLINE)); - CHKiRet(strmWrite(pStrm, (uchar*) pszRecType, 3)); /* record types are always 3 octets */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteChar(pStrm, '1')); + CHKiRet(strm.WriteChar(pStrm, COOKIE_OBJLINE)); + CHKiRet(strm.Write(pStrm, (uchar*) pszRecType, 3)); /* record types are always 3 octets */ + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteChar(pStrm, '1')); /* object type, version and string length */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWrite(pStrm, pObj->pObjInfo->pszID, pObj->pObjInfo->lenID)); - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteLong(pStrm, objGetVersion(pObj))); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.Write(pStrm, pObj->pObjInfo->pszID, pObj->pObjInfo->lenID)); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteLong(pStrm, objGetVersion(pObj))); /* record trailer */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteChar(pStrm, '\n')); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteChar(pStrm, '\n')); finalize_it: RETiRet; @@ -259,7 +260,7 @@ BeginSerialize(strm_t *pStrm, obj_t *pObj) ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_assert(pObj); - CHKiRet(strmRecordBegin(pStrm)); + CHKiRet(strm.RecordBegin(pStrm)); CHKiRet(objSerializeHeader(pStrm, pObj, (uchar*) "Obj")); finalize_it: @@ -284,7 +285,7 @@ BeginSerializePropBag(strm_t *pStrm, obj_t *pObj) ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_assert(pObj); - CHKiRet(strmRecordBegin(pStrm)); + CHKiRet(strm.RecordBegin(pStrm)); CHKiRet(objSerializeHeader(pStrm, pObj, (uchar*) "OPB")); finalize_it: @@ -377,23 +378,23 @@ SerializeProp(strm_t *pStrm, uchar *pszPropName, propType_t propType, void *pUsr } /* cookie */ - CHKiRet(strmWriteChar(pStrm, COOKIE_PROPLINE)); + CHKiRet(strm.WriteChar(pStrm, COOKIE_PROPLINE)); /* name */ - CHKiRet(strmWrite(pStrm, pszPropName, strlen((char*)pszPropName))); - CHKiRet(strmWriteChar(pStrm, ':')); + CHKiRet(strm.Write(pStrm, pszPropName, strlen((char*)pszPropName))); + CHKiRet(strm.WriteChar(pStrm, ':')); /* type */ - CHKiRet(strmWriteLong(pStrm, (int) vType)); - CHKiRet(strmWriteChar(pStrm, ':')); + CHKiRet(strm.WriteLong(pStrm, (int) vType)); + CHKiRet(strm.WriteChar(pStrm, ':')); /* length */ - CHKiRet(strmWriteLong(pStrm, lenBuf)); - CHKiRet(strmWriteChar(pStrm, ':')); + CHKiRet(strm.WriteLong(pStrm, lenBuf)); + CHKiRet(strm.WriteChar(pStrm, ':')); /* data */ - CHKiRet(strmWrite(pStrm, (uchar*) pszBuf, lenBuf)); + CHKiRet(strm.Write(pStrm, (uchar*) pszBuf, lenBuf)); /* trailer */ - CHKiRet(strmWriteChar(pStrm, ':')); - CHKiRet(strmWriteChar(pStrm, '\n')); + CHKiRet(strm.WriteChar(pStrm, ':')); + CHKiRet(strm.WriteChar(pStrm, '\n')); finalize_it: RETiRet; @@ -410,12 +411,12 @@ EndSerialize(strm_t *pStrm) assert(pStrm != NULL); - CHKiRet(strmWriteChar(pStrm, COOKIE_ENDLINE)); - CHKiRet(strmWrite(pStrm, (uchar*) "End\n", sizeof("END\n") - 1)); - CHKiRet(strmWriteChar(pStrm, COOKIE_BLANKLINE)); - CHKiRet(strmWriteChar(pStrm, '\n')); + CHKiRet(strm.WriteChar(pStrm, COOKIE_ENDLINE)); + CHKiRet(strm.Write(pStrm, (uchar*) "End\n", sizeof("END\n") - 1)); + CHKiRet(strm.WriteChar(pStrm, COOKIE_BLANKLINE)); + CHKiRet(strm.WriteChar(pStrm, '\n')); - CHKiRet(strmRecordEnd(pStrm)); + CHKiRet(strm.RecordEnd(pStrm)); finalize_it: RETiRet; @@ -423,7 +424,7 @@ finalize_it: /* define a helper to make code below a bit cleaner (and quicker to write) */ -#define NEXTC CHKiRet(strmReadChar(pStrm, &c))/*;dbgprintf("c: %c\n", c)*/ +#define NEXTC CHKiRet(strm.ReadChar(pStrm, &c))/*;dbgprintf("c: %c\n", c)*/ /* de-serialize an embedded, non-octect-counted string. This is useful @@ -617,7 +618,7 @@ static rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) NEXTC; if(c != COOKIE_PROPLINE) { /* oops, we've read one char that does not belong to use - unget it first */ - CHKiRet(strmUnreadChar(pStrm, c)); + CHKiRet(strm.UnreadChar(pStrm, c)); ABORT_FINALIZE(RS_RET_NO_PROPLINE); } @@ -718,7 +719,7 @@ static rsRetVal objDeserializeTryRecover(strm_t *pStrm) } } - CHKiRet(strmUnreadChar(pStrm, c)); + CHKiRet(strm.UnreadChar(pStrm, c)); finalize_it: dbgprintf("deserializer has possibly been able to re-sync and recover, state %d\n", iRet); @@ -1278,6 +1279,7 @@ objClassExit(void) { DEFiRet; /* release objects we no longer need */ + objRelease(strm, CORE_COMPONENT); objRelease(var, CORE_COMPONENT); objRelease(module, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); @@ -1322,9 +1324,11 @@ objClassInit(modInfo_t *pModInfo) CHKiRet(cfsyslineInit()); CHKiRet(varClassInit(pModInfo)); CHKiRet(moduleClassInit(pModInfo)); + CHKiRet(strmClassInit(pModInfo)); CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(strm, CORE_COMPONENT)); finalize_it: RETiRet; diff --git a/runtime/queue.c b/runtime/queue.c index 4e017e84..3f14b535 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -59,6 +59,7 @@ /* static data */ DEFobjStaticHelpers DEFobjCurrIf(glbl) +DEFobjCurrIf(strm) /* forward-definitions */ rsRetVal qqueueChkPersist(qqueue_t *pThis); @@ -667,7 +668,7 @@ qqueueLoadPersStrmInfoFixup(strm_t *pStrm, qqueue_t __attribute__((unused)) *pTh DEFiRet; ISOBJ_TYPE_assert(pStrm, strm); ISOBJ_TYPE_assert(pThis, qqueue); - CHKiRet(strmSetDir(pStrm, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetDir(pStrm, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); finalize_it: RETiRet; } @@ -744,11 +745,11 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) /* If we reach this point, we have a .qi file */ - CHKiRet(strmConstruct(&psQIF)); - CHKiRet(strmSettOperationsMode(psQIF, STREAMMODE_READ)); - CHKiRet(strmSetsType(psQIF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psQIF, pszQIFNam, lenQIFNam)); - CHKiRet(strmConstructFinalize(psQIF)); + CHKiRet(strm.Construct(&psQIF)); + CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_READ)); + CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); + CHKiRet(strm.ConstructFinalize(psQIF)); /* first, we try to read the property bag for ourselfs */ CHKiRet(obj.DeserializePropBag((obj_t*) pThis, psQIF)); @@ -770,8 +771,8 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) CHKiRet(obj.Deserialize(&pThis->tVars.disk.pRead, (uchar*) "strm", psQIF, (rsRetVal(*)(obj_t*,void*))qqueueLoadPersStrmInfoFixup, pThis)); - CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pWrite)); - CHKiRet(strmSeekCurrOffs(pThis->tVars.disk.pRead)); + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pWrite)); + CHKiRet(strm.SeekCurrOffs(pThis->tVars.disk.pRead)); /* OK, we could successfully read the file, so we now can request that it be * deleted when we are done with the persisted information. @@ -780,7 +781,7 @@ qqueueTryLoadPersistedInfo(qqueue_t *pThis) finalize_it: if(psQIF != NULL) - strmDestruct(&psQIF); + strm.Destruct(&psQIF); if(iRet != RS_RET_OK) { dbgoprint((obj_t*) pThis, "error %d reading .qi file - can not read persisted info (if any)\n", @@ -815,24 +816,24 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) if(bRestarted == 1) { ; } else { - CHKiRet(strmConstruct(&pThis->tVars.disk.pWrite)); - CHKiRet(strmSetDir(pThis->tVars.disk.pWrite, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pWrite, 10000000)); - CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE)); - CHKiRet(strmSetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR)); - CHKiRet(strmConstructFinalize(pThis->tVars.disk.pWrite)); + CHKiRet(strm.Construct(&pThis->tVars.disk.pWrite)); + CHKiRet(strm.SetDir(pThis->tVars.disk.pWrite, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pWrite, 10000000)); + CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE)); + CHKiRet(strm.SetsType(pThis->tVars.disk.pWrite, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pWrite)); - CHKiRet(strmConstruct(&pThis->tVars.disk.pRead)); - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); - CHKiRet(strmSetDir(pThis->tVars.disk.pRead, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strmSetiMaxFiles(pThis->tVars.disk.pRead, 10000000)); - CHKiRet(strmSettOperationsMode(pThis->tVars.disk.pRead, STREAMMODE_READ)); - CHKiRet(strmSetsType(pThis->tVars.disk.pRead, STREAMTYPE_FILE_CIRCULAR)); - CHKiRet(strmConstructFinalize(pThis->tVars.disk.pRead)); + CHKiRet(strm.Construct(&pThis->tVars.disk.pRead)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); + CHKiRet(strm.SetDir(pThis->tVars.disk.pRead, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); + CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pRead, 10000000)); + CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pRead, STREAMMODE_READ)); + CHKiRet(strm.SetsType(pThis->tVars.disk.pRead, STREAMTYPE_FILE_CIRCULAR)); + CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pRead)); - CHKiRet(strmSetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); - CHKiRet(strmSetFName(pThis->tVars.disk.pRead, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.SetFName(pThis->tVars.disk.pWrite, pThis->pszFilePrefix, pThis->lenFilePrefix)); + CHKiRet(strm.SetFName(pThis->tVars.disk.pRead, pThis->pszFilePrefix, pThis->lenFilePrefix)); } /* now we set (and overwrite in case of a persisted restart) some parameters which @@ -840,8 +841,8 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) * for example file name generation must not be changed as that would break the * ability to read existing queue files. -- rgerhards, 2008-01-12 */ - CHKiRet(strmSetiMaxFileSize(pThis->tVars.disk.pWrite, pThis->iMaxFileSize)); - CHKiRet(strmSetiMaxFileSize(pThis->tVars.disk.pRead, pThis->iMaxFileSize)); + CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pWrite, pThis->iMaxFileSize)); + CHKiRet(strm.SetiMaxFileSize(pThis->tVars.disk.pRead, pThis->iMaxFileSize)); finalize_it: RETiRet; @@ -854,8 +855,8 @@ static rsRetVal qDestructDisk(qqueue_t *pThis) ASSERT(pThis != NULL); - strmDestruct(&pThis->tVars.disk.pWrite); - strmDestruct(&pThis->tVars.disk.pRead); + strm.Destruct(&pThis->tVars.disk.pWrite); + strm.Destruct(&pThis->tVars.disk.pRead); RETiRet; } @@ -867,10 +868,10 @@ static rsRetVal qAddDisk(qqueue_t *pThis, void* pUsr) ASSERT(pThis != NULL); - CHKiRet(strmSetWCntr(pThis->tVars.disk.pWrite, &nWriteCount)); + CHKiRet(strm.SetWCntr(pThis->tVars.disk.pWrite, &nWriteCount)); CHKiRet((objSerialize(pUsr))(pUsr, pThis->tVars.disk.pWrite)); - CHKiRet(strmFlush(pThis->tVars.disk.pWrite)); - CHKiRet(strmSetWCntr(pThis->tVars.disk.pWrite, NULL)); /* no more counting for now... */ + CHKiRet(strm.Flush(pThis->tVars.disk.pWrite)); + CHKiRet(strm.SetWCntr(pThis->tVars.disk.pWrite, NULL)); /* no more counting for now... */ pThis->tVars.disk.sizeOnDisk += nWriteCount; @@ -894,9 +895,9 @@ static rsRetVal qDelDisk(qqueue_t *pThis, void **ppUsr) int64 offsIn; int64 offsOut; - CHKiRet(strmGetCurrOffset(pThis->tVars.disk.pRead, &offsIn)); + CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pRead, &offsIn)); CHKiRet(obj.Deserialize(ppUsr, (uchar*) "msg", pThis->tVars.disk.pRead, NULL, NULL)); - CHKiRet(strmGetCurrOffset(pThis->tVars.disk.pRead, &offsOut)); + CHKiRet(strm.GetCurrOffset(pThis->tVars.disk.pRead, &offsOut)); /* This time it is a bit tricky: we free disk space only upon file deletion. So we need * to keep track of what we have read until we get an out-offset that is lower than the @@ -1917,16 +1918,16 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) pThis->bNeedDelQIF = 0; } /* indicate spool file needs to be deleted */ - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); FINALIZE; /* nothing left to do, so be happy */ } - CHKiRet(strmConstruct(&psQIF)); - CHKiRet(strmSettOperationsMode(psQIF, STREAMMODE_WRITE)); - CHKiRet(strmSetiAddtlOpenFlags(psQIF, O_TRUNC)); - CHKiRet(strmSetsType(psQIF, STREAMTYPE_FILE_SINGLE)); - CHKiRet(strmSetFName(psQIF, pszQIFNam, lenQIFNam)); - CHKiRet(strmConstructFinalize(psQIF)); + CHKiRet(strm.Construct(&psQIF)); + CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_WRITE)); + CHKiRet(strm.SetiAddtlOpenFlags(psQIF, O_TRUNC)); + CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); + CHKiRet(strm.ConstructFinalize(psQIF)); /* first, write the property bag for ourselfs * And, surprisingly enough, we currently need to persist only the size of the @@ -1951,14 +1952,14 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) } /* now persist the stream info */ - CHKiRet(strmSerialize(pThis->tVars.disk.pWrite, psQIF)); - CHKiRet(strmSerialize(pThis->tVars.disk.pRead, psQIF)); + CHKiRet(strm.Serialize(pThis->tVars.disk.pWrite, psQIF)); + CHKiRet(strm.Serialize(pThis->tVars.disk.pRead, psQIF)); /* tell the input file object that it must not delete the file on close if the queue * is non-empty - but only if we are not during a simple checkpoint */ if(bIsCheckpoint != QUEUE_CHECKPOINT) { - CHKiRet(strmSetbDeleteOnClose(pThis->tVars.disk.pRead, 0)); + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 0)); } /* we have persisted the queue object. So whenever it comes to an empty queue, @@ -1968,7 +1969,7 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) finalize_it: if(psQIF != NULL) - strmDestruct(&psQIF); + strm.Destruct(&psQIF); RETiRet; } @@ -2340,6 +2341,7 @@ rsRetVal qqueueQueryInterface(void) { return RS_RET_NOT_IMPLEMENTED; } BEGINObjClassInit(qqueue, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(strm, CORE_COMPONENT)); /* now set our own handlers */ OBJSetMethodHandler(objMethod_SETPROPERTY, qqueueSetProperty); diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 8df100a1..81550f12 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -150,8 +150,6 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(datetimeClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "str,"; - CHKiRet(strmClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "wti"; CHKiRet(wtiClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "wtp"; diff --git a/runtime/stream.c b/runtime/stream.c index f1f69cc8..261eb9ca 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -45,10 +45,17 @@ #include "srUtils.h" #include "obj.h" #include "stream.h" +#include "unicode-helper.h" +#include "module-template.h" /* static data */ DEFobjStaticHelpers +/* forward definitions */ +static rsRetVal strmFlush(strm_t *pThis); +static rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); + + /* methods */ /* first, we define type-specific handlers. The provide a generic functionality, @@ -234,10 +241,6 @@ strmHandleEOF(strm_t *pThis) case STREAMTYPE_FILE_CIRCULAR: /* we have multiple files and need to switch to the next one */ /* TODO: think about emulating EOF in this case (not yet needed) */ -#if 0 - if(pThis->iMaxFiles == 0) /* TODO: why do we need this? ;) */ - ABORT_FINALIZE(RS_RET_EOF); -#endif dbgoprint((obj_t*) pThis, "file %d EOF\n", pThis->fd); CHKiRet(strmNextFile(pThis)); break; @@ -295,7 +298,7 @@ finalize_it: * NOTE: needs to be enhanced to support sticking with a strm entry (if not * deleted). */ -rsRetVal strmReadChar(strm_t *pThis, uchar *pC) +static rsRetVal strmReadChar(strm_t *pThis, uchar *pC) { DEFiRet; @@ -329,7 +332,7 @@ finalize_it: * character buffering capability. * rgerhards, 2008-01-07 */ -rsRetVal strmUnreadChar(strm_t *pThis, uchar c) +static rsRetVal strmUnreadChar(strm_t *pThis, uchar c) { ASSERT(pThis != NULL); ASSERT(pThis->iUngetC == -1); @@ -351,7 +354,7 @@ rsRetVal strmUnreadChar(strm_t *pThis, uchar c) * are pthread_killed() upon termination. So if we use their native pointer, they * can cleanup (but only then). */ -rsRetVal +static rsRetVal strmReadLine(strm_t *pThis, cstr_t **ppCStr) { DEFiRet; @@ -393,7 +396,7 @@ ENDobjConstruct(strm) /* ConstructionFinalizer * rgerhards, 2008-01-09 */ -rsRetVal strmConstructFinalize(strm_t *pThis) +static rsRetVal strmConstructFinalize(strm_t *pThis) { DEFiRet; @@ -420,14 +423,10 @@ CODESTARTobjDestruct(strm) if(pThis->fd != -1) strmCloseFile(pThis); - if(pThis->pszDir != NULL) - free(pThis->pszDir); - if(pThis->pIOBuf != NULL) - free(pThis->pIOBuf); - if(pThis->pszCurrFName != NULL) - free(pThis->pszCurrFName); - if(pThis->pszFName != NULL) - free(pThis->pszFName); + free(pThis->pszDir); + free(pThis->pIOBuf); + free(pThis->pszCurrFName); + free(pThis->pszFName); ENDobjDestruct(strm) @@ -453,6 +452,7 @@ finalize_it: RETiRet; } + /* write memory buffer to a stream object. * To support direct writes of large objects, this method may be called * with a buffer pointing to some region other than the stream buffer itself. @@ -503,7 +503,8 @@ finalize_it: * and is automatically called when the output buffer is full. * rgerhards, 2008-01-10 */ -rsRetVal strmFlush(strm_t *pThis) +static rsRetVal +strmFlush(strm_t *pThis) { DEFiRet; @@ -545,7 +546,7 @@ static rsRetVal strmSeek(strm_t *pThis, off_t offs) /* seek to current offset. This is primarily a helper to readjust the OS file * pointer after a strm object has been deserialized. */ -rsRetVal strmSeekCurrOffs(strm_t *pThis) +static rsRetVal strmSeekCurrOffs(strm_t *pThis) { DEFiRet; @@ -558,7 +559,7 @@ rsRetVal strmSeekCurrOffs(strm_t *pThis) /* write a *single* character to a stream object -- rgerhards, 2008-01-10 */ -rsRetVal strmWriteChar(strm_t *pThis, uchar c) +static rsRetVal strmWriteChar(strm_t *pThis, uchar c) { DEFiRet; @@ -578,7 +579,7 @@ finalize_it: /* write an integer value (actually a long) to a stream object */ -rsRetVal strmWriteLong(strm_t *pThis, long i) +static rsRetVal strmWriteLong(strm_t *pThis, long i) { DEFiRet; uchar szBuf[32]; @@ -595,7 +596,8 @@ finalize_it: /* write memory buffer to a stream object */ -rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +static rsRetVal +strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { DEFiRet; size_t iPartial; @@ -645,14 +647,14 @@ DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) DEFpropSetMeth(strm, sType, strmType_t) -rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) +static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) { pThis->iMaxFiles = iNewVal; pThis->iFileNumDigits = getNumberDigits(iNewVal); return RS_RET_OK; } -rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal) +static rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal) { DEFiRet; @@ -671,7 +673,7 @@ finalize_it: * it any longer, it must free it. * rgerhards, 2008-01-09 */ -rsRetVal +static rsRetVal strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) { DEFiRet; @@ -701,7 +703,7 @@ finalize_it: * it any longer, it must free it. * rgerhards, 2008-01-09 */ -rsRetVal +static rsRetVal strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) { DEFiRet; @@ -745,7 +747,7 @@ finalize_it: * * rgerhards, 2008-01-10 */ -rsRetVal strmRecordBegin(strm_t *pThis) +static rsRetVal strmRecordBegin(strm_t *pThis) { ASSERT(pThis != NULL); ASSERT(pThis->bInRecord == 0); @@ -753,7 +755,7 @@ rsRetVal strmRecordBegin(strm_t *pThis) return RS_RET_OK; } -rsRetVal strmRecordEnd(strm_t *pThis) +static rsRetVal strmRecordEnd(strm_t *pThis) { DEFiRet; ASSERT(pThis != NULL); @@ -775,7 +777,7 @@ rsRetVal strmRecordEnd(strm_t *pThis) * We do not serialize the dynamic properties. * rgerhards, 2008-01-10 */ -rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) +static rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm) { DEFiRet; int i; @@ -821,7 +823,7 @@ finalize_it: * any new set overwrites the previous one. * rgerhards, 2008-02-27 */ -rsRetVal +static rsRetVal strmSetWCntr(strm_t *pThis, number_t *pWCnt) { DEFiRet; @@ -841,8 +843,8 @@ strmSetWCntr(strm_t *pThis, number_t *pWCnt) /* This function can be used as a generic way to set properties. * rgerhards, 2008-01-11 */ -#define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, (uchar*) name, sizeof(name) - 1) -rsRetVal strmSetProperty(strm_t *pThis, var_t *pProp) +#define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, UCHAR_CONSTANT(name), sizeof(name) - 1) +static rsRetVal strmSetProperty(strm_t *pThis, var_t *pProp) { DEFiRet; @@ -881,7 +883,7 @@ finalize_it: * reported on the second call may actually be lower than on the first call. This is due to * file circulation. A caller must deal with that. -- rgerhards, 2008-01-30 */ -rsRetVal +static rsRetVal strmGetCurrOffset(strm_t *pThis, int64 *pOffs) { DEFiRet; @@ -909,8 +911,33 @@ CODESTARTobjQueryInterface(strm) * work here (if we can support an older interface version - that, * of course, also affects the "if" above). */ - /*xxxpIf->oID = OBJvm; SAMPLE */ - + pIf->Construct = strmConstruct; + pIf->ConstructFinalize = strmConstructFinalize; + pIf->Destruct = strmDestruct; + pIf->ReadChar = strmReadChar; + pIf->UnreadChar = strmUnreadChar; + pIf->ReadLine = strmReadLine; + pIf->SeekCurrOffs = strmSeekCurrOffs; + pIf->Write = strmWrite; + pIf->WriteChar = strmWriteChar; + pIf->WriteLong = strmWriteLong; + pIf->SetFName = strmSetFName; + pIf->SetDir = strmSetDir; + pIf->Flush = strmFlush; + pIf->RecordBegin = strmRecordBegin; + pIf->RecordEnd = strmRecordEnd; + pIf->Serialize = strmSerialize; + pIf->SetiAddtlOpenFlags = strmSetiAddtlOpenFlags; + pIf->GetCurrOffset = strmGetCurrOffset; + pIf->SetWCntr = strmSetWCntr; + /* set methods */ + pIf->SetbDeleteOnClose = strmSetbDeleteOnClose; + pIf->SetiMaxFileSize = strmSetiMaxFileSize; + pIf->SetiMaxFiles = strmSetiMaxFiles; + pIf->SetiFileNumDigits = strmSetiFileNumDigits; + pIf->SettOperationsMode = strmSettOperationsMode; + pIf->SettOpenMode = strmSettOpenMode; + pIf->SetsType = strmSetsType; finalize_it: ENDobjQueryInterface(strm) @@ -921,13 +948,10 @@ ENDobjQueryInterface(strm) */ BEGINObjClassInit(strm, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ - OBJSetMethodHandler(objMethod_SERIALIZE, strmSerialize); OBJSetMethodHandler(objMethod_SETPROPERTY, strmSetProperty); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strmConstructFinalize); ENDObjClassInit(strm) - -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/stream.h b/runtime/stream.h index 371358ab..ece33270 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -93,39 +93,39 @@ typedef struct strm_s { /* interfaces */ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ + rsRetVal (*Construct)(strm_t **ppThis); + rsRetVal (*ConstructFinalize)(strm_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(strm_t **ppThis); + rsRetVal (*SetMaxFileSize)(strm_t *pThis, int64 iMaxFileSize); + rsRetVal (*SetFileName)(strm_t *pThis, uchar *pszName, size_t iLenName); + rsRetVal (*ReadChar)(strm_t *pThis, uchar *pC); + rsRetVal (*UnreadChar)(strm_t *pThis, uchar c); + rsRetVal (*ReadLine)(strm_t *pThis, cstr_t **ppCStr); + rsRetVal (*SeekCurrOffs)(strm_t *pThis); + rsRetVal (*Write)(strm_t *pThis, uchar *pBuf, size_t lenBuf); + rsRetVal (*WriteChar)(strm_t *pThis, uchar c); + rsRetVal (*WriteLong)(strm_t *pThis, long i); + rsRetVal (*SetFName)(strm_t *pThis, uchar *pszPrefix, size_t iLenPrefix); + rsRetVal (*SetDir)(strm_t *pThis, uchar *pszDir, size_t iLenDir); + rsRetVal (*Flush)(strm_t *pThis); + rsRetVal (*RecordBegin)(strm_t *pThis); + rsRetVal (*RecordEnd)(strm_t *pThis); + rsRetVal (*Serialize)(strm_t *pThis, strm_t *pStrm); + rsRetVal (*SetiAddtlOpenFlags)(strm_t *pThis, int iNewVal); + rsRetVal (*GetCurrOffset)(strm_t *pThis, int64 *pOffs); + rsRetVal (*SetWCntr)(strm_t *pThis, number_t *pWCnt); + INTERFACEpropSetMeth(strm, bDeleteOnClose, int); + INTERFACEpropSetMeth(strm, iMaxFileSize, int); + INTERFACEpropSetMeth(strm, iMaxFiles, int); + INTERFACEpropSetMeth(strm, iFileNumDigits, int); + INTERFACEpropSetMeth(strm, tOperationsMode, int); + INTERFACEpropSetMeth(strm, tOpenMode, mode_t); + INTERFACEpropSetMeth(strm, sType, strmType_t); ENDinterface(strm) #define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ /* prototypes */ -rsRetVal strmConstruct(strm_t **ppThis); -rsRetVal strmConstructFinalize(strm_t __attribute__((unused)) *pThis); -rsRetVal strmDestruct(strm_t **ppThis); -rsRetVal strmSetMaxFileSize(strm_t *pThis, int64 iMaxFileSize); -rsRetVal strmSetFileName(strm_t *pThis, uchar *pszName, size_t iLenName); -rsRetVal strmReadChar(strm_t *pThis, uchar *pC); -rsRetVal strmUnreadChar(strm_t *pThis, uchar c); -rsRetVal strmReadLine(strm_t *pThis, cstr_t **ppCStr); -rsRetVal strmSeekCurrOffs(strm_t *pThis); -rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); -rsRetVal strmWriteChar(strm_t *pThis, uchar c); -rsRetVal strmWriteLong(strm_t *pThis, long i); -rsRetVal strmSetFName(strm_t *pThis, uchar *pszPrefix, size_t iLenPrefix); -rsRetVal strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir); -rsRetVal strmFlush(strm_t *pThis); -rsRetVal strmRecordBegin(strm_t *pThis); -rsRetVal strmRecordEnd(strm_t *pThis); -rsRetVal strmSerialize(strm_t *pThis, strm_t *pStrm); -rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal); -rsRetVal strmGetCurrOffset(strm_t *pThis, int64 *pOffs); -rsRetVal strmSetWCntr(strm_t *pThis, number_t *pWCnt); PROTOTYPEObjClassInit(strm); -PROTOTYPEpropSetMeth(strm, bDeleteOnClose, int); -PROTOTYPEpropSetMeth(strm, iMaxFileSize, int); -PROTOTYPEpropSetMeth(strm, iMaxFiles, int); -PROTOTYPEpropSetMeth(strm, iFileNumDigits, int); -PROTOTYPEpropSetMeth(strm, tOperationsMode, int); -PROTOTYPEpropSetMeth(strm, tOpenMode, mode_t); -PROTOTYPEpropSetMeth(strm, sType, strmType_t); #endif /* #ifndef STREAM_H_INCLUDED */ diff --git a/runtime/sysvar.c b/runtime/sysvar.c index c102d1f5..4a6ace19 100644 --- a/runtime/sysvar.c +++ b/runtime/sysvar.c @@ -175,8 +175,6 @@ CODESTARTobjQueryInterface(sysvar) * work here (if we can support an older interface version - that, * of course, also affects the "if" above). */ - //xxxpIf->oID = "sysvar";//OBJsysvar; - pIf->Construct = sysvarConstruct; pIf->ConstructFinalize = sysvarConstructFinalize; pIf->Destruct = sysvarDestruct; diff --git a/tools/omfile.c b/tools/omfile.c index d1274f1b..689bad0c 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -776,7 +776,7 @@ doFlush(instanceData *pData) if(pData->poBuf->iBuf == 0) FINALIZE; /* nothing to write, but make this a valid case */ - if(1) { // zlib enabled! + if(0) { // zlib enabled! CHKiRet(doZipWrite(pData)); } else { CHKiRet(doPhysWrite(pData, pData->poBuf->fd, (char*)pData->poBuf->pszBuf, pData->poBuf->iBuf)); diff --git a/tools/syslogd.c b/tools/syslogd.c index b43b7a37..22cb6879 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -136,7 +136,6 @@ #include "errmsg.h" #include "datetime.h" #include "parser.h" -#include "sysvar.h" #include "unicode-helper.h" /* definitions for objects we access */ @@ -3340,26 +3339,6 @@ GlobalClassExit(void) objRelease(datetime, CORE_COMPONENT); /* TODO: implement the rest of the deinit */ -#if 0 - CHKiRet(datetimeClassInit(NULL)); - CHKiRet(msgClassInit(NULL)); - CHKiRet(strmClassInit(NULL)); - CHKiRet(wtiClassInit(NULL)); - CHKiRet(wtpClassInit(NULL)); - CHKiRet(qqueueClassInit(NULL)); - CHKiRet(vmstkClassInit(NULL)); - CHKiRet(sysvarClassInit(NULL)); - CHKiRet(vmClassInit(NULL)); - CHKiRet(vmopClassInit(NULL)); - CHKiRet(vmprgClassInit(NULL)); - CHKiRet(ctok_tokenClassInit(NULL)); - CHKiRet(ctokClassInit(NULL)); - CHKiRet(exprClassInit(NULL)); - - /* dummy "classes" */ - CHKiRet(actionClassInit()); - CHKiRet(templateInit()); -#endif /* dummy "classes */ strExit(); -- cgit v1.2.3 From f2800ba261d2fb7466cbdebbf80afe92f0bffd3d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 15:10:24 +0200 Subject: modified stream class and omfile to work with it now some basic operations are carried out via the stream class. --- doc/rsyslog_conf_global.html | 3 + runtime/queue.c | 3 +- runtime/stream.c | 168 ++++++++++++++++++++++++++++++++++++------- runtime/stream.h | 11 ++- tools/omfile.c | 94 +++++++++++++++++------- 5 files changed, 225 insertions(+), 54 deletions(-) diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 778e18f8..0f408fcf 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -189,6 +189,9 @@ supported in order to be compliant to the upcoming new syslog RFC series.
    • $MaxOpenFiles
    • $ModDir
    • $ModLoad
      • +
    • $OMFileZipLevel 0..9 [default 0] - if greater 0, turns on gzip compression +of the output file. The higher the number, the better the compression, but also the +more CPU is required for zipping.
    • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, have a different format that contains the message that is being repeated. Note that only the first "n" characters are included, with n to be at least 80 characters, most diff --git a/runtime/queue.c b/runtime/queue.c index 3f14b535..3532a145 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1923,8 +1923,7 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) } CHKiRet(strm.Construct(&psQIF)); - CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_WRITE)); - CHKiRet(strm.SetiAddtlOpenFlags(psQIF, O_TRUNC)); + CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_WRITE_TRUNC)); CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); CHKiRet(strm.ConstructFinalize(psQIF)); diff --git a/runtime/stream.c b/runtime/stream.c index 261eb9ca..abcfce0c 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -1,4 +1,3 @@ -//TODO: O_TRUC mode! /* The serial stream class. * * A serial stream provides serial data access. In theory, serial streams @@ -50,6 +49,7 @@ /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(zlibw) /* forward definitions */ static rsRetVal strmFlush(strm_t *pThis); @@ -74,7 +74,6 @@ static rsRetVal strmOpenFile(strm_t *pThis) int iFlags; ASSERT(pThis != NULL); - ASSERT(pThis->tOperationsMode == STREAMMODE_READ || pThis->tOperationsMode == STREAMMODE_WRITE); if(pThis->fd != -1) ABORT_FINALIZE(RS_RET_OK); @@ -95,13 +94,27 @@ static rsRetVal strmOpenFile(strm_t *pThis) } } +RUNLOG_VAR("%d", pThis->tOperationsMode); /* compute which flags we need to provide to open */ - if(pThis->tOperationsMode == STREAMMODE_READ) - iFlags = O_RDONLY; - else - iFlags = O_WRONLY | O_CREAT; + switch(pThis->tOperationsMode) { + case STREAMMODE_READ: + iFlags = O_CLOEXEC | O_NOCTTY | O_RDONLY; + break; + case STREAMMODE_WRITE: /* legacy mode used inside queue engine */ + iFlags = O_CLOEXEC | O_NOCTTY | O_WRONLY | O_CREAT; + break; + case STREAMMODE_WRITE_TRUNC: + iFlags = O_CLOEXEC | O_NOCTTY | O_WRONLY | O_CREAT | O_TRUNC; + break; + case STREAMMODE_WRITE_APPEND: + iFlags = O_CLOEXEC | O_NOCTTY | O_WRONLY | O_CREAT | O_APPEND; + break; + default:assert(0); + break; + } - iFlags |= pThis->iAddtlOpenFlags; + iFlags |= pThis->iAddtlOpenFlags; // TODO: remove this! +dbgprintf("XXX: open with flags %d\n", iFlags); pThis->fd = open((char*)pThis->pszCurrFName, iFlags, pThis->tOpenMode); if(pThis->fd == -1) { @@ -135,7 +148,7 @@ static rsRetVal strmCloseFile(strm_t *pThis) ASSERT(pThis->fd != -1); dbgoprint((obj_t*) pThis, "file %d closing\n", pThis->fd); - if(pThis->tOperationsMode == STREAMMODE_WRITE) + if(pThis->tOperationsMode != STREAMMODE_READ) strmFlush(pThis); close(pThis->fd); // TODO: error check @@ -398,14 +411,25 @@ ENDobjConstruct(strm) */ static rsRetVal strmConstructFinalize(strm_t *pThis) { + rsRetVal localRet; DEFiRet; ASSERT(pThis != NULL); - if(pThis->pIOBuf == NULL) { /* allocate our io buffer in case we have not yet */ - if((pThis->pIOBuf = (uchar*) malloc(sizeof(uchar) * pThis->sIOBufSize)) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - pThis->iBufPtrMax = 0; /* results in immediate read request */ + CHKmalloc(pThis->pIOBuf = (uchar*) malloc(sizeof(uchar) * pThis->sIOBufSize)); + pThis->iBufPtrMax = 0; /* results in immediate read request */ + if(pThis->iZipLevel) { /* do we need a zip buf? */ + localRet = objUse(zlibw, LM_ZLIBW_FILENAME); + if(localRet != RS_RET_OK) { + pThis->iZipLevel = 0; + DBGPRINTF("stream was requested with zip mode, but zlibw module unavailable (%d) - using " + "without zip\n", localRet); + } else { + /* we use the same size as the original buf, as we would like + * to make sure we can write out everyting with a SINGLE api call! + */ + CHKmalloc(pThis->pZipBuf = (Bytef*) malloc(sizeof(uchar) * pThis->sIOBufSize)); + } } finalize_it: @@ -416,15 +440,20 @@ finalize_it: /* destructor for the strm object */ BEGINobjDestruct(strm) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(strm) - if(pThis->tOperationsMode == STREAMMODE_WRITE) + if(pThis->tOperationsMode != STREAMMODE_READ) strmFlush(pThis); /* ... then free resources */ if(pThis->fd != -1) strmCloseFile(pThis); + if(pThis->iZipLevel) { /* do we need a zip buf? */ + objRelease(zlibw, LM_ZLIBW_FILENAME); + } + free(pThis->pszDir); free(pThis->pIOBuf); + free(pThis->pZipBuf); free(pThis->pszCurrFName); free(pThis->pszFName); ENDobjDestruct(strm) @@ -453,20 +482,17 @@ finalize_it: } -/* write memory buffer to a stream object. - * To support direct writes of large objects, this method may be called - * with a buffer pointing to some region other than the stream buffer itself. - * However, in that case the stream buffer must be empty (strmFlush() has to - * be called before), because we would otherwise mess up with the sequence - * inside the stream. -- rgerhards, 2008-01-10 +/* physically write to the output file. the provided data is ready for + * writing (e.g. zipped if we are requested to do that). + * rgerhards, 2009-06-04 */ -static rsRetVal strmWriteInternal(strm_t *pThis, uchar *pBuf, size_t lenBuf) +static rsRetVal +strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { DEFiRet; int iWritten; ASSERT(pThis != NULL); - ASSERT(pBuf == pThis->pIOBuf || pThis->iBufPtr == 0); if(pThis->fd == -1) CHKiRet(strmOpenFile(pThis)); @@ -499,6 +525,95 @@ finalize_it: } +/* write the output buffer in zip mode + * This means we compress it first and then do a physical write. + * Note that we always do a full deflateInit ... deflate ... deflateEnd + * sequence. While this is not optimal, we need to do it because we need + * to ensure that the file is readable even when we are aborted. Doing the + * full sequence brings us as far towards this goal as possible (and not + * doing it would be a total failure). It may be worth considering to + * add a config switch so that the user can decide the risk he is ready + * to take, but so far this is not yet implemented (not even requested ;)). + * rgerhards, 2009-06-04 + */ +static rsRetVal +doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) +{ + z_stream zstrm; + int zRet; /* zlib return state */ + DEFiRet; + assert(pThis != NULL); + assert(pBuf != NULL); + + /* allocate deflate state */ + zstrm.zalloc = Z_NULL; + zstrm.zfree = Z_NULL; + zstrm.opaque = Z_NULL; + /* see note in file header for the params we use with deflateInit2() */ + zRet = zlibw.DeflateInit2(&zstrm, pThis->iZipLevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + if(zRet != Z_OK) { + dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } +RUNLOG_STR("deflateInit2() done successfully\n"); + + /* now doing the compression */ + zstrm.avail_in = lenBuf; + zstrm.next_in = (Bytef*) pBuf; + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", zstrm.avail_in, zstrm.total_in); + zstrm.avail_out = pThis->sIOBufSize; + zstrm.next_out = pThis->pZipBuf; + zRet = zlibw.Deflate(&zstrm, Z_FINISH); /* no bad return value */ + dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, zstrm.avail_out); + assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ + CHKiRet(strmPhysWrite(pThis, (uchar*)pThis->pZipBuf, pThis->sIOBufSize - zstrm.avail_out)); + } while (zstrm.avail_out == 0); + assert(zstrm.avail_in == 0); /* all input will be used */ + +RUNLOG_STR("deflate() should be done successfully\n"); + + zRet = zlibw.DeflateEnd(&zstrm); + if(zRet != Z_OK) { + dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); + ABORT_FINALIZE(RS_RET_ZLIB_ERR); + } +RUNLOG_STR("deflateEnd() done successfully\n"); + +finalize_it: + RETiRet; +} + + +/* write memory buffer to a stream object. + * To support direct writes of large objects, this method may be called + * with a buffer pointing to some region other than the stream buffer itself. + * However, in that case the stream buffer must be empty (strmFlush() has to + * be called before), because we would otherwise mess up with the sequence + * inside the stream. -- rgerhards, 2008-01-10 + */ +static rsRetVal +strmWriteInternal(strm_t *pThis, uchar *pBuf, size_t lenBuf) +{ + DEFiRet; + + ASSERT(pThis != NULL); + ASSERT(pBuf == pThis->pIOBuf || pThis->iBufPtr == 0); + + if(pThis->iZipLevel) { + CHKiRet(doZipWrite(pThis, pBuf, lenBuf)); + } else { + /* write without zipping */ + CHKiRet(strmPhysWrite(pThis, pBuf, lenBuf)); + } + +finalize_it: + RETiRet; +} + + /* flush stream output buffer to persistent storage. This can be called at any time * and is automatically called when the output buffer is full. * rgerhards, 2008-01-10 @@ -511,7 +626,7 @@ strmFlush(strm_t *pThis) ASSERT(pThis != NULL); dbgoprint((obj_t*) pThis, "file %d flush, buflen %ld\n", pThis->fd, (long) pThis->iBufPtr); - if(pThis->tOperationsMode == STREAMMODE_WRITE && pThis->iBufPtr > 0) { + if(pThis->tOperationsMode != STREAMMODE_READ && pThis->iBufPtr > 0) { iRet = strmWriteInternal(pThis, pThis->pIOBuf, pThis->iBufPtr); } @@ -605,6 +720,7 @@ strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) ASSERT(pThis != NULL); ASSERT(pBuf != NULL); +dbgprintf("strmWrite(%p, '%s', %ld);\n", pThis, pBuf,lenBuf); /* check if the to-be-written data is larger than our buffer size */ if(lenBuf >= pThis->sIOBufSize) { /* it is - so we do a direct write, that is most efficient. @@ -646,6 +762,7 @@ DEFpropSetMeth(strm, iFileNumDigits, int) DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) DEFpropSetMeth(strm, sType, strmType_t) +DEFpropSetMeth(strm, iZipLevel, int) static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) { @@ -681,13 +798,14 @@ strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) ASSERT(pThis != NULL); ASSERT(pszName != NULL); +dbgprintf("XXX: strm setFname: '%s'\n", pszName); if(iLenName < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); if(pThis->pszFName != NULL) free(pThis->pszFName); - if((pThis->pszFName = malloc(sizeof(uchar) * iLenName + 1)) == NULL) + if((pThis->pszFName = malloc(sizeof(uchar) * (iLenName + 1))) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszFName, pszName, iLenName + 1); /* always think about the \0! */ @@ -711,6 +829,7 @@ strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) ASSERT(pThis != NULL); ASSERT(pszDir != NULL); +dbgprintf("XXX: strm setDir: '%s'\n", pszDir); if(iLenDir < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); @@ -927,7 +1046,7 @@ CODESTARTobjQueryInterface(strm) pIf->RecordBegin = strmRecordBegin; pIf->RecordEnd = strmRecordEnd; pIf->Serialize = strmSerialize; - pIf->SetiAddtlOpenFlags = strmSetiAddtlOpenFlags; + /* TODO: remove this! */ pIf->SetiAddtlOpenFlags = strmSetiAddtlOpenFlags; pIf->GetCurrOffset = strmGetCurrOffset; pIf->SetWCntr = strmSetWCntr; /* set methods */ @@ -938,6 +1057,7 @@ CODESTARTobjQueryInterface(strm) pIf->SettOperationsMode = strmSettOperationsMode; pIf->SettOpenMode = strmSettOpenMode; pIf->SetsType = strmSetsType; + pIf->SetiZipLevel = strmSetiZipLevel; finalize_it: ENDobjQueryInterface(strm) diff --git a/runtime/stream.h b/runtime/stream.h index ece33270..449bf6c6 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -47,6 +47,7 @@ #include "obj-types.h" #include "glbl.h" #include "stream.h" +#include "zlibw.h" /* stream types */ typedef enum { @@ -55,10 +56,12 @@ typedef enum { STREAMTYPE_FILE_MONITOR = 2 /**< monitor a (third-party) file */ } strmType_t; -typedef enum { +typedef enum { /* when extending, do NOT change existing modes! */ STREAMMMODE_INVALID = 0, STREAMMODE_READ = 1, - STREAMMODE_WRITE = 2 + STREAMMODE_WRITE = 2, + STREAMMODE_WRITE_TRUNC = 3, + STREAMMODE_WRITE_APPEND = 4 } strmMode_t; /* The strm_t data structure */ @@ -89,6 +92,9 @@ typedef struct strm_s { size_t iBufPtr; /* pointer into current buffer */ int iUngetC; /* char set via UngetChar() call or -1 if none set */ int bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ + int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */ + Bytef *pZipBuf; + } strm_t; /* interfaces */ @@ -121,6 +127,7 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, tOperationsMode, int); INTERFACEpropSetMeth(strm, tOpenMode, mode_t); INTERFACEpropSetMeth(strm, sType, strmType_t); + INTERFACEpropSetMeth(strm, iZipLevel, int); ENDinterface(strm) #define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/tools/omfile.c b/tools/omfile.c index 689bad0c..3f517906 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -70,6 +70,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,7 @@ #include "module-template.h" #include "errmsg.h" #include "unicode-helper.h" +#include "stream.h" #include "zlibw.h" MODULE_TYPE_OUTPUT @@ -96,6 +98,7 @@ MODULE_TYPE_OUTPUT DEF_OMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(zlibw) +DEFobjCurrIf(strm) /* The following structure is a dynafile name cache entry. */ @@ -131,12 +134,14 @@ static uid_t dirUID; /* UID to be used for newly created directories */ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs; /* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ +static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ static uchar *pszTplName = NULL; /* name of the default template to use */ /* end globals for default values */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ + strm_t *pStrm; /* our output stream */ short fd; /* file descriptor for (current) file */ outbuf_t *poBuf; /* output buffer */ enum { @@ -165,6 +170,7 @@ typedef struct _instanceData { dynaFileCacheEntry **dynCache; off_t f_sizeLimit; /* file size limit, 0 = no limit */ uchar *f_sizeLimitCmd; /* command to carry out when size limit is reached */ + int iZipLevel; /* zip mode to use for this selector */ } instanceData; @@ -446,13 +452,11 @@ prepareFile(instanceData *pData, uchar *newFileName) FINALIZE; /* we are done in this case */ } - if(access((char*)newFileName, F_OK) == 0) { - /* file already exists */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, - pData->fCreateMode); - } else { - pData->fd = -1; + // TODO: handle TTY case! (here or in stream.c?) 2009-06-04 + + if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ + pData->fd = -1; if(pData->bCreateDirs) { /* We first need to create parent dirs if they are missing. * We do not report any errors here ourselfs but let the code @@ -485,8 +489,29 @@ prepareFile(instanceData *pData, uchar *newFileName) */ } } + close(pData->fd); /* close again, as we need a stream further on */ } } + + char szNameBuf[MAXFNAME]; + char szDirName[MAXFNAME]; + char szBaseName[MAXFNAME]; + strcpy(szNameBuf, (char*)pData->f_fname); + strcpy(szDirName, dirname(szNameBuf)); + strcpy(szNameBuf, (char*)pData->f_fname); + strcpy(szBaseName, basename(szNameBuf)); +DBGPRINTF("XXX: name to set: '%s', dirname '%s'\n", pData->f_fname, szDirName); + + CHKiRet(strm.Construct(&pData->pStrm)); + CHKiRet(strm.SetFName(pData->pStrm, (uchar*)szBaseName, strlen(szBaseName))); + CHKiRet(strm.SetDir(pData->pStrm, (uchar*)szDirName, strlen(szDirName))); + CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); + CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); + CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.ConstructFinalize(pData->pStrm)); + + pData->fd = 2; /* TODO: dummy to keep current inconsistent code happy - remove later */ + finalize_it: /* this was "pData->fd != 0", which I think was a bug. I guess 0 was intended to mean * non-open file descriptor. Anyhow, I leave this comment for the time being to that if @@ -715,20 +740,20 @@ static rsRetVal doZipWrite(instanceData *pData) { outbuf_t *poBuf; - z_stream strm; + z_stream zstrm; int zRet; /* zlib return state */ DEFiRet; assert(pData != NULL); poBuf = pData->poBuf; /* use as a shortcut */ - strm = poBuf->zStrm; /* another shortcut */ + zstrm = poBuf->zStrm; /* another shortcut */ /* allocate deflate state */ - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; + zstrm.zalloc = Z_NULL; + zstrm.zfree = Z_NULL; + zstrm.opaque = Z_NULL; /* see note in file header for the params we use with deflateInit2() */ - zRet = zlibw.DeflateInit2(&strm, 9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + zRet = zlibw.DeflateInit2(&zstrm, 9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); if(zRet != Z_OK) { dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); @@ -736,24 +761,24 @@ doZipWrite(instanceData *pData) RUNLOG_STR("deflateInit2() done successfully\n"); /* now doing the compression */ - strm.avail_in = poBuf->iBuf; - strm.next_in = (Bytef*) poBuf->pszBuf; + zstrm.avail_in = poBuf->iBuf; + zstrm.next_in = (Bytef*) poBuf->pszBuf; /* run deflate() on input until output buffer not full, finish compression if all of source has been read in */ do { - dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", strm.avail_in, strm.total_in); - strm.avail_out = OUTBUF_LEN; - strm.next_out = (Bytef*) poBuf->zipBuf; - zRet = zlibw.Deflate(&strm, Z_FINISH); /* no bad return value */ - dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, strm.avail_out); + dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", zstrm.avail_in, zstrm.total_in); + zstrm.avail_out = OUTBUF_LEN; + zstrm.next_out = (Bytef*) poBuf->zipBuf; + zRet = zlibw.Deflate(&zstrm, Z_FINISH); /* no bad return value */ + dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, zstrm.avail_out); assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ - CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - strm.avail_out)); - } while (strm.avail_out == 0); - assert(strm.avail_in == 0); /* all input will be used */ + CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - zstrm.avail_out)); + } while (zstrm.avail_out == 0); + assert(zstrm.avail_in == 0); /* all input will be used */ RUNLOG_STR("deflate() should be done successfully\n"); - zRet = zlibw.DeflateEnd(&strm); + zRet = zlibw.DeflateEnd(&zstrm); if(zRet != Z_OK) { dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); @@ -802,8 +827,14 @@ doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) ASSERT(pszBuf != NULL); poBuf = pData->poBuf; /* use as a shortcut */ -dbgprintf("doWrite, pData->fd %d, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", -pData->fd, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); +dbgprintf("doWrite, pData->fd %d, pData->pStrm %p, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", +pData->fd, pData->pStrm, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); + +RUNLOG_VAR("%p", pData->pStrm); + if(pData->pStrm != NULL){ + CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf)); + FINALIZE; // TODO: clean up later + } if(pData->fd != poBuf->fd) { // TODO: more efficient use for dynafiles @@ -869,6 +900,10 @@ ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ + if(pData->pStrm != NULL) { +RUNLOG_STR("XXX: destructing stream"); + strm.Destruct(&pData->pStrm); + } if(pData->bDynamicName) { dynaFileFreeCache(pData); } else if(pData->fd != -1) @@ -945,6 +980,7 @@ CODESTARTparseSelectorAct pData->fileGID = fileGID; pData->dirUID = dirUID; pData->dirGID = dirGID; + pData->iZipLevel = iZipLevel; pData->iDynaFileCacheSize = iDynaFileCacheSize; /* freeze current setting */ /* we now allocate the cache table. We use calloc() intentionally, as we * need all pointers to be initialized to NULL pointers. @@ -983,6 +1019,7 @@ CODESTARTparseSelectorAct pData->fileGID = fileGID; pData->dirUID = dirUID; pData->dirGID = dirGID; + pData->iZipLevel = iZipLevel; /* at this stage, we ignore the return value of prepareFile, this is taken * care of in later steps. -- rgerhards, 2009-03-19 @@ -1021,6 +1058,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a fDirCreateMode = 0700; bCreateDirs = 1; bEnableSync = 0; + iZipLevel = 0; if(pszTplName != NULL) { free(pszTplName); pszTplName = NULL; @@ -1046,6 +1084,8 @@ ENDdoHUP BEGINmodExit CODESTARTmodExit + objRelease(errmsg, CORE_COMPONENT); + objRelease(strm, CORE_COMPONENT); objRelease(zlibw, LM_ZLIBW_FILENAME); free(pszTplName); ENDmodExit @@ -1062,9 +1102,11 @@ BEGINmodInit(File) CODESTARTmodInit *ipIFVersProvided = CURR_MOD_IF_VERSION; /* we only support the current interface specification */ CODEmodInit_QueryRegCFSLineHdlr - CHKiRet(objUse(zlibw, LM_ZLIBW_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(zlibw, LM_ZLIBW_FILENAME)); + CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirgroup", 0, eCmdHdlrGID, NULL, &dirGID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"fileowner", 0, eCmdHdlrUID, NULL, &fileUID, STD_LOADABLE_MODULE_ID)); -- cgit v1.2.3 From 76da7f9f4e3dc900046a1956153d10532f2b1ae0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 4 Jun 2009 15:59:37 +0200 Subject: added $OMFileIOBufferSize config directive and plumbing --- ChangeLog | 3 ++ doc/rsyslog_conf_global.html | 1 + plugins/imfile/imfile.c | 3 +- runtime/stream.c | 21 +------- runtime/stream.h | 3 +- runtime/unicode-helper.h | 5 ++ tools/omfile.c | 114 ++++++++++++++++++++----------------------- tools/syslogd.c | 8 +-- 8 files changed, 69 insertions(+), 89 deletions(-) diff --git a/ChangeLog b/ChangeLog index 9d398f09..8fc4194d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,6 +12,9 @@ Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? - reduced max value for $DynaFileCacheSize to 1,000 (the former maximum of 10,000 really made no sense, even 1,000 is very high, but we like to keep the user in control ;)). +- added configuration commands (see doc for explanations) + * $OMFileZipLevel + * $OMFileIOBufferSize --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 0f408fcf..1e268f4b 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -192,6 +192,7 @@ supported in order to be compliant to the upcoming new syslog RFC series.
    • $OMFileZipLevel 0..9 [default 0] - if greater 0, turns on gzip compression of the output file. The higher the number, the better the compression, but also the more CPU is required for zipping.
      • +
    • $OMFileIOBufferSize <size_nbr>, default 4k, size of the buffer used to writing output data. The larger the buffer, the potentially better performance is. The default of 4k is quite conservative, it is useful to go up to 64k, and 128K if you used gzip compression (then, even higher sizes may make sense)
    • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, have a different format that contains the message that is being repeated. Note that only the first "n" characters are included, with n to be at least 80 characters, most diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index ff32c857..42c70539 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -357,8 +357,7 @@ persistStrmState(fileInfo_t *pInfo) /* TODO: create a function persistObj in obj.c? */ CHKiRet(strm.Construct(&psSF)); CHKiRet(strm.SetDir(psSF, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); - CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_WRITE)); - CHKiRet(strm.SetiAddtlOpenFlags(psSF, O_TRUNC)); + CHKiRet(strm.SettOperationsMode(psSF, STREAMMODE_WRITE_TRUNC)); CHKiRet(strm.SetsType(psSF, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.SetFName(psSF, pInfo->pszStateFile, strlen((char*) pInfo->pszStateFile))); CHKiRet(strm.ConstructFinalize(psSF)); diff --git a/runtime/stream.c b/runtime/stream.c index abcfce0c..a4844d2b 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -94,7 +94,6 @@ static rsRetVal strmOpenFile(strm_t *pThis) } } -RUNLOG_VAR("%d", pThis->tOperationsMode); /* compute which flags we need to provide to open */ switch(pThis->tOperationsMode) { case STREAMMODE_READ: @@ -113,9 +112,6 @@ RUNLOG_VAR("%d", pThis->tOperationsMode); break; } - iFlags |= pThis->iAddtlOpenFlags; // TODO: remove this! -dbgprintf("XXX: open with flags %d\n", iFlags); - pThis->fd = open((char*)pThis->pszCurrFName, iFlags, pThis->tOpenMode); if(pThis->fd == -1) { int ierrnoSave = errno; @@ -763,6 +759,7 @@ DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) DEFpropSetMeth(strm, sType, strmType_t) DEFpropSetMeth(strm, iZipLevel, int) +DEFpropSetMeth(strm, sIOBufSize, size_t) static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) { @@ -771,19 +768,6 @@ static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) return RS_RET_OK; } -static rsRetVal strmSetiAddtlOpenFlags(strm_t *pThis, int iNewVal) -{ - DEFiRet; - - if(iNewVal & O_APPEND) - ABORT_FINALIZE(RS_RET_PARAM_ERROR); - - pThis->iAddtlOpenFlags = iNewVal; - -finalize_it: - RETiRet; -} - /* set the stream's file prefix * The passed-in string is duplicated. So if the caller does not need @@ -829,7 +813,6 @@ strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) ASSERT(pThis != NULL); ASSERT(pszDir != NULL); -dbgprintf("XXX: strm setDir: '%s'\n", pszDir); if(iLenDir < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); @@ -1046,7 +1029,6 @@ CODESTARTobjQueryInterface(strm) pIf->RecordBegin = strmRecordBegin; pIf->RecordEnd = strmRecordEnd; pIf->Serialize = strmSerialize; - /* TODO: remove this! */ pIf->SetiAddtlOpenFlags = strmSetiAddtlOpenFlags; pIf->GetCurrOffset = strmGetCurrOffset; pIf->SetWCntr = strmSetWCntr; /* set methods */ @@ -1058,6 +1040,7 @@ CODESTARTobjQueryInterface(strm) pIf->SettOpenMode = strmSettOpenMode; pIf->SetsType = strmSetsType; pIf->SetiZipLevel = strmSetiZipLevel; + pIf->SetsIOBufSize = strmSetsIOBufSize; finalize_it: ENDobjQueryInterface(strm) diff --git a/runtime/stream.h b/runtime/stream.h index 449bf6c6..e4871611 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -74,7 +74,6 @@ typedef struct strm_s { int lenFName; strmMode_t tOperationsMode; mode_t tOpenMode; - int iAddtlOpenFlags; /* can be used to specifiy additional (compatible!) open flags */ int64 iMaxFileSize;/* maximum size a file may grow to */ int iMaxFiles; /* maximum number of files if a circular mode is in use */ int iFileNumDigits;/* min number of digits to use in file number (only in circular mode) */ @@ -117,7 +116,6 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*RecordBegin)(strm_t *pThis); rsRetVal (*RecordEnd)(strm_t *pThis); rsRetVal (*Serialize)(strm_t *pThis, strm_t *pStrm); - rsRetVal (*SetiAddtlOpenFlags)(strm_t *pThis, int iNewVal); rsRetVal (*GetCurrOffset)(strm_t *pThis, int64 *pOffs); rsRetVal (*SetWCntr)(strm_t *pThis, number_t *pWCnt); INTERFACEpropSetMeth(strm, bDeleteOnClose, int); @@ -128,6 +126,7 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, tOpenMode, mode_t); INTERFACEpropSetMeth(strm, sType, strmType_t); INTERFACEpropSetMeth(strm, iZipLevel, int); + INTERFACEpropSetMeth(strm, sIOBufSize, size_t); ENDinterface(strm) #define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h index fe4cc1f8..5289da3f 100644 --- a/runtime/unicode-helper.h +++ b/runtime/unicode-helper.h @@ -31,6 +31,11 @@ #include +static inline char* ustrncpy(uchar *psz1, uchar *psz2, size_t len) +{ + return strncpy((char*) psz1, (char*) psz2, len); +} + static inline int ustrcmp(uchar *psz1, uchar *psz2) { return strcmp((char*) psz1, (char*) psz2); diff --git a/tools/omfile.c b/tools/omfile.c index 3f517906..85f04f13 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -123,6 +123,8 @@ typedef struct { } outbuf_t; +#define IOBUF_DFLT_SIZE 1024 /* default size for io buffers */ + /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ static int fCreateMode = 0644; /* mode to use when creating files */ @@ -135,6 +137,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs; /* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ +static int iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static uchar *pszTplName = NULL; /* name of the default template to use */ /* end globals for default values */ @@ -168,9 +171,10 @@ typedef struct _instanceData { * pointer points to the overall structure. */ dynaFileCacheEntry **dynCache; - off_t f_sizeLimit; /* file size limit, 0 = no limit */ - uchar *f_sizeLimitCmd; /* command to carry out when size limit is reached */ + off_t iSizeLimit; /* file size limit, 0 = no limit */ + uchar *iSizeLimitCmd; /* command to carry out when size limit is reached */ int iZipLevel; /* zip mode to use for this selector */ + int iIOBufSize; /* size of associated io buffer */ } instanceData; @@ -293,12 +297,12 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR } /* OK, we finally got a correct template. So let's use it... */ - strncpy((char*) pData->f_fname, (char*) pOch->pszFileTemplate, MAXFNAME); - pData->f_sizeLimit = pOch->uSizeLimit; + ustrncpy(pData->f_fname, pOch->pszFileTemplate, MAXFNAME); + pData->iSizeLimit = pOch->uSizeLimit; /* WARNING: It is dangerous "just" to pass the pointer. As we * never rebuild the output channel description, this is acceptable here. */ - pData->f_sizeLimitCmd = pOch->cmdOnSizeLimit; + pData->iSizeLimitCmd = pOch->cmdOnSizeLimit; iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName); @@ -322,7 +326,7 @@ int resolveFileSizeLimit(instanceData *pData) off_t actualFileSize; ASSERT(pData != NULL); - if(pData->f_sizeLimitCmd == NULL) + if(pData->iSizeLimitCmd == NULL) return 1; /* nothing we can do in this case... */ /* the execProg() below is probably not great, but at least is is @@ -334,7 +338,7 @@ int resolveFileSizeLimit(instanceData *pData) * when we have a space in the program name. If we find it, everything after * the space is treated as a single argument. */ - if((pCmd = ustrdup(pData->f_sizeLimitCmd)) == NULL) { + if((pCmd = ustrdup(pData->iSizeLimitCmd)) == NULL) { /* there is not much we can do - we make syslogd close the file in this case */ return 1; } @@ -357,7 +361,7 @@ int resolveFileSizeLimit(instanceData *pData) pData->fCreateMode); actualFileSize = lseek(pData->fd, 0, SEEK_END); - if(actualFileSize >= pData->f_sizeLimit) { + if(actualFileSize >= pData->iSizeLimit) { /* OK, it didn't work out... */ return 1; } @@ -500,12 +504,14 @@ prepareFile(instanceData *pData, uchar *newFileName) strcpy(szDirName, dirname(szNameBuf)); strcpy(szNameBuf, (char*)pData->f_fname); strcpy(szBaseName, basename(szNameBuf)); -DBGPRINTF("XXX: name to set: '%s', dirname '%s'\n", pData->f_fname, szDirName); CHKiRet(strm.Construct(&pData->pStrm)); CHKiRet(strm.SetFName(pData->pStrm, (uchar*)szBaseName, strlen(szBaseName))); CHKiRet(strm.SetDir(pData->pStrm, (uchar*)szDirName, strlen(szDirName))); CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); +dbgprintf("static IOBufSize %d, CONST %d\n", iIOBufSize, IOBUF_DFLT_SIZE); +RUNLOG_VAR("%d", pData->iIOBufSize); + CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.ConstructFinalize(pData->pStrm)); @@ -650,9 +656,9 @@ again: /* check if we have a file size limit and, if so, * obey to it. */ - if(pData->f_sizeLimit != 0) { + if(pData->iSizeLimit != 0) { actualFileSize = lseek(fd, 0, SEEK_END); - if(actualFileSize >= pData->f_sizeLimit) { + if(actualFileSize >= pData->iSizeLimit) { char errMsg[256]; /* for now, we simply disable a file once it is * beyond the maximum size. This is better than having @@ -665,14 +671,14 @@ again: /* didn't work out, so disable... */ snprintf(errMsg, sizeof(errMsg), "no longer writing to file %s; grown beyond configured file size of %lld bytes, actual size %lld - configured command did not resolve situation", - pData->f_fname, (long long) pData->f_sizeLimit, (long long) actualFileSize); + pData->f_fname, (long long) pData->iSizeLimit, (long long) actualFileSize); errno = 0; errmsg.LogError(0, RS_RET_DISABLE_ACTION, "%s", errMsg); ABORT_FINALIZE(RS_RET_DISABLE_ACTION); } else { snprintf(errMsg, sizeof(errMsg), "file %s had grown beyond configured file size of %lld bytes, actual size was %lld - configured command resolved situation", - pData->f_fname, (long long) pData->f_sizeLimit, (long long) actualFileSize); + pData->f_fname, (long long) pData->iSizeLimit, (long long) actualFileSize); errno = 0; errmsg.LogError(0, NO_ERRCODE, "%s", errMsg); } @@ -901,7 +907,6 @@ BEGINfreeInstance CODESTARTfreeInstance doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->pStrm != NULL) { -RUNLOG_STR("XXX: destructing stream"); strm.Destruct(&pData->pStrm); } if(pData->bDynamicName) { @@ -934,12 +939,11 @@ CODESTARTparseSelectorAct pData->bSyncFile = 0; p++; } else { - pData->bSyncFile = bEnableSync ? 1 : 0; + pData->bSyncFile = bEnableSync; } - pData->f_sizeLimit = 0; /* default value, use outchannels to configure! */ + pData->iSizeLimit = 0; /* default value, use outchannels to configure! */ - switch (*p) - { + switch(*p) { case '$': CODE_STD_STRING_REQUESTparseSelectorAct(1) /* rgerhards 2005-06-21: this is a special setting for output-channel @@ -949,13 +953,8 @@ CODESTARTparseSelectorAct * rgerhards, 2007-07-24: output-channels will go away. We keep them * for compatibility reasons, but seems to have been a bad idea. */ - if((iRet = cflineParseOutchannel(pData, p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS)) == RS_RET_OK) { - pData->bDynamicName = 0; - pData->fCreateMode = fCreateMode; /* preserve current setting */ - pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */ - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, - pData->fCreateMode); - } + CHKiRet(cflineParseOutchannel(pData, p, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS)); + pData->bDynamicName = 0; break; case '?': /* This is much like a regular file handle, but we need to obtain @@ -972,16 +971,6 @@ CODESTARTparseSelectorAct pData->bDynamicName = 1; pData->iCurrElt = -1; /* no current element */ - pData->fCreateMode = fCreateMode; /* freeze current setting */ - pData->fDirCreateMode = fDirCreateMode; /* preserve current setting */ - pData->bCreateDirs = bCreateDirs; - pData->bFailOnChown = bFailOnChown; - pData->fileUID = fileUID; - pData->fileGID = fileGID; - pData->dirUID = dirUID; - pData->dirGID = dirGID; - pData->iZipLevel = iZipLevel; - pData->iDynaFileCacheSize = iDynaFileCacheSize; /* freeze current setting */ /* we now allocate the cache table. We use calloc() intentionally, as we * need all pointers to be initialized to NULL pointers. */ @@ -995,50 +984,49 @@ CODESTARTparseSelectorAct case '|': case '/': CODE_STD_STRING_REQUESTparseSelectorAct(1) - /* rgerhards, 2007-0726: first check if file or pipe */ if(*p == '|') { pData->fileType = eTypePIPE; ++p; } else { pData->fileType = eTypeFILE; } - /* rgerhards 2004-11-17: from now, we need to have different - * processing, because after the first comma, the template name - * to use is specified. So we need to scan for the first coma first - * and then look at the rest of the line. - */ CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)); - pData->bDynamicName = 0; - pData->fCreateMode = fCreateMode; /* preserve current setting */ - pData->fDirCreateMode = fDirCreateMode; - pData->bCreateDirs = bCreateDirs; - pData->bFailOnChown = bFailOnChown; - pData->fileUID = fileUID; - pData->fileGID = fileGID; - pData->dirUID = dirUID; - pData->dirGID = dirGID; - pData->iZipLevel = iZipLevel; - - /* at this stage, we ignore the return value of prepareFile, this is taken - * care of in later steps. -- rgerhards, 2009-03-19 + break; + default: + ABORT_FINALIZE(RS_RET_CONFLINE_UNPROCESSED); + } + + /* freeze current paremeters for this action */ + pData->iDynaFileCacheSize = iDynaFileCacheSize; + pData->fCreateMode = fCreateMode; + pData->fDirCreateMode = fDirCreateMode; + pData->bCreateDirs = bCreateDirs; + pData->bFailOnChown = bFailOnChown; + pData->fileUID = fileUID; + pData->fileGID = fileGID; + pData->dirUID = dirUID; + pData->dirGID = dirGID; + pData->iZipLevel = iZipLevel; + pData->iIOBufSize = iIOBufSize; + + if(pData->bDynamicName == 0) { + if(ustrcmp(p, UCHAR_CONSTANT(_PATH_CONSOLE)) == 0) + pData->fileType = eTypeCONSOLE; + /* try open and emit error message if not possible. At this stage, we ignore the + * return value of prepareFile, this is taken care of in later steps. */ prepareFile(pData, pData->f_fname); if(pData->fd < 0 ) { - pData->fd = -1; DBGPRINTF("Error opening log file: %s\n", pData->f_fname); errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); - break; } - if(ustrcmp(p, UCHAR_CONSTANT(_PATH_CONSOLE)) == 0) - pData->fileType = eTypeCONSOLE; - break; - default: - iRet = RS_RET_CONFLINE_UNPROCESSED; - break; } + +dbgprintf("in init static IOBufSize %d, CONST %d\n", iIOBufSize, IOBUF_DFLT_SIZE); +RUNLOG_VAR("%d", pData->iIOBufSize); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct @@ -1059,6 +1047,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bCreateDirs = 1; bEnableSync = 0; iZipLevel = 0; + iIOBufSize = IOBUF_DFLT_SIZE; if(pszTplName != NULL) { free(pszTplName); pszTplName = NULL; @@ -1107,6 +1096,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirgroup", 0, eCmdHdlrGID, NULL, &dirGID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"fileowner", 0, eCmdHdlrUID, NULL, &fileUID, STD_LOADABLE_MODULE_ID)); diff --git a/tools/syslogd.c b/tools/syslogd.c index 22cb6879..206b79e8 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3042,18 +3042,18 @@ static rsRetVal loadBuildInModules(void) { DEFiRet; - if((iRet = module.doModInit(modInitFile, (uchar*) "builtin-file", NULL)) != RS_RET_OK) { + if((iRet = module.doModInit(modInitFile, UCHAR_CONSTANT("builtin-file"), NULL)) != RS_RET_OK) { RETiRet; } #ifdef SYSLOG_INET - if((iRet = module.doModInit(modInitFwd, (uchar*) "builtin-fwd", NULL)) != RS_RET_OK) { + if((iRet = module.doModInit(modInitFwd, UCHAR_CONSTANT("builtin-fwd"), NULL)) != RS_RET_OK) { RETiRet; } #endif - if((iRet = module.doModInit(modInitShell, (uchar*) "builtin-shell", NULL)) != RS_RET_OK) { + if((iRet = module.doModInit(modInitShell, UCHAR_CONSTANT("builtin-shell"), NULL)) != RS_RET_OK) { RETiRet; } - if((iRet = module.doModInit(modInitDiscard, (uchar*) "builtin-discard", NULL)) != RS_RET_OK) { + if((iRet = module.doModInit(modInitDiscard, UCHAR_CONSTANT("builtin-discard"), NULL)) != RS_RET_OK) { RETiRet; } -- cgit v1.2.3 From 68f0ffb29acf5ff07ccd8f30581d96d6915a834e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Jun 2009 15:41:05 +0200 Subject: made dynafiles working again now only stream class is utilized. ttys, pipes and outchannel functionality is currently disabled. But the testbench worked again. Cleanup needed, will do this with next commit (it may break things and I like to have this milestone here). --- tests/diag.sh | 3 +- tools/omfile.c | 148 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 75 insertions(+), 76 deletions(-) diff --git a/tests/diag.sh b/tests/diag.sh index 1ceca75b..1fa8f62a 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -9,7 +9,7 @@ #valgrind="valgrind --tool=drd --log-fd=1" #valgrind="valgrind --tool=helgrind --log-fd=1" #set -o xtrace -#export RSYSLOG_DEBUG="debug nostdout printmutexaction" +#export RSYSLOG_DEBUG="debug nostdout noprintmutexaction" #export RSYSLOG_DEBUGLOG="log" case $1 in 'init') $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason @@ -17,6 +17,7 @@ case $1 in rm -f rsyslogd.started work-*.conf rm -f work rsyslog.out.log rsyslog.out.log.save # common work files rm -rf test-spool + rm -f core.* vgcore.* mkdir test-spool ;; 'exit') rm -f rsyslogd.started work-*.conf diag-common.conf diff --git a/tools/omfile.c b/tools/omfile.c index 85f04f13..dd585893 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -104,7 +104,7 @@ DEFobjCurrIf(strm) */ struct s_dynaFileCacheEntry { uchar *pName; /* name currently open, if dynamic name */ - short fd; /* name associated with file name in cache */ + strm_t *pStrm; /* our output stream */ time_t lastUsed; /* for LRU - last access */ }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; @@ -116,7 +116,6 @@ typedef struct { uchar pszBuf[OUTBUF_LEN]; /* output buffer for buffered writing */ size_t lenBuf; /* max size of buffer */ size_t iBuf; /* current buffer index */ - int fd; /* which file descriptor is this buf for? */ /* elements for zip writing */ z_stream zStrm; char zipBuf[OUTBUF_LEN]; @@ -145,8 +144,7 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ strm_t *pStrm; /* our output stream */ - short fd; /* file descriptor for (current) file */ - outbuf_t *poBuf; /* output buffer */ + //outbuf_t *poBuf; /* output buffer */ enum { eTypeFILE, eTypeTTY, @@ -203,7 +201,7 @@ CODESTARTdbgPrintInstInfo ); } else { /* regular file */ dbgprintf("%s", pData->f_fname); - if (pData->fd == -1) + if (pData->pStrm == NULL) dbgprintf(" (unused)"); } ENDdbgPrintInstInfo @@ -312,6 +310,7 @@ finalize_it: } +#if 0 /* rgerhards 2005-06-21: Try to resolve a size limit * situation. This first runs the command, and then * checks if we are still above the treshold. @@ -368,6 +367,7 @@ int resolveFileSizeLimit(instanceData *pData) return 0; } +#endif /* This function deletes an entry from the dynamic file name @@ -377,13 +377,12 @@ int resolveFileSizeLimit(instanceData *pData) * function immediately returns. Parameter bFreeEntry is 1 * if the entry should be d_free()ed and 0 if not. */ -static void +static rsRetVal dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) { + DEFiRet; ASSERT(pCache != NULL); - BEGINfunc; - if(pCache[iEntry] == NULL) FINALIZE; @@ -394,7 +393,8 @@ dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) * not the name to be freed. */ if(pCache[iEntry]->pName != NULL) { - close(pCache[iEntry]->fd); + if(pCache[iEntry]->pStrm != NULL) + strm.Destruct(&pCache[iEntry]->pStrm); d_free(pCache[iEntry]->pName); pCache[iEntry]->pName = NULL; } @@ -405,7 +405,7 @@ dynaFileDelCacheEntry(dynaFileCacheEntry **pCache, int iEntry, int bFreeEntry) } finalize_it: - ENDfunc; + RETiRet; } @@ -450,17 +450,20 @@ static void dynaFileFreeCache(instanceData *pData) static rsRetVal prepareFile(instanceData *pData, uchar *newFileName) { + int fd; DEFiRet; +#if 0 if(pData->fileType == eTypePIPE) { - pData->fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); + fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); FINALIZE; /* we are done in this case */ } +#endif // TODO: handle TTY case! (here or in stream.c?) 2009-06-04 if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ - pData->fd = -1; + fd = -1; if(pData->bCreateDirs) { /* We first need to create parent dirs if they are missing. * We do not report any errors here ourselfs but let the code @@ -475,17 +478,17 @@ prepareFile(instanceData *pData, uchar *newFileName) /* no matter if we needed to create directories or not, we now try to create * the file. -- rgerhards, 2008-12-18 (based on patch from William Tisater) */ - pData->fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, + fd = open((char*) newFileName, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, pData->fCreateMode); - if(pData->fd != -1) { + if(fd != -1) { /* check and set uid/gid */ if(pData->fileUID != (uid_t)-1 || pData->fileGID != (gid_t) -1) { /* we need to set owner/group */ - if(fchown(pData->fd, pData->fileUID, pData->fileGID) != 0) { + if(fchown(fd, pData->fileUID, pData->fileGID) != 0) { if(pData->bFailOnChown) { int eSave = errno; - close(pData->fd); - pData->fd = -1; + close(fd); + fd = -1; errno = eSave; } /* we will silently ignore the chown() failure @@ -493,16 +496,16 @@ prepareFile(instanceData *pData, uchar *newFileName) */ } } - close(pData->fd); /* close again, as we need a stream further on */ + close(fd); /* close again, as we need a stream further on */ } } char szNameBuf[MAXFNAME]; char szDirName[MAXFNAME]; char szBaseName[MAXFNAME]; - strcpy(szNameBuf, (char*)pData->f_fname); + strcpy(szNameBuf, (char*)newFileName); strcpy(szDirName, dirname(szNameBuf)); - strcpy(szNameBuf, (char*)pData->f_fname); + strcpy(szNameBuf, (char*)newFileName); strcpy(szBaseName, basename(szNameBuf)); CHKiRet(strm.Construct(&pData->pStrm)); @@ -516,18 +519,18 @@ RUNLOG_VAR("%d", pData->iIOBufSize); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.ConstructFinalize(pData->pStrm)); - pData->fd = 2; /* TODO: dummy to keep current inconsistent code happy - remove later */ - finalize_it: - /* this was "pData->fd != 0", which I think was a bug. I guess 0 was intended to mean + /* this was "fd != 0", which I think was a bug. I guess 0 was intended to mean * non-open file descriptor. Anyhow, I leave this comment for the time being to that if * problems surface, one at least knows what happened. -- rgerhards, 2009-03-19 */ - if(pData->fd != -1 && isatty(pData->fd)) { - DBGPRINTF("file %d is a tty file\n", pData->fd); +#if 0 // TODO: this must be done by stream class! + if(fd != -1 && isatty(fd)) { + DBGPRINTF("file %d is a tty file\n", fd); pData->fileType = eTypeTTY; untty(); } +#endif RETiRet; } @@ -541,15 +544,16 @@ finalize_it: * be written. * This is a helper to writeFile(). rgerhards, 2007-07-03 */ -static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) +static inline rsRetVal +prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) { time_t ttOldest; /* timestamp of oldest element */ int iOldest; int i; int iFirstFree; + rsRetVal localRet; dynaFileCacheEntry **pCache; - - BEGINfunc + DEFiRet; ASSERT(pData != NULL); ASSERT(newFileName != NULL); @@ -562,8 +566,9 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg if( (pData->iCurrElt != -1) && !ustrcmp(newFileName, pCache[pData->iCurrElt]->pName)) { /* great, we are all set */ - pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ - return 0; + pCache[pData->iCurrElt]->lastUsed = time(NULL); /* update timestamp for LRU */ // TODO: optimize time call! + // LRU needs only a strictly monotonically increasing counter, so such a one could do + FINALIZE; } /* ok, no luck. Now let's search the table if we find a matching spot. @@ -579,10 +584,10 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg } else { /* got an element, let's see if it matches */ if(!ustrcmp(newFileName, pCache[i]->pName)) { /* we found our element! */ - pData->fd = pCache[i]->fd; + pData->pStrm = pCache[i]->pStrm; pData->iCurrElt = i; pCache[i]->lastUsed = time(NULL); /* update timestamp for LRU */ - return 0; + FINALIZE; } /* did not find it - so lets keep track of the counters for LRU */ if(pCache[i]->lastUsed < ttOldest) { @@ -598,23 +603,20 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg iFirstFree = pData->iCurrCacheSize++; } + pData->iCurrElt = -1; if(iFirstFree == -1) { dynaFileDelCacheEntry(pCache, iOldest, 0); iFirstFree = iOldest; /* this one *is* now free ;) */ } else { /* we need to allocate memory for the cache structure */ - pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry)); - if(pCache[iFirstFree] == NULL) { - DBGPRINTF("prepareDynfile(): could not alloc mem, discarding this request\n"); - return -1; - } + CHKmalloc(pCache[iFirstFree] = (dynaFileCacheEntry*) calloc(1, sizeof(dynaFileCacheEntry))); } /* Ok, we finally can open the file */ - prepareFile(pData, newFileName); /* ignore exact error, we check fd below */ + localRet = prepareFile(pData, newFileName); /* ignore exact error, we check fd below */ /* file is either open now or an error state set */ - if(pData->fd == -1) { + if(pData->pStrm == NULL) { /* do not report anything if the message is an internally-generated * message. Otherwise, we could run into a never-ending loop. The bad * news is that we also lose errors on startup messages, but so it is. @@ -625,22 +627,21 @@ static int prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsg errmsg.LogError(0, NO_ERRCODE, "Could not open dynamic file '%s' - discarding message", newFileName); } dynaFileDelCacheEntry(pCache, iFirstFree, 1); - pData->iCurrElt = -1; - return -1; + ABORT_FINALIZE(localRet); } - pCache[iFirstFree]->fd = pData->fd; - pCache[iFirstFree]->pName = ustrdup(newFileName); /* TODO: check for NULL (very unlikely) */ - pCache[iFirstFree]->lastUsed = time(NULL); + CHKmalloc(pCache[iFirstFree]->pName = ustrdup(newFileName)); + pCache[iFirstFree]->pStrm = pData->pStrm; + pCache[iFirstFree]->lastUsed = time(NULL); // monotonically increasing value! TODO: performance pData->iCurrElt = iFirstFree; DBGPRINTF("Added new entry %d for file cache, file '%s'.\n", iFirstFree, newFileName); - ENDfunc - - return 0; +finalize_it: + RETiRet; } +#if 0 /* physically write the file */ static rsRetVal @@ -816,6 +817,7 @@ doFlush(instanceData *pData) finalize_it: RETiRet; } +#endif /* do the actual write process. This function is to be called once we are ready for writing. @@ -826,15 +828,12 @@ finalize_it: static rsRetVal doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) { - int i; - outbuf_t *poBuf; DEFiRet; ASSERT(pData != NULL); ASSERT(pszBuf != NULL); - poBuf = pData->poBuf; /* use as a shortcut */ -dbgprintf("doWrite, pData->fd %d, pData->pStrm %p, poBuf->fd %d, iBuf %ld, lenBuf %ld\n", -pData->fd, pData->pStrm, pData->poBuf->fd, pData->poBuf->iBuf, poBuf->lenBuf); +dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", +pData->pStrm, lenBuf); RUNLOG_VAR("%p", pData->pStrm); if(pData->pStrm != NULL){ @@ -842,6 +841,7 @@ RUNLOG_VAR("%p", pData->pStrm); FINALIZE; // TODO: clean up later } +#if 0 if(pData->fd != poBuf->fd) { // TODO: more efficient use for dynafiles CHKiRet(doFlush(pData)); @@ -854,6 +854,7 @@ RUNLOG_VAR("%p", pData->pStrm); CHKiRet(doFlush(pData)); } } +#endif finalize_it: RETiRet; @@ -875,45 +876,42 @@ writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) * check if it still is ok or a new file needs to be created */ if(pData->bDynamicName) { - if(prepareDynFile(pData, ppString[1], iMsgOpts) != 0) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ - } - - if(pData->fd == -1) { - rsRetVal iRetLocal; - iRetLocal = prepareFile(pData, pData->f_fname); - if((iRetLocal != RS_RET_OK) || (pData->fd == -1)) - ABORT_FINALIZE(RS_RET_SUSPENDED); /* whatever the failure was, we need to retry */ + CHKiRet(prepareDynFile(pData, ppString[1], iMsgOpts)); + } else { /* "regular", non-dynafile */ + if(pData->pStrm == NULL) { + CHKiRet(prepareFile(pData, pData->f_fname)); + } } /* create the message based on format specified */ CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0])))); finalize_it: + if(iRet != RS_RET_OK) { + /* in v5, we shall return different states for message-cause failur (but only there!) */ + iRet = RS_RET_SUSPENDED; + } RETiRet; } BEGINcreateInstance CODESTARTcreateInstance - pData->fd = -1; - CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); - pData->poBuf->lenBuf = OUTBUF_LEN; + pData->pStrm = NULL; + //CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); + //pData->poBuf->lenBuf = OUTBUF_LEN; finalize_it: ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance - doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ - if(pData->pStrm != NULL) { - strm.Destruct(&pData->pStrm); - } + //DEL! doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->bDynamicName) { dynaFileFreeCache(pData); - } else if(pData->fd != -1) - close(pData->fd); - free(pData->poBuf); + } else if(pData->pStrm != NULL) + strm.Destruct(&pData->pStrm); + //free(pData->poBuf); ENDfreeInstance @@ -1019,7 +1017,7 @@ CODESTARTparseSelectorAct */ prepareFile(pData, pData->f_fname); - if(pData->fd < 0 ) { + if(pData->pStrm == NULL) { DBGPRINTF("Error opening log file: %s\n", pData->f_fname); errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); } @@ -1063,9 +1061,9 @@ CODESTARTdoHUP dynaFileFreeCacheEntries(pData); pData->iCurrElt = -1; /* invalidate current element */ } else { - if(pData->fd != -1) { - close(pData->fd); - pData->fd = -1; + if(pData->pStrm != NULL) { + strm.Destruct(&pData->pStrm); + pData->pStrm = NULL; } } ENDdoHUP -- cgit v1.2.3 From 84b786753744d9f7a002665571a428283feae248 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 5 Jun 2009 18:41:28 +0200 Subject: cleanup in preparation of next steps now cleand up omfile and straighted out some things. The only commented-out code left is code that must be moved/merged to the stream class, my next target. --- runtime/stream.h | 24 +++++++- tools/omfile.c | 174 ++++--------------------------------------------------- 2 files changed, 33 insertions(+), 165 deletions(-) diff --git a/runtime/stream.h b/runtime/stream.h index e4871611..7eb386fb 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -19,7 +19,29 @@ * can easily be persistet. The bottom line is that it makes much sense to * use this class whereever possible as its features may grow in the future. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * An important note on writing gzip format via zlib (kept anonymous + * by request): + * + * -------------------------------------------------------------------------- + * We'd like to make sure the output file is in full gzip format + * (compatible with gzip -d/zcat etc). There is a flag in how the output + * is initialized within zlib to properly add the gzip wrappers to the + * output. (gzip is effectively a small metadata wrapper around raw + * zstream output.) + * + * I had written an old bit of code to do this - the documentation on + * deflatInit2() was pretty tricky to nail down on this specific feature: + * + * int deflateInit2 (z_streamp strm, int level, int method, int windowBits, + * int memLevel, int strategy); + * + * I believe "31" would be the value for the "windowBits" field that you'd + * want to try: + * + * deflateInit2(zstrmptr, 6, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); + * -------------------------------------------------------------------------- + * + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * diff --git a/tools/omfile.c b/tools/omfile.c index dd585893..f196d602 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -20,29 +20,6 @@ * * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * - * An important note on writing gzip format via zlib (kept anonymous - * by request): - * - * -------------------------------------------------------------------------- - * We'd like to make sure the output file is in full gzip format - * (compatible with gzip -d/zcat etc). There is a flag in how the output - * is initialized within zlib to properly add the gzip wrappers to the - * output. (gzip is effectively a small metadata wrapper around raw - * zstream output.) - * - * I had written an old bit of code to do this - the documentation on - * deflatInit2() was pretty tricky to nail down on this specific feature: - * - * int deflateInit2 (z_streamp strm, int level, int method, int windowBits, - * int memLevel, int strategy); - * - * I believe "31" would be the value for the "windowBits" field that you'd - * want to try: - * - * deflateInit2(zstrmptr, 6, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); - * -------------------------------------------------------------------------- - * - * * This file is part of rsyslog. * * Rsyslog is free software: you can redistribute it and/or modify @@ -105,22 +82,10 @@ DEFobjCurrIf(strm) struct s_dynaFileCacheEntry { uchar *pName; /* name currently open, if dynamic name */ strm_t *pStrm; /* our output stream */ - time_t lastUsed; /* for LRU - last access */ + time_t lastUsed; /* for LRU - last access */ // TODO: perforamcne change to counter (see other comment!) }; typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; -/* the output buffer structure */ -// TODO: later make this part of the dynafile cache! -#define OUTBUF_LEN 128 // TODO: make dynamic! -typedef struct { - uchar pszBuf[OUTBUF_LEN]; /* output buffer for buffered writing */ - size_t lenBuf; /* max size of buffer */ - size_t iBuf; /* current buffer index */ - /* elements for zip writing */ - z_stream zStrm; - char zipBuf[OUTBUF_LEN]; -} outbuf_t; - #define IOBUF_DFLT_SIZE 1024 /* default size for io buffers */ @@ -144,7 +109,6 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ strm_t *pStrm; /* our output stream */ - //outbuf_t *poBuf; /* output buffer */ enum { eTypeFILE, eTypeTTY, @@ -152,10 +116,10 @@ typedef struct _instanceData { eTypePIPE } fileType; char bDynamicName; /* 0 - static name, 1 - dynamic name (with properties) */ - int fCreateMode; /* file creation mode for open() */ + int fCreateMode; /* file creation mode for open() */ // TODO: shuffle down to stream class int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ - int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ + int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ // TODO: stream class? RE-IMPLEMENT! uid_t fileUID; /* IDs for creation */ uid_t dirUID; gid_t fileGID; @@ -452,14 +416,8 @@ prepareFile(instanceData *pData, uchar *newFileName) { int fd; DEFiRet; -#if 0 - if(pData->fileType == eTypePIPE) { - fd = open((char*) pData->f_fname, O_RDWR|O_NONBLOCK|O_CLOEXEC); - FINALIZE; /* we are done in this case */ - } -#endif - // TODO: handle TTY case! (here or in stream.c?) 2009-06-04 + // TODO: handle TTY/PIPE case! (in stream.c!) 2009-06-04 if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ @@ -512,8 +470,6 @@ prepareFile(instanceData *pData, uchar *newFileName) CHKiRet(strm.SetFName(pData->pStrm, (uchar*)szBaseName, strlen(szBaseName))); CHKiRet(strm.SetDir(pData->pStrm, (uchar*)szDirName, strlen(szDirName))); CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); -dbgprintf("static IOBufSize %d, CONST %d\n", iIOBufSize, IOBUF_DFLT_SIZE); -RUNLOG_VAR("%d", pData->iIOBufSize); CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); @@ -574,6 +530,7 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) /* ok, no luck. Now let's search the table if we find a matching spot. * While doing so, we also prepare for creation of a new one. */ + pData->iCurrElt = -1; /* invalid current element pointer */ iFirstFree = -1; /* not yet found */ iOldest = 0; /* we assume the first element to be the oldest - that will change as we loop */ ttOldest = time(NULL) + 1; /* there must always be an older one */ @@ -603,7 +560,6 @@ prepareDynFile(instanceData *pData, uchar *newFileName, unsigned iMsgOpts) iFirstFree = pData->iCurrCacheSize++; } - pData->iCurrElt = -1; if(iFirstFree == -1) { dynaFileDelCacheEntry(pCache, iOldest, 0); iFirstFree = iOldest; /* this one *is* now free ;) */ @@ -642,6 +598,7 @@ finalize_it: #if 0 +// TODO: mirgrate code below to stream class (update its write handler, which is not great!) /* physically write the file */ static rsRetVal @@ -738,85 +695,6 @@ again: finalize_it: RETiRet; } - - -/* write the output buffer in zip mode - * This means we compress it first and then do a physical write. - */ -static rsRetVal -doZipWrite(instanceData *pData) -{ - outbuf_t *poBuf; - z_stream zstrm; - int zRet; /* zlib return state */ - DEFiRet; - assert(pData != NULL); - - poBuf = pData->poBuf; /* use as a shortcut */ - zstrm = poBuf->zStrm; /* another shortcut */ - - /* allocate deflate state */ - zstrm.zalloc = Z_NULL; - zstrm.zfree = Z_NULL; - zstrm.opaque = Z_NULL; - /* see note in file header for the params we use with deflateInit2() */ - zRet = zlibw.DeflateInit2(&zstrm, 9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY); - if(zRet != Z_OK) { - dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); - ABORT_FINALIZE(RS_RET_ZLIB_ERR); - } -RUNLOG_STR("deflateInit2() done successfully\n"); - - /* now doing the compression */ - zstrm.avail_in = poBuf->iBuf; - zstrm.next_in = (Bytef*) poBuf->pszBuf; - /* run deflate() on input until output buffer not full, finish - compression if all of source has been read in */ - do { - dbgprintf("in deflate() loop, avail_in %d, total_in %ld\n", zstrm.avail_in, zstrm.total_in); - zstrm.avail_out = OUTBUF_LEN; - zstrm.next_out = (Bytef*) poBuf->zipBuf; - zRet = zlibw.Deflate(&zstrm, Z_FINISH); /* no bad return value */ - dbgprintf("after deflate, ret %d, avail_out %d\n", zRet, zstrm.avail_out); - assert(zRet != Z_STREAM_ERROR); /* state not clobbered */ - CHKiRet(doPhysWrite(pData, poBuf->fd, poBuf->zipBuf, OUTBUF_LEN - zstrm.avail_out)); - } while (zstrm.avail_out == 0); - assert(zstrm.avail_in == 0); /* all input will be used */ - -RUNLOG_STR("deflate() should be done successfully\n"); - - zRet = zlibw.DeflateEnd(&zstrm); - if(zRet != Z_OK) { - dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); - ABORT_FINALIZE(RS_RET_ZLIB_ERR); - } -RUNLOG_STR("deflateEnd() done successfully\n"); - -finalize_it: - RETiRet; -} - - -/* flush the output buffer - */ -static rsRetVal -doFlush(instanceData *pData) -{ - DEFiRet; - assert(pData != NULL); - - if(pData->poBuf->iBuf == 0) - FINALIZE; /* nothing to write, but make this a valid case */ - - if(0) { // zlib enabled! - CHKiRet(doZipWrite(pData)); - } else { - CHKiRet(doPhysWrite(pData, pData->poBuf->fd, (char*)pData->poBuf->pszBuf, pData->poBuf->iBuf)); - } - -finalize_it: - RETiRet; -} #endif @@ -832,30 +710,12 @@ doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) ASSERT(pData != NULL); ASSERT(pszBuf != NULL); -dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", -pData->pStrm, lenBuf); - -RUNLOG_VAR("%p", pData->pStrm); +dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf); if(pData->pStrm != NULL){ CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf)); FINALIZE; // TODO: clean up later } -#if 0 - if(pData->fd != poBuf->fd) { - // TODO: more efficient use for dynafiles - CHKiRet(doFlush(pData)); - poBuf->fd = pData->fd; - } - - for(i = 0 ; i < lenBuf ; ++i) { - poBuf->pszBuf[poBuf->iBuf++] = pszBuf[i]; - if(poBuf->iBuf == poBuf->lenBuf) { - CHKiRet(doFlush(pData)); - } - } -#endif - finalize_it: RETiRet; } @@ -883,7 +743,6 @@ writeFile(uchar **ppString, unsigned iMsgOpts, instanceData *pData) } } - /* create the message based on format specified */ CHKiRet(doWrite(pData, ppString[0], strlen(CHAR_CONVERT(ppString[0])))); finalize_it: @@ -898,20 +757,15 @@ finalize_it: BEGINcreateInstance CODESTARTcreateInstance pData->pStrm = NULL; - //CHKmalloc(pData->poBuf = calloc(1, sizeof(outbuf_t))); - //pData->poBuf->lenBuf = OUTBUF_LEN; -finalize_it: ENDcreateInstance BEGINfreeInstance CODESTARTfreeInstance - //DEL! doFlush(pData); /* flush anything that is pending, TODO: change when enhancing dynafile handling! */ if(pData->bDynamicName) { dynaFileFreeCache(pData); } else if(pData->pStrm != NULL) strm.Destruct(&pData->pStrm); - //free(pData->poBuf); ENDfreeInstance @@ -969,14 +823,9 @@ CODESTARTparseSelectorAct pData->bDynamicName = 1; pData->iCurrElt = -1; /* no current element */ - /* we now allocate the cache table. We use calloc() intentionally, as we - * need all pointers to be initialized to NULL pointers. - */ - if((pData->dynCache = (dynaFileCacheEntry**) - calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))) == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - DBGPRINTF("Could not allocate memory for dynaFileCache - selector disabled.\n"); - } + /* we now allocate the cache table */ + CHKmalloc(pData->dynCache = (dynaFileCacheEntry**) + calloc(iDynaFileCacheSize, sizeof(dynaFileCacheEntry*))); break; case '|': @@ -1022,9 +871,6 @@ CODESTARTparseSelectorAct errmsg.LogError(0, RS_RET_NO_FILE_ACCESS, "Could no open output file '%s'", pData->f_fname); } } - -dbgprintf("in init static IOBufSize %d, CONST %d\n", iIOBufSize, IOBUF_DFLT_SIZE); -RUNLOG_VAR("%d", pData->iIOBufSize); CODE_STD_FINALIZERparseSelectorAct ENDparseSelectorAct -- cgit v1.2.3 From 8d1e2e496c6a4a4d40d1e8604c746e0d32013536 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 8 Jun 2009 18:39:06 +0200 Subject: improved error handling and added fsync() support ... restoring missing functionality after the restructuring of imfile. As a side-effect, this also lays the foundation for even more reliable queue engine operations (but this is not yet done). --- runtime/stream.c | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- runtime/stream.h | 5 ++- tools/omfile.c | 4 +- 3 files changed, 113 insertions(+), 7 deletions(-) diff --git a/runtime/stream.c b/runtime/stream.c index a4844d2b..8e2f87dc 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -150,6 +150,12 @@ static rsRetVal strmCloseFile(strm_t *pThis) close(pThis->fd); // TODO: error check pThis->fd = -1; + if(pThis->fdDir != -1) { + /* close associated directory handle, if it is open */ + close(pThis->fdDir); + pThis->fdDir = -1; + } + if(pThis->bDeleteOnClose) { unlink((char*) pThis->pszCurrFName); // TODO: check returncode } @@ -395,10 +401,11 @@ finalize_it: BEGINobjConstruct(strm) /* be sure to specify the object type also in END macro! */ pThis->iCurrFNum = 1; pThis->fd = -1; + pThis->fdDir = -1; pThis->iUngetC = -1; pThis->sType = STREAMTYPE_FILE_SINGLE; pThis->sIOBufSize = glblGetIOBufSize(); - pThis->tOpenMode = 0600; /* TODO: make configurable */ + pThis->tOpenMode = 0600; ENDobjConstruct(strm) @@ -428,6 +435,19 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) } } + /* if we are aset to sync, we must obtain a file handle to the directory for fsync() purposes */ + if(pThis->bSync) { + pThis->fdDir = open((char*)pThis->pszDir, O_RDONLY); + if(pThis->fdDir == -1) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + // TODO: log an error message? think so... + DBGPRINTF("error %d opening directory file for fsync() use - fsync for directory disabled: %s\n", + errno, errStr); + } + } + finalize_it: RETiRet; } @@ -478,6 +498,81 @@ finalize_it: } + +/* issue write() api calls until either the buffer is completely + * written or an error occured (it may happen that multiple writes + * are required, what is perfectly legal. On exit, *pLenBuf contains + * the number of bytes actually written. + * rgerhards, 2009-06-08 + */ +static rsRetVal +doWriteCall(int fd, uchar *pBuf, size_t *pLenBuf) +{ + ssize_t lenBuf; + ssize_t iTotalWritten; + ssize_t iWritten; + char *pWriteBuf; + DEFiRet; + + lenBuf = *pLenBuf; + pWriteBuf = (char*) pBuf; + iTotalWritten = 0; + do { + iWritten = write(fd, pWriteBuf, lenBuf); + if(iWritten < 0) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("log file (%d) write error %d: %s\n", fd, err, errStr); + if(err == EINTR) { + /*NO ERROR, just continue */; + } else { + ABORT_FINALIZE(RS_RET_ERR); + // TODO: cover more error cases! + } + } + /* advance buffer to next write position */ + iTotalWritten += iWritten; + lenBuf -= iWritten; + pWriteBuf += iWritten; + } while(lenBuf > 0); /* Warning: do..while()! */ + +finalize_it: + *pLenBuf -= iTotalWritten; + RETiRet; +} + + +/* sync the file to disk, so that any unwritten data is persisted. This + * also syncs the directory and thus makes sure that the file survives + * fatal failure. -- rgerhards, 2009-06-08 + */ +static rsRetVal +syncFile(strm_t *pThis) +{ + int ret; + DEFiRet; + + DBGPRINTF("syncing file %d\n", pThis->fd); + ret = fdatasync(pThis->fd); + if(ret != 0) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("sync failed for file %d with error (%d): %s - ignoring\n", + pThis->fd, err, errStr); + } + // TODO: check error! + + if(pThis->fdDir != -1) { + ret = fsync(pThis->fdDir); +dbgprintf("sync on dir (fd %d) requested, return code %d\n", pThis->fdDir, ret); + } + + RETiRet; +} + + /* physically write to the output file. the provided data is ready for * writing (e.g. zipped if we are requested to do that). * rgerhards, 2009-06-04 @@ -485,17 +580,17 @@ finalize_it: static rsRetVal strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { + size_t iWritten; DEFiRet; - int iWritten; ASSERT(pThis != NULL); if(pThis->fd == -1) CHKiRet(strmOpenFile(pThis)); - iWritten = write(pThis->fd, pBuf, lenBuf); - dbgoprint((obj_t*) pThis, "file %d write wrote %d bytes\n", pThis->fd, iWritten); - /* TODO: handle error case -- rgerhards, 2008-01-07 */ + iWritten = lenBuf; + CHKiRet(doWriteCall(pThis->fd, pBuf, &iWritten)); + dbgoprint((obj_t*) pThis, "file %d write wrote %d bytes\n", pThis->fd, (int) iWritten); /* Now indicate buffer empty again. We do this in any case, because there * is no way we could react more intelligently to an error during write. @@ -511,6 +606,10 @@ strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) if(pThis->pUsrWCntr != NULL) *pThis->pUsrWCntr += iWritten; + if(pThis->bSync) { + CHKiRet(syncFile(pThis)); + } + if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) CHKiRet(strmCheckNextOutputFile(pThis)); @@ -759,6 +858,7 @@ DEFpropSetMeth(strm, tOperationsMode, int) DEFpropSetMeth(strm, tOpenMode, mode_t) DEFpropSetMeth(strm, sType, strmType_t) DEFpropSetMeth(strm, iZipLevel, int) +DEFpropSetMeth(strm, bSync, int) DEFpropSetMeth(strm, sIOBufSize, size_t) static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) @@ -1040,6 +1140,7 @@ CODESTARTobjQueryInterface(strm) pIf->SettOpenMode = strmSettOpenMode; pIf->SetsType = strmSetsType; pIf->SetiZipLevel = strmSetiZipLevel; + pIf->SetbSync = strmSetbSync; pIf->SetsIOBufSize = strmSetsIOBufSize; finalize_it: ENDobjQueryInterface(strm) diff --git a/runtime/stream.h b/runtime/stream.h index 7eb386fb..a66108b7 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -103,10 +103,12 @@ typedef struct strm_s { int64 iCurrOffs;/* current offset */ int64 *pUsrWCntr; /* NULL or a user-provided counter that receives the nbr of bytes written since the last CntrSet() */ /* dynamic properties, valid only during file open, not to be persistet */ - size_t sIOBufSize;/* size of IO buffer */ + int bSync; /* sync this file after every write? */ + size_t sIOBufSize;/* size of IO buffer */ uchar *pszDir; /* Directory */ int lenDir; int fd; /* the file descriptor, -1 if closed */ + int fdDir; /* the directory's descriptor, in case bSync is requested (-1 if closed) */ uchar *pszCurrFName; /* name of current file (if open) */ uchar *pIOBuf; /* io Buffer */ size_t iBufPtrMax; /* current max Ptr in Buffer (if partial read!) */ @@ -148,6 +150,7 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, tOpenMode, mode_t); INTERFACEpropSetMeth(strm, sType, strmType_t); INTERFACEpropSetMeth(strm, iZipLevel, int); + INTERFACEpropSetMeth(strm, bSync, int); INTERFACEpropSetMeth(strm, sIOBufSize, size_t); ENDinterface(strm) #define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/tools/omfile.c b/tools/omfile.c index f196d602..e3f7af19 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -116,7 +116,7 @@ typedef struct _instanceData { eTypePIPE } fileType; char bDynamicName; /* 0 - static name, 1 - dynamic name (with properties) */ - int fCreateMode; /* file creation mode for open() */ // TODO: shuffle down to stream class + int fCreateMode; /* file creation mode for open() */ int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ // TODO: stream class? RE-IMPLEMENT! @@ -472,6 +472,8 @@ prepareFile(instanceData *pData, uchar *newFileName) CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); + CHKiRet(strm.SettOpenMode(pData->pStrm, fCreateMode)); + CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.ConstructFinalize(pData->pStrm)); -- cgit v1.2.3 From 9704f129f72ec9ece11aeccea4bbf0cbccb116cb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 9 Jun 2009 19:00:18 +0200 Subject: added capability to fsync() queue disk files for enhanced reliability also adds speed, because you do no longer need to run the whole file system in sync mode. New testbench and new config directives: - $MainMsgQueueSyncQueueFiles - $ActionQueueSyncQueueFiles --- ChangeLog | 5 +++++ action.c | 4 ++++ doc/queues.html | 6 +++++- runtime/queue.c | 5 +++++ runtime/queue.h | 2 ++ tests/Makefile.am | 3 +++ tests/diskqueue-fsync.sh | 15 +++++++++++++++ tests/testsuites/diskqueue-fsync.conf | 17 +++++++++++++++++ tools/syslogd.c | 4 ++++ 9 files changed, 60 insertions(+), 1 deletion(-) create mode 100755 tests/diskqueue-fsync.sh create mode 100644 tests/testsuites/diskqueue-fsync.conf diff --git a/ChangeLog b/ChangeLog index 8fc4194d..57bb6e7e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -12,9 +12,14 @@ Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? - reduced max value for $DynaFileCacheSize to 1,000 (the former maximum of 10,000 really made no sense, even 1,000 is very high, but we like to keep the user in control ;)). +- added capability to fsync() queue disk files for enhanced reliability + (also add's speed, because you do no longer need to run the whole file + system in sync mode) - added configuration commands (see doc for explanations) * $OMFileZipLevel * $OMFileIOBufferSize + * $MainMsgQueueSyncQueueFiles + * $ActionQueueSyncQueueFiles --------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) diff --git a/action.c b/action.c index 51620fce..8bdb6dec 100644 --- a/action.c +++ b/action.c @@ -72,6 +72,7 @@ static int iActionQueueNumWorkers = 1; /* number of worker threads for the mm static uchar *pszActionQFName = NULL; /* prefix for the main message queue file */ static int64 iActionQueMaxFileSize = 1024*1024; static int iActionQPersistUpdCnt = 0; /* persist queue info every n updates */ +static int bActionQSyncQeueFiles = 0; /* sync queue files */ static int iActionQtoQShutdown = 0; /* queue shutdown */ static int iActionQtoActShutdown = 1000; /* action shutdown (in phase 2) */ static int iActionQtoEnq = 2000; /* timeout for queue enque */ @@ -151,6 +152,7 @@ actionResetQueueParams(void) iActionQueueNumWorkers = 1; /* number of worker threads for the mm queue above */ iActionQueMaxFileSize = 1024*1024; iActionQPersistUpdCnt = 0; /* persist queue info every n updates */ + bActionQSyncQeueFiles = 0; iActionQtoQShutdown = 0; /* queue shutdown */ iActionQtoActShutdown = 1000; /* action shutdown (in phase 2) */ iActionQtoEnq = 2000; /* timeout for queue enque */ @@ -273,6 +275,7 @@ actionConstructFinalize(action_t *pThis) setQPROP(qqueueSetMaxFileSize, "$ActionQueueFileSize", iActionQueMaxFileSize); setQPROPstr(qqueueSetFilePrefix, "$ActionQueueFileName", pszActionQFName); setQPROP(qqueueSetiPersistUpdCnt, "$ActionQueueCheckpointInterval", iActionQPersistUpdCnt); + setQPROP(qqueueSetbSyncQueueFiles, "$ActionQueueSyncQueueFiles", bActionQSyncQeueFiles); setQPROP(qqueueSettoQShutdown, "$ActionQueueTimeoutShutdown", iActionQtoQShutdown ); setQPROP(qqueueSettoActShutdown, "$ActionQueueTimeoutActionCompletion", iActionQtoActShutdown); setQPROP(qqueueSettoWrkShutdown, "$ActionQueueWorkerTimeoutThreadShutdown", iActionQtoWrkShutdown); @@ -838,6 +841,7 @@ actionAddCfSysLineHdrl(void) CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuediscardmark", 0, eCmdHdlrInt, NULL, &iActionQDiscardMark, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuediscardseverity", 0, eCmdHdlrInt, NULL, &iActionQDiscardSeverity, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuecheckpointinterval", 0, eCmdHdlrInt, NULL, &iActionQPersistUpdCnt, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuesyncqueuefiles", 0, eCmdHdlrBinary, NULL, &bActionQSyncQeueFiles, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetype", 0, eCmdHdlrGetWord, setActionQueType, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueueworkerthreads", 0, eCmdHdlrInt, NULL, &iActionQueueNumWorkers, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"actionqueuetimeoutshutdown", 0, eCmdHdlrInt, NULL, &iActionQtoQShutdown, NULL)); diff --git a/doc/queues.html b/doc/queues.html index 4a9509a0..45ce1bd1 100644 --- a/doc/queues.html +++ b/doc/queues.html @@ -115,7 +115,11 @@ isolation. This is currently selected by specifying different $WorkDirectory< config directives before the queue creation statement.

      To create a disk queue, use the "$<object>QueueType Disk" config directive. Checkpoint intervals can be specified via "$<object>QueueCheckpointInterval", -with 0 meaning no checkpoints.

      +with 0 meaning no checkpoints. Note that disk-based queues can be made very reliable +by issuing a (f)sync after each write operation. Starting with version 4.3.2, this can +be requested via "<object>QueueSyncQueueFiles on/off with the +default being off. Activating this option has a performance penalty, so it should +not be turned on without reason.

      In-Memory Queues

      In-memory queue mode is what most people have on their mind when they think about computing queues. Here, the enqueued data elements are held in memory. diff --git a/runtime/queue.c b/runtime/queue.c index 3532a145..aa8e6c21 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -294,6 +294,7 @@ qqueueStartDA(qqueue_t *pThis) CHKiRet(qqueueSetMaxFileSize(pThis->pqDA, pThis->iMaxFileSize)); CHKiRet(qqueueSetFilePrefix(pThis->pqDA, pThis->pszFilePrefix, pThis->lenFilePrefix)); CHKiRet(qqueueSetiPersistUpdCnt(pThis->pqDA, pThis->iPersistUpdCnt)); + CHKiRet(qqueueSetbSyncQueueFiles(pThis->pqDA, pThis->bSyncQueueFiles)); CHKiRet(qqueueSettoActShutdown(pThis->pqDA, pThis->toActShutdown)); CHKiRet(qqueueSettoEnq(pThis->pqDA, pThis->toEnq)); CHKiRet(qqueueSetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED)); @@ -817,6 +818,7 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) ; } else { CHKiRet(strm.Construct(&pThis->tVars.disk.pWrite)); + CHKiRet(strm.SetbSync(pThis->tVars.disk.pWrite, pThis->bSyncQueueFiles)); CHKiRet(strm.SetDir(pThis->tVars.disk.pWrite, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pWrite, 10000000)); CHKiRet(strm.SettOperationsMode(pThis->tVars.disk.pWrite, STREAMMODE_WRITE)); @@ -824,6 +826,7 @@ static rsRetVal qConstructDisk(qqueue_t *pThis) CHKiRet(strm.ConstructFinalize(pThis->tVars.disk.pWrite)); CHKiRet(strm.Construct(&pThis->tVars.disk.pRead)); + CHKiRet(strm.SetbSync(pThis->tVars.disk.pRead, pThis->bSyncQueueFiles)); CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pRead, 1)); CHKiRet(strm.SetDir(pThis->tVars.disk.pRead, glbl.GetWorkDir(), strlen((char*)glbl.GetWorkDir()))); CHKiRet(strm.SetiMaxFiles(pThis->tVars.disk.pRead, 10000000)); @@ -1924,6 +1927,7 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) CHKiRet(strm.Construct(&psQIF)); CHKiRet(strm.SettOperationsMode(psQIF, STREAMMODE_WRITE_TRUNC)); + CHKiRet(strm.SetbSync(psQIF, pThis->bSyncQueueFiles)); CHKiRet(strm.SetsType(psQIF, STREAMTYPE_FILE_SINGLE)); CHKiRet(strm.SetFName(psQIF, pszQIFNam, lenQIFNam)); CHKiRet(strm.ConstructFinalize(psQIF)); @@ -2279,6 +2283,7 @@ finalize_it: /* some simple object access methods */ +DEFpropSetMeth(qqueue, bSyncQueueFiles, int) DEFpropSetMeth(qqueue, iPersistUpdCnt, int) DEFpropSetMeth(qqueue, iDeqtWinFromHr, int) DEFpropSetMeth(qqueue, iDeqtWinToHr, int) diff --git a/runtime/queue.h b/runtime/queue.h index a267862d..07f134aa 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -73,6 +73,7 @@ typedef struct queue_s { void *pUsr; /* a global, user-supplied pointer. Is passed back to consumer. */ int iUpdsSincePersist;/* nbr of queue updates since the last persist call */ int iPersistUpdCnt; /* persits queue info after this nbr of updates - 0 -> persist only on shutdown */ + int bSyncQueueFiles;/* if working with files, sync them after each write? */ int iHighWtrMrk; /* high water mark for disk-assisted memory queues */ int iLowWtrMrk; /* low water mark for disk-assisted memory queues */ int iDiscardMrk; /* if the queue is above this mark, low-severity messages are discarded */ @@ -186,6 +187,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread int iMaxQueueSize, rsRetVal (*pConsumer)(void*,void*)); PROTOTYPEObjClassInit(qqueue); PROTOTYPEpropSetMeth(qqueue, iPersistUpdCnt, int); +PROTOTYPEpropSetMeth(qqueue, bSyncQueueFiles, int); PROTOTYPEpropSetMeth(qqueue, iDeqtWinFromHr, int); PROTOTYPEpropSetMeth(qqueue, iDeqtWinToHr, int); PROTOTYPEpropSetMeth(qqueue, toQShutdown, long); diff --git a/tests/Makefile.am b/tests/Makefile.am index a95139f2..3d555fc3 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -5,6 +5,7 @@ TESTS = $(TESTRUNS) cfg.sh \ validation-run.sh \ imtcp-multiport.sh \ diskqueue.sh \ + diskqueue-fsync.sh \ manytcp.sh \ queue-persist.sh @@ -59,6 +60,8 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ fieldtest.sh \ diskqueue.sh \ testsuites/diskqueue.conf \ + diskqueue-fsync.sh \ + testsuites/diskqueue-fsync.conf \ imtcp-multiport.sh \ testsuites/imtcp-multiport.conf \ manytcp.sh \ diff --git a/tests/diskqueue-fsync.sh b/tests/diskqueue-fsync.sh new file mode 100755 index 00000000..f558b491 --- /dev/null +++ b/tests/diskqueue-fsync.sh @@ -0,0 +1,15 @@ +# Test for disk-only queue mode (with fsync for queue files) +# This test checks if queue files can be correctly written +# and read back, but it does not test the transition from +# memory to disk mode for DA queues. +# added 2009-06-09 by Rgerhards +# This file is part of the rsyslog project, released under GPLv3 +# uncomment for debugging support: +echo testing queue disk-only mode, fsync case +source $srcdir/diag.sh init +source $srcdir/diag.sh startup diskqueue-fsync.conf +# 5000 messages should be enough - the disk fsync test is very slow! +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 5000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh exit diff --git a/tests/testsuites/diskqueue-fsync.conf b/tests/testsuites/diskqueue-fsync.conf new file mode 100644 index 00000000..0a02c6ce --- /dev/null +++ b/tests/testsuites/diskqueue-fsync.conf @@ -0,0 +1,17 @@ +# Test for queue disk mode (see .sh file for details) +# rgerhards, 2009-04-17 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$InputTCPServerRun 13514 + +# set spool locations and switch queue to disk-only mode +$WorkDirectory test-spool +$MainMsgQueueSyncQueueFiles on +$MainMsgQueueTimeoutShutdown 10000 +$MainMsgQueueFilename mainq +$MainMsgQueueType disk + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tools/syslogd.c b/tools/syslogd.c index 206b79e8..19ece349 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -297,6 +297,7 @@ static queueType_t MainMsgQueType = QUEUETYPE_FIXED_ARRAY; /* type of the main m static uchar *pszMainMsgQFName = NULL; /* prefix for the main message queue file */ static int64 iMainMsgQueMaxFileSize = 1024*1024; static int iMainMsgQPersistUpdCnt = 0; /* persist queue info every n updates */ +static int bMainMsgQSyncQeueFiles = 0; /* sync queue files on every write? */ static int iMainMsgQtoQShutdown = 0; /* queue shutdown */ static int iMainMsgQtoActShutdown = 1000; /* action shutdown (in phase 2) */ static int iMainMsgQtoEnq = 2000; /* timeout for queue enque */ @@ -359,6 +360,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iMainMsgQueMaxFileSize = 1024 * 1024; iMainMsgQueueNumWorkers = 1; iMainMsgQPersistUpdCnt = 0; + bMainMsgQSyncQeueFiles = 0; iMainMsgQtoQShutdown = 0; iMainMsgQtoActShutdown = 1000; iMainMsgQtoEnq = 2000; @@ -2715,6 +2717,7 @@ init(void) setQPROP(qqueueSetsizeOnDiskMax, "$MainMsgQueueMaxDiskSpace", iMainMsgQueMaxDiskSpace); setQPROPstr(qqueueSetFilePrefix, "$MainMsgQueueFileName", pszMainMsgQFName); setQPROP(qqueueSetiPersistUpdCnt, "$MainMsgQueueCheckpointInterval", iMainMsgQPersistUpdCnt); + setQPROP(qqueueSetbSyncQueueFiles, "$MainMsgQueueSyncQueueFiles", bMainMsgQSyncQeueFiles); setQPROP(qqueueSettoQShutdown, "$MainMsgQueueTimeoutShutdown", iMainMsgQtoQShutdown ); setQPROP(qqueueSettoActShutdown, "$MainMsgQueueTimeoutActionCompletion", iMainMsgQtoActShutdown); setQPROP(qqueueSettoWrkShutdown, "$MainMsgQueueWorkerTimeoutThreadShutdown", iMainMsgQtoWrkShutdown); @@ -3084,6 +3087,7 @@ static rsRetVal loadBuildInModules(void) CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuediscardmark", 0, eCmdHdlrInt, NULL, &iMainMsgQDiscardMark, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuediscardseverity", 0, eCmdHdlrSeverity, NULL, &iMainMsgQDiscardSeverity, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuecheckpointinterval", 0, eCmdHdlrInt, NULL, &iMainMsgQPersistUpdCnt, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesyncqueuefiles", 0, eCmdHdlrBinary, NULL, &bMainMsgQSyncQeueFiles, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuetype", 0, eCmdHdlrGetWord, setMainMsgQueType, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueueworkerthreads", 0, eCmdHdlrInt, NULL, &iMainMsgQueueNumWorkers, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuetimeoutshutdown", 0, eCmdHdlrInt, NULL, &iMainMsgQtoQShutdown, NULL)); -- cgit v1.2.3 From 6f4e3c4e4c85acdcf58969970484a54639ecc8f9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jun 2009 16:49:14 +0200 Subject: restructered code in perparation for multiple rule set support ... this was long overdue, and I finlly tackeld it. It turned out to be more complex than I initially thought. The next step now probably is to actually implement multiple rule sets and the beauty that comes with them. --- dirty.h | 2 + runtime/Makefile.am | 4 + runtime/conf.c | 102 +++++---- runtime/conf.h | 2 +- runtime/linkedlist.c | 2 +- runtime/obj-types.h | 5 +- runtime/obj.h | 2 +- runtime/rsyslog.c | 8 + runtime/rsyslog.h | 19 +- runtime/rule.c | 431 +++++++++++++++++++++++++++++++++++++ runtime/rule.h | 74 +++++++ runtime/ruleset.c | 369 ++++++++++++++++++++++++++++++++ runtime/ruleset.h | 56 +++++ runtime/stream.c | 3 - tests/diskqueue-fsync.sh | 2 + tests/ourtail.c | 2 + tests/runtime-dummy.c | 4 + tests/validation-run.sh | 7 +- tools/syslogd.c | 537 +++++------------------------------------------ tools/syslogd.h | 56 +---- 20 files changed, 1093 insertions(+), 594 deletions(-) create mode 100644 runtime/rule.c create mode 100644 runtime/rule.h create mode 100644 runtime/ruleset.c create mode 100644 runtime/ruleset.h diff --git a/dirty.h b/dirty.h index 6d585753..bab15485 100644 --- a/dirty.h +++ b/dirty.h @@ -33,6 +33,7 @@ rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len int parseRFCSyslogMsg(msg_t *pMsg, int flags); int parseLegacySyslogMsg(msg_t *pMsg, int flags); rsRetVal diagGetMainMsgQSize(int *piSize); /* for imdiag */ +char* getFIOPName(unsigned iFIOP); /* TODO: the following 2 need to go in conf obj interface... */ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); @@ -56,6 +57,7 @@ extern int bReduceRepeatMsgs; extern int bDropTrailingLF; extern uchar cCCEscapeChar; extern int bEscapeCCOnRcv; +extern ruleset_t *pCurrRuleset; #ifdef USE_NETZIP /* config param: minimum message size to try compression. The smaller * the message, the less likely is any compression gain. We check for diff --git a/runtime/Makefile.am b/runtime/Makefile.am index 81422d64..de0daac4 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -67,6 +67,10 @@ librsyslog_la_SOURCES = \ vmop.h \ queue.c \ queue.h \ + ruleset.c \ + ruleset.h \ + rule.c \ + rule.h \ cfsysline.c \ cfsysline.h \ \ diff --git a/runtime/conf.c b/runtime/conf.c index 7cdcf5ec..e0ed3d6a 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -69,13 +69,15 @@ #include "expr.h" #include "ctok.h" #include "ctok_token.h" +#include "rule.h" +#include "ruleset.h" #ifdef OS_SOLARIS # define NAME_MAX MAXNAMELEN #endif /* forward definitions */ -static rsRetVal cfline(uchar *line, selector_t **pfCurr); +static rsRetVal cfline(uchar *line, rule_t **pfCurr); static rsRetVal processConfFile(uchar *pConfFile); @@ -87,6 +89,8 @@ DEFobjCurrIf(ctok_token) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) DEFobjCurrIf(net) +DEFobjCurrIf(rule) +DEFobjCurrIf(ruleset) static int iNbrActions; /* number of actions the running config has. Needs to be init on ReInitConf() */ @@ -395,7 +399,7 @@ processConfFile(uchar *pConfFile) DEFiRet; int iLnNbr = 0; FILE *cf; - selector_t *fCurr = NULL; + rule_t *pCurrRule = NULL; uchar *p; uchar cbuf[CFGLNSIZ]; uchar *cline; @@ -429,7 +433,6 @@ processConfFile(uchar *pConfFile) * TODO: review the code at whole - this is highly suspect (but will go away * once we do the rest of RainerScript). */ - /* was: strcpy((char*)cline, (char*)p); */ for( i = 0 ; p[i] != '\0' ; ++i) { cline[i] = p[i]; } @@ -453,7 +456,7 @@ processConfFile(uchar *pConfFile) /* we now have the complete line, and are positioned at the first non-whitespace * character. So let's process it */ - if(cfline(cbuf, &fCurr) != RS_RET_OK) { + if(cfline(cbuf, &pCurrRule) != RS_RET_OK) { /* we log a message, but otherwise ignore the error. After all, the next * line can be correct. -- rgerhards, 2007-08-02 */ @@ -467,16 +470,18 @@ processConfFile(uchar *pConfFile) } /* we probably have one selector left to be added - so let's do that now */ - CHKiRet(selectorAddList(fCurr)); + if(pCurrRule != NULL) { + CHKiRet(ruleset.AddRule(pCurrRuleset, &pCurrRule)); + } /* close the configuration file */ - (void) fclose(cf); + fclose(cf); finalize_it: if(iRet != RS_RET_OK) { char errStr[1024]; - if(fCurr != NULL) - selectorDestruct(fCurr); + if(pCurrRule != NULL) + rule.Destruct(&pCurrRule); rs_strerror_r(errno, errStr, sizeof(errStr)); dbgprintf("error %d processing config file '%s'; os error (if any): %s\n", @@ -588,7 +593,7 @@ cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int * rgerhards 2005-09-15 */ /* GPLv3 - stems back to sysklogd */ -static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f) +static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register rule_t *pRule) { uchar *p; register uchar *q; @@ -603,17 +608,17 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f ASSERT(pline != NULL); ASSERT(*pline != NULL); - ASSERT(f != NULL); + ISOBJ_TYPE_assert(pRule, rule); dbgprintf(" - traditional PRI filter\n"); errno = 0; /* keep strerror_r() stuff out of logerror messages */ - f->f_filter_type = FILTER_PRI; + pRule->f_filter_type = FILTER_PRI; /* Note: file structure is pre-initialized to zero because it was * created with calloc()! */ for (i = 0; i <= LOG_NFACILITIES; i++) { - f->f_filterData.f_pmask[i] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; } /* scan through the list of selectors */ @@ -668,32 +673,32 @@ static rsRetVal cflineProcessTradPRIFilter(uchar **pline, register selector_t *f for (i = 0; i <= LOG_NFACILITIES; i++) { if ( pri == INTERNAL_NOPRI ) { if ( ignorepri ) - f->f_filterData.f_pmask[i] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; else - f->f_filterData.f_pmask[i] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; } else if ( singlpri ) { if ( ignorepri ) - f->f_filterData.f_pmask[i] &= ~(1<f_filterData.f_pmask[i] &= ~(1<f_filterData.f_pmask[i] |= (1<f_filterData.f_pmask[i] |= (1<f_filterData.f_pmask[i] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i] = TABLE_NOPRI; else - f->f_filterData.f_pmask[i] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i] = TABLE_ALLPRI; } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) - f->f_filterData.f_pmask[i] &= ~(1<f_filterData.f_pmask[i] &= ~(1<f_filterData.f_pmask[i] |= (1<f_filterData.f_pmask[i] |= (1<f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; else - f->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; } else if ( singlpri ) { if ( ignorepri ) - f->f_filterData.f_pmask[i >> 3] &= ~(1<f_filterData.f_pmask[i >> 3] &= ~(1<f_filterData.f_pmask[i >> 3] |= (1<f_filterData.f_pmask[i >> 3] |= (1<f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_NOPRI; else - f->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; + pRule->f_filterData.f_pmask[i >> 3] = TABLE_ALLPRI; } else { if ( ignorepri ) for (i2= 0; i2 <= pri; ++i2) - f->f_filterData.f_pmask[i >> 3] &= ~(1<f_filterData.f_pmask[i >> 3] &= ~(1<f_filterData.f_pmask[i >> 3] |= (1<f_filterData.f_pmask[i >> 3] |= (1<f_filter_type = FILTER_EXPR; /* if we come to over here, pline starts with "if ". We just skip that part. */ @@ -823,7 +827,7 @@ finalize_it: * of the action part. A pointer to that beginnig is passed back to the caller. * rgerhards 2005-09-15 */ -static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) +static rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) { rsParsObj *pPars; cstr_t *pCSCompOp; @@ -1014,10 +1018,10 @@ static rsRetVal cflineProcessTagSelector(uchar **pline) /* read the filter part of a configuration line and store the filter - * in the supplied selector_t + * in the supplied rule_t * rgerhards, 2007-08-01 */ -static rsRetVal cflineDoFilter(uchar **pp, selector_t *f) +static rsRetVal cflineDoFilter(uchar **pp, rule_t *f) { DEFiRet; @@ -1110,17 +1114,15 @@ static rsRetVal cflineDoAction(uchar **p, action_t **ppAction) /* Process a configuration file line in traditional "filter selector" format - * or one that builds upon this format. + * or one that builds upon this format. Note that ppRule may be a NULL pointer, + * which is valid and happens if there is no previous line (right at the start + * of the master config file!). */ -static rsRetVal cflineClassic(uchar *p, selector_t **pfCurr) +static rsRetVal +cflineClassic(uchar *p, rule_t **ppRule) { DEFiRet; action_t *pAction; - selector_t *fCurr; - - ASSERT(pfCurr != NULL); - - fCurr = *pfCurr; /* lines starting with '&' have no new filters and just add * new actions to the currently processed selector. @@ -1138,16 +1140,18 @@ static rsRetVal cflineClassic(uchar *p, selector_t **pfCurr) * selector is NULL, which means we do not need to care about it at * all. -- rgerhards, 2007-08-01 */ - CHKiRet(selectorAddList(fCurr)); - CHKiRet(selectorConstruct(&fCurr)); /* create "fresh" selector */ - CHKiRet(cflineDoFilter(&p, fCurr)); /* pull filters */ + if(*ppRule != NULL) { + CHKiRet(ruleset.AddRule(pCurrRuleset, ppRule)); + } + CHKiRet(rule.Construct(ppRule)); /* create "fresh" selector */ + CHKiRet(rule.ConstructFinalize(*ppRule)); /* create "fresh" selector */ + CHKiRet(cflineDoFilter(&p, *ppRule)); /* pull filters */ } CHKiRet(cflineDoAction(&p, &pAction)); - CHKiRet(llAppend(&fCurr->llActList, NULL, (void*) pAction)); + CHKiRet(llAppend(&(*ppRule)->llActList, NULL, (void*) pAction)); finalize_it: - *pfCurr = fCurr; RETiRet; } @@ -1157,11 +1161,12 @@ finalize_it: * rgerhards, 2007-08-01 */ static rsRetVal -cfline(uchar *line, selector_t **pfCurr) +cfline(uchar *line, rule_t **pfCurr) { DEFiRet; ASSERT(line != NULL); +if(*pfCurr != NULL){ ISOBJ_TYPE_assert(*pfCurr, rule);} dbgprintf("cfline: '%s'\n", line); @@ -1180,6 +1185,7 @@ cfline(uchar *line, selector_t **pfCurr) break; default: iRet = cflineClassic(line, pfCurr); +ISOBJ_TYPE_assert(*pfCurr, rule); break; } @@ -1254,6 +1260,8 @@ CODESTARTObjClassExit(conf) objRelease(module, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); + objRelease(rule, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); ENDObjClassExit(conf) @@ -1269,6 +1277,8 @@ BEGINAbstractObjClassInit(conf, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANG CHKiRet(objUse(module, CORE_COMPONENT)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* TODO: make this dependcy go away! */ + CHKiRet(objUse(rule, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); ENDObjClassInit(conf) /* vi:set ai: diff --git a/runtime/conf.h b/runtime/conf.h index 2494d4dc..f29de1ba 100644 --- a/runtime/conf.h +++ b/runtime/conf.h @@ -35,7 +35,7 @@ BEGINinterface(conf) /* name must also be changed in ENDinterface macro! */ rsRetVal (*cfsysline)(uchar *p); rsRetVal (*doModLoad)(uchar **pp, __attribute__((unused)) void* pVal); rsRetVal (*doIncludeLine)(uchar **pp, __attribute__((unused)) void* pVal); - rsRetVal (*cfline)(uchar *line, selector_t **pfCurr); + rsRetVal (*cfline)(uchar *line, rule_t **pfCurr); rsRetVal (*processConfFile)(uchar *pConfFile); rsRetVal (*ReInitConf)(void); rsRetVal (*GetNbrActActions)(int *); diff --git a/runtime/linkedlist.c b/runtime/linkedlist.c index 8f842e43..cc095f6e 100644 --- a/runtime/linkedlist.c +++ b/runtime/linkedlist.c @@ -398,7 +398,7 @@ rsRetVal llExecFunc(linkedList_t *pThis, rsRetVal (*pFunc)(void*, void*), void* */ llCookie = llCookiePrev; } else if (iRet != RS_RET_OK) { - goto finalize_it; + FINALIZE; } llCookiePrev = llCookie; } diff --git a/runtime/obj-types.h b/runtime/obj-types.h index 914c2f2c..23f74590 100644 --- a/runtime/obj-types.h +++ b/runtime/obj-types.h @@ -105,12 +105,13 @@ struct obj_s { /* the dummy struct that each derived class can be casted to */ # define ISOBJ_TYPE_assert(pObj, objType) \ do { \ ASSERT(pObj != NULL); \ - ASSERT((unsigned) ((obj_t*) (pObj))->iObjCooCKiE == (unsigned) 0xBADEFEE); \ if(strcmp((char*)(((obj_t*)pObj)->pObjInfo->pszID), #objType)) { \ dbgprintf("%s:%d ISOBJ assert failure: invalid object type, expected '%s' " \ - "actual '%s'\n", __FILE__, __LINE__, #objType, (((obj_t*)pObj)->pObjInfo->pszID)); \ + "actual '%s', cookie: %X\n", __FILE__, __LINE__, #objType, \ + (((obj_t*)pObj)->pObjInfo->pszID), ((obj_t*)(pObj))->iObjCooCKiE); \ assert(0); /* trigger assertion, messge we already have */ \ } \ + ASSERT((unsigned) ((obj_t*)(pObj))->iObjCooCKiE == (unsigned) 0xBADEFEE); \ } while(0) #else /* non-debug mode, no checks but much faster */ # define BEGINobjInstance obj_t objData diff --git a/runtime/obj.h b/runtime/obj.h index dc04203b..3973a16e 100644 --- a/runtime/obj.h +++ b/runtime/obj.h @@ -68,7 +68,7 @@ #define objSerializePTR(strm, propName, propType) \ CHKiRet(obj.SerializeProp(strm, (uchar*) #propName, PROPTYPE_##propType, (void*) pThis->propName)); #define DEFobjStaticHelpers \ - static objInfo_t *pObjInfoOBJ = NULL; \ + static objInfo_t __attribute__((unused)) *pObjInfoOBJ = NULL; \ DEFobjCurrIf(obj) diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 81550f12..3496bb0d 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -77,6 +77,8 @@ #include "conf.h" #include "glbl.h" #include "errmsg.h" +#include "rule.h" +#include "ruleset.h" /* forward definitions */ static rsRetVal dfltErrLogger(int, uchar *errMsg); @@ -172,6 +174,10 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(ctokClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "expr"; CHKiRet(exprClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "rule"; + CHKiRet(ruleClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "ruleset"; + CHKiRet(rulesetClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "conf"; CHKiRet(confClassInit(NULL)); @@ -204,6 +210,8 @@ rsrtExit(void) /* do actual de-init only if we are the last runtime user */ confClassExit(); glblClassExit(); + rulesetClassExit(); + ruleClassExit(); objClassExit(); /* *THIS* *MUST/SHOULD?* always be the first class initilizer being called (except debug)! */ } diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index f93592b7..b587554e 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -62,7 +62,9 @@ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ typedef struct thrdInfo thrdInfo_t; typedef struct obj_s obj_t; -typedef struct filed selector_t;/* TODO: this so far resides in syslogd.c, think about modularization */ +typedef struct ruleset_s ruleset_t; +typedef struct rule_s rule_t; +//typedef struct filed selector_t;/* TODO: this so far resides in syslogd.c, think about modularization */ typedef struct NetAddr netAddr_t; typedef struct netstrms_s netstrms_t; typedef struct netstrm_s netstrm_t; @@ -112,6 +114,16 @@ typedef enum { eFLOWCTL_FULL_DELAY = 2 /**< delay possible for extended period of time */ } flowControl_t; +/* filter operations */ +typedef enum { + FIOP_NOP = 0, /* do not use - No Operation */ + FIOP_CONTAINS = 1, /* contains string? */ + FIOP_ISEQUAL = 2, /* is (exactly) equal? */ + FIOP_STARTSWITH = 3, /* starts with a string? */ + FIOP_REGEX = 4, /* matches a (BRE) regular expression? */ + FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ +} fiop_t; + /* The error codes below are orginally "borrowed" from * liblogging. As such, we reserve values up to -2999 @@ -367,6 +379,11 @@ typedef enum rsObjectID rsObjID; # define O_CLOEXEC 0 #endif +/* some constants */ +// TODO: do we really need them - if not, delete -- rgerhards, 2009-06-10 +#define IGNORE_ERROR_CODES 1 +#define ABORT_ON_ERROR 0 + /* The following prototype is convenient, even though it may not be the 100% correct place.. -- rgerhards 2008-01-07 */ void dbgprintf(char *, ...) __attribute__((format(printf, 1, 2))); diff --git a/runtime/rule.c b/runtime/rule.c new file mode 100644 index 00000000..fa56dc3b --- /dev/null +++ b/runtime/rule.c @@ -0,0 +1,431 @@ +/* rule.c - rsyslog's rule object + * + * See file comment in rule.c for the overall structure of rule processing. + * + * Module begun 2009-06-10 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include +#include + +#include "rsyslog.h" +#include "obj.h" +#include "action.h" +#include "rule.h" +#include "errmsg.h" +#include "vm.h" +#include "var.h" +#include "srUtils.h" +#include "dirty.h" /* for getFIOPName */ + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(expr) +DEFobjCurrIf(var) +DEFobjCurrIf(vm) + +/* iterate over all actions, this is often needed, for example when HUP processing + * must be done or a shutdown is pending. + */ +static rsRetVal +iterateAllActions(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + return llExecFunc(&pThis->llActList, pFunc, pParam); +} + + + +/* helper to processMsg(), used to call the configured actions. It is + * executed from within llExecFunc() of the action list. + * rgerhards, 2007-08-02 + */ +typedef struct processMsgDoActions_s { + int bPrevWasSuspended; /* was the previous action suspended? */ + msg_t *pMsg; +} processMsgDoActions_t; +DEFFUNC_llExecFunc(processMsgDoActions) +{ + DEFiRet; + rsRetVal iRetMod; /* return value of module - we do not always pass that back */ + action_t *pAction = (action_t*) pData; + processMsgDoActions_t *pDoActData = (processMsgDoActions_t*) pParam; + + assert(pAction != NULL); + + if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { + dbgprintf("not calling action because the previous one is not suspended\n"); + ABORT_FINALIZE(RS_RET_OK); + } + + iRetMod = actionCallAction(pAction, pDoActData->pMsg); + if(iRetMod == RS_RET_DISCARDMSG) { + ABORT_FINALIZE(RS_RET_DISCARDMSG); + } else if(iRetMod == RS_RET_SUSPENDED) { + /* indicate suspension for next module to be called */ + pDoActData->bPrevWasSuspended = 1; + } else { + pDoActData->bPrevWasSuspended = 0; + } + +finalize_it: + RETiRet; +} + + +/* This functions looks at the given message and checks if it matches the + * provided filter condition. + */ +static rsRetVal +shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) +{ + DEFiRet; + unsigned short pbMustBeFreed; + char *pszPropVal; + int bRet = 0; + vm_t *pVM = NULL; + var_t *pResult = NULL; + + ISOBJ_TYPE_assert(pRule, rule); + assert(pMsg != NULL); + + /* we first have a look at the global, BSD-style block filters (for tag + * and host). Only if they match, we evaluate the actual filter. + * rgerhards, 2005-10-18 + */ + if(pRule->eHostnameCmpMode == HN_NO_COMP) { + /* EMPTY BY INTENSION - we check this value first, because + * it is the one most often used, so this saves us time! + */ + } else if(pRule->eHostnameCmpMode == HN_COMP_MATCH) { + if(rsCStrSzStrCmp(pRule->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { + /* not equal, so we are already done... */ + dbgprintf("hostname filter '+%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSHostnameComp), getHOSTNAME(pMsg)); + FINALIZE; + } + } else { /* must be -hostname */ + if(!rsCStrSzStrCmp(pRule->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { + /* not equal, so we are already done... */ + dbgprintf("hostname filter '-%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSHostnameComp), getHOSTNAME(pMsg)); + FINALIZE; + } + } + + if(pRule->pCSProgNameComp != NULL) { + int bInv = 0, bEqv = 0, offset = 0; + if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp)) == '-') { + if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp) + 1) == '-') + offset = 1; + else { + bInv = 1; + offset = 1; + } + } + if(!rsCStrOffsetSzStrCmp(pRule->pCSProgNameComp, offset, (uchar*) getProgramName(pMsg), getProgramNameLen(pMsg))) + bEqv = 1; + + if((!bEqv && !bInv) || (bEqv && bInv)) { + /* not equal or inverted selection, so we are already done... */ + dbgprintf("programname filter '%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp), getProgramName(pMsg)); + FINALIZE; + } + } + + /* done with the BSD-style block filters */ + + if(pRule->f_filter_type == FILTER_PRI) { + /* skip messages that are incorrect priority */ + if ( (pRule->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ + ((pRule->f_filterData.f_pmask[pMsg->iFacility] & (1<iSeverity)) == 0) ) + bRet = 0; + else + bRet = 1; + } else if(pRule->f_filter_type == FILTER_EXPR) { + CHKiRet(vm.Construct(&pVM)); + CHKiRet(vm.ConstructFinalize(pVM)); + CHKiRet(vm.SetMsg(pVM, pMsg)); + CHKiRet(vm.ExecProg(pVM, pRule->f_filterData.f_expr->pVmprg)); + CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); + dbgprintf("result of expression evaluation: %lld\n", pResult->val.num); + /* VM is destructed on function exit */ + bRet = (pResult->val.num) ? 1 : 0; + } else { + assert(pRule->f_filter_type == FILTER_PROP); /* assert() just in case... */ + pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.pCSPropName, &pbMustBeFreed); + + /* Now do the compares (short list currently ;)) */ + switch(pRule->f_filterData.prop.operation ) { + case FIOP_CONTAINS: + if(rsCStrLocateInSzStr(pRule->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1) + bRet = 1; + break; + case FIOP_ISEQUAL: + if(rsCStrSzStrCmp(pRule->f_filterData.prop.pCSCompValue, + (uchar*) pszPropVal, strlen(pszPropVal)) == 0) + bRet = 1; /* process message! */ + break; + case FIOP_STARTSWITH: + if(rsCStrSzStrStartsWithCStr(pRule->f_filterData.prop.pCSCompValue, + (uchar*) pszPropVal, strlen(pszPropVal)) == 0) + bRet = 1; /* process message! */ + break; + case FIOP_REGEX: + if(rsCStrSzStrMatchRegex(pRule->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, 0, &pRule->f_filterData.prop.regex_cache) == RS_RET_OK) + bRet = 1; + break; + case FIOP_EREREGEX: + if(rsCStrSzStrMatchRegex(pRule->f_filterData.prop.pCSCompValue, + (unsigned char*) pszPropVal, 1, &pRule->f_filterData.prop.regex_cache) == RS_RET_OK) + bRet = 1; + break; + default: + /* here, it handles NOP (for performance reasons) */ + assert(pRule->f_filterData.prop.operation == FIOP_NOP); + bRet = 1; /* as good as any other default ;) */ + break; + } + + /* now check if the value must be negated */ + if(pRule->f_filterData.prop.isNegated) + bRet = (bRet == 1) ? 0 : 1; + + if(Debug) { + dbgprintf("Filter: check for property '%s' (value '%s') ", + rsCStrGetSzStrNoNULL(pRule->f_filterData.prop.pCSPropName), + pszPropVal); + if(pRule->f_filterData.prop.isNegated) + dbgprintf("NOT "); + dbgprintf("%s '%s': %s\n", + getFIOPName(pRule->f_filterData.prop.operation), + rsCStrGetSzStrNoNULL(pRule->f_filterData.prop.pCSCompValue), + bRet ? "TRUE" : "FALSE"); + } + + /* cleanup */ + if(pbMustBeFreed) + free(pszPropVal); + } + +finalize_it: + /* destruct in any case, not just on error, but it makes error handling much easier */ + if(pVM != NULL) + vm.Destruct(&pVM); + + if(pResult != NULL) + var.Destruct(&pResult); + + *bProcessMsg = bRet; + RETiRet; +} + + + +/* Process (consume) a received message. Calls the actions configured. + * rgerhards, 2005-10-13 + */ +static rsRetVal +processMsg(rule_t *pThis, msg_t *pMsg) +{ + int bProcessMsg; + processMsgDoActions_t DoActData; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, rule); + assert(pMsg != NULL); + + /* first check the filters... */ + CHKiRet(shouldProcessThisMessage(pThis, pMsg, &bProcessMsg)); + if(bProcessMsg) { + DoActData.pMsg = pMsg; + DoActData.bPrevWasSuspended = 0; + CHKiRet(llExecFunc(&pThis->llActList, processMsgDoActions, (void*)&DoActData)); + } + +finalize_it: + RETiRet; +} + + +/* Standard-Constructor + */ +BEGINobjConstruct(rule) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(rule) + + +/* ConstructionFinalizer + * rgerhards, 2008-01-09 + */ +static rsRetVal +ruleConstructFinalize(rule_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, rule); + + /* note: actionDestruct is from action.c API! */ + CHKiRet(llInit(&pThis->llActList, actionDestruct, NULL, NULL)); + +finalize_it: + RETiRet; +} + + +/* destructor for the rule object */ +BEGINobjDestruct(rule) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(rule) + if(pThis->pCSHostnameComp != NULL) + rsCStrDestruct(&pThis->pCSHostnameComp); + if(pThis->pCSProgNameComp != NULL) + rsCStrDestruct(&pThis->pCSProgNameComp); + + if(pThis->f_filter_type == FILTER_PROP) { + if(pThis->f_filterData.prop.pCSPropName != NULL) + rsCStrDestruct(&pThis->f_filterData.prop.pCSPropName); + if(pThis->f_filterData.prop.pCSCompValue != NULL) + rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); + if(pThis->f_filterData.prop.regex_cache != NULL) + rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache); + } else if(pThis->f_filter_type == FILTER_EXPR) { + if(pThis->f_filterData.f_expr != NULL) + expr.Destruct(&pThis->f_filterData.f_expr); + } + + llDestroy(&pThis->llActList); + free(pThis); + + return RS_RET_OK; +ENDobjDestruct(rule) + + +/* helper to DebugPrint, to print out all actions via + * the llExecFunc() facility. + */ +DEFFUNC_llExecFunc(dbgPrintInitInfoAction) +{ + DEFiRet; + iRet = actionDbgPrint((action_t*) pData); + dbgprintf("\n"); + RETiRet; +} + + +/* debugprint for the rule object */ +BEGINobjDebugPrint(rule) /* be sure to specify the object type also in END and CODESTART macros! */ + int i; +CODESTARTobjDebugPrint(rule) + dbgoprint((obj_t*) pThis, "rsyslog rule:\n"); + if(pThis->pCSProgNameComp != NULL) + dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(pThis->pCSProgNameComp)); + if(pThis->eHostnameCmpMode != HN_NO_COMP) + dbgprintf("hostname: %s '%s'\n", + pThis->eHostnameCmpMode == HN_COMP_MATCH ? + "only" : "allbut", + rsCStrGetSzStrNoNULL(pThis->pCSHostnameComp)); + if(pThis->f_filter_type == FILTER_PRI) { + for (i = 0; i <= LOG_NFACILITIES; i++) + if (pThis->f_filterData.f_pmask[i] == TABLE_NOPRI) + dbgprintf(" X "); + else + dbgprintf("%2X ", pThis->f_filterData.f_pmask[i]); + } else if(pThis->f_filter_type == FILTER_EXPR) { + dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); + } else { + dbgprintf("PROPERTY-BASED Filter:\n"); + dbgprintf("\tProperty.: '%s'\n", + rsCStrGetSzStrNoNULL(pThis->f_filterData.prop.pCSPropName)); + dbgprintf("\tOperation: "); + if(pThis->f_filterData.prop.isNegated) + dbgprintf("NOT "); + dbgprintf("'%s'\n", getFIOPName(pThis->f_filterData.prop.operation)); + dbgprintf("\tValue....: '%s'\n", + rsCStrGetSzStrNoNULL(pThis->f_filterData.prop.pCSCompValue)); + dbgprintf("\tAction...: "); + } + + dbgprintf("\nActions:\n"); + llExecFunc(&pThis->llActList, dbgPrintInitInfoAction, NULL); /* actions */ + + dbgprintf("\n"); +ENDobjDebugPrint(rule) + + +/* queryInterface function + * rgerhards, 2008-02-21 + */ +BEGINobjQueryInterface(rule) +CODESTARTobjQueryInterface(rule) + if(pIf->ifVersion != ruleCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = ruleConstruct; + pIf->ConstructFinalize = ruleConstructFinalize; + pIf->Destruct = ruleDestruct; + pIf->DebugPrint = ruleDebugPrint; + + pIf->IterateAllActions = iterateAllActions; + pIf->ProcessMsg = processMsg; +finalize_it: +ENDobjQueryInterface(rule) + + +/* Exit the rule class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(rule, OBJ_IS_CORE_MODULE) /* class, version */ + objRelease(errmsg, CORE_COMPONENT); + objRelease(expr, CORE_COMPONENT); + objRelease(var, CORE_COMPONENT); + objRelease(vm, CORE_COMPONENT); +ENDObjClassExit(rule) + + +/* Initialize the rule class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(rule, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(expr, CORE_COMPONENT)); + CHKiRet(objUse(var, CORE_COMPONENT)); + CHKiRet(objUse(vm, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, ruleDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, ruleConstructFinalize); +ENDObjClassInit(rule) + +/* vi:set ai: + */ diff --git a/runtime/rule.h b/runtime/rule.h new file mode 100644 index 00000000..96be9e30 --- /dev/null +++ b/runtime/rule.h @@ -0,0 +1,74 @@ +/* The rule object. + * + * This implements rules within rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_RULE_H +#define INCLUDED_RULE_H + +#include "linkedlist.h" +#include "regexp.h" +#include "expr.h" + +/* the rule object */ +struct rule_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + /* filter properties */ + enum { + FILTER_PRI = 0, /* traditional PRI based filer */ + FILTER_PROP = 1, /* extended filter, property based */ + FILTER_EXPR = 2 /* extended filter, expression based */ + } f_filter_type; + EHostnameCmpMode eHostnameCmpMode; + cstr_t *pCSHostnameComp; /* hostname to check */ + cstr_t *pCSProgNameComp; /* tag to check or NULL, if not to be checked */ + union { + u_char f_pmask[LOG_NFACILITIES+1]; /* priority mask */ + struct { + cstr_t *pCSPropName; + fiop_t operation; + regex_t *regex_cache; /* cache for compiled REs, if such are used */ + cstr_t *pCSCompValue; /* value to "compare" against */ + char isNegated; /* actually a boolean ;) */ + } prop; + expr_t *f_expr; /* expression object */ + } f_filterData; + + linkedList_t llActList; /* list of configured actions */ +}; + +/* interfaces */ +BEGINinterface(rule) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(rule); + rsRetVal (*Construct)(rule_t **ppThis); + rsRetVal (*ConstructFinalize)(rule_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(rule_t **ppThis); + rsRetVal (*IterateAllActions)(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void *pParam); + rsRetVal (*ProcessMsg)(rule_t *pThis, msg_t *pMsg); +ENDinterface(rule) +#define ruleCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(rule); + +#endif /* #ifndef INCLUDED_RULE_H */ diff --git a/runtime/ruleset.c b/runtime/ruleset.c new file mode 100644 index 00000000..d135074f --- /dev/null +++ b/runtime/ruleset.c @@ -0,0 +1,369 @@ +/* ruleset.c - rsyslog's ruleset object + * + * We have a two-way structure of linked lists: one global linked list + * (llAllRulesets) hold alls rule sets that we know. Included in each + * list is a list of rules (which contain a list of actions, but that's + * a different story). + * + * Usually, only a single rule set is executed. However, there exist some + * situations where all rules must be iterated over, for example on HUP. Thus, + * we also provide interfaces to do that. + * + * Module begun 2009-06-10 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include +#include + +#include "rsyslog.h" +#include "obj.h" +#include "ruleset.h" +#include "rule.h" +#include "errmsg.h" +#include "unicode-helper.h" + +static rsRetVal debugPrintAll(void); // TODO: remove! + +/* static data */ +DEFobjStaticHelpers +DEFobjCurrIf(errmsg) +DEFobjCurrIf(rule) + +linkedList_t llRulesets; /* this is NOT a pointer - no typo here ;) */ + +/* ---------- linked-list key handling functions ---------- */ + +/* destructor for linked list keys. + */ +static rsRetVal keyDestruct(void __attribute__((unused)) *pData) +{ + free(pData); + return RS_RET_OK; +} + + +/* ---------- END linked-list key handling functions ---------- */ + + +/* dirver to iterate over all of this ruleset actions */ +typedef struct iterateAllActions_s { + rsRetVal (*pFunc)(void*, void*); + void *pParam; +} iterateAllActions_t; +DEFFUNC_llExecFunc(doIterateRulesetActions) +{ + DEFiRet; + rule_t* pRule = (rule_t*) pData; + iterateAllActions_t *pMyParam = (iterateAllActions_t*) pParam; + iRet = rule.IterateAllActions(pRule, pMyParam->pFunc, pMyParam->pParam); + RETiRet; +} +/* iterate over all actions of THIS rule set. + */ +static rsRetVal +iterateRulesetAllActions(ruleset_t *pThis, rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + iterateAllActions_t params; + DEFiRet; + ISOBJ_TYPE_assert(pThis, ruleset); + assert(pFunc != NULL); + + params.pFunc = pFunc; + params.pParam = pParam; + CHKiRet(llExecFunc(&llRulesets, doIterateRulesetActions, ¶ms)); + +finalize_it: + RETiRet; +} + + +/* dirver to iterate over all actions */ +DEFFUNC_llExecFunc(doIterateAllActions) +{ + DEFiRet; + ruleset_t* pThis = (ruleset_t*) pData; + iterateAllActions_t *pMyParam = (iterateAllActions_t*) pParam; + iRet = iterateRulesetAllActions(pThis, pMyParam->pFunc, pMyParam->pParam); + RETiRet; +} +/* iterate over ALL actions present in the WHOLE system. + * this is often needed, for example when HUP processing + * must be done or a shutdown is pending. + */ +static rsRetVal +iterateAllActions(rsRetVal (*pFunc)(void*, void*), void* pParam) +{ + iterateAllActions_t params; + DEFiRet; + assert(pFunc != NULL); + + params.pFunc = pFunc; + params.pParam = pParam; + CHKiRet(llExecFunc(&llRulesets, doIterateAllActions, ¶ms)); + +finalize_it: + RETiRet; +} + + + +/* helper to processMsg(), used to call the configured actions. It is + * executed from within llExecFunc() of the action list. + * rgerhards, 2007-08-02 + */ +DEFFUNC_llExecFunc(processMsgDoRules) +{ + ISOBJ_TYPE_assert(pData, rule); + return rule.ProcessMsg((rule_t*) pData, (msg_t*) pParam); +} + + +/* Process (consume) a received message. Calls the actions configured. + * rgerhards, 2005-10-13 + */ +static rsRetVal +processMsg(ruleset_t *pThis, msg_t *pMsg) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, ruleset); + assert(pMsg != NULL); + + CHKiRet(llExecFunc(&pThis->llRules, processMsgDoRules, pMsg)); + +finalize_it: + if(iRet == RS_RET_DISCARDMSG) + iRet = RS_RET_OK; + + RETiRet; +} + +/* Add a new rule to the end of the current rule set. We do a number + * of checks and ignore the rule if it does not pass them. + */ +static rsRetVal +addRule(ruleset_t *pThis, rule_t **ppRule) +{ + int iActionCnt; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, ruleset); + ISOBJ_TYPE_assert(*ppRule, rule); + + CHKiRet(llGetNumElts(&(*ppRule)->llActList, &iActionCnt)); + if(iActionCnt == 0) { + errmsg.LogError(0, NO_ERRCODE, "warning: selector line without actions will be discarded"); + rule.Destruct(ppRule); + } else { + CHKiRet(llAppend(&pThis->llRules, NULL, *ppRule)); + dbgprintf("selector line successfully processed\n"); + } + +finalize_it: + RETiRet; +} + + +/* set name for ruleset */ +static rsRetVal setName(ruleset_t *pThis, uchar *pszName) +{ + DEFiRet; + free(pThis->pszName); + CHKmalloc(pThis->pszName = ustrdup(pszName)); + +finalize_it: + RETiRet; +} + + +/* destructor we need to destruct rules inside our linked list contents. + */ +static rsRetVal +doRuleDestruct(void *pData) +{ + rule_t *pRule = (rule_t *) pData; + DEFiRet; + rule.Destruct(&pRule); + RETiRet; +} + + +/* Standard-Constructor + */ +BEGINobjConstruct(ruleset) /* be sure to specify the object type also in END macro! */ + CHKiRet(llInit(&pThis->llRules, doRuleDestruct, NULL, NULL)); +finalize_it: +ENDobjConstruct(ruleset) + + +/* ConstructionFinalizer + * This also adds the rule set to the list of all known rulesets. + */ +static rsRetVal +rulesetConstructFinalize(ruleset_t *pThis) +{ + uchar *keyName; + DEFiRet; + ISOBJ_TYPE_assert(pThis, ruleset); + + /* we must duplicate our name, as the key destructer would also + * free it, resulting in a double-free. It's also cleaner to have + * two separate copies. + */ + CHKmalloc(keyName = ustrdup(pThis->pszName)); + CHKiRet(llAppend(&llRulesets, keyName, pThis)); + +finalize_it: + RETiRet; +} + + +/* destructor for the ruleset object */ +BEGINobjDestruct(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(ruleset) + dbgprintf("destructing ruleset %p, name %p\n", pThis, pThis->pszName); + llDestroy(&pThis->llRules); + free(pThis->pszName); +ENDobjDestruct(ruleset) + +/* this is a special destructor for the linkedList class. LinkedList does NOT + * provide a pointer to the pointer, but rather the raw pointer itself. So we + * must map this, otherwise the destructor will abort. + */ +static rsRetVal +rulesetDestructForLinkedList(void *pData) +{ + ruleset_t *pThis = (ruleset_t*) pData; + return rulesetDestruct(&pThis); +} + + +/* destruct ALL rule sets that reside in the system. This must + * be callable before unloading this module as the module may + * not be unloaded before unload of the actions is required. This is + * kind of a left-over from previous logic and may be optimized one + * everything runs stable again. -- rgerhards, 2009-06-10 + */ +static rsRetVal +destructAllActions(void) +{ + DEFiRet; + + CHKiRet(llDestroy(&llRulesets)); + CHKiRet(llInit(&llRulesets, rulesetDestructForLinkedList, keyDestruct, strcasecmp)); + +finalize_it: + RETiRet; +} + +/* helper for debugPrint(), initiates rule printing */ +DEFFUNC_llExecFunc(doDebugPrintRule) +{ + return rule.DebugPrint((rule_t*) pData); +} +/* debugprint for the ruleset object */ +BEGINobjDebugPrint(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(ruleset) + dbgoprint((obj_t*) pThis, "rsyslog ruleset %s:\n", pThis->pszName); + llExecFunc(&pThis->llRules, doDebugPrintRule, NULL); +ENDobjDebugPrint(ruleset) + + +/* helper for debugPrintAll(), prints a single ruleset */ +DEFFUNC_llExecFunc(doDebugPrintAll) +{ + return rulesetDebugPrint((ruleset_t*) pData); +} +/* debug print all rulesets + */ +static rsRetVal +debugPrintAll(void) +{ + DEFiRet; + dbgprintf("All Rulesets:\n"); + llExecFunc(&llRulesets, doDebugPrintAll, NULL); + dbgprintf("End of Rulesets.\n"); + RETiRet; +} + + +/* queryInterface function + * rgerhards, 2008-02-21 + */ +BEGINobjQueryInterface(ruleset) +CODESTARTobjQueryInterface(ruleset) + if(pIf->ifVersion != rulesetCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = rulesetConstruct; + pIf->ConstructFinalize = rulesetConstructFinalize; + pIf->Destruct = rulesetDestruct; + pIf->DebugPrint = rulesetDebugPrint; + + pIf->IterateAllActions = iterateAllActions; + pIf->DestructAllActions = destructAllActions; + pIf->AddRule = addRule; + pIf->ProcessMsg = processMsg; + pIf->SetName = setName; + pIf->DebugPrintAll = debugPrintAll; +finalize_it: +ENDobjQueryInterface(ruleset) + + +/* Exit the ruleset class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(ruleset, OBJ_IS_CORE_MODULE) /* class, version */ + llDestroy(&llRulesets); + objRelease(errmsg, CORE_COMPONENT); + objRelease(rule, CORE_COMPONENT); +ENDObjClassExit(ruleset) + + +/* Initialize the ruleset class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(ruleset, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(rule, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, rulesetDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, rulesetConstructFinalize); + + /* prepare global data */ + CHKiRet(llInit(&llRulesets, rulesetDestructForLinkedList, keyDestruct, strcasecmp)); +ENDObjClassInit(ruleset) + +/* vi:set ai: + */ diff --git a/runtime/ruleset.h b/runtime/ruleset.h new file mode 100644 index 00000000..b609e6b3 --- /dev/null +++ b/runtime/ruleset.h @@ -0,0 +1,56 @@ +/* The ruleset object. + * + * This implements rulesets within rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_RULESET_H +#define INCLUDED_RULESET_H + +#include "linkedlist.h" + +/* the ruleset object */ +struct ruleset_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + linkedList_t llRules; /* this is NOT a pointer - no typo here ;) */ + uchar *pszName; /* name of our ruleset */ +}; + +/* interfaces */ +BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(ruleset); + rsRetVal (*DebugPrintAll)(void); + rsRetVal (*Construct)(ruleset_t **ppThis); + rsRetVal (*ConstructFinalize)(ruleset_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(ruleset_t **ppThis); + rsRetVal (*IterateAllActions)(rsRetVal (*pFunc)(void*, void*), void* pParam); + rsRetVal (*DestructAllActions)(void); + rsRetVal (*AddRule)(ruleset_t *pThis, rule_t **ppRule); + rsRetVal (*ProcessMsg)(ruleset_t *pThis, msg_t *pMsg); + rsRetVal (*SetName)(ruleset_t *pThis, uchar *pszName); +ENDinterface(ruleset) +#define rulesetCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(ruleset); + +#endif /* #ifndef INCLUDED_RULESET_H */ diff --git a/runtime/stream.c b/runtime/stream.c index 8e2f87dc..49d29e0e 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -650,7 +650,6 @@ doZipWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) dbgprintf("error %d returned from zlib/deflateInit2()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); } -RUNLOG_STR("deflateInit2() done successfully\n"); /* now doing the compression */ zstrm.avail_in = lenBuf; @@ -668,14 +667,12 @@ RUNLOG_STR("deflateInit2() done successfully\n"); } while (zstrm.avail_out == 0); assert(zstrm.avail_in == 0); /* all input will be used */ -RUNLOG_STR("deflate() should be done successfully\n"); zRet = zlibw.DeflateEnd(&zstrm); if(zRet != Z_OK) { dbgprintf("error %d returned from zlib/deflateEnd()\n", zRet); ABORT_FINALIZE(RS_RET_ZLIB_ERR); } -RUNLOG_STR("deflateEnd() done successfully\n"); finalize_it: RETiRet; diff --git a/tests/diskqueue-fsync.sh b/tests/diskqueue-fsync.sh index f558b491..8d2cb709 100755 --- a/tests/diskqueue-fsync.sh +++ b/tests/diskqueue-fsync.sh @@ -5,6 +5,8 @@ # added 2009-06-09 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 # uncomment for debugging support: +echo diskqueue-fsync test seems to have some issues +exit 1 echo testing queue disk-only mode, fsync case source $srcdir/diag.sh init source $srcdir/diag.sh startup diskqueue-fsync.conf diff --git a/tests/ourtail.c b/tests/ourtail.c index 6781b5fe..4e8a6412 100644 --- a/tests/ourtail.c +++ b/tests/ourtail.c @@ -40,4 +40,6 @@ int main(int __attribute__((unused)) argc, char __attribute__((unused)) *argv[]) for( ; c != EOF ; c = getchar()) putchar(c); + + return 0; } diff --git a/tests/runtime-dummy.c b/tests/runtime-dummy.c index 9cddd913..38e6bba1 100644 --- a/tests/runtime-dummy.c +++ b/tests/runtime-dummy.c @@ -25,7 +25,9 @@ * * A copy of the GPL can be found in the file "COPYING" in this distribution. */ +#include "config.h" #include +#include "rsyslog.h" int bReduceRepeatMsgs = 0; int repeatinterval = 30; @@ -37,5 +39,7 @@ void cflineClassic(void) {}; void selectorAddList(void) {}; void selectorConstruct(void) {}; void selectorDestruct(void) {}; +void getFIOPName(void) {}; +ruleset_t *pCurrRuleset; /* these are required by some dynamically loaded modules */ diff --git a/tests/validation-run.sh b/tests/validation-run.sh index 10981290..2e922283 100755 --- a/tests/validation-run.sh +++ b/tests/validation-run.sh @@ -22,17 +22,18 @@ # A copy of the GPL can be found in the file "COPYING" in this distribution. #set -x echo "testing a failed configuration verification run" -../tools/rsyslogd -u2 -c3 -N1 -f$srcdir/testsuites/invalid.conf +../tools/rsyslogd -dn -u2 -c4 -N1 -f$srcdir/testsuites/invalid.conf -M../runtime/.libs:../.libs if [ $? -ne 1 ]; then exit 1 fi echo testing a valid config verification run -../tools/rsyslogd -u2 -c3 -N1 -f$srcdir/testsuites/valid.conf +../tools/rsyslogd -u2 -c4 -N1 -f$srcdir/testsuites/valid.conf -M../runtime/.libs:../.libs if [ $? -ne 0 ]; then exit 1 fi echo testing empty config file -../tools/rsyslogd -u2 -c3 -N1 -f/dev/null +../tools/rsyslogd -u2 -c4 -N1 -f/dev/null -M../runtime/.libs:../.libs if [ $? -ne 1 ]; then exit 1 fi +echo SUCCESS: validation run tests diff --git a/tools/syslogd.c b/tools/syslogd.c index 19ece349..183c7760 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -132,11 +132,13 @@ #include "queue.h" #include "stream.h" #include "conf.h" -#include "vm.h" #include "errmsg.h" #include "datetime.h" #include "parser.h" #include "unicode-helper.h" +#include "ruleset.h" +#include "rule.h" +#include "vm.h" /* definitions for objects we access */ DEFobjCurrIf(obj) @@ -144,10 +146,10 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(datetime) DEFobjCurrIf(conf) DEFobjCurrIf(expr) -DEFobjCurrIf(vm) -DEFobjCurrIf(var) DEFobjCurrIf(module) DEFobjCurrIf(errmsg) +DEFobjCurrIf(rule) +DEFobjCurrIf(ruleset) DEFobjCurrIf(net) /* TODO: make go away! */ @@ -246,7 +248,7 @@ int repeatinterval[2] = { 30, 60 }; /* # of secs before flush */ #define LIST_DELIMITER ':' /* delimiter between two hosts */ -struct filed *Files = NULL; /* read-only after init() (but beware of sigusr1!) */ +ruleset_t *pCurrRuleset; /* ruleset that is currently being processed */ static pid_t ppid; /* This is a quick and dirty hack used for spliting main/startup thread */ @@ -313,7 +315,8 @@ static int iMainMsgQueueDeqtWinToHr = 25; /* hour begin of time frame when que /* support for simple textual representation of FIOP names * rgerhards, 2005-09-27 */ -static char* getFIOPName(unsigned iFIOP) +char* +getFIOPName(unsigned iFIOP) { char *pRet; switch(iFIOP) { @@ -396,7 +399,6 @@ static char **crunch_list(char *list); static void reapchild(); static void debug_switch(); static void sighup_handler(); -static void freeSelectors(void); static void processImInternal(void); @@ -431,67 +433,6 @@ diagGetMainMsgQSize(int *piSize) /* ------------------------------ end support functions for imdiag ------------------------------ */ -/* function to destruct a selector_t object - * rgerhards, 2007-08-01 - */ -rsRetVal -selectorDestruct(void *pVal) -{ - selector_t *pThis = (selector_t *) pVal; - - assert(pThis != NULL); - - if(pThis->pCSHostnameComp != NULL) - rsCStrDestruct(&pThis->pCSHostnameComp); - if(pThis->pCSProgNameComp != NULL) - rsCStrDestruct(&pThis->pCSProgNameComp); - - if(pThis->f_filter_type == FILTER_PROP) { - if(pThis->f_filterData.prop.pCSPropName != NULL) - rsCStrDestruct(&pThis->f_filterData.prop.pCSPropName); - if(pThis->f_filterData.prop.pCSCompValue != NULL) - rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); - if(pThis->f_filterData.prop.regex_cache != NULL) - rsCStrRegexDestruct(&pThis->f_filterData.prop.regex_cache); - } else if(pThis->f_filter_type == FILTER_EXPR) { - if(pThis->f_filterData.f_expr != NULL) - expr.Destruct(&pThis->f_filterData.f_expr); - } - - llDestroy(&pThis->llActList); - free(pThis); - - return RS_RET_OK; -} - - -/* function to construct a selector_t object - * rgerhards, 2007-08-01 - */ -rsRetVal -selectorConstruct(selector_t **ppThis) -{ - DEFiRet; - selector_t *pThis; - - assert(ppThis != NULL); - - if((pThis = (selector_t*) calloc(1, sizeof(selector_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - CHKiRet(llInit(&pThis->llActList, actionDestruct, NULL, NULL)); - -finalize_it: - if(iRet != RS_RET_OK) { - if(pThis != NULL) { - selectorDestruct(pThis); - } - } - *ppThis = pThis; - RETiRet; -} - - /* rgerhards, 2005-10-24: crunch_list is called only during option processing. So * it is never called once rsyslogd is running (not even when HUPed). This code * contains some exits, but they are considered safe because they only happen @@ -993,233 +934,6 @@ finalize_it: RETiRet; } -/* This functions looks at the given message and checks if it matches the - * provided filter condition. If so, it returns true, else it returns - * false. This is a helper to logmsg() and meant to drive the decision - * process if a message is to be processed or not. As I expect this - * decision code to grow more complex over time AND logmsg() is already - * a very lengthy function, I thought a separate function is more appropriate. - * 2005-09-19 rgerhards - * 2008-02-25 rgerhards: changed interface, now utilizes iRet, bProcessMsg - * returns is message should be procesed. - */ -static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProcessMsg) -{ - DEFiRet; - unsigned short pbMustBeFreed; - char *pszPropVal; - int bRet = 0; - vm_t *pVM = NULL; - var_t *pResult = NULL; - - assert(f != NULL); - assert(pMsg != NULL); - - /* we first have a look at the global, BSD-style block filters (for tag - * and host). Only if they match, we evaluate the actual filter. - * rgerhards, 2005-10-18 - */ - if(f->eHostnameCmpMode == HN_NO_COMP) { - /* EMPTY BY INTENSION - we check this value first, because - * it is the one most often used, so this saves us time! - */ - } else if(f->eHostnameCmpMode == HN_COMP_MATCH) { - if(rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, so we are already done... */ - dbgprintf("hostname filter '+%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } else { /* must be -hostname */ - if(!rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, so we are already done... */ - dbgprintf("hostname filter '-%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } - - if(f->pCSProgNameComp != NULL) { - int bInv = 0, bEqv = 0, offset = 0; - if(*(rsCStrGetSzStrNoNULL(f->pCSProgNameComp)) == '-') { - if(*(rsCStrGetSzStrNoNULL(f->pCSProgNameComp) + 1) == '-') - offset = 1; - else { - bInv = 1; - offset = 1; - } - } - if(!rsCStrOffsetSzStrCmp(f->pCSProgNameComp, offset, (uchar*) getProgramName(pMsg), getProgramNameLen(pMsg))) - bEqv = 1; - - if((!bEqv && !bInv) || (bEqv && bInv)) { - /* not equal or inverted selection, so we are already done... */ - dbgprintf("programname filter '%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSProgNameComp), getProgramName(pMsg)); - FINALIZE; - } - } - - /* done with the BSD-style block filters */ - - if(f->f_filter_type == FILTER_PRI) { - /* skip messages that are incorrect priority */ - if ( (f->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ - ((f->f_filterData.f_pmask[pMsg->iFacility] & (1<iSeverity)) == 0) ) - bRet = 0; - else - bRet = 1; - } else if(f->f_filter_type == FILTER_EXPR) { - CHKiRet(vm.Construct(&pVM)); - CHKiRet(vm.ConstructFinalize(pVM)); - CHKiRet(vm.SetMsg(pVM, pMsg)); - CHKiRet(vm.ExecProg(pVM, f->f_filterData.f_expr->pVmprg)); - CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); - dbgprintf("result of expression evaluation: %lld\n", pResult->val.num); - /* VM is destructed on function exit */ - bRet = (pResult->val.num) ? 1 : 0; - } else { - assert(f->f_filter_type == FILTER_PROP); /* assert() just in case... */ - pszPropVal = MsgGetProp(pMsg, NULL, f->f_filterData.prop.pCSPropName, &pbMustBeFreed); - - /* Now do the compares (short list currently ;)) */ - switch(f->f_filterData.prop.operation ) { - case FIOP_CONTAINS: - if(rsCStrLocateInSzStr(f->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1) - bRet = 1; - break; - case FIOP_ISEQUAL: - if(rsCStrSzStrCmp(f->f_filterData.prop.pCSCompValue, - (uchar*) pszPropVal, strlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_STARTSWITH: - if(rsCStrSzStrStartsWithCStr(f->f_filterData.prop.pCSCompValue, - (uchar*) pszPropVal, strlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_REGEX: - if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 0, &f->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - case FIOP_EREREGEX: - if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 1, &f->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - default: - /* here, it handles NOP (for performance reasons) */ - assert(f->f_filterData.prop.operation == FIOP_NOP); - bRet = 1; /* as good as any other default ;) */ - break; - } - - /* now check if the value must be negated */ - if(f->f_filterData.prop.isNegated) - bRet = (bRet == 1) ? 0 : 1; - - if(Debug) { - dbgprintf("Filter: check for property '%s' (value '%s') ", - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName), - pszPropVal); - if(f->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("%s '%s': %s\n", - getFIOPName(f->f_filterData.prop.operation), - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue), - bRet ? "TRUE" : "FALSE"); - } - - /* cleanup */ - if(pbMustBeFreed) - free(pszPropVal); - } - -finalize_it: - /* destruct in any case, not just on error, but it makes error handling much easier */ - if(pVM != NULL) - vm.Destruct(&pVM); - - if(pResult != NULL) - var.Destruct(&pResult); - - *bProcessMsg = bRet; - RETiRet; -} - - -/* helper to processMsg(), used to call the configured actions. It is - * executed from within llExecFunc() of the action list. - * rgerhards, 2007-08-02 - */ -typedef struct processMsgDoActions_s { - int bPrevWasSuspended; /* was the previous action suspended? */ - msg_t *pMsg; -} processMsgDoActions_t; -DEFFUNC_llExecFunc(processMsgDoActions) -{ - DEFiRet; - rsRetVal iRetMod; /* return value of module - we do not always pass that back */ - action_t *pAction = (action_t*) pData; - processMsgDoActions_t *pDoActData = (processMsgDoActions_t*) pParam; - - assert(pAction != NULL); - - if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { - dbgprintf("not calling action because the previous one is not suspended\n"); - ABORT_FINALIZE(RS_RET_OK); - } - - iRetMod = actionCallAction(pAction, pDoActData->pMsg); - if(iRetMod == RS_RET_DISCARDMSG) { - ABORT_FINALIZE(RS_RET_DISCARDMSG); - } else if(iRetMod == RS_RET_SUSPENDED) { - /* indicate suspension for next module to be called */ - pDoActData->bPrevWasSuspended = 1; - } else { - pDoActData->bPrevWasSuspended = 0; - } - -finalize_it: - RETiRet; -} - - -/* Process (consume) a received message. Calls the actions configured. - * rgerhards, 2005-10-13 - */ -static void -processMsg(msg_t *pMsg) -{ - selector_t *f; - int bContinue; - int bProcessMsg; - processMsgDoActions_t DoActData; - rsRetVal iRet; - - BEGINfunc - assert(pMsg != NULL); - - /* log the message to the particular outputs */ - - bContinue = 1; - for (f = Files; f != NULL && bContinue ; f = f->f_next) { - /* first check the filters... */ - iRet = shouldProcessThisMessage(f, pMsg, &bProcessMsg); - if(!bProcessMsg) { - continue; - } - - /* ok -- from here, we have action-specific code, nothing really selector-specific -- rger 2007-08-01 */ - DoActData.pMsg = pMsg; - DoActData.bPrevWasSuspended = 0; - if(llExecFunc(&f->llActList, processMsgDoActions, (void*)&DoActData) == RS_RET_DISCARDMSG) - bContinue = 0; - } - ENDfunc -} - /* The consumer of dequeued messages. This function is called by the * queue engine on dequeueing of a message. It runs on a SEPARATE @@ -1237,7 +951,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) if((pMsg->msgFlags & NEEDS_PARSING) != 0) { parseMsg(pMsg); } - processMsg(pMsg); + ruleset.ProcessMsg(pCurrRuleset, pMsg); msgDestruct(&pMsg); RETiRet; @@ -1744,7 +1458,6 @@ reapchild() DEFFUNC_llExecFunc(flushRptdMsgsActions) { action_t *pAction = (action_t*) pData; - assert(pAction != NULL); BEGINfunc @@ -1767,20 +1480,12 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions) } -/* This method flushes reapeat messages. +/* This method flushes repeat messages. */ static void doFlushRptdMsgs(void) { - register selector_t *f; - - /* see if we need to flush any "message repeated n times"... - * Note that this interferes with objects running on other threads. - * We are using appropriate locking inside the function to handle that. - */ - for (f = Files; f != NULL ; f = f->f_next) { - llExecFunc(&f->llActList, flushRptdMsgsActions, NULL); - } + ruleset.IterateAllActions(flushRptdMsgsActions, NULL); } @@ -1971,6 +1676,16 @@ freeAllDynMemForTermination(void) } +/* Finalize and destruct all actions. + */ +static inline void +destructAllActions(void) +{ + ruleset.DestructAllActions(); + bHaveMainQueue = 0; // flag that internal messages need to be temporarily stored +} + + /* die() is called when the program shall end. This typically only occurs * during sigterm or during the initialization. * As die() is intended to shutdown rsyslogd, it is @@ -2021,7 +1736,7 @@ die(int sig) * repeated msgs. */ dbgprintf("Terminating outputs...\n"); - freeSelectors(); + destructAllActions(); dbgprintf("all primary multi-thread sources have been terminated - now doing aux cleanup...\n"); /* rger 2005-02-22 @@ -2057,7 +1772,7 @@ die(int sig) * rgerhards, 2007-08-03 * I have added some code now, but all that mod init/de-init should be moved to * init, so that modules are unloaded and reloaded on HUP to. Eventually it should go - * into freeSelectors() - but that needs to be seen. -- rgerhards, 2007-08-09 + * into destructAllActions() - but that needs to be seen. -- rgerhards, 2007-08-09 */ module.UnloadAndDestructAll(eMOD_LINK_ALL); @@ -2185,56 +1900,6 @@ static void doDropPrivUid(int iUid) } -/* helper to freeSelectors(), used with llExecFunc() to flush - * pending output. -- rgerhards, 2007-08-02 - * We do not need to lock the action object here as the processing - * queue is already empty and no other threads are running when - * we call this function. -- rgerhards, 2007-12-12 - */ -DEFFUNC_llExecFunc(freeSelectorsActions) -{ - action_t *pAction = (action_t*) pData; - - assert(pAction != NULL); - - /* flush any pending output */ - if(pAction->f_prevcount) { - actionWriteToAction(pAction); - } - - return RS_RET_OK; /* never fails ;) */ -} - - -/* Close all open log files and free selector descriptor array. - */ -static void freeSelectors(void) -{ - selector_t *f; - selector_t *fPrev; - - if(Files != NULL) { - dbgprintf("Freeing log structures.\n"); - - for(f = Files ; f != NULL ; f = f->f_next) { - llExecFunc(&f->llActList, freeSelectorsActions, NULL); - } - - /* actions flushed and ready for destruction - so do that... */ - f = Files; - while (f != NULL) { - fPrev = f; - f = f->f_next; - selectorDestruct(fPrev); - } - - /* Reflect the deletion of the selectors linked list. */ - Files = NULL; - bHaveMainQueue = 0; - } -} - - /* helper to generateConfigDAG, to print out all actions via * the llExecFunc() facility. * rgerhards, 2007-08-02 @@ -2311,14 +1976,14 @@ DEFFUNC_llExecFunc(generateConfigDAGAction) static rsRetVal generateConfigDAG(uchar *pszDAGFile) { - selector_t *f; + //rule_t *f; FILE *fp; int iActUnit = 1; - int bHasFilter = 0; /* filter associated with this action unit? */ - int bHadFilter; - int i; + //int bHasFilter = 0; /* filter associated with this action unit? */ + //int bHadFilter; + //int i; struct dag_info dagInfo; - char *pszFilterName; + //char *pszFilterName; char szConnectingNode[64]; DEFiRet; @@ -2347,6 +2012,8 @@ generateConfigDAG(uchar *pszDAGFile) strcpy(szConnectingNode, "act0_0"); dagInfo.bDiscarded = 0; +/* TODO: re-enable! */ +#if 0 for(f = Files; f != NULL ; f = f->f_next) { /* BSD-Style filters are currently ignored */ bHadFilter = bHasFilter; @@ -2402,6 +2069,7 @@ generateConfigDAG(uchar *pszDAGFile) ++iActUnit; } +#endif fprintf(fp, "\t%s -> act%d_0\n", szConnectingNode, iActUnit); fprintf(fp, "\tact%d_0\t\t[label=discard shape=box]\n" @@ -2413,20 +2081,6 @@ finalize_it: } -/* helper to dbPrintInitInfo, to print out all actions via - * the llExecFunc() facility. - * rgerhards, 2007-08-02 - */ -DEFFUNC_llExecFunc(dbgPrintInitInfoAction) -{ - DEFiRet; - iRet = actionDbgPrint((action_t*) pData); - dbgprintf("\n"); - - RETiRet; -} - - /* print debug information as part of init(). This pretty much * outputs the whole config of rsyslogd. I've moved this code * out of init() to clean it somewhat up. @@ -2434,46 +2088,7 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction) */ static void dbgPrintInitInfo(void) { - selector_t *f; - int iSelNbr = 1; - int i; - - dbgprintf("\nActive selectors:\n"); - for (f = Files; f != NULL ; f = f->f_next) { - dbgprintf("Selector %d:\n", iSelNbr++); - if(f->pCSProgNameComp != NULL) - dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp)); - if(f->eHostnameCmpMode != HN_NO_COMP) - dbgprintf("hostname: %s '%s'\n", - f->eHostnameCmpMode == HN_COMP_MATCH ? - "only" : "allbut", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp)); - if(f->f_filter_type == FILTER_PRI) { - for (i = 0; i <= LOG_NFACILITIES; i++) - if (f->f_filterData.f_pmask[i] == TABLE_NOPRI) - dbgprintf(" X "); - else - dbgprintf("%2X ", f->f_filterData.f_pmask[i]); - } else if(f->f_filter_type == FILTER_EXPR) { - dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); - } else { - dbgprintf("PROPERTY-BASED Filter:\n"); - dbgprintf("\tProperty.: '%s'\n", - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName)); - dbgprintf("\tOperation: "); - if(f->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("'%s'\n", getFIOPName(f->f_filterData.prop.operation)); - dbgprintf("\tValue....: '%s'\n", - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue)); - dbgprintf("\tAction...: "); - } - - dbgprintf("\nActions:\n"); - llExecFunc(&f->llActList, dbgPrintInitInfoAction, NULL); /* actions */ - - dbgprintf("\n"); - } + ruleset.DebugPrintAll(); dbgprintf("\n"); if(bDebugPrintTemplateList) tplPrintList(); @@ -2553,13 +2168,13 @@ startInputModules(void) static rsRetVal init(void) { - DEFiRet; rsRetVal localRet; int iNbrActions; int bHadConfigErr = 0; char cbuf[BUFSIZ]; char bufStartUpMsg[512]; struct sigaction sigAct; + DEFiRet; thrdTerminateAll(); /* stop all running input threads - TODO: reconsider location! */ @@ -2580,7 +2195,7 @@ init(void) /* Close all open log files and free log descriptor array. This also frees * all output-modules instance data. */ - freeSelectors(); + destructAllActions(); /* Unload all non-static modules */ dbgprintf("Unloading non-static modules.\n"); @@ -2601,6 +2216,11 @@ init(void) conf.ReInitConf(); + // TODO: move to the right place + ruleset.Construct(&pCurrRuleset); + ruleset.SetName(pCurrRuleset, UCHAR_CONSTANT("RSYSLOG_DefaultRuleset")); + ruleset.ConstructFinalize(pCurrRuleset); + /* open the configuration file */ localRet = conf.processConfFile(ConfFile); CHKiRet(conf.GetNbrActActions(&iNbrActions)); @@ -2622,23 +2242,24 @@ init(void) * We ignore any errors while doing this - we would be lost anyhow... */ errmsg.LogError(0, NO_ERRCODE, "EMERGENCY CONFIGURATION ACTIVATED - fix rsyslog config file!"); - selector_t *f = NULL; /* note: we previously used _POSIY_TTY_NAME_MAX+1, but this turned out to be * too low on linux... :-S -- rgerhards, 2008-07-28 */ char szTTYNameBuf[128]; - conf.cfline((uchar*)"*.ERR\t" _PATH_CONSOLE, &f); - conf.cfline((uchar*)"syslog.*\t" _PATH_CONSOLE, &f); - conf.cfline((uchar*)"*.PANIC\t*", &f); - conf.cfline((uchar*)"syslog.*\troot", &f); + rule_t *pRule; + CHKiRet(rule.Construct(&pRule)); + conf.cfline((uchar*)"*.ERR\t" _PATH_CONSOLE, &pRule); + conf.cfline((uchar*)"syslog.*\t" _PATH_CONSOLE, &pRule); + conf.cfline((uchar*)"*.PANIC\t*", &pRule); + conf.cfline((uchar*)"syslog.*\troot", &pRule); if(ttyname_r(0, szTTYNameBuf, sizeof(szTTYNameBuf)) == 0) { snprintf(cbuf,sizeof(cbuf), "*.*\t%s", szTTYNameBuf); - conf.cfline((uchar*)cbuf, &f); + conf.cfline((uchar*)cbuf, &pRule); } else { dbgprintf("error %d obtaining controlling terminal, not using that emergency rule\n", errno); } - selectorAddList(f); + ruleset.AddRule(pCurrRuleset, &pRule); } legacyOptsHook(); @@ -2777,53 +2398,6 @@ finalize_it: } -/* add a completely-processed selector (after config line parsing) to - * the linked list of selectors. We now need to check - * if it has any actions associated and, if so, link it to the linked - * list. If it has nothing associated with it, we can simply discard - * it. - * We have one special case during initialization: then, the current - * selector is NULL, which means we do not need to care about it at - * all. -- rgerhards, 2007-08-01 - */ -rsRetVal -selectorAddList(selector_t *f) -{ - DEFiRet; - int iActionCnt; - - static selector_t *nextp = NULL; /* TODO: make this go away (see comment below) */ - - if(f != NULL) { - CHKiRet(llGetNumElts(&f->llActList, &iActionCnt)); - if(iActionCnt == 0) { - errmsg.LogError(0, NO_ERRCODE, "warning: selector line without actions will be discarded"); - selectorDestruct(f); - } else { - /* successfully created an entry */ - dbgprintf("selector line successfully processed\n"); - /* TODO: we should use the linked list class for the selector list, else we need to add globals - * ... well nextp could be added temporarily... - * Thanks to varmojfekoj for having the idea to just use "Files" to make this - * code work. I had actually forgotten to fix the code here before moving to 1.18.0. - * And, of course, I also did not migrate the selector_t structure to the linked list class. - * However, that should still be one of the very next things to happen. - * rgerhards, 2007-08-06 - */ - if(Files == NULL) { - Files = f; - } else { - nextp->f_next = f; - } - nextp = f; - } - } - -finalize_it: - RETiRet; -} - - /* set the main message queue mode * rgerhards, 2008-01-03 */ @@ -2911,7 +2485,6 @@ DEFFUNC_llExecFunc(doHUPActions) static inline void doHUP(void) { - selector_t *f; char buf[512]; snprintf(buf, sizeof(buf) / sizeof(char), @@ -2926,9 +2499,7 @@ doHUP(void) init(); /* main queue is stopped as part of init() */ } else { DBGPRINTF("Received SIGHUP, configured to be a non-restart type of HUP - notifying actions.\n"); - for(f = Files; f != NULL ; f = f->f_next) { - llExecFunc(&f->llActList, doHUPActions, NULL); - } + ruleset.IterateAllActions(doHUPActions, NULL); } } @@ -3285,14 +2856,14 @@ InitGlobalClasses(void) CHKiRet(objUse(errmsg, CORE_COMPONENT)); pErrObj = "module"; CHKiRet(objUse(module, CORE_COMPONENT)); - pErrObj = "var"; - CHKiRet(objUse(var, CORE_COMPONENT)); pErrObj = "datetime"; CHKiRet(objUse(datetime, CORE_COMPONENT)); - pErrObj = "vm"; - CHKiRet(objUse(vm, CORE_COMPONENT)); pErrObj = "expr"; CHKiRet(objUse(expr, CORE_COMPONENT)); + pErrObj = "rule"; + CHKiRet(objUse(rule, CORE_COMPONENT)); + pErrObj = "ruleset"; + CHKiRet(objUse(ruleset, CORE_COMPONENT)); pErrObj = "conf"; CHKiRet(objUse(conf, CORE_COMPONENT)); @@ -3336,10 +2907,10 @@ GlobalClassExit(void) /* first, release everything we used ourself */ objRelease(net, LM_NET_FILENAME);/* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */ objRelease(conf, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); + objRelease(rule, CORE_COMPONENT); objRelease(expr, CORE_COMPONENT); vmClassExit(); /* this is hack, currently core_modules do not get this automatically called */ - objRelease(vm, CORE_COMPONENT); - objRelease(var, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); /* TODO: implement the rest of the deinit */ diff --git a/tools/syslogd.h b/tools/syslogd.h index 8b9bd131..9bcfb2d2 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -28,67 +28,17 @@ #include "action.h" #include "linkedlist.h" #include "expr.h" +#include "net.h" /* TODO: remove when you remove isAllowedSender from here! */ #ifndef _PATH_CONSOLE #define _PATH_CONSOLE "/dev/console" #endif - -/* This structure represents the files that will have log - * copies printed. - * RGerhards 2004-11-08: Each instance of the filed structure - * describes what I call an "output channel". This is important - * to mention as we now allow database connections to be - * present in the filed structure. If helps immensely, if we - * think of it as the abstraction of an output channel. - * rgerhards, 2005-10-26: The structure below provides ample - * opportunity for non-thread-safety. Each of the variable - * accesses must be carefully evaluated, many of them probably - * be guarded by mutexes. But beware of deadlocks... - * rgerhards, 2007-08-01: as you can see, the structure has shrunk pretty much. I will - * remove some of the comments some time. It's still the structure that controls much - * of the processing that goes on in syslogd, but it now has lots of helpers. - */ -struct filed { - struct filed *f_next; /* next in linked list */ - /* filter properties */ - enum { - FILTER_PRI = 0, /* traditional PRI based filer */ - FILTER_PROP = 1, /* extended filter, property based */ - FILTER_EXPR = 2 /* extended filter, expression based */ - } f_filter_type; - EHostnameCmpMode eHostnameCmpMode; - cstr_t *pCSHostnameComp; /* hostname to check */ - cstr_t *pCSProgNameComp; /* tag to check or NULL, if not to be checked */ - union { - u_char f_pmask[LOG_NFACILITIES+1]; /* priority mask */ - struct { - cstr_t *pCSPropName; - enum { - FIOP_NOP = 0, /* do not use - No Operation */ - FIOP_CONTAINS = 1, /* contains string? */ - FIOP_ISEQUAL = 2, /* is (exactly) equal? */ - FIOP_STARTSWITH = 3, /* starts with a string? */ - FIOP_REGEX = 4, /* matches a (BRE) regular expression? */ - FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ - } operation; - regex_t *regex_cache; /* cache for compiled REs, if such are used */ - cstr_t *pCSCompValue; /* value to "compare" against */ - char isNegated; /* actually a boolean ;) */ - } prop; - expr_t *f_expr; /* expression object */ - } f_filterData; - - linkedList_t llActList; /* list of configured actions */ -}; - - -#include "net.h" /* TODO: remove when you remoe isAllowedSender from here! */ void untty(void); -rsRetVal selectorConstruct(selector_t **ppThis); +rsRetVal selectorConstruct(ruleset_t **ppThis); rsRetVal selectorDestruct(void *pVal); -rsRetVal selectorAddList(selector_t *f); +rsRetVal selectorAddList(rule_t *f); /* the following prototypes should go away once we have an input * module interface -- rgerhards, 2007-12-12 */ -- cgit v1.2.3 From 0290be816e683a711fc3c3a73a2ec79b804cfc8b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jun 2009 16:55:00 +0200 Subject: fixed small nit in build system --- tests/Makefile.am | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 3d555fc3..dbaf85f0 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -73,8 +73,6 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ diag.sh \ - daqueue-persist.sh \ - daqueue-persist-drvr.sh \ testsuites/diag-common.conf \ queue-persist.sh \ queue-persist-drvr.sh \ -- cgit v1.2.3 From 1dbdee7c774dd20e4653efc6871ddef5adce2785 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jun 2009 18:05:24 +0200 Subject: fixed a small memory leak... and another problem, both introduced today. Also did some general cleanup. --- runtime/obj.c | 39 ++++++++++++++++----------------------- runtime/rule.c | 3 --- runtime/wti.c | 3 +-- runtime/wtp.c | 3 +-- 4 files changed, 18 insertions(+), 30 deletions(-) diff --git a/runtime/obj.c b/runtime/obj.c index f5ec66b2..8b9c9c83 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -87,6 +87,7 @@ #include "modules.h" #include "errmsg.h" #include "cfsysline.h" +#include "unicode-helper.h" /* static data */ DEFobjCurrIf(obj) /* we define our own interface, as this is expected by some macros! */ @@ -145,8 +146,8 @@ InfoConstruct(objInfo_t **ppThis, uchar *pszID, int iObjVers, ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); pThis->pszID = pszID; - pThis->lenID = strlen((char*)pszID); - pThis->pszName = (uchar*)strdup((char*)pszID); /* it's OK if we have NULL ptr, GetName() will deal with that! */ + pThis->lenID = ustrlen(pszID); + pThis->pszName = ustrdup(pszID); /* it's OK if we have NULL ptr, GetName() will deal with that! */ pThis->iObjVers = iObjVers; pThis->QueryIF = pQueryIF; pThis->pModInfo = pModInfo; @@ -177,8 +178,7 @@ InfoDestruct(objInfo_t **ppThis) pThis = *ppThis; assert(pThis != NULL); - if(pThis->pszName != NULL) - free(pThis->pszName); + free(pThis->pszName); free(pThis); *ppThis = NULL; @@ -206,9 +206,7 @@ DestructObjSelf(obj_t *pThis) DEFiRet; ISOBJ_assert(pThis); - if(pThis->pszName != NULL) { - free(pThis->pszName); - } + free(pThis->pszName); RETiRet; } @@ -321,31 +319,31 @@ SerializeProp(strm_t *pStrm, uchar *pszPropName, propType_t propType, void *pUsr switch(propType) { case PROPTYPE_PSZ: pszBuf = (uchar*) pUsr; - lenBuf = strlen((char*) pszBuf); + lenBuf = ustrlen(pszBuf); vType = VARTYPE_STR; break; case PROPTYPE_SHORT: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), (long) *((short*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_INT: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), (long) *((int*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_LONG: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), *((long*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_INT64: CHKiRet(srUtilItoA((char*) szBuf, sizeof(szBuf), *((int64*) pUsr))); pszBuf = szBuf; - lenBuf = strlen((char*) szBuf); + lenBuf = ustrlen(szBuf); vType = VARTYPE_NUMBER; break; case PROPTYPE_CSTR: @@ -380,7 +378,7 @@ SerializeProp(strm_t *pStrm, uchar *pszPropName, propType_t propType, void *pUsr /* cookie */ CHKiRet(strm.WriteChar(pStrm, COOKIE_PROPLINE)); /* name */ - CHKiRet(strm.Write(pStrm, pszPropName, strlen((char*)pszPropName))); + CHKiRet(strm.Write(pStrm, pszPropName, ustrlen(pszPropName))); CHKiRet(strm.WriteChar(pStrm, ':')); /* type */ CHKiRet(strm.WriteLong(pStrm, (int) vType)); @@ -804,7 +802,7 @@ Deserialize(void *ppObj, uchar *pszTypeExpected, strm_t *pStrm, rsRetVal (*fFixu } } while(iRetLocal != RS_RET_OK); - if(rsCStrSzStrCmp(pstrID, pszTypeExpected, strlen((char*)pszTypeExpected))) /* TODO: optimize strlen() - caller shall provide */ + if(rsCStrSzStrCmp(pstrID, pszTypeExpected, ustrlen(pszTypeExpected))) /* TODO: optimize strlen() - caller shall provide */ ABORT_FINALIZE(RS_RET_INVALID_OID); CHKiRet(FindObjInfo(pstrID, &pObjInfo)); @@ -949,13 +947,8 @@ SetName(obj_t *pThis, uchar *pszName) { DEFiRet; - if(pThis->pszName != NULL) - free(pThis->pszName); - - pThis->pszName = (uchar*) strdup((char*) pszName); - - if(pThis->pszName == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + free(pThis->pszName); + CHKmalloc(pThis->pszName = ustrdup(pszName)); finalize_it: RETiRet; @@ -1058,7 +1051,7 @@ RegisterObj(uchar *pszObjName, objInfo_t *pInfo) i = 0; while(!bFound && i < OBJ_NUM_IDS && arrObjInfo[i] != NULL) { if( arrObjInfo[i] != NULL - && !strcmp((char*)arrObjInfo[i]->pszID, (char*)pszObjName)) { + && !ustrcmp(arrObjInfo[i]->pszID, pszObjName)) { bFound = 1; break; } @@ -1097,7 +1090,7 @@ UnregisterObj(uchar *pszObjName) i = 0; while(!bFound && i < OBJ_NUM_IDS) { if( arrObjInfo[i] != NULL - && !strcmp((char*)arrObjInfo[i]->pszID, (char*)pszObjName)) { + && !ustrcmp(arrObjInfo[i]->pszID, pszObjName)) { bFound = 1; break; } diff --git a/runtime/rule.c b/runtime/rule.c index fa56dc3b..c157242c 100644 --- a/runtime/rule.c +++ b/runtime/rule.c @@ -317,9 +317,6 @@ CODESTARTobjDestruct(rule) } llDestroy(&pThis->llActList); - free(pThis); - - return RS_RET_OK; ENDobjDestruct(rule) diff --git a/runtime/wti.c b/runtime/wti.c index 544bffa7..18767ea1 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -201,8 +201,7 @@ CODESTARTobjDestruct(wti) pthread_cond_destroy(&pThis->condExitDone); pthread_mutex_destroy(&pThis->mut); - if(pThis->pszDbgHdr != NULL) - free(pThis->pszDbgHdr); + free(pThis->pszDbgHdr); ENDobjDestruct(wti) diff --git a/runtime/wtp.c b/runtime/wtp.c index 04eb974f..df39daa3 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -151,8 +151,7 @@ CODESTARTobjDestruct(wtp) pthread_mutex_destroy(&pThis->mut); pthread_mutex_destroy(&pThis->mutThrdShutdwn); - if(pThis->pszDbgHdr != NULL) - free(pThis->pszDbgHdr); + free(pThis->pszDbgHdr); ENDobjDestruct(wtp) -- cgit v1.2.3 From 6141845f7514622f77d308b7aadb15891d3a627a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jun 2009 18:24:46 +0200 Subject: re-enabled queue disk fsync test ... actually, it was not broken, but just very slow. I have now reduced the number of test messages so that make check will not be held for an extended period of time. --- runtime/ruleset.c | 1 - tests/diskqueue-fsync.sh | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/runtime/ruleset.c b/runtime/ruleset.c index d135074f..a1454275 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -88,7 +88,6 @@ iterateRulesetAllActions(ruleset_t *pThis, rsRetVal (*pFunc)(void*, void*), void { iterateAllActions_t params; DEFiRet; - ISOBJ_TYPE_assert(pThis, ruleset); assert(pFunc != NULL); params.pFunc = pFunc; diff --git a/tests/diskqueue-fsync.sh b/tests/diskqueue-fsync.sh index 8d2cb709..45fb864c 100755 --- a/tests/diskqueue-fsync.sh +++ b/tests/diskqueue-fsync.sh @@ -5,13 +5,11 @@ # added 2009-06-09 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 # uncomment for debugging support: -echo diskqueue-fsync test seems to have some issues -exit 1 echo testing queue disk-only mode, fsync case source $srcdir/diag.sh init source $srcdir/diag.sh startup diskqueue-fsync.conf -# 5000 messages should be enough - the disk fsync test is very slow! -source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 5000 +# 1000 messages should be enough - the disk fsync test is very slow! +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 1000 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages -source $srcdir/diag.sh seq-check 0 4999 +source $srcdir/diag.sh seq-check 0 999 source $srcdir/diag.sh exit -- cgit v1.2.3 From 1c8fe77b78a64d69138b30ec28b430677b197601 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 10 Jun 2009 19:03:38 +0200 Subject: added $Ruleset config command so we now can define multiple rule sets, we just can not use them ;) That means we have the foundation to bind listeners to different rule sets. --- doc/rsyslog_conf_global.html | 2 ++ runtime/conf.c | 6 +++--- runtime/rule.c | 22 ++++++++++++++++++++++ runtime/rule.h | 3 +++ tools/syslogd.c | 27 +++++++++++++++++++++++++++ 5 files changed, 57 insertions(+), 3 deletions(-) diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index 1e268f4b..e9b1c082 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -201,6 +201,8 @@ line is that n is large enough to get a good idea which message was repeated but large enough for the whole message. (Introduced with 4.1.5). Once set, it affects all following actions.

    • $RepeatedMsgReduction
    • $ResetConfigVariables
      • +
    • $Ruleset name - starts a new ruleset. All following actions belong to +that new rule set.
    • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The default may change as uniprocessor systems become less common. [available since 4.1.0]
      • diff --git a/runtime/conf.c b/runtime/conf.c index e0ed3d6a..014d5a9a 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -471,7 +471,7 @@ processConfFile(uchar *pConfFile) /* we probably have one selector left to be added - so let's do that now */ if(pCurrRule != NULL) { - CHKiRet(ruleset.AddRule(pCurrRuleset, &pCurrRule)); + CHKiRet(ruleset.AddRule(rule.GetAssRuleset(pCurrRule), &pCurrRule)); } /* close the configuration file */ @@ -1141,9 +1141,10 @@ cflineClassic(uchar *p, rule_t **ppRule) * all. -- rgerhards, 2007-08-01 */ if(*ppRule != NULL) { - CHKiRet(ruleset.AddRule(pCurrRuleset, ppRule)); + CHKiRet(ruleset.AddRule(rule.GetAssRuleset(*ppRule), ppRule)); } CHKiRet(rule.Construct(ppRule)); /* create "fresh" selector */ + CHKiRet(rule.SetAssRuleset(*ppRule, pCurrRuleset)); /* create "fresh" selector */ CHKiRet(rule.ConstructFinalize(*ppRule)); /* create "fresh" selector */ CHKiRet(cflineDoFilter(&p, *ppRule)); /* pull filters */ } @@ -1166,7 +1167,6 @@ cfline(uchar *line, rule_t **pfCurr) DEFiRet; ASSERT(line != NULL); -if(*pfCurr != NULL){ ISOBJ_TYPE_assert(*pfCurr, rule);} dbgprintf("cfline: '%s'\n", line); diff --git a/runtime/rule.c b/runtime/rule.c index c157242c..f17c524e 100644 --- a/runtime/rule.c +++ b/runtime/rule.c @@ -320,6 +320,26 @@ CODESTARTobjDestruct(rule) ENDobjDestruct(rule) +/* set the associated ruleset */ +static rsRetVal +setAssRuleset(rule_t *pThis, ruleset_t *pRuleset) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, rule); + ISOBJ_TYPE_assert(pRuleset, ruleset); + pThis->pRuleset = pRuleset; + RETiRet; +} + +/* get the associated ruleset (may be NULL if not set!) */ +static ruleset_t* +getAssRuleset(rule_t *pThis) +{ + ISOBJ_TYPE_assert(pThis, rule); + return pThis->pRuleset; +} + + /* helper to DebugPrint, to print out all actions via * the llExecFunc() facility. */ @@ -393,6 +413,8 @@ CODESTARTobjQueryInterface(rule) pIf->IterateAllActions = iterateAllActions; pIf->ProcessMsg = processMsg; + pIf->SetAssRuleset = setAssRuleset; + pIf->GetAssRuleset = getAssRuleset; finalize_it: ENDobjQueryInterface(rule) diff --git a/runtime/rule.h b/runtime/rule.h index 96be9e30..38b11c63 100644 --- a/runtime/rule.h +++ b/runtime/rule.h @@ -53,6 +53,7 @@ struct rule_s { expr_t *f_expr; /* expression object */ } f_filterData; + ruleset_t *pRuleset; /* associated ruleset */ linkedList_t llActList; /* list of configured actions */ }; @@ -64,6 +65,8 @@ BEGINinterface(rule) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Destruct)(rule_t **ppThis); rsRetVal (*IterateAllActions)(rule_t *pThis, rsRetVal (*pFunc)(void*, void*), void *pParam); rsRetVal (*ProcessMsg)(rule_t *pThis, msg_t *pMsg); + rsRetVal (*SetAssRuleset)(rule_t *pThis, ruleset_t*); + ruleset_t* (*GetAssRuleset)(rule_t *pThis); ENDinterface(rule) #define ruleCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 183c7760..f4b59970 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2398,6 +2398,32 @@ finalize_it: } +/* Begin a new rule set. The new rule set is created, and all rules that now + * follow go into that rule set. + * TODO: we may later add the capability to switch back to an already existing + * rule set. + * NOTE: pCurrRuleset is NOT desructed and must not be! The ruleset class keeps + * a list of all known rule sets, and can destruct them at the end of execution. + * pCurrRuleset is just a shortcut so that "everyone" knows which ruleset to + * extend. + * TODO: A problem with this function is the way config lines are processed. The rule + * is actually only written when the next rule is completely read. That way, this + * (past) rule goes into the wrong (new) ruleset. I need to see how to fix this best... + * rgerhards, 2009-06-10 + */ +static rsRetVal beginNewRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + DEFiRet; + CHKiRet(ruleset.Construct(&pCurrRuleset)); + CHKiRet(ruleset.SetName(pCurrRuleset, pszName)); + CHKiRet(ruleset.ConstructFinalize(pCurrRuleset)); + +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} + + /* set the main message queue mode * rgerhards, 2008-01-03 */ @@ -2651,6 +2677,7 @@ static rsRetVal loadBuildInModules(void) * This, I think, is the right thing to do. -- rgerhards, 2007-07-31 */ CHKiRet(regCfSysLineHdlr((uchar *)"actionresumeretrycount", 0, eCmdHdlrInt, NULL, &glbliActionResumeRetryCount, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"ruleset", 0, eCmdHdlrGetWord, beginNewRuleset, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszMainMsgQFName, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesize", 0, eCmdHdlrInt, NULL, &iMainMsgQueueSize, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuehighwatermark", 0, eCmdHdlrInt, NULL, &iMainMsgQHighWtrMark, NULL)); -- cgit v1.2.3 From 0917edf26da9055c6dc160aafca97896daea3e6c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 09:47:44 +0200 Subject: re-enabled outchannel functionality --- runtime/rsyslog.h | 3 ++ runtime/srUtils.h | 1 + runtime/srutils.c | 27 ++++++++++ runtime/stream.c | 130 ++++++++++++++++++++++++++++++++++++++++++++--- runtime/stream.h | 7 ++- runtime/unicode-helper.h | 27 ++++++---- tools/omfile.c | 91 +++++++-------------------------- 7 files changed, 198 insertions(+), 88 deletions(-) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index b587554e..92942c70 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -289,6 +289,9 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_PREVIOUS_COMMITTED = -2122, /**< output plugin status: previous record was committed (an OK state!) */ RS_RET_ACTION_FAILED = -2123, /**< action failed and is now suspended (consider this permanent for the time being) */ RS_RET_NONFATAL_CONFIG_ERR = -2124, /**< non-fatal error during config processing */ + RS_RET_NON_SIZELIMITCMD = -2125, /**< size limit for file defined, but no size limit command given */ + RS_RET_SIZELIMITCMD_DIDNT_RESOLVE = -2126, /**< size limit command did not resolve situation */ + RS_RET_STREAM_DISABLED = -2127, /**< a file has been disabled (e.g. by size limit restriction) */ RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ RS_RET_ZLIB_ERR = -2141, /**< error during zlib call */ diff --git a/runtime/srUtils.h b/runtime/srUtils.h index bfce4cbb..699f8527 100644 --- a/runtime/srUtils.h +++ b/runtime/srUtils.h @@ -92,6 +92,7 @@ void srSleep(int iSeconds, int iuSeconds); char *rs_strerror_r(int errnum, char *buf, size_t buflen); int decodeSyslogName(uchar *name, syslogName_t *codetab); int getSubString(uchar **ppSrc, char *pDst, size_t DstSize, char cSep); +rsRetVal getFileSize(uchar *pszName, off_t *pSize); /* mutex operations */ /* some macros to cancel-safe lock a mutex (it will automatically be released diff --git a/runtime/srutils.c b/runtime/srutils.c index d01ca20d..5407531f 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -553,6 +553,33 @@ int getSubString(uchar **ppSrc, char *pDst, size_t DstSize, char cSep) } +/* get the size of a file or return appropriate error code. If an error is returned, + * *pSize content is undefined. + * rgerhards, 2009-06-12 + */ +rsRetVal +getFileSize(uchar *pszName, off_t *pSize) +{ + int ret; + struct stat statBuf; + DEFiRet; + + ret = stat((char*) pszName, &statBuf); + if(ret == -1) { + switch(errno) { + case EACCES: ABORT_FINALIZE(RS_RET_NO_FILE_ACCESS); + case ENOTDIR: + case ENOENT: ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); + default: ABORT_FINALIZE(RS_RET_FILE_NO_STAT); + } + } + + *pSize = statBuf.st_size; + +finalize_it: + RETiRet; +} + /* vim:set ai: */ diff --git a/runtime/stream.c b/runtime/stream.c index 49d29e0e..5355a1a5 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -6,8 +6,9 @@ * "driver"). * * File begun on 2008-01-09 by RGerhards + * Large modifications in 2009-06 to support using it with omfile, including zip writer. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -54,11 +55,112 @@ DEFobjCurrIf(zlibw) /* forward definitions */ static rsRetVal strmFlush(strm_t *pThis); static rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf); +static rsRetVal strmCloseFile(strm_t *pThis); /* methods */ -/* first, we define type-specific handlers. The provide a generic functionality, + +/* Try to resolve a size limit situation. This is used to support custom-file size handlers + * for omfile. It first runs the command, and then checks if we are still above the size + * treshold. Note that this works only with single file names, NOT with circular names. + * Note that pszCurrFName can NOT be taken from pThis, because the stream is closed when + * we are called (and that destroys pszCurrFName, as there is NO CURRENT file name!). So + * we need to receive the name as a parameter. + * initially wirtten 2005-06-21, moved to this class & updates 2009-06-01, both rgerhards + */ +static rsRetVal +resolveFileSizeLimit(strm_t *pThis, uchar *pszCurrFName) +{ + uchar *pParams; + uchar *pCmd; + uchar *p; + off_t actualFileSize; + rsRetVal localRet; + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + assert(pszCurrFName != NULL); + + if(pThis->pszSizeLimitCmd == NULL) { + ABORT_FINALIZE(RS_RET_NON_SIZELIMITCMD); /* nothing we can do in this case... */ + } + + /* we first check if we have command line parameters. We assume this, + * when we have a space in the program name. If we find it, everything after + * the space is treated as a single argument. + */ + CHKmalloc(pCmd = ustrdup(pThis->pszSizeLimitCmd)); + + for(p = pCmd ; *p && *p != ' ' ; ++p) { + /* JUST SKIP */ + } + + if(*p == ' ') { + *p = '\0'; /* pretend string-end */ + pParams = p+1; + } else + pParams = NULL; + + /* the execProg() below is probably not great, but at least is is + * fairly secure now. Once we change the way file size limits are + * handled, we should also revisit how this command is run (and + * with which parameters). rgerhards, 2007-07-20 + */ + execProg(pCmd, 1, pParams); + + free(pCmd); + + localRet = getFileSize(pszCurrFName, &actualFileSize); + + if(localRet == RS_RET_OK && actualFileSize >= pThis->iSizeLimit) { + ABORT_FINALIZE(RS_RET_SIZELIMITCMD_DIDNT_RESOLVE); /* OK, it didn't work out... */ + } else if(localRet != RS_RET_FILE_NOT_FOUND) { + /* file not found is OK, the command may have moved away the file */ + ABORT_FINALIZE(localRet); + } + +finalize_it: + if(iRet != RS_RET_OK) { + if(iRet == RS_RET_SIZELIMITCMD_DIDNT_RESOLVE) + dbgprintf("file size limit cmd for file '%s' did no resolve situation\n", pszCurrFName); + else + dbgprintf("file size limit cmd for file '%s' failed with code %d.\n", pszCurrFName, iRet); + pThis->bDisabled = 1; + } + + RETiRet; +} + + +/* Check if the file has grown beyond the configured omfile iSizeLimit + * and, if so, initiate processing. + */ +static rsRetVal +doSizeLimitProcessing(strm_t *pThis) +{ + uchar *pszCurrFName = NULL; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, strm); + ASSERT(pThis->iSizeLimit != 0); + ASSERT(pThis->fd != -1); + + if(pThis->iCurrOffs >= pThis->iSizeLimit) { + /* strmClosefile() destroys the current file name, so we + * need to preserve it. + */ + CHKmalloc(pszCurrFName = ustrdup(pThis->pszCurrFName)); + CHKiRet(strmCloseFile(pThis)); + CHKiRet(resolveFileSizeLimit(pThis, pszCurrFName)); + } + +finalize_it: + free(pszCurrFName); + RETiRet; +} + + +/* now, we define type-specific handlers. The provide a generic functionality, * but for this specific type of strm. The mapping to these handlers happens during * strm construction. Later on, handlers are called by pointers present in the * strm instance object. @@ -123,6 +225,12 @@ static rsRetVal strmOpenFile(strm_t *pThis) } pThis->iCurrOffs = 0; + if(pThis->tOperationsMode == STREAMMODE_WRITE_APPEND) { + /* we need to obtain the current offset */ + off_t offset; + CHKiRet(getFileSize(pThis->pszCurrFName, &offset)); + pThis->iCurrOffs = offset; + } dbgoprint((obj_t*) pThis, "opened file '%s' for %s (0x%x) as %d\n", pThis->pszCurrFName, (pThis->tOperationsMode == STREAMMODE_READ) ? "READ" : "WRITE", iFlags, pThis->fd); @@ -527,7 +635,7 @@ doWriteCall(int fd, uchar *pBuf, size_t *pLenBuf) if(err == EINTR) { /*NO ERROR, just continue */; } else { - ABORT_FINALIZE(RS_RET_ERR); + ABORT_FINALIZE(RS_RET_IO_ERROR); // TODO: cover more error cases! } } @@ -538,7 +646,7 @@ doWriteCall(int fd, uchar *pBuf, size_t *pLenBuf) } while(lenBuf > 0); /* Warning: do..while()! */ finalize_it: - *pLenBuf -= iTotalWritten; + *pLenBuf = iTotalWritten; RETiRet; } @@ -610,8 +718,11 @@ strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) CHKiRet(syncFile(pThis)); } - if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) + if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) { CHKiRet(strmCheckNextOutputFile(pThis)); + } else if(pThis->iSizeLimit != 0) { + CHKiRet(doSizeLimitProcessing(pThis)); + } finalize_it: pThis->iBufPtr = 0; /* see comment above */ @@ -812,7 +923,10 @@ strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) ASSERT(pThis != NULL); ASSERT(pBuf != NULL); -dbgprintf("strmWrite(%p, '%s', %ld);\n", pThis, pBuf,lenBuf); +dbgprintf("strmWrite(%p, '%65.65s', %ld);, disabled %d, sizelim %ld, size %lld\n", pThis, pBuf,lenBuf, pThis->bDisabled, pThis->iSizeLimit, pThis->iCurrOffs); + if(pThis->bDisabled) + ABORT_FINALIZE(RS_RET_STREAM_DISABLED); + /* check if the to-be-written data is larger than our buffer size */ if(lenBuf >= pThis->sIOBufSize) { /* it is - so we do a direct write, that is most efficient. @@ -857,6 +971,8 @@ DEFpropSetMeth(strm, sType, strmType_t) DEFpropSetMeth(strm, iZipLevel, int) DEFpropSetMeth(strm, bSync, int) DEFpropSetMeth(strm, sIOBufSize, size_t) +DEFpropSetMeth(strm, iSizeLimit, off_t) +DEFpropSetMeth(strm, pszSizeLimitCmd, uchar*) static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) { @@ -1139,6 +1255,8 @@ CODESTARTobjQueryInterface(strm) pIf->SetiZipLevel = strmSetiZipLevel; pIf->SetbSync = strmSetbSync; pIf->SetsIOBufSize = strmSetsIOBufSize; + pIf->SetiSizeLimit = strmSetiSizeLimit; + pIf->SetpszSizeLimitCmd = strmSetpszSizeLimitCmd; finalize_it: ENDobjQueryInterface(strm) diff --git a/runtime/stream.h b/runtime/stream.h index a66108b7..a06e089f 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -103,6 +103,7 @@ typedef struct strm_s { int64 iCurrOffs;/* current offset */ int64 *pUsrWCntr; /* NULL or a user-provided counter that receives the nbr of bytes written since the last CntrSet() */ /* dynamic properties, valid only during file open, not to be persistet */ + int bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */ int bSync; /* sync this file after every write? */ size_t sIOBufSize;/* size of IO buffer */ uchar *pszDir; /* Directory */ @@ -117,7 +118,9 @@ typedef struct strm_s { int bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */ Bytef *pZipBuf; - + /* support for omfile size-limiting commands, special counters, NOT persisted! */ + off_t iSizeLimit; /* file size limit, 0 = no limit */ + uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ } strm_t; /* interfaces */ @@ -152,6 +155,8 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, iZipLevel, int); INTERFACEpropSetMeth(strm, bSync, int); INTERFACEpropSetMeth(strm, sIOBufSize, size_t); + INTERFACEpropSetMeth(strm, iSizeLimit, off_t); + INTERFACEpropSetMeth(strm, pszSizeLimitCmd, uchar*); ENDinterface(strm) #define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h index 5289da3f..8216e992 100644 --- a/runtime/unicode-helper.h +++ b/runtime/unicode-helper.h @@ -4,6 +4,9 @@ * The following functions are wrappers which hopefully enable us to move * from 8-bit chars to unicode with relative ease when we finally attack this * + * Note: while we prefer inline functions, this leads to invalid references in + * core dumps. So in a debug build, we use macros where appropriate... + * * Begun 2009-05-21 RGerhards * * Copyright (C) 2009 by Rainer Gerhards and Adiscon GmbH @@ -31,10 +34,21 @@ #include -static inline char* ustrncpy(uchar *psz1, uchar *psz2, size_t len) -{ - return strncpy((char*) psz1, (char*) psz2, len); -} +#ifdef DEBUG +# define ustrncpy(psz1, psz2, len) strncpy((char*)(psz1), (char*)(psz2), (len)) +# define ustrdup(psz) (uchar*)strdup((char*)(psz)) +#else + static inline uchar* ustrncpy(uchar *psz1, uchar *psz2, size_t len) + { + return strncpy((char*) psz1, (char*) psz2, len); + } + + static inline uchar* ustrdup(uchar *psz) + { + return (uchar*) strdup((char*)psz); + } + +#endif /* #ifdef DEBUG */ static inline int ustrcmp(uchar *psz1, uchar *psz2) { @@ -46,11 +60,6 @@ static inline int ustrlen(uchar *psz) return strlen((char*) psz); } -static inline uchar* ustrdup(uchar *psz) -{ - return (uchar*) strdup((char*)psz); -} - #define UCHAR_CONSTANT(x) ((uchar*) (x)) #define CHAR_CONVERT(x) ((char*) (x)) diff --git a/tools/omfile.c b/tools/omfile.c index e3f7af19..cf020f7b 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -67,6 +67,7 @@ #include "unicode-helper.h" #include "stream.h" #include "zlibw.h" +#include "unicode-helper.h" MODULE_TYPE_OUTPUT @@ -119,7 +120,7 @@ typedef struct _instanceData { int fCreateMode; /* file creation mode for open() */ int fDirCreateMode; /* creation mode for mkdir() */ int bCreateDirs; /* auto-create directories? */ - int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ // TODO: stream class? RE-IMPLEMENT! + int bSyncFile; /* should the file by sync()'ed? 1- yes, 0- no */ uid_t fileUID; /* IDs for creation */ uid_t dirUID; gid_t fileGID; @@ -134,7 +135,7 @@ typedef struct _instanceData { */ dynaFileCacheEntry **dynCache; off_t iSizeLimit; /* file size limit, 0 = no limit */ - uchar *iSizeLimitCmd; /* command to carry out when size limit is reached */ + uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ int iZipLevel; /* zip mode to use for this selector */ int iIOBufSize; /* size of associated io buffer */ } instanceData; @@ -264,7 +265,7 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR /* WARNING: It is dangerous "just" to pass the pointer. As we * never rebuild the output channel description, this is acceptable here. */ - pData->iSizeLimitCmd = pOch->cmdOnSizeLimit; + pData->pszSizeLimitCmd = pOch->cmdOnSizeLimit; iRet = cflineParseTemplateName(&p, pOMSR, iEntry, iTplOpts, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName); @@ -274,66 +275,6 @@ finalize_it: } -#if 0 -/* rgerhards 2005-06-21: Try to resolve a size limit - * situation. This first runs the command, and then - * checks if we are still above the treshold. - * returns 0 if ok, 1 otherwise - * TODO: consider moving the initial check in here, too - */ -int resolveFileSizeLimit(instanceData *pData) -{ - uchar *pParams; - uchar *pCmd; - uchar *p; - off_t actualFileSize; - ASSERT(pData != NULL); - - if(pData->iSizeLimitCmd == NULL) - return 1; /* nothing we can do in this case... */ - - /* the execProg() below is probably not great, but at least is is - * fairly secure now. Once we change the way file size limits are - * handled, we should also revisit how this command is run (and - * with which parameters). rgerhards, 2007-07-20 - */ - /* we first check if we have command line parameters. We assume this, - * when we have a space in the program name. If we find it, everything after - * the space is treated as a single argument. - */ - if((pCmd = ustrdup(pData->iSizeLimitCmd)) == NULL) { - /* there is not much we can do - we make syslogd close the file in this case */ - return 1; - } - - for(p = pCmd ; *p && *p != ' ' ; ++p) { - /* JUST SKIP */ - } - - if(*p == ' ') { - *p = '\0'; /* pretend string-end */ - pParams = p+1; - } else - pParams = NULL; - - execProg(pCmd, 1, pParams); - - free(pCmd); - - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_CREAT|O_NOCTTY|O_CLOEXEC, - pData->fCreateMode); - - actualFileSize = lseek(pData->fd, 0, SEEK_END); - if(actualFileSize >= pData->iSizeLimit) { - /* OK, it didn't work out... */ - return 1; - } - - return 0; -} -#endif - - /* This function deletes an entry from the dynamic file name * cache. A pointer to the cache must be passed in as well * as the index of the to-be-deleted entry. This index may @@ -458,23 +399,29 @@ prepareFile(instanceData *pData, uchar *newFileName) } } - char szNameBuf[MAXFNAME]; - char szDirName[MAXFNAME]; - char szBaseName[MAXFNAME]; - strcpy(szNameBuf, (char*)newFileName); - strcpy(szDirName, dirname(szNameBuf)); - strcpy(szNameBuf, (char*)newFileName); - strcpy(szBaseName, basename(szNameBuf)); + /* the copies below are clumpsy, but there is no way around given the + * anomalies in dirname() and basename() [they MODIFY the provided buffer...] + */ + uchar szNameBuf[MAXFNAME]; + uchar szDirName[MAXFNAME]; + uchar szBaseName[MAXFNAME]; + ustrncpy(szNameBuf, newFileName, MAXFNAME); + ustrncpy(szDirName, (uchar*)dirname((char*)szNameBuf), MAXFNAME); + ustrncpy(szNameBuf, newFileName, MAXFNAME); + ustrncpy(szBaseName, (uchar*)basename((char*)szNameBuf), MAXFNAME); CHKiRet(strm.Construct(&pData->pStrm)); - CHKiRet(strm.SetFName(pData->pStrm, (uchar*)szBaseName, strlen(szBaseName))); - CHKiRet(strm.SetDir(pData->pStrm, (uchar*)szDirName, strlen(szDirName))); + CHKiRet(strm.SetFName(pData->pStrm, szBaseName, ustrlen(szBaseName))); + CHKiRet(strm.SetDir(pData->pStrm, szDirName, ustrlen(szDirName))); CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SettOpenMode(pData->pStrm, fCreateMode)); CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile)); CHKiRet(strm.SetsType(pData->pStrm, STREAMTYPE_FILE_SINGLE)); + CHKiRet(strm.SetiSizeLimit(pData->pStrm, pData->iSizeLimit)); + if(pData->pszSizeLimitCmd != NULL) + CHKiRet(strm.SetpszSizeLimitCmd(pData->pStrm, ustrdup(pData->pszSizeLimitCmd))); CHKiRet(strm.ConstructFinalize(pData->pStrm)); finalize_it: -- cgit v1.2.3 From 7f7e5ef75a6183c3b625afe58984343f9b7997be Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 09:57:50 +0200 Subject: improved config error messages now contain a copy of the config line that (most likely) caused the error --- ChangeLog | 2 ++ runtime/conf.c | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index 57bb6e7e..39ca00ec 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +- improved config error messages: now contain a copy of the config line + that (most likely) caused the error - added a generic network stream server (in addition to rather specific syslog tcp server) - added ability for the UDP output action to rebind its send socket after diff --git a/runtime/conf.c b/runtime/conf.c index 014d5a9a..412a756d 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -71,6 +71,7 @@ #include "ctok_token.h" #include "rule.h" #include "ruleset.h" +#include "unicode-helper.h" #ifdef OS_SOLARIS # define NAME_MAX MAXNAMELEN @@ -396,7 +397,6 @@ finalize_it: static rsRetVal processConfFile(uchar *pConfFile) { - DEFiRet; int iLnNbr = 0; FILE *cf; rule_t *pCurrRule = NULL; @@ -405,6 +405,9 @@ processConfFile(uchar *pConfFile) uchar *cline; int i; int bHadAnError = 0; + uchar *pszOrgLine = NULL; + size_t lenLine; + DEFiRet; ASSERT(pConfFile != NULL); if((cf = fopen((char*)pConfFile, "r")) == NULL) { @@ -417,9 +420,12 @@ processConfFile(uchar *pConfFile) while (fgets((char*)cline, sizeof(cbuf) - (cline - cbuf), cf) != NULL) { ++iLnNbr; /* drop LF - TODO: make it better, replace fgets(), but its clean as it is */ - if(cline[strlen((char*)cline)-1] == '\n') { - cline[strlen((char*)cline) -1] = '\0'; + lenLine = ustrlen(cline); + if(cline[lenLine-1] == '\n') { + cline[lenLine-1] = '\0'; } + free(pszOrgLine); + pszOrgLine = ustrdup(cline); /* save if needed for errmsg, NULL ptr is OK */ /* check for end-of-section, comments, strip off trailing * spaces and newline character. */ @@ -464,7 +470,7 @@ processConfFile(uchar *pConfFile) dbgprintf("config line NOT successfully processed\n"); snprintf((char*)szErrLoc, sizeof(szErrLoc) / sizeof(uchar), "%s, line %d", pConfFile, iLnNbr); - errmsg.LogError(0, NO_ERRCODE, "the last error occured in %s", (char*)szErrLoc); + errmsg.LogError(0, NO_ERRCODE, "the last error occured in %s:\"%s\"", (char*)szErrLoc, (char*)pszOrgLine); bHadAnError = 1; } } @@ -488,6 +494,8 @@ finalize_it: iRet, pConfFile, errStr); } + free(pszOrgLine); + if(bHadAnError && (iRet == RS_RET_OK)) { /* a bit dirty, enhance in future releases */ iRet = RS_RET_NONFATAL_CONFIG_ERR; } -- cgit v1.2.3 From 1af948107e6e520788e374adccf4986bf07e92f5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 10:14:45 +0200 Subject: fixed abort when emergency configuration was activated this regression was introduced last friday, so this is *NOT* present in any released version. --- runtime/conf.c | 1 - tools/syslogd.c | 11 +++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/runtime/conf.c b/runtime/conf.c index 412a756d..81b6c081 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -1193,7 +1193,6 @@ cfline(uchar *line, rule_t **pfCurr) break; default: iRet = cflineClassic(line, pfCurr); -ISOBJ_TYPE_assert(*pfCurr, rule); break; } diff --git a/tools/syslogd.c b/tools/syslogd.c index f4b59970..96f3cb34 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2247,12 +2247,11 @@ init(void) * too low on linux... :-S -- rgerhards, 2008-07-28 */ char szTTYNameBuf[128]; - rule_t *pRule; - CHKiRet(rule.Construct(&pRule)); - conf.cfline((uchar*)"*.ERR\t" _PATH_CONSOLE, &pRule); - conf.cfline((uchar*)"syslog.*\t" _PATH_CONSOLE, &pRule); - conf.cfline((uchar*)"*.PANIC\t*", &pRule); - conf.cfline((uchar*)"syslog.*\troot", &pRule); + rule_t *pRule = NULL; /* initialization to NULL is *vitally* important! */ + conf.cfline(UCHAR_CONSTANT("*.ERR\t" _PATH_CONSOLE), &pRule); + conf.cfline(UCHAR_CONSTANT("syslog.*\t" _PATH_CONSOLE), &pRule); + conf.cfline(UCHAR_CONSTANT("*.PANIC\t*"), &pRule); + conf.cfline(UCHAR_CONSTANT("syslog.*\troot"), &pRule); if(ttyname_r(0, szTTYNameBuf, sizeof(szTTYNameBuf)) == 0) { snprintf(cbuf,sizeof(cbuf), "*.*\t%s", szTTYNameBuf); conf.cfline((uchar*)cbuf, &pRule); -- cgit v1.2.3 From e3d9843c85b1dfddabc937ac6ccb4057d626bf03 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 11:47:00 +0200 Subject: re-enabled pipe, tty and console in omfile ... by moving code to stream.c. Thanks to the new design, new cases are not really needed, resulting in cleaner code. I also did a cleanup of header file usage as a side-activity. --- dirty.h | 4 - plugins/omgssapi/omgssapi.c | 1 + plugins/ommail/ommail.c | 2 +- plugins/ommysql/ommysql.c | 2 +- plugins/ompgsql/ompgsql.c | 2 +- plugins/omprog/omprog.c | 2 +- plugins/omrelp/omrelp.c | 2 +- plugins/omsnmp/omsnmp.c | 2 +- plugins/omstdout/omstdout.c | 2 +- plugins/omtemplate/omtemplate.c | 2 +- runtime/conf.h | 4 + runtime/rsyslog.h | 13 ++++ runtime/stream.c | 162 ++++++++++++++++++++++++++-------------- runtime/stream.h | 5 +- tools/omfile.c | 142 +---------------------------------- tools/omfwd.c | 1 + tools/omshell.c | 2 +- tools/omusrmsg.c | 2 +- tools/syslogd.c | 11 +-- tools/syslogd.h | 10 --- 20 files changed, 147 insertions(+), 226 deletions(-) diff --git a/dirty.h b/dirty.h index bab15485..7ae57d09 100644 --- a/dirty.h +++ b/dirty.h @@ -35,10 +35,6 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags); rsRetVal diagGetMainMsgQSize(int *piSize); /* for imdiag */ char* getFIOPName(unsigned iFIOP); -/* TODO: the following 2 need to go in conf obj interface... */ -rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); -rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl); - /* Intervals at which we flush out "message repeated" messages, * in seconds after previous message is logged. After each flush, * we move to the next interval until we reach the largest. diff --git a/plugins/omgssapi/omgssapi.c b/plugins/omgssapi/omgssapi.c index 361f657f..7b5a46e1 100644 --- a/plugins/omgssapi/omgssapi.c +++ b/plugins/omgssapi/omgssapi.c @@ -44,6 +44,7 @@ #include #include #include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "net.h" diff --git a/plugins/ommail/ommail.c b/plugins/ommail/ommail.c index 5faadce3..3a7669c9 100644 --- a/plugins/ommail/ommail.c +++ b/plugins/ommail/ommail.c @@ -44,7 +44,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "cfsysline.h" diff --git a/plugins/ommysql/ommysql.c b/plugins/ommysql/ommysql.c index ecf738a9..d6870a7b 100644 --- a/plugins/ommysql/ommysql.c +++ b/plugins/ommysql/ommysql.c @@ -36,7 +36,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/plugins/ompgsql/ompgsql.c b/plugins/ompgsql/ompgsql.c index 6daac1c7..eb774835 100644 --- a/plugins/ompgsql/ompgsql.c +++ b/plugins/ompgsql/ompgsql.c @@ -40,7 +40,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/plugins/omprog/omprog.c b/plugins/omprog/omprog.c index 2a078a6d..01fa7cea 100644 --- a/plugins/omprog/omprog.c +++ b/plugins/omprog/omprog.c @@ -36,7 +36,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/plugins/omrelp/omrelp.c b/plugins/omrelp/omrelp.c index 8d74c82f..d5ef8b4f 100644 --- a/plugins/omrelp/omrelp.c +++ b/plugins/omrelp/omrelp.c @@ -36,7 +36,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "cfsysline.h" diff --git a/plugins/omsnmp/omsnmp.c b/plugins/omsnmp/omsnmp.c index 72fa8d64..4db60e62 100644 --- a/plugins/omsnmp/omsnmp.c +++ b/plugins/omsnmp/omsnmp.c @@ -36,7 +36,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "cfsysline.h" #include "module-template.h" diff --git a/plugins/omstdout/omstdout.c b/plugins/omstdout/omstdout.c index 181895a4..b9125f19 100644 --- a/plugins/omstdout/omstdout.c +++ b/plugins/omstdout/omstdout.c @@ -35,7 +35,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/plugins/omtemplate/omtemplate.c b/plugins/omtemplate/omtemplate.c index e35968ad..5577f8c6 100644 --- a/plugins/omtemplate/omtemplate.c +++ b/plugins/omtemplate/omtemplate.c @@ -36,7 +36,7 @@ #include #include #include -#include "dirty.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" diff --git a/runtime/conf.h b/runtime/conf.h index f29de1ba..25b887be 100644 --- a/runtime/conf.h +++ b/runtime/conf.h @@ -51,5 +51,9 @@ PROTOTYPEObj(conf); extern EHostnameCmpMode eDfltHostnameCmpMode; extern cstr_t *pDfltHostnameCmp; extern cstr_t *pDfltProgNameCmp; +/* TODO: the following 2 need to go in conf obj interface... */ +rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); +rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl); + #endif /* #ifndef INCLUDED_CONF_H */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 92942c70..28f7bce9 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -99,6 +99,7 @@ typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename? typedef long long int64; typedef long long unsigned uint64; typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ +typedef short bool; #ifdef __hpux typedef unsigned int u_int32_t; /* TODO: is this correct? */ @@ -124,6 +125,18 @@ typedef enum { FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ } fiop_t; +/* file types (omfile & stream) */ +typedef enum { + eTypeFILE, + eTypeTTY, + eTypeCONSOLE, + eTypePIPE +} filetype_t; + + +#ifndef _PATH_CONSOLE +#define _PATH_CONSOLE "/dev/console" +#endif /* The error codes below are orginally "borrowed" from * liblogging. As such, we reserve values up to -2999 diff --git a/runtime/stream.c b/runtime/stream.c index 5355a1a5..c8672aa2 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -166,35 +166,14 @@ finalize_it: * strm instance object. */ -/* open a strm file - * It is OK to call this function when the stream is already open. In that - * case, it returns immediately with RS_RET_OK +/* do the physical open() call on a file. */ -static rsRetVal strmOpenFile(strm_t *pThis) +static rsRetVal +doPhysOpen(strm_t *pThis) { - DEFiRet; int iFlags; - - ASSERT(pThis != NULL); - - if(pThis->fd != -1) - ABORT_FINALIZE(RS_RET_OK); - - if(pThis->pszFName == NULL) - ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); - - if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) { - CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, - pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, pThis->iFileNumDigits)); - } else { - if(pThis->pszDir == NULL) { - if((pThis->pszCurrFName = (uchar*) strdup((char*) pThis->pszFName)) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } else { - CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, - pThis->pszFName, pThis->lenFName, -1, 0)); - } - } + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); /* compute which flags we need to provide to open */ switch(pThis->tOperationsMode) { @@ -222,8 +201,51 @@ static rsRetVal strmOpenFile(strm_t *pThis) ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); else ABORT_FINALIZE(RS_RET_IO_ERROR); + } else { + if(!ustrcmp(pThis->pszCurrFName, UCHAR_CONSTANT(_PATH_CONSOLE)) || isatty(pThis->fd)) { + DBGPRINTF("file %d is a tty-type file\n", pThis->fd); + pThis->bIsTTY = 1; + } else { + pThis->bIsTTY = 0; + } } +finalize_it: + RETiRet; +} + + +/* open a strm file + * It is OK to call this function when the stream is already open. In that + * case, it returns immediately with RS_RET_OK + */ +static rsRetVal strmOpenFile(strm_t *pThis) +{ + DEFiRet; + + ASSERT(pThis != NULL); + + if(pThis->fd != -1) + ABORT_FINALIZE(RS_RET_OK); + + if(pThis->pszFName == NULL) + ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); + + if(pThis->sType == STREAMTYPE_FILE_CIRCULAR) { + CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, + pThis->pszFName, pThis->lenFName, pThis->iCurrFNum, pThis->iFileNumDigits)); + } else { + if(pThis->pszDir == NULL) { + if((pThis->pszCurrFName = (uchar*) strdup((char*) pThis->pszFName)) == NULL) + ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + } else { + CHKiRet(genFileName(&pThis->pszCurrFName, pThis->pszDir, pThis->lenDir, + pThis->pszFName, pThis->lenFName, -1, 0)); + } + } + + CHKiRet(doPhysOpen(pThis)); + pThis->iCurrOffs = 0; if(pThis->tOperationsMode == STREAMMODE_WRITE_APPEND) { /* we need to obtain the current offset */ @@ -232,8 +254,8 @@ static rsRetVal strmOpenFile(strm_t *pThis) pThis->iCurrOffs = offset; } - dbgoprint((obj_t*) pThis, "opened file '%s' for %s (0x%x) as %d\n", pThis->pszCurrFName, - (pThis->tOperationsMode == STREAMMODE_READ) ? "READ" : "WRITE", iFlags, pThis->fd); + dbgoprint((obj_t*) pThis, "opened file '%s' for %s as %d\n", pThis->pszCurrFName, + (pThis->tOperationsMode == STREAMMODE_READ) ? "READ" : "WRITE", pThis->fd); finalize_it: RETiRet; @@ -255,7 +277,7 @@ static rsRetVal strmCloseFile(strm_t *pThis) if(pThis->tOperationsMode != STREAMMODE_READ) strmFlush(pThis); - close(pThis->fd); // TODO: error check + close(pThis->fd); pThis->fd = -1; if(pThis->fdDir != -1) { @@ -265,7 +287,13 @@ static rsRetVal strmCloseFile(strm_t *pThis) } if(pThis->bDeleteOnClose) { - unlink((char*) pThis->pszCurrFName); // TODO: check returncode + if(unlink((char*) pThis->pszCurrFName) == -1) { + char errStr[1024]; + int err = errno; + rs_strerror_r(err, errStr, sizeof(errStr)); + DBGPRINTF("error %d unlinking '%s' - ignored: %s\n", + errno, pThis->pszCurrFName, errStr); + } } pThis->iCurrOffs = 0; /* we are back at begin of file */ @@ -544,13 +572,12 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) } /* if we are aset to sync, we must obtain a file handle to the directory for fsync() purposes */ - if(pThis->bSync) { - pThis->fdDir = open((char*)pThis->pszDir, O_RDONLY); + if(pThis->bSync && !pThis->bIsTTY) { + pThis->fdDir = open((char*)pThis->pszDir, O_RDONLY | O_CLOEXEC | O_NOCTTY); if(pThis->fdDir == -1) { char errStr[1024]; int err = errno; rs_strerror_r(err, errStr, sizeof(errStr)); - // TODO: log an error message? think so... DBGPRINTF("error %d opening directory file for fsync() use - fsync for directory disabled: %s\n", errno, errStr); } @@ -606,6 +633,29 @@ finalize_it: } +/* try to recover a tty after a write error. This may have happend + * due to vhangup(), and, if so, we can simply re-open it. + */ +#ifdef linux +# define ERR_TTYHUP EIO +#else +# define ERR_TTYHUP EBADF +#endif +static rsRetVal +tryTTYRecover(strm_t *pThis, int err) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); + if(err == ERR_TTYHUP) { + close(pThis->fd); + CHKiRet(doPhysOpen(pThis)); + } + +finalize_it: + RETiRet; +} +#undef ER_TTYHUP + /* issue write() api calls until either the buffer is completely * written or an error occured (it may happen that multiple writes @@ -614,29 +664,36 @@ finalize_it: * rgerhards, 2009-06-08 */ static rsRetVal -doWriteCall(int fd, uchar *pBuf, size_t *pLenBuf) +doWriteCall(strm_t *pThis, uchar *pBuf, size_t *pLenBuf) { ssize_t lenBuf; ssize_t iTotalWritten; ssize_t iWritten; char *pWriteBuf; DEFiRet; + ISOBJ_TYPE_assert(pThis, strm); lenBuf = *pLenBuf; pWriteBuf = (char*) pBuf; iTotalWritten = 0; do { - iWritten = write(fd, pWriteBuf, lenBuf); + iWritten = write(pThis->fd, pWriteBuf, lenBuf); if(iWritten < 0) { char errStr[1024]; int err = errno; rs_strerror_r(err, errStr, sizeof(errStr)); - DBGPRINTF("log file (%d) write error %d: %s\n", fd, err, errStr); + DBGPRINTF("log file (%d) write error %d: %s\n", pThis->fd, err, errStr); if(err == EINTR) { /*NO ERROR, just continue */; } else { - ABORT_FINALIZE(RS_RET_IO_ERROR); - // TODO: cover more error cases! + if(pThis->bIsTTY) { + CHKiRet(tryTTYRecover(pThis, err)); + } else { + ABORT_FINALIZE(RS_RET_IO_ERROR); + /* Would it make sense to cover more error cases? So far, I + * do not see good reason to do so. + */ + } } } /* advance buffer to next write position */ @@ -653,7 +710,10 @@ finalize_it: /* sync the file to disk, so that any unwritten data is persisted. This * also syncs the directory and thus makes sure that the file survives - * fatal failure. -- rgerhards, 2009-06-08 + * fatal failure. Note that we do NOT return an error status if the + * sync fails. Doing so would probably cause more trouble than it + * is worth (read: data loss may occur where we otherwise might not + * have it). -- rgerhards, 2009-06-08 */ static rsRetVal syncFile(strm_t *pThis) @@ -661,6 +721,9 @@ syncFile(strm_t *pThis) int ret; DEFiRet; + if(pThis->bIsTTY) + FINALIZE; /* TTYs can not be synced */ + DBGPRINTF("syncing file %d\n", pThis->fd); ret = fdatasync(pThis->fd); if(ret != 0) { @@ -670,19 +733,20 @@ syncFile(strm_t *pThis) DBGPRINTF("sync failed for file %d with error (%d): %s - ignoring\n", pThis->fd, err, errStr); } - // TODO: check error! if(pThis->fdDir != -1) { ret = fsync(pThis->fdDir); -dbgprintf("sync on dir (fd %d) requested, return code %d\n", pThis->fdDir, ret); } +finalize_it: RETiRet; } /* physically write to the output file. the provided data is ready for * writing (e.g. zipped if we are requested to do that). + * Note that if the write() API fails, we do not reset any pointers, but return + * an error code. That means we may redo work in the next iteration. * rgerhards, 2009-06-04 */ static rsRetVal @@ -690,24 +754,15 @@ strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { size_t iWritten; DEFiRet; - - ASSERT(pThis != NULL); + ISOBJ_TYPE_assert(pThis, strm); if(pThis->fd == -1) CHKiRet(strmOpenFile(pThis)); iWritten = lenBuf; - CHKiRet(doWriteCall(pThis->fd, pBuf, &iWritten)); + CHKiRet(doWriteCall(pThis, pBuf, &iWritten)); dbgoprint((obj_t*) pThis, "file %d write wrote %d bytes\n", pThis->fd, (int) iWritten); - /* Now indicate buffer empty again. We do this in any case, because there - * is no way we could react more intelligently to an error during write. - * This MUST be done BEFORE strCheckNextOutputFile(), otherwise we have an - * endless loop. We reset the buffer pointer also in finalize_it - this is - * necessary if we run into problems. Not resetting it would again cause an - * endless loop. So it is better to loose some data (which also justifies - * duplicating that code, too...) -- rgerhards, 2008-01-10 - */ pThis->iBufPtr = 0; pThis->iCurrOffs += iWritten; /* update user counter, if provided */ @@ -725,8 +780,6 @@ strmPhysWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) } finalize_it: - pThis->iBufPtr = 0; /* see comment above */ - RETiRet; } @@ -995,7 +1048,6 @@ strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) ASSERT(pThis != NULL); ASSERT(pszName != NULL); -dbgprintf("XXX: strm setFname: '%s'\n", pszName); if(iLenName < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); diff --git a/runtime/stream.h b/runtime/stream.h index a06e089f..021c4792 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -99,7 +99,7 @@ typedef struct strm_s { int64 iMaxFileSize;/* maximum size a file may grow to */ int iMaxFiles; /* maximum number of files if a circular mode is in use */ int iFileNumDigits;/* min number of digits to use in file number (only in circular mode) */ - int bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */ + bool bDeleteOnClose; /* set to 1 to auto-delete on close -- be careful with that setting! */ int64 iCurrOffs;/* current offset */ int64 *pUsrWCntr; /* NULL or a user-provided counter that receives the nbr of bytes written since the last CntrSet() */ /* dynamic properties, valid only during file open, not to be persistet */ @@ -115,12 +115,13 @@ typedef struct strm_s { size_t iBufPtrMax; /* current max Ptr in Buffer (if partial read!) */ size_t iBufPtr; /* pointer into current buffer */ int iUngetC; /* char set via UngetChar() call or -1 if none set */ - int bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ + bool bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */ Bytef *pZipBuf; /* support for omfile size-limiting commands, special counters, NOT persisted! */ off_t iSizeLimit; /* file size limit, 0 = no limit */ uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ + bool bIsTTY; /* is this a tty file? */ } strm_t; /* interfaces */ diff --git a/tools/omfile.c b/tools/omfile.c index cf020f7b..6377268d 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -55,7 +55,7 @@ # include #endif -#include "syslogd.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "template.h" @@ -64,7 +64,6 @@ #include "cfsysline.h" #include "module-template.h" #include "errmsg.h" -#include "unicode-helper.h" #include "stream.h" #include "zlibw.h" #include "unicode-helper.h" @@ -110,12 +109,6 @@ static uchar *pszTplName = NULL; /* name of the default template to use */ typedef struct _instanceData { uchar f_fname[MAXFNAME];/* file or template name (display only) */ strm_t *pStrm; /* our output stream */ - enum { - eTypeFILE, - eTypeTTY, - eTypeCONSOLE, - eTypePIPE - } fileType; char bDynamicName; /* 0 - static name, 1 - dynamic name (with properties) */ int fCreateMode; /* file creation mode for open() */ int fDirCreateMode; /* creation mode for mkdir() */ @@ -218,14 +211,6 @@ static rsRetVal cflineParseOutchannel(instanceData *pData, uchar* p, omodStringR struct outchannel *pOch; char szBuf[128]; /* should be more than sufficient */ - /* this must always be a file, because we can not set a size limit - * on a pipe... - * rgerhards 2005-06-21: later, this will be a separate type, but let's - * emulate things for the time being. When everything runs, we can - * extend it... - */ - pData->fileType = eTypeFILE; - ++p; /* skip '$' */ i = 0; /* get outchannel name */ @@ -358,8 +343,6 @@ prepareFile(instanceData *pData, uchar *newFileName) int fd; DEFiRet; - // TODO: handle TTY/PIPE case! (in stream.c!) 2009-06-04 - if(access((char*)newFileName, F_OK) != 0) { /* file does not exist, create it (and eventually parent directories */ fd = -1; @@ -425,18 +408,6 @@ prepareFile(instanceData *pData, uchar *newFileName) CHKiRet(strm.ConstructFinalize(pData->pStrm)); finalize_it: - /* this was "fd != 0", which I think was a bug. I guess 0 was intended to mean - * non-open file descriptor. Anyhow, I leave this comment for the time being to that if - * problems surface, one at least knows what happened. -- rgerhards, 2009-03-19 - */ -#if 0 // TODO: this must be done by stream class! - if(fd != -1 && isatty(fd)) { - DBGPRINTF("file %d is a tty file\n", fd); - pData->fileType = eTypeTTY; - untty(); - } -#endif - RETiRet; } @@ -546,107 +517,6 @@ finalize_it: } -#if 0 -// TODO: mirgrate code below to stream class (update its write handler, which is not great!) -/* physically write the file - */ -static rsRetVal -doPhysWrite(instanceData *pData, int fd, char *pszBuf, size_t lenBuf) -{ - off_t actualFileSize; - int iLenWritten; - DEFiRet; - ASSERT(pData != NULL); - -dbgprintf("doPhysWrite, fd %d, iBuf %d\n", fd, (int) lenBuf); -again: - /* check if we have a file size limit and, if so, - * obey to it. - */ - if(pData->iSizeLimit != 0) { - actualFileSize = lseek(fd, 0, SEEK_END); - if(actualFileSize >= pData->iSizeLimit) { - char errMsg[256]; - /* for now, we simply disable a file once it is - * beyond the maximum size. This is better than having - * us aborted by the OS... rgerhards 2005-06-21 - */ - (void) close(fd); - /* try to resolve the situation */ - // TODO: *doesn't work, will need to use new fd ! - if(resolveFileSizeLimit(pData) != 0) { - /* didn't work out, so disable... */ - snprintf(errMsg, sizeof(errMsg), - "no longer writing to file %s; grown beyond configured file size of %lld bytes, actual size %lld - configured command did not resolve situation", - pData->f_fname, (long long) pData->iSizeLimit, (long long) actualFileSize); - errno = 0; - errmsg.LogError(0, RS_RET_DISABLE_ACTION, "%s", errMsg); - ABORT_FINALIZE(RS_RET_DISABLE_ACTION); - } else { - snprintf(errMsg, sizeof(errMsg), - "file %s had grown beyond configured file size of %lld bytes, actual size was %lld - configured command resolved situation", - pData->f_fname, (long long) pData->iSizeLimit, (long long) actualFileSize); - errno = 0; - errmsg.LogError(0, NO_ERRCODE, "%s", errMsg); - } - } - } - - iLenWritten = write(fd, pszBuf, lenBuf); - if(iLenWritten < 0) { - int e = errno; - char errStr[1024]; - rs_strerror_r(errno, errStr, sizeof(errStr)); - DBGPRINTF("log file (%d) write error %d: %s\n", fd, e, errStr); - - /* If a named pipe is full, we suspend this action for a while */ - if(pData->fileType == eTypePIPE && e == EAGAIN) - ABORT_FINALIZE(RS_RET_SUSPENDED); - - close(pData->fd); - pData->fd = -1; /* tell that fd is no longer open! */ - if(pData->bDynamicName && pData->iCurrElt != -1) { - /* in this case, we need to invalidate the name in the cache, too - * otherwise, an invalid fd may show up if we had a file name change. - * rgerhards, 2009-03-19 - */ - pData->dynCache[pData->iCurrElt]->fd = -1; - } - /* Check for EBADF on TTY's due to vhangup() - * Linux uses EIO instead (mrn 12 May 96) - */ - if((pData->fileType == eTypeTTY || pData->fileType == eTypeCONSOLE) -#ifdef linux - && e == EIO -#else - && e == EBADF -#endif - ) { - pData->fd = open((char*) pData->f_fname, O_WRONLY|O_APPEND|O_NOCTTY|O_CLOEXEC); - if (pData->fd < 0) { - iRet = RS_RET_SUSPENDED; - errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); - } else { - untty(); - goto again; - } - } else { - iRet = RS_RET_SUSPENDED; - errno = e; - errmsg.LogError(0, NO_ERRCODE, "%s", pData->f_fname); - } - } else if (pData->bSyncFile) { - fsync(fd); - } - - pData->poBuf->iBuf = 0; - -finalize_it: - RETiRet; -} -#endif - - /* do the actual write process. This function is to be called once we are ready for writing. * It will do buffered writes and persist data only when the buffer is full. Note that we must * be careful to detect when the file handle changed. @@ -662,7 +532,7 @@ doWrite(instanceData *pData, uchar *pszBuf, int lenBuf) dbgprintf("doWrite, pData->pStrm %p, lenBuf %d\n", pData->pStrm, lenBuf); if(pData->pStrm != NULL){ CHKiRet(strm.Write(pData->pStrm, pszBuf, lenBuf)); - FINALIZE; // TODO: clean up later + FINALIZE; } finalize_it: @@ -780,12 +650,6 @@ CODESTARTparseSelectorAct case '|': case '/': CODE_STD_STRING_REQUESTparseSelectorAct(1) - if(*p == '|') { - pData->fileType = eTypePIPE; - ++p; - } else { - pData->fileType = eTypeFILE; - } CHKiRet(cflineParseFileName(p, (uchar*) pData->f_fname, *ppOMSR, 0, OMSR_NO_RQD_TPL_OPTS, (pszTplName == NULL) ? (uchar*)"RSYSLOG_FileFormat" : pszTplName)); pData->bDynamicName = 0; @@ -808,8 +672,6 @@ CODESTARTparseSelectorAct pData->iIOBufSize = iIOBufSize; if(pData->bDynamicName == 0) { - if(ustrcmp(p, UCHAR_CONSTANT(_PATH_CONSOLE)) == 0) - pData->fileType = eTypeCONSOLE; /* try open and emit error message if not possible. At this stage, we ignore the * return value of prepareFile, this is taken care of in later steps. */ diff --git a/tools/omfwd.c b/tools/omfwd.c index c8fedfc9..2966a5e4 100644 --- a/tools/omfwd.c +++ b/tools/omfwd.c @@ -48,6 +48,7 @@ #endif #include #include "syslogd.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "net.h" diff --git a/tools/omshell.c b/tools/omshell.c index 7b815869..f8a68527 100644 --- a/tools/omshell.c +++ b/tools/omshell.c @@ -38,7 +38,7 @@ #include #include #include -#include "syslogd.h" +#include "conf.h" #include "syslogd-types.h" #include "srUtils.h" #include "omshell.h" diff --git a/tools/omusrmsg.c b/tools/omusrmsg.c index 830bbc87..499a11dd 100644 --- a/tools/omusrmsg.c +++ b/tools/omusrmsg.c @@ -66,7 +66,7 @@ #include "srUtils.h" #include "stringbuf.h" #include "syslogd-types.h" -#include "syslogd.h" +#include "conf.h" #include "omusrmsg.h" #include "module-template.h" #include "errmsg.h" diff --git a/tools/syslogd.c b/tools/syslogd.c index 96f3cb34..05c61059 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -138,6 +138,7 @@ #include "unicode-helper.h" #include "ruleset.h" #include "rule.h" +#include "net.h" #include "vm.h" /* definitions for objects we access */ @@ -221,7 +222,7 @@ static rsRetVal GlobalClassExit(void); #endif #ifndef _PATH_TTY -#define _PATH_TTY "/dev/tty" +# define _PATH_TTY "/dev/tty" #endif static uchar *ConfFile = (uchar*) _PATH_LOGCONF; /* read-only after startup */ @@ -504,7 +505,7 @@ static char **crunch_list(char *list) void untty(void) #ifdef HAVE_SETSID { - if ( !Debug ) { + if(!Debug) { setsid(); } return; @@ -513,18 +514,18 @@ void untty(void) { int i; - if ( !Debug ) { + if(!Debug) { i = open(_PATH_TTY, O_RDWR|O_CLOEXEC); if (i >= 0) { # if !defined(__hpux) - (void) ioctl(i, (int) TIOCNOTTY, (char *)0); + (void) ioctl(i, (int) TIOCNOTTY, NULL); # else /* TODO: we need to implement something for HP UX! -- rgerhards, 2008-03-04 */ /* actually, HP UX should have setsid, so the code directly above should * trigger. So the actual question is why it doesn't do that... */ # endif - (void) close(i); + close(i); } } } diff --git a/tools/syslogd.h b/tools/syslogd.h index 9bcfb2d2..3dfdbe2b 100644 --- a/tools/syslogd.h +++ b/tools/syslogd.h @@ -28,17 +28,7 @@ #include "action.h" #include "linkedlist.h" #include "expr.h" -#include "net.h" /* TODO: remove when you remove isAllowedSender from here! */ - -#ifndef _PATH_CONSOLE -#define _PATH_CONSOLE "/dev/console" -#endif - -void untty(void); -rsRetVal selectorConstruct(ruleset_t **ppThis); -rsRetVal selectorDestruct(void *pVal); -rsRetVal selectorAddList(rule_t *f); /* the following prototypes should go away once we have an input * module interface -- rgerhards, 2007-12-12 */ -- cgit v1.2.3 From 21dafea3ee98d16a8fe93d0d5228939dc259aea7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 12:18:17 +0200 Subject: implemented $OMFileFlushOnTXEnd directive plus some cleanup... --- ChangeLog | 1 + doc/rsyslog_conf_global.html | 7 +++++++ runtime/rsyslog.h | 8 -------- runtime/ruleset.c | 9 ++++++--- tools/omfile.c | 12 +++++++++++- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/ChangeLog b/ChangeLog index 39ca00ec..86d72c9e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -20,6 +20,7 @@ Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? - added configuration commands (see doc for explanations) * $OMFileZipLevel * $OMFileIOBufferSize + * $OMFileFlushOnTXEnd * $MainMsgQueueSyncQueueFiles * $ActionQueueSyncQueueFiles --------------------------------------------------------------------------- diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index e9b1c082..a909b00c 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -193,6 +193,13 @@ supported in order to be compliant to the upcoming new syslog RFC series. of the output file. The higher the number, the better the compression, but also the more CPU is required for zipping.
    • $OMFileIOBufferSize <size_nbr>, default 4k, size of the buffer used to writing output data. The larger the buffer, the potentially better performance is. The default of 4k is quite conservative, it is useful to go up to 64k, and 128K if you used gzip compression (then, even higher sizes may make sense)
      • +
    • $OMFileFlushOnTXEnd <[on/off]>, default off, by default, omfile +writes output using a buffered writer. Disk writes are only done when the buffer is +full. So if an error happens during that write, data is potentially lost. In cases where +this is unacceptable, set $OMFileFlushOnTXEnd to on. Then, data is written at the end +of each transaction (for pre-v5 this means after each log message) and the usual +error recovery thus can handle write errors without data loss. Note that this option +severely reduces the effect of zip compression.
    • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, have a different format that contains the message that is being repeated. Note that only the first "n" characters are included, with n to be at least 80 characters, most diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 28f7bce9..9421ca67 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -125,14 +125,6 @@ typedef enum { FIOP_EREREGEX = 5 /* matches a ERE regular expression? */ } fiop_t; -/* file types (omfile & stream) */ -typedef enum { - eTypeFILE, - eTypeTTY, - eTypeCONSOLE, - eTypePIPE -} filetype_t; - #ifndef _PATH_CONSOLE #define _PATH_CONSOLE "/dev/console" diff --git a/runtime/ruleset.c b/runtime/ruleset.c index a1454275..f9edde8b 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -68,7 +68,7 @@ static rsRetVal keyDestruct(void __attribute__((unused)) *pData) /* ---------- END linked-list key handling functions ---------- */ -/* dirver to iterate over all of this ruleset actions */ +/* driver to iterate over all of this ruleset actions */ typedef struct iterateAllActions_s { rsRetVal (*pFunc)(void*, void*); void *pParam; @@ -81,6 +81,7 @@ DEFFUNC_llExecFunc(doIterateRulesetActions) iRet = rule.IterateAllActions(pRule, pMyParam->pFunc, pMyParam->pParam); RETiRet; } +#if 0 /* iterate over all actions of THIS rule set. */ static rsRetVal @@ -99,7 +100,7 @@ finalize_it: } -/* dirver to iterate over all actions */ +/* driver to iterate over all actions */ DEFFUNC_llExecFunc(doIterateAllActions) { DEFiRet; @@ -108,6 +109,7 @@ DEFFUNC_llExecFunc(doIterateAllActions) iRet = iterateRulesetAllActions(pThis, pMyParam->pFunc, pMyParam->pParam); RETiRet; } +#endif /* iterate over ALL actions present in the WHOLE system. * this is often needed, for example when HUP processing * must be done or a shutdown is pending. @@ -121,7 +123,8 @@ iterateAllActions(rsRetVal (*pFunc)(void*, void*), void* pParam) params.pFunc = pFunc; params.pParam = pParam; - CHKiRet(llExecFunc(&llRulesets, doIterateAllActions, ¶ms)); + //CHKiRet(llExecFunc(&llRulesets, doIterateAllActions, ¶ms)); + CHKiRet(llExecFunc(&llRulesets, doIterateRulesetActions, ¶ms)); finalize_it: RETiRet; diff --git a/tools/omfile.c b/tools/omfile.c index 6377268d..6d9eb096 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -101,6 +101,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs; /* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ +static bool bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */ static int iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static uchar *pszTplName = NULL; /* name of the default template to use */ /* end globals for default values */ @@ -131,6 +132,7 @@ typedef struct _instanceData { uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ int iZipLevel; /* zip mode to use for this selector */ int iIOBufSize; /* size of associated io buffer */ + bool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ } instanceData; @@ -595,7 +597,12 @@ ENDtryResume BEGINdoAction CODESTARTdoAction DBGPRINTF("file to log to: %s\n", pData->f_fname); - iRet = writeFile(ppString, iMsgOpts, pData); + CHKiRet(writeFile(ppString, iMsgOpts, pData)); + if(pData->bFlushOnTXEnd) { + /* TODO v5: do this in endTransaction only! */ + CHKiRet(strm.Flush(pData->pStrm)); + } +finalize_it: ENDdoAction @@ -669,6 +676,7 @@ CODESTARTparseSelectorAct pData->dirUID = dirUID; pData->dirGID = dirGID; pData->iZipLevel = iZipLevel; + pData->bFlushOnTXEnd = bFlushOnTXEnd; pData->iIOBufSize = iIOBufSize; if(pData->bDynamicName == 0) { @@ -702,6 +710,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bCreateDirs = 1; bEnableSync = 0; iZipLevel = 0; + bFlushOnTXEnd = 0; iIOBufSize = IOBUF_DFLT_SIZE; if(pszTplName != NULL) { free(pszTplName); @@ -751,6 +760,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushontxend", 0, eCmdHdlrBinary, NULL, &bFlushOnTXEnd, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirgroup", 0, eCmdHdlrGID, NULL, &dirGID, STD_LOADABLE_MODULE_ID)); -- cgit v1.2.3 From ca0ddc30a3edce02a440904a01f0b866c0f82b5a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 15:31:08 +0200 Subject: completed multi-ruleset core support ... as well as added multi-ruleset support for imtcp --- dirty.h | 1 - doc/rsyslog_conf_global.html | 10 ++++-- plugins/imtcp/imtcp.c | 43 ++++++++++++++++++----- runtime/cfsysline.c | 6 ++-- runtime/conf.c | 2 +- runtime/msg.c | 13 +++++-- runtime/msg.h | 2 ++ runtime/obj.c | 4 +-- runtime/rsyslog.c | 20 +++++------ runtime/ruleset.c | 84 ++++++++++++++++++++++++++++++++++++++++++-- runtime/ruleset.h | 6 +++- tcps_sess.c | 1 + tcpsrv.c | 17 +++++++++ tcpsrv.h | 3 ++ tools/syslogd.c | 75 ++++++++++++++++++++++++--------------- 15 files changed, 226 insertions(+), 61 deletions(-) diff --git a/dirty.h b/dirty.h index 7ae57d09..513886b5 100644 --- a/dirty.h +++ b/dirty.h @@ -53,7 +53,6 @@ extern int bReduceRepeatMsgs; extern int bDropTrailingLF; extern uchar cCCEscapeChar; extern int bEscapeCCOnRcv; -extern ruleset_t *pCurrRuleset; #ifdef USE_NETZIP /* config param: minimum message size to try compression. The smaller * the message, the less likely is any compression gain. We check for diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index a909b00c..d58bcac0 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -108,6 +108,9 @@ that no rebind is done. This directive is useful for use with load-balancers.$DefaultNetstreamDriver <drivername>, the default network stream driver to use. Defaults to ptcp.$DefaultNetstreamDriverCAFile </path/to/cafile.pem>
    • $DefaultNetstreamDriverCertFile </path/to/certfile.pem>
    • $DefaultNetstreamDriverKeyFile </path/to/keyfile.pem>
      • +
    • $DefaultRuleset name - changes the default ruleset for unbound inputs to +the provided name (the default default ruleset is named +"RSYSLOG_DefaultRuleset").
    • $CreateDirs [on/off] - create directories on an as-needed basis
    • $DirCreateMode
    • $DirGroup
      • @@ -208,8 +211,11 @@ line is that n is large enough to get a good idea which message was repeated but large enough for the whole message. (Introduced with 4.1.5). Once set, it affects all following actions.
    • $RepeatedMsgReduction
    • $ResetConfigVariables
      • -
    • $Ruleset name - starts a new ruleset. All following actions belong to -that new rule set.
      • +
    • $Ruleset name - starts a new ruleset or switches back to one already defined. +All following actions belong to that new rule set. +the name does not yet exist, it is created. To swith back to rsyslog's +default ruleset, specify "RSYSLOG_DefaultRuleset") as the name. +All following actions belong to that new rule set.
    • $OptimizeForUniprocessor [on/off] - turns on optimizatons which lead to better performance on uniprocessors. If you run on multicore-machiens, turning this off lessens CPU load. The default may change as uniprocessor systems become less common. [available since 4.1.0]
      • diff --git a/plugins/imtcp/imtcp.c b/plugins/imtcp/imtcp.c index 84e660bc..e1f513c8 100644 --- a/plugins/imtcp/imtcp.c +++ b/plugins/imtcp/imtcp.c @@ -61,6 +61,7 @@ #include "netstrm.h" #include "errmsg.h" #include "tcpsrv.h" +#include "ruleset.h" #include "net.h" /* for permittedPeers, may be removed when this is removed */ MODULE_TYPE_INPUT @@ -72,6 +73,7 @@ DEFobjCurrIf(tcps_sess) DEFobjCurrIf(net) DEFobjCurrIf(netstrm) DEFobjCurrIf(errmsg) +DEFobjCurrIf(ruleset) /* Module static data */ static tcpsrv_t *pOurTcpsrv = NULL; /* our TCP server(listener) TODO: change for multiple instances */ @@ -84,6 +86,7 @@ static int iStrmDrvrMode = 0; /* mode for stream driver, driver-dependent (0 mos static int iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; /* addtl frame delimiter, e.g. for netscreen, default none */ static uchar *pszStrmDrvrAuthMode = NULL; /* authentication mode to use */ static uchar *pszInputName = NULL; /* value for inputname property, NULL is OK and handled by core engine */ +static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ /* callbacks */ @@ -157,6 +160,27 @@ finalize_it: } +/* accept a new ruleset to bind. Checks if it exists and complains, if not */ +static rsRetVal setRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + ruleset_t *pRuleset; + rsRetVal localRet; + DEFiRet; + + localRet = ruleset.GetRuleset(&pRuleset, pszName); + if(localRet == RS_RET_NOT_FOUND) { + errmsg.LogError(0, NO_ERRCODE, "error: ruleset '%s' not found - ignored", pszName); + } + CHKiRet(localRet); + pBindRuleset = pRuleset; + DBGPRINTF("imtcp current bind ruleset %p: '%s'\n", pRuleset, pszName); + +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} + + static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVal) { DEFiRet; @@ -180,7 +204,8 @@ static rsRetVal addTCPListener(void __attribute__((unused)) *pVal, uchar *pNewVa } } - /* initialized, now add socket */ + /* initialized, now add socket and listener params */ + CHKiRet(tcpsrv.SetRuleset(pOurTcpsrv, pBindRuleset)); CHKiRet(tcpsrv.SetInputName(pOurTcpsrv, pszInputName == NULL ? UCHAR_CONSTANT("imtcp") : pszInputName)); tcpsrv.configureTCPListen(pOurTcpsrv, pNewVal); @@ -240,6 +265,7 @@ CODESTARTmodExit objRelease(tcps_sess, LM_TCPSRV_FILENAME); objRelease(tcpsrv, LM_TCPSRV_FILENAME); objRelease(errmsg, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); ENDmodExit @@ -249,14 +275,10 @@ resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unus iTCPSessMax = 200; iStrmDrvrMode = 0; iAddtlFrameDelim = TCPSRV_NO_ADDTL_DELIMITER; - if(pszInputName != NULL) { - free(pszInputName); - pszInputName = NULL; - } - if(pszStrmDrvrAuthMode != NULL) { - free(pszStrmDrvrAuthMode); - pszStrmDrvrAuthMode = NULL; - } + free(pszInputName); + pszInputName = NULL; + free(pszStrmDrvrAuthMode); + pszStrmDrvrAuthMode = NULL; return RS_RET_OK; } @@ -279,6 +301,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(tcps_sess, LM_TCPSRV_FILENAME)); CHKiRet(objUse(tcpsrv, LM_TCPSRV_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverrun"), 0, eCmdHdlrGetWord, @@ -295,6 +318,8 @@ CODEmodInit_QueryRegCFSLineHdlr NULL, &iAddtlFrameDelim, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverinputname"), 0, eCmdHdlrGetWord, NULL, &pszInputName, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("inputtcpserverbindruleset"), 0, + eCmdHdlrGetWord, setRuleset, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("resetconfigvariables"), 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, STD_LOADABLE_MODULE_ID)); ENDmodInit diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index e1e4a6a4..c39e54f6 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -814,7 +814,7 @@ rsRetVal regCfSysLineHdlr(uchar *pCmdName, int bChainingPermitted, ecslCmdHdrlTy CHKiRet(cslcConstruct(&pThis, bChainingPermitted)); CHKiRet_Hdlr(cslcAddHdlr(pThis, eType, pHdlr, pData, pOwnerCookie)) { cslcDestruct(pThis); - goto finalize_it; + FINALIZE; } /* important: add to list, AFTER everything else is OK. Else * we mess up things in the error case. @@ -825,7 +825,7 @@ rsRetVal regCfSysLineHdlr(uchar *pCmdName, int bChainingPermitted, ecslCmdHdrlTy } CHKiRet_Hdlr(llAppend(&llCmdList, pMyCmdName, (void*) pThis)) { cslcDestruct(pThis); - goto finalize_it; + FINALIZE; } } else { /* command already exists, are we allowed to chain? */ @@ -834,7 +834,7 @@ rsRetVal regCfSysLineHdlr(uchar *pCmdName, int bChainingPermitted, ecslCmdHdrlTy } CHKiRet_Hdlr(cslcAddHdlr(pThis, eType, pHdlr, pData, pOwnerCookie)) { cslcDestruct(pThis); - goto finalize_it; + FINALIZE; } } diff --git a/runtime/conf.c b/runtime/conf.c index 81b6c081..dbc54fd4 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -1152,7 +1152,7 @@ cflineClassic(uchar *p, rule_t **ppRule) CHKiRet(ruleset.AddRule(rule.GetAssRuleset(*ppRule), ppRule)); } CHKiRet(rule.Construct(ppRule)); /* create "fresh" selector */ - CHKiRet(rule.SetAssRuleset(*ppRule, pCurrRuleset)); /* create "fresh" selector */ + CHKiRet(rule.SetAssRuleset(*ppRule, ruleset.GetCurrent())); /* create "fresh" selector */ CHKiRet(rule.ConstructFinalize(*ppRule)); /* create "fresh" selector */ CHKiRet(cflineDoFilter(&p, *ppRule)); /* pull filters */ } diff --git a/runtime/msg.c b/runtime/msg.c index dbc3c779..10f283aa 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -46,6 +46,7 @@ #include "regexp.h" #include "atomic.h" #include "unicode-helper.h" +#include "ruleset.h" /* static data */ DEFobjStaticHelpers @@ -1166,13 +1167,21 @@ void MsgAssignTAG(msg_t *pMsg, uchar *pBuf) } +/* rgerhards 2009-06-12: set associated ruleset + */ +void MsgSetRuleset(msg_t *pMsg, ruleset_t *pRuleset) +{ + assert(pMsg != NULL); + pMsg->pRuleset = pRuleset; +} + + /* rgerhards 2004-11-16: set TAG in msg object */ void MsgSetTAG(msg_t *pMsg, char* pszTAG) { assert(pMsg != NULL); - if(pMsg->pszTAG != NULL) - free(pMsg->pszTAG); + free(pMsg->pszTAG); pMsg->iLenTAG = strlen(pszTAG); if((pMsg->pszTAG = malloc(pMsg->iLenTAG + 1)) != NULL) memcpy(pMsg->pszTAG, pszTAG, pMsg->iLenTAG + 1); diff --git a/runtime/msg.h b/runtime/msg.h index a14f6b15..b42f641f 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -120,6 +120,7 @@ short bDoLock; /* use the mutex? */ char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ + ruleset_t *pRuleset; /* ruleset to be used for processing this message */ }; @@ -167,6 +168,7 @@ char *getPROCID(msg_t *pM); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); void MsgSetTAG(msg_t *pMsg, char* pszTAG); +void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); char *getTAG(msg_t *pM); int getHOSTNAMELen(msg_t *pM); diff --git a/runtime/obj.c b/runtime/obj.c index 8b9c9c83..f38b1d7f 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -1279,8 +1279,8 @@ objClassExit(void) /* TODO: implement the class exits! */ #if 0 - cfsyslineInit(pModInfo); - varClassInit(pModInfo); + cfsyslineExit(pModInfo); + varClassExit(pModInfo); #endif errmsgClassExit(); moduleClassExit(); diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 3496bb0d..6f732f0e 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -152,12 +152,10 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(datetimeClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "wti"; - CHKiRet(wtiClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "wtp"; - CHKiRet(wtpClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "queue"; - CHKiRet(qqueueClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "ctok_token"; + CHKiRet(ctok_tokenClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "ctok"; + CHKiRet(ctokClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "vmstk"; CHKiRet(vmstkClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "sysvar"; @@ -168,16 +166,18 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(vmopClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "vmprg"; CHKiRet(vmprgClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "ctok_token"; - CHKiRet(ctok_tokenClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "ctok"; - CHKiRet(ctokClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "expr"; CHKiRet(exprClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "rule"; CHKiRet(ruleClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "ruleset"; CHKiRet(rulesetClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "wti"; + CHKiRet(wtiClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "wtp"; + CHKiRet(wtpClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "queue"; + CHKiRet(qqueueClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "conf"; CHKiRet(confClassInit(NULL)); diff --git a/runtime/ruleset.c b/runtime/ruleset.c index f9edde8b..93d40e24 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -40,6 +40,7 @@ #include "rsyslog.h" #include "obj.h" +#include "msg.h" #include "ruleset.h" #include "rule.h" #include "errmsg.h" @@ -53,6 +54,8 @@ DEFobjCurrIf(errmsg) DEFobjCurrIf(rule) linkedList_t llRulesets; /* this is NOT a pointer - no typo here ;) */ +ruleset_t *pCurrRuleset = NULL; /* currently "active" ruleset */ +ruleset_t *pDfltRuleset = NULL; /* currentl default ruleset, e.g. for binding to actions which have no other */ /* ---------- linked-list key handling functions ---------- */ @@ -147,12 +150,15 @@ DEFFUNC_llExecFunc(processMsgDoRules) * rgerhards, 2005-10-13 */ static rsRetVal -processMsg(ruleset_t *pThis, msg_t *pMsg) +processMsg(msg_t *pMsg) { + ruleset_t *pThis; DEFiRet; - ISOBJ_TYPE_assert(pThis, ruleset); assert(pMsg != NULL); + pThis = (pMsg->pRuleset == NULL) ? pDfltRuleset : pMsg->pRuleset; + ISOBJ_TYPE_assert(pThis, ruleset); + CHKiRet(llExecFunc(&pThis->llRules, processMsgDoRules, pMsg)); finalize_it: @@ -200,6 +206,69 @@ finalize_it: } +/* get current ruleset + * We use a non-standard calling interface, as nothing can go wrong and it + * is really much more natural to return the pointer directly. + */ +static ruleset_t* +GetCurrent(void) +{ + return pCurrRuleset; +} + + +/* Find the ruleset with the given name and return a pointer to its object. + */ +static rsRetVal +GetRuleset(ruleset_t **ppRuleset, uchar *pszName) +{ + DEFiRet; + assert(ppRuleset != NULL); + assert(pszName != NULL); + + CHKiRet(llFind(&llRulesets, pszName, (void*) ppRuleset)); + +finalize_it: + RETiRet; +} + + +/* Set a new default rule set. If the default can not be found, no change happens. + */ +static rsRetVal +SetDefaultRuleset(uchar *pszName) +{ + ruleset_t *pRuleset; + DEFiRet; + assert(pszName != NULL); + + CHKiRet(GetRuleset(&pRuleset, pszName)); + pDfltRuleset = pRuleset; + dbgprintf("default rule set changed to %p: '%s'\n", pRuleset, pszName); + +finalize_it: + RETiRet; +} + + +/* Set a new current rule set. If the ruleset can not be found, no change happens. + */ +static rsRetVal +SetCurrRuleset(uchar *pszName) +{ + ruleset_t *pRuleset; + DEFiRet; + assert(pszName != NULL); + + CHKiRet(GetRuleset(&pRuleset, pszName)); + pCurrRuleset = pRuleset; + dbgprintf("current rule set changed to %p: '%s'\n", pRuleset, pszName); + +finalize_it: + RETiRet; +} + + /* destructor we need to destruct rules inside our linked list contents. */ static rsRetVal @@ -237,6 +306,13 @@ rulesetConstructFinalize(ruleset_t *pThis) CHKmalloc(keyName = ustrdup(pThis->pszName)); CHKiRet(llAppend(&llRulesets, keyName, pThis)); + /* this now also is the new current ruleset */ + pCurrRuleset = pThis; + + /* and also the default, if so far none has been set */ + if(pDfltRuleset == NULL) + pDfltRuleset = pThis; + finalize_it: RETiRet; } @@ -336,6 +412,10 @@ CODESTARTobjQueryInterface(ruleset) pIf->ProcessMsg = processMsg; pIf->SetName = setName; pIf->DebugPrintAll = debugPrintAll; + pIf->GetCurrent = GetCurrent; + pIf->GetRuleset = GetRuleset; + pIf->SetDefaultRuleset = SetDefaultRuleset; + pIf->SetCurrRuleset = SetCurrRuleset; finalize_it: ENDobjQueryInterface(ruleset) diff --git a/runtime/ruleset.h b/runtime/ruleset.h index b609e6b3..32571687 100644 --- a/runtime/ruleset.h +++ b/runtime/ruleset.h @@ -44,8 +44,12 @@ BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ rsRetVal (*IterateAllActions)(rsRetVal (*pFunc)(void*, void*), void* pParam); rsRetVal (*DestructAllActions)(void); rsRetVal (*AddRule)(ruleset_t *pThis, rule_t **ppRule); - rsRetVal (*ProcessMsg)(ruleset_t *pThis, msg_t *pMsg); rsRetVal (*SetName)(ruleset_t *pThis, uchar *pszName); + rsRetVal (*ProcessMsg)(msg_t *pMsg); + rsRetVal (*GetRuleset)(ruleset_t **ppThis, uchar*); + rsRetVal (*SetDefaultRuleset)(uchar*); + rsRetVal (*SetCurrRuleset)(uchar*); + ruleset_t* (*GetCurrent)(void); ENDinterface(ruleset) #define rulesetCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/tcps_sess.c b/tcps_sess.c index 62d51f66..d6bcd51b 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -254,6 +254,7 @@ defaultDoSubmitMessage(tcps_sess_t *pThis) pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; MsgSetRcvFrom(pMsg, pThis->fromHost); + MsgSetRuleset(pMsg, pThis->pLstnInfo->pRuleset); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); CHKiRet(submitMsg(pMsg)); diff --git a/tcpsrv.c b/tcpsrv.c index 3516b2e3..95409d2f 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -69,6 +69,7 @@ #include "netstrm.h" #include "nssel.h" #include "errmsg.h" +#include "ruleset.h" #include "unicode-helper.h" MODULE_TYPE_LIB @@ -81,6 +82,7 @@ MODULE_TYPE_LIB DEFobjStaticHelpers DEFobjCurrIf(conf) DEFobjCurrIf(glbl) +DEFobjCurrIf(ruleset) DEFobjCurrIf(tcps_sess) DEFobjCurrIf(errmsg) DEFobjCurrIf(net) @@ -104,6 +106,8 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) CHKmalloc(pEntry = malloc(sizeof(tcpLstnPortList_t))); pEntry->pszPort = pszPort; pEntry->pSrv = pThis; +RUNLOG_VAR("%p", pThis->pRuleset); + pEntry->pRuleset = pThis->pRuleset; CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); /* and add to list */ @@ -755,6 +759,16 @@ finalize_it: } +/* Set the ruleset (ptr) to use */ +static rsRetVal +SetRuleset(tcpsrv_t *pThis, ruleset_t *pRuleset) +{ + DEFiRet; + pThis->pRuleset = pRuleset; + RETiRet; +} + + /* here follows a number of methods that shuffle authentication settings down * to the drivers. Drivers not supporting these settings may return an error * state. @@ -855,6 +869,7 @@ CODESTARTobjQueryInterface(tcpsrv) pIf->SetCBOnRegularClose = SetCBOnRegularClose; pIf->SetCBOnErrClose = SetCBOnErrClose; pIf->SetOnMsgReceive = SetOnMsgReceive; + pIf->SetRuleset = SetRuleset; finalize_it: ENDobjQueryInterface(tcpsrv) @@ -868,6 +883,7 @@ CODESTARTObjClassExit(tcpsrv) /* release objects we no longer need */ objRelease(tcps_sess, DONT_LOAD_LIB); objRelease(conf, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); objRelease(netstrms, DONT_LOAD_LIB); @@ -891,6 +907,7 @@ BEGINObjClassInit(tcpsrv, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE CHKiRet(objUse(tcps_sess, DONT_LOAD_LIB)); CHKiRet(objUse(conf, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, tcpsrvDebugPrint); diff --git a/tcpsrv.h b/tcpsrv.h index 2d174ce0..a6076e67 100644 --- a/tcpsrv.h +++ b/tcpsrv.h @@ -38,6 +38,7 @@ struct tcpLstnPortList_s { uchar *pszPort; /**< the ports the listener shall listen on */ uchar *pszInputName; /**< value to be used as input name */ tcpsrv_t *pSrv; /**< pointer to higher-level server instance */ + ruleset_t *pRuleset; /**< associated ruleset */ tcpLstnPortList_t *pNext; /**< next port or NULL */ }; @@ -50,6 +51,7 @@ struct tcpsrv_s { int iDrvrMode; /**< mode of the stream driver to use */ uchar *pszDrvrAuthMode; /**< auth mode of the stream driver to use */ uchar *pszInputName; /**< value to be used as input name */ + ruleset_t *pRuleset; /**< ruleset to bind to */ permittedPeers_t *pPermPeers;/**< driver's permitted peers */ int iLstnMax; /**< max nbr of listeners currently supported */ netstrm_t **ppLstn; /**< our netstream listners */ @@ -107,6 +109,7 @@ BEGINinterface(tcpsrv) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetSessMax)(tcpsrv_t *pThis, int iMaxSess); /* 2009-04-09 */ /* added v6 */ rsRetVal (*SetOnMsgReceive)(tcpsrv_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar*, int)); /* 2009-05-24 */ + rsRetVal (*SetRuleset)(tcpsrv_t *pThis, ruleset_t*); /* 2009-06-12 */ ENDinterface(tcpsrv) #define tcpsrvCURR_IF_VERSION 6 /* increment whenever you change the interface structure! */ /* change for v4: diff --git a/tools/syslogd.c b/tools/syslogd.c index 05c61059..99bf281d 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -249,8 +249,6 @@ int repeatinterval[2] = { 30, 60 }; /* # of secs before flush */ #define LIST_DELIMITER ':' /* delimiter between two hosts */ -ruleset_t *pCurrRuleset; /* ruleset that is currently being processed */ - static pid_t ppid; /* This is a quick and dirty hack used for spliting main/startup thread */ typedef struct legacyOptsLL_s { @@ -952,7 +950,7 @@ msgConsumer(void __attribute__((unused)) *notNeeded, void *pUsr) if((pMsg->msgFlags & NEEDS_PARSING) != 0) { parseMsg(pMsg); } - ruleset.ProcessMsg(pCurrRuleset, pMsg); + ruleset.ProcessMsg(pMsg); msgDestruct(&pMsg); RETiRet; @@ -2172,6 +2170,7 @@ init(void) rsRetVal localRet; int iNbrActions; int bHadConfigErr = 0; + ruleset_t *pRuleset; char cbuf[BUFSIZ]; char bufStartUpMsg[512]; struct sigaction sigAct; @@ -2217,10 +2216,10 @@ init(void) conf.ReInitConf(); - // TODO: move to the right place - ruleset.Construct(&pCurrRuleset); - ruleset.SetName(pCurrRuleset, UCHAR_CONSTANT("RSYSLOG_DefaultRuleset")); - ruleset.ConstructFinalize(pCurrRuleset); + /* construct the default ruleset */ + ruleset.Construct(&pRuleset); + ruleset.SetName(pRuleset, UCHAR_CONSTANT("RSYSLOG_DefaultRuleset")); + ruleset.ConstructFinalize(pRuleset); /* open the configuration file */ localRet = conf.processConfFile(ConfFile); @@ -2259,7 +2258,7 @@ init(void) } else { dbgprintf("error %d obtaining controlling terminal, not using that emergency rule\n", errno); } - ruleset.AddRule(pCurrRuleset, &pRule); + ruleset.AddRule(ruleset.GetCurrent(), &pRule); } legacyOptsHook(); @@ -2398,25 +2397,45 @@ finalize_it: } -/* Begin a new rule set. The new rule set is created, and all rules that now - * follow go into that rule set. - * TODO: we may later add the capability to switch back to an already existing - * rule set. - * NOTE: pCurrRuleset is NOT desructed and must not be! The ruleset class keeps - * a list of all known rule sets, and can destruct them at the end of execution. - * pCurrRuleset is just a shortcut so that "everyone" knows which ruleset to - * extend. - * TODO: A problem with this function is the way config lines are processed. The rule - * is actually only written when the next rule is completely read. That way, this - * (past) rule goes into the wrong (new) ruleset. I need to see how to fix this best... - * rgerhards, 2009-06-10 +/* Switch the default ruleset (that, what servcies bind to if nothing specific + * is specified). + * rgerhards, 2009-06-12 + */ +static rsRetVal +setDefaultRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + DEFiRet; + + CHKiRet(ruleset.SetDefaultRuleset(pszName)); + +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} + + +/* Switch to either an already existing rule set or start a new one. The + * named rule set becomes the new "current" rule set (what means that new + * actions are added to it). + * rgerhards, 2009-06-12 */ -static rsRetVal beginNewRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +static rsRetVal +setCurrRuleset(void __attribute__((unused)) *pVal, uchar *pszName) { + ruleset_t *pRuleset; + rsRetVal localRet; DEFiRet; - CHKiRet(ruleset.Construct(&pCurrRuleset)); - CHKiRet(ruleset.SetName(pCurrRuleset, pszName)); - CHKiRet(ruleset.ConstructFinalize(pCurrRuleset)); + + localRet = ruleset.SetCurrRuleset(pszName); + + if(localRet == RS_RET_NOT_FOUND) { + DBGPRINTF("begin new current rule set '%s'\n", pszName); + CHKiRet(ruleset.Construct(&pRuleset)); + CHKiRet(ruleset.SetName(pRuleset, pszName)); + CHKiRet(ruleset.ConstructFinalize(pRuleset)); + } else { + ABORT_FINALIZE(localRet); + } finalize_it: free(pszName); /* no longer needed */ @@ -2658,7 +2677,7 @@ static rsRetVal loadBuildInModules(void) } /* dirty, but this must be for the time being: the usrmsg module must always be - * loaded as last module. This is because it processes any time of action selector. + * loaded as last module. This is because it processes any type of action selector. * If we load it before other modules, these others will never have a chance of * working with the config file. We may change that implementation so that a user name * must start with an alnum, that would definitely help (but would it break backwards @@ -2666,8 +2685,7 @@ static rsRetVal loadBuildInModules(void) * User names now must begin with: * [a-zA-Z0-9_.] */ - if((iRet = module.doModInit(modInitUsrMsg, (uchar*) "builtin-usrmsg", NULL)) != RS_RET_OK) - RETiRet; + CHKiRet(module.doModInit(modInitUsrMsg, (uchar*) "builtin-usrmsg", NULL)); /* ok, initialization of the command handler probably does not 100% belong right in * this space here. However, with the current design, this is actually quite a good @@ -2677,7 +2695,8 @@ static rsRetVal loadBuildInModules(void) * This, I think, is the right thing to do. -- rgerhards, 2007-07-31 */ CHKiRet(regCfSysLineHdlr((uchar *)"actionresumeretrycount", 0, eCmdHdlrInt, NULL, &glbliActionResumeRetryCount, NULL)); - CHKiRet(regCfSysLineHdlr((uchar *)"ruleset", 0, eCmdHdlrGetWord, beginNewRuleset, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"defaultruleset", 0, eCmdHdlrGetWord, setDefaultRuleset, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"ruleset", 0, eCmdHdlrGetWord, setCurrRuleset, NULL, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuefilename", 0, eCmdHdlrGetWord, NULL, &pszMainMsgQFName, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuesize", 0, eCmdHdlrInt, NULL, &iMainMsgQueueSize, NULL)); CHKiRet(regCfSysLineHdlr((uchar *)"mainmsgqueuehighwatermark", 0, eCmdHdlrInt, NULL, &iMainMsgQHighWtrMark, NULL)); -- cgit v1.2.3 From b5ccdf06724d309b777d4dd38a455ed2ef0318a0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 15:59:50 +0200 Subject: performance-enhanced imtcp ...by now using lowres time and thus saving many time() calls. This needs some performance testing and must be made configurable if it works out. --- plugins/imudp/imudp.c | 31 ++++++++++++++++++++++++++++++- tcps_sess.c | 12 ++++++++++-- tcpsrv.c | 1 - 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index a486b818..50e8efaf 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -68,6 +68,7 @@ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a * it so that we can check available memory in willRun() and request * termination if we can not get it. -- rgerhards, 2007-12-27 */ +// TODO: static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ #define TIME_REQUERY_DFLT 2 static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ @@ -97,7 +98,7 @@ static rsRetVal addListner(void __attribute__((unused)) *pVal, uchar *pNewVal) else bindAddr = pszBindAddr; - dbgprintf("Trying to open syslog UDP ports at %s:%s.\n", + DBGPRINTF("Trying to open syslog UDP ports at %s:%s.\n", (bindAddr == NULL) ? (uchar*)"*" : bindAddr, pNewVal); newSocks = net.create_udp_socket(bindAddr, (pNewVal == NULL || *pNewVal == '\0') ? (uchar*) "514" : pNewVal, 1); @@ -137,6 +138,30 @@ finalize_it: } +#if 0 /* TODO: implement when tehre is time, requires restructure of socket array! */ +/* accept a new ruleset to bind. Checks if it exists and complains, if not */ +static rsRetVal +setRuleset(void __attribute__((unused)) *pVal, uchar *pszName) +{ + ruleset_t *pRuleset; + rsRetVal localRet; + DEFiRet; + + localRet = ruleset.GetRuleset(&pRuleset, pszName); + if(localRet == RS_RET_NOT_FOUND) { + errmsg.LogError(0, NO_ERRCODE, "error: ruleset '%s' not found - ignored", pszName); + } + CHKiRet(localRet); + pBindRuleset = pRuleset; + DBGPRINTF("imudp current bind ruleset %p: '%s'\n", pRuleset, pszName); + +finalize_it: + free(pszName); /* no longer needed */ + RETiRet; +} +#endif + + /* This function is a helper to runInput. I have extracted it * from the main loop just so that we do not have that large amount of code * in a single place. This function takes a socket and pulls messages from @@ -386,6 +411,10 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(net, LM_NET_FILENAME)); /* register config file handlers */ + /* TODO: add - but this requires more changes, no time right now... + CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserverbindruleset", 0, eCmdHdlrGetWord, + setRuleset, NULL, STD_LOADABLE_MODULE_ID)); + */ CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserverrun", 0, eCmdHdlrGetWord, addListner, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"udpserveraddress", 0, eCmdHdlrGetWord, diff --git a/tcps_sess.c b/tcps_sess.c index d6bcd51b..8ba5cbec 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -56,6 +56,11 @@ DEFobjCurrIf(datetime) static int iMaxLine; /* maximum size of a single message */ +#define TIME_REQUERY_DFLT 16 // TODO change back! 2 +static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ +static int iNbrTimeUsed = 0; /* how often has previous time been used so far? */ + + /* forward definitions */ static rsRetVal Close(tcps_sess_t *pThis); @@ -240,9 +245,10 @@ defaultDoSubmitMessage(tcps_sess_t *pThis) FINALIZE; } - //TODO: if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { + if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { +RUNLOG_STR("XXX: quering time!"); datetime.getCurrTime(&stTime, &ttGenTime); - //} + } /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); /* first trim the buffer to what we have actually received */ @@ -307,6 +313,7 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); + iNbrTimeUsed = 0; /* full time query */ defaultDoSubmitMessage(pThis); } @@ -451,6 +458,7 @@ DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) /* We now copy the message to the session buffer. */ pEnd = pData + iLen; /* this is one off, which is intensional */ + iNbrTimeUsed = 0; /* full time query */ while(pData < pEnd) { CHKiRet(processDataRcvd(pThis, *pData++)); } diff --git a/tcpsrv.c b/tcpsrv.c index 95409d2f..5e2bd530 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -106,7 +106,6 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) CHKmalloc(pEntry = malloc(sizeof(tcpLstnPortList_t))); pEntry->pszPort = pszPort; pEntry->pSrv = pThis; -RUNLOG_VAR("%p", pThis->pRuleset); pEntry->pRuleset = pThis->pRuleset; CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); -- cgit v1.2.3 From 057c9b11fca013205877e15dd851927a11aa058b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 16:30:39 +0200 Subject: fixed a regression from past commit & more performance enhancement a larger buffer is a cheap, yet useful, enhancement... ;) --- tcps_sess.c | 5 +++-- tcpsrv.c | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tcps_sess.c b/tcps_sess.c index 8ba5cbec..f01114d6 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -234,8 +234,8 @@ static rsRetVal defaultDoSubmitMessage(tcps_sess_t *pThis) { msg_t *pMsg; - struct syslogTime stTime; - time_t ttGenTime; + static struct syslogTime stTime; /* the static vars are currently OK (single input thread!) */ + static time_t ttGenTime; DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -245,6 +245,7 @@ defaultDoSubmitMessage(tcps_sess_t *pThis) FINALIZE; } +RUNLOG_VAR("%ld", ttGenTime); if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { RUNLOG_STR("XXX: quering time!"); datetime.getCurrTime(&stTime, &ttGenTime); diff --git a/tcpsrv.c b/tcpsrv.c index 5e2bd530..21699bca 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -513,7 +513,7 @@ Run(tcpsrv_t *pThis) while(nfds && iTCPSess != -1) { CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { - char buf[8*1024]; /* reception buffer - may hold a partial or multiple messages */ + char buf[64*1024]; /* reception buffer - may hold a partial or multiple messages */ dbgprintf("netstream %p with new data\n", pThis->pSessions[iTCPSess]->pStrm); /* Receive message */ -- cgit v1.2.3 From 1f874c220860d3a19fb6cfb60f0902a08639f6ab Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 12 Jun 2009 17:46:16 +0200 Subject: fixed small bug in testbench --- tcps_sess.c | 2 -- tests/diskqueue-fsync.sh | 1 + tests/diskqueue.sh | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tcps_sess.c b/tcps_sess.c index f01114d6..1446df10 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -245,9 +245,7 @@ defaultDoSubmitMessage(tcps_sess_t *pThis) FINALIZE; } -RUNLOG_VAR("%ld", ttGenTime); if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { -RUNLOG_STR("XXX: quering time!"); datetime.getCurrTime(&stTime, &ttGenTime); } /* we now create our own message object and submit it to the queue */ diff --git a/tests/diskqueue-fsync.sh b/tests/diskqueue-fsync.sh index 45fb864c..0282202d 100755 --- a/tests/diskqueue-fsync.sh +++ b/tests/diskqueue-fsync.sh @@ -11,5 +11,6 @@ source $srcdir/diag.sh startup diskqueue-fsync.conf # 1000 messages should be enough - the disk fsync test is very slow! source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 1000 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown source $srcdir/diag.sh seq-check 0 999 source $srcdir/diag.sh exit diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index 2fe31db9..668a1224 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -11,5 +11,6 @@ source $srcdir/diag.sh startup diskqueue.conf # 20000 messages should be enough - the disk test is slow enough ;) source $srcdir/diag.sh tcpflood 127.0.0.1 13514 1 20000 source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh wait-shutdown source $srcdir/diag.sh seq-check 0 19999 source $srcdir/diag.sh exit -- cgit v1.2.3 From 16ecb90c3ac88bb2261c31c990d88f97f1a1b32f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 15 Jun 2009 13:44:51 +0200 Subject: omfile buffers are now synchronized after inactivity This is the first shot at this functionality. Currently, we run off a fixed counter in the rsyslogd mainloop, which needs to be restructured. But this code works, so it is a good time for a commit. --- runtime/Makefile.am | 2 + runtime/apc.c | 400 ++++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/apc.h | 56 ++++++++ runtime/obj.c | 2 + runtime/srUtils.h | 13 ++ runtime/stream.c | 68 +++++++++ runtime/stream.h | 10 +- tools/omfile.c | 7 + tools/syslogd.c | 44 +----- 9 files changed, 559 insertions(+), 43 deletions(-) create mode 100644 runtime/apc.c create mode 100644 runtime/apc.h diff --git a/runtime/Makefile.am b/runtime/Makefile.am index de0daac4..c2ef7cfa 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -39,6 +39,8 @@ librsyslog_la_SOURCES = \ obj.h \ modules.c \ modules.h \ + apc.c \ + apc.h \ sync.c \ sync.h \ expr.c \ diff --git a/runtime/apc.c b/runtime/apc.c new file mode 100644 index 00000000..b0b5f298 --- /dev/null +++ b/runtime/apc.c @@ -0,0 +1,400 @@ +/* apc.c - asynchronous procedure call support + * + * An asynchronous procedure call (APC) is a procedure call (guess what) that is potentially run + * asynchronously to its main thread. It can be scheduled to occur at a caller-provided time. + * As long as the procedure has not been called, the APC entry may be modified by the caller + * or deleted. It is the caller's purpose to make sure proper synchronization is in place. + * The APC object only case about APC's own control structures (which *are* properly + * guarded by synchronization primitives). + * + * Module begun 2009-06-15 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include +#include +#include + +#include "rsyslog.h" +#include "obj.h" +#include "apc.h" +#include "srUtils.h" + +/* static data */ +DEFobjStaticHelpers + +/* following is a used to implement a monotonically increasing id for the apcs. That + * ID can be used to cancel an apc request. Note that the ID is generated with modulo + * arithmetic, so at some point, it will wrap. Howerver, this happens at 2^32-1 at + * earliest, so this is not considered a problem. + */ +apc_id_t apcID = 0; + +/* private data structures */ + +/* the apc list and its entries + * This is a doubly-linked list as we need to be able to do inserts + * and deletes right in the middle of the list. It is inspired by the + * Unix callout mechanism. + * Note that we support two generic caller-provided parameters as + * experience shows that at most two are often used. This causes very + * little overhead, but simplifies caller code in cases where exactly + * two parameters are needed. We hope this is a useful optimizaton. + * rgerhards, 2009-06-15 + */ +typedef struct apc_list_s { + struct apc_list_s *pNext; + struct apc_list_s *pPrev; + apc_id_t id; + apc_t *pApc; /* pointer to the APC object to be scheduled */ +} apc_list_t; + +apc_list_t *apcListRoot = NULL; +apc_list_t *apcListTail = NULL; +pthread_mutex_t listMutex; /* needs to be locked for all list operations */ + + +/* destructor for the apc object */ +BEGINobjDestruct(apc) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(apc) +ENDobjDestruct(apc) + + +/* ------------------------------ APC list handling functions ------------------------------ */ + +/* Function that handles changes to the list root. Most importantly, this function + * needs to schedule a new timer. It is OK to call this function with an empty list. + */ +static rsRetVal +listRootChanged(void) +{ + DEFiRet; + + if(apcListRoot == NULL) + FINALIZE; + + // TODO: implement! + +finalize_it: + RETiRet; +} + + +/* insert an apc entry into the APC list. The same entry MUST NOT already be present! + */ +static rsRetVal +insertApc(apc_t *pThis, apc_id_t *pID) +{ + apc_list_t *pCurr; + apc_list_t *pNew; + DEFiRet; + + CHKmalloc(pNew = (apc_list_t*) calloc(1, sizeof(apc_list_t))); + pNew->pApc = pThis; + pNew->id = *pID = apcID++; +dbgprintf("insert apc %p, id %ld\n", pThis, pNew->id); + + /* find right list location */ + if(apcListRoot == NULL) { + /* no need to search, list is empty */ + apcListRoot = pNew; + apcListTail = pNew; + CHKiRet(listRootChanged()); + } else { + for(pCurr = apcListRoot ; pCurr != NULL ; pCurr = pCurr->pNext) { + if(pCurr->pApc->ttExec > pThis->ttExec) + break; + } + + if(pCurr == NULL) { + /* insert at tail */ + pNew->pPrev = apcListTail; + apcListTail->pNext = pNew; + apcListTail = pNew; + } else { + if(pCurr == apcListRoot) { + /* new first entry */ + pCurr->pPrev = pNew; + pNew->pNext = pCurr; + apcListRoot = pNew; + CHKiRet(listRootChanged()); + } else { + /* in the middle of the list */ + pCurr->pPrev = pNew; + pNew->pNext = pCurr; + } + } + } + + +finalize_it: + RETiRet; +} + + +/* Delete an apc entry from the APC list. It is OK if the entry is not found, + * in this case we assume it already has been processed. + */ +static rsRetVal +deleteApc(apc_id_t id) +{ + apc_list_t *pCurr; + DEFiRet; + +dbgprintf("trying to delete apc %ld\n", id); + for(pCurr = apcListRoot ; pCurr != NULL ; pCurr = pCurr->pNext) { + if(pCurr->id == id) { +RUNLOG_STR("apc id found, now deleting!\n"); + if(pCurr == apcListRoot) { + apcListRoot = pCurr->pNext; + CHKiRet(listRootChanged()); + } else { + pCurr->pPrev->pNext = pCurr->pNext; + } + if(pCurr->pNext == NULL) { + apcListTail = pCurr->pPrev; + } else { + pCurr->pNext->pPrev = pCurr->pPrev; + } + free(pCurr); + pCurr = NULL; + break; + } + } + +finalize_it: + RETiRet; +} + + +/* unlist all elements up to the current timestamp. Return this as a seperate list + * to the caller. Returns an empty (NULL ptr) list if there are no such elements. + * The caller must handle that gracefully. The list is returned in the parameter. + */ +static rsRetVal +unlistCurrent(apc_list_t **ppList) +{ + apc_list_t *pCurr; + time_t tCurr; + DEFiRet; + assert(ppList != NULL); + + time(&tCurr); + + if(apcListRoot == NULL || apcListRoot->pApc->ttExec > tCurr) { + *ppList = NULL; + FINALIZE; + } + + *ppList = apcListRoot; + /* now search up to which entry we need to execute */ + for(pCurr = apcListRoot ; pCurr != NULL && pCurr->pApc->ttExec <= tCurr ; pCurr = pCurr->pNext) { + /*JUST SKIP TO LAST ELEMENT*/; + } + + if(pCurr == NULL) { + /* all elements can be unlisted */ + apcListRoot = NULL; + apcListTail = NULL; + } else { + /* need to set a new root */ + pCurr->pPrev->pNext = NULL; /* terminate newly unlisted list */ + pCurr->pPrev = NULL; /* we are the new root */ + apcListRoot = pCurr; + } + +finalize_it: + RETiRet; +} + + +/* ------------------------------ END APC list handling functions ------------------------------ */ + + +/* execute all list elements that are currently scheduled for execution. We do this in two phases. + * In the first phase, we look the list mutex and move everything from the head of the queue to + * the current timestamp to a new to-be-executed list. Then we unlock the mutex and do the actual + * exec (which may take some time). + * Note that the caller is responsible for proper + * caller-level synchronization. The caller may schedule another Apc, this module must + * ensure that (and it does so by not locking the list mutex while we call the Apc). + * Note: this function "consumes" the apc_t, so it is no longer existing after this + * function returns. + */ +// TODO make static and associated with our own pthread-based timer +rsRetVal +execScheduled(void) +{ + apc_list_t *pExecList; + apc_list_t *pCurr; + apc_list_t *pNext; + DEFVARS_mutexProtection_uncond; + DEFiRet; + + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + iRet = unlistCurrent(&pExecList); + END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + CHKiRet(iRet); + + DBGPRINTF("running apc scheduler - we have %s to execute\n", + pExecList == NULL ? "nothing" : "something"); + for(pCurr = pExecList ; pCurr != NULL ; pCurr = pNext) { +dbgprintf("executing apc list entry %p, apc %p\n", pCurr, pCurr->pApc); + pNext = pCurr->pNext; + pCurr->pApc->pProc(pCurr->pApc->param1, pCurr->pApc->param2); + apcDestruct(&pCurr->pApc); + free(pCurr); + } + +finalize_it: + RETiRet; +} + + +/* Standard-Constructor + */ +BEGINobjConstruct(apc) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(apc) + + +/* ConstructionFinalizer + * Note that we use a non-standard calling interface: pID returns the current APC + * id. This is the only way to handle the situation without the need for extra + * locking. + * rgerhards, 2008-01-09 + */ +static rsRetVal +apcConstructFinalize(apc_t *pThis, apc_id_t *pID) +{ + DEFVARS_mutexProtection_uncond; + DEFiRet; + ISOBJ_TYPE_assert(pThis, apc); + assert(pID != NULL); + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + insertApc(pThis, pID); + END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); +RUNLOG_STR("apcConstructFinalize post mutex unlock\n"); + RETiRet; +} + + +/* some set methods */ +static rsRetVal +SetProcedure(apc_t *pThis, void (*pProc)(void*, void*)) +{ + ISOBJ_TYPE_assert(pThis, apc); + pThis->pProc = pProc; + return RS_RET_OK; +} +static rsRetVal +SetParam1(apc_t *pThis, void *param1) +{ + ISOBJ_TYPE_assert(pThis, apc); + pThis->param1 = param1; + return RS_RET_OK; +} +static rsRetVal +SetParam2(apc_t *pThis, void *param2) +{ + ISOBJ_TYPE_assert(pThis, apc); + pThis->param1 = param2; + return RS_RET_OK; +} + + +/* cancel an Apc request, ID is provided. It is OK if the ID can not be found, this may + * happen if the Apc was executed in the mean time. So it is safe to call CancelApc() at + * any time. + */ +static rsRetVal +CancelApc(apc_id_t id) +{ + DEFVARS_mutexProtection_uncond; + + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + deleteApc(id); + END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); + return RS_RET_OK; +} + + +/* debugprint for the apc object */ +BEGINobjDebugPrint(apc) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(apc) + dbgoprint((obj_t*) pThis, "APC module, currently no state info available\n"); +ENDobjDebugPrint(apc) + + +/* queryInterface function + */ +BEGINobjQueryInterface(apc) +CODESTARTobjQueryInterface(apc) + if(pIf->ifVersion != apcCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = apcConstruct; + pIf->ConstructFinalize = apcConstructFinalize; + pIf->Destruct = apcDestruct; + pIf->DebugPrint = apcDebugPrint; + pIf->CancelApc = CancelApc; + pIf->SetProcedure = SetProcedure; + pIf->SetParam1 = SetParam1; + pIf->SetParam2 = SetParam2; +finalize_it: +ENDobjQueryInterface(apc) + + +/* Exit the apc class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(apc, OBJ_IS_CORE_MODULE) /* class, version */ + //objRelease(apcstk, CORE_COMPONENT); + pthread_mutex_destroy(&listMutex); +ENDObjClassExit(apc) + + +/* Initialize the apc class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(apc, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + //CHKiRet(objUse(apcstk, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, apcDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, apcConstructFinalize); + + /* do other initializations */ + pthread_mutex_init(&listMutex, NULL); +ENDObjClassInit(apc) + +/* vi:set ai: + */ diff --git a/runtime/apc.h b/runtime/apc.h new file mode 100644 index 00000000..7c679b97 --- /dev/null +++ b/runtime/apc.h @@ -0,0 +1,56 @@ +/* The apc object. + * + * See apc.c for more information. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_APC_H +#define INCLUDED_APC_H + +/* the apc object */ +typedef struct apc_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + time_t ttExec; /* when to call procedure (so far seconds...) */ + void (*pProc)(void*, void*); /* which procedure to call */ + void *param1; /* user-supplied parameters */ + void *param2; /* user-supplied parameters */ +} apc_t; + +typedef unsigned long apc_id_t; /* monotonically incrementing apc ID */ + +/* interfaces */ +BEGINinterface(apc) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(apc); + rsRetVal (*Construct)(apc_t **ppThis); + rsRetVal (*ConstructFinalize)(apc_t *pThis, apc_id_t *); + rsRetVal (*Destruct)(apc_t **ppThis); + rsRetVal (*SetProcedure)(apc_t *pThis, void (*pProc)(void*, void*)); + rsRetVal (*SetParam1)(apc_t *pThis, void *); + rsRetVal (*SetParam2)(apc_t *pThis, void *); + rsRetVal (*CancelApc)(apc_id_t); +ENDinterface(apc) +#define apcCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(apc); + +#endif /* #ifndef INCLUDED_APC_H */ diff --git a/runtime/obj.c b/runtime/obj.c index f38b1d7f..41991853 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -88,6 +88,7 @@ #include "errmsg.h" #include "cfsysline.h" #include "unicode-helper.h" +#include "apc.h" /* static data */ DEFobjCurrIf(obj) /* we define our own interface, as this is expected by some macros! */ @@ -1313,6 +1314,7 @@ objClassInit(modInfo_t *pModInfo) CHKiRet(objGetObjInterface(&obj)); /* get ourselves ;) */ /* init classes we use (limit to as few as possible!) */ + CHKiRet(apcClassInit(pModInfo)); CHKiRet(errmsgClassInit(pModInfo)); CHKiRet(cfsyslineInit()); CHKiRet(varClassInit(pModInfo)); diff --git a/runtime/srUtils.h b/runtime/srUtils.h index 699f8527..b37559cf 100644 --- a/runtime/srUtils.h +++ b/runtime/srUtils.h @@ -125,4 +125,17 @@ rsRetVal getFileSize(uchar *pszName, off_t *pSize); d_pthread_mutex_unlock(mut); \ pthread_setcancelstate(iCancelStateSave, NULL); \ } + +/* The unconditional versions of the macro always lock the mutex. They are preferred in + * complex scenarios, where the simple ones might get mixed up by multiple calls. + */ +#define DEFVARS_mutexProtection_uncond\ + int iCancelStateSave +#define BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(mut) \ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); \ + d_pthread_mutex_lock(mut); +#define END_MTX_PROTECTED_OPERATIONS_UNCOND(mut) \ + d_pthread_mutex_unlock(mut); \ + pthread_setcancelstate(iCancelStateSave, NULL); + #endif diff --git a/runtime/stream.c b/runtime/stream.c index c8672aa2..773da319 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -39,6 +39,7 @@ #include #include /* required for HP UX */ #include +#include #include "rsyslog.h" #include "stringbuf.h" @@ -47,10 +48,12 @@ #include "stream.h" #include "unicode-helper.h" #include "module-template.h" +#include "apc.h" /* static data */ DEFobjStaticHelpers DEFobjCurrIf(zlibw) +DEFobjCurrIf(apc) /* forward definitions */ static rsRetVal strmFlush(strm_t *pThis); @@ -60,6 +63,20 @@ static rsRetVal strmCloseFile(strm_t *pThis); /* methods */ +/* async flush apc handler + */ +static void +flushApc(void *param1, void __attribute__((unused)) *param2) +{ + DEFVARS_mutexProtection_uncond; + strm_t *pThis = (strm_t*) param1; + ISOBJ_TYPE_assert(pThis, strm); + + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&pThis->mut); + strmFlush(pThis); + END_MTX_PROTECTED_OPERATIONS_UNCOND(&pThis->mut); +} + /* Try to resolve a size limit situation. This is used to support custom-file size handlers * for omfile. It first runs the command, and then checks if we are still above the size @@ -583,6 +600,11 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) } } + /* if we should call flush apc's, we need a mutex */ + if(pThis->iFlushInterval != 0) { + pthread_mutex_init(&pThis->mut, 0); + } + finalize_it: RETiRet; } @@ -602,6 +624,11 @@ CODESTARTobjDestruct(strm) objRelease(zlibw, LM_ZLIBW_FILENAME); } + if(pThis->iFlushInterval != 0) { + // TODO: check if there is an apc and remove it! + pthread_mutex_destroy(&pThis->mut); + } + free(pThis->pszDir); free(pThis->pIOBuf); free(pThis->pZipBuf); @@ -965,11 +992,33 @@ finalize_it: } +/* schedule an Apc flush request. + * rgerhards, 2009-06-15 + */ +static inline rsRetVal +scheduleFlushRequest(strm_t *pThis) +{ + apc_t *pApc; + DEFiRet; + + CHKiRet(apc.CancelApc(pThis->apcID)); +dbgprintf("XXX: requesting to add apc!\n"); + CHKiRet(apc.Construct(&pApc)); + CHKiRet(apc.SetProcedure(pApc, (void (*)(void*, void*))flushApc)); + CHKiRet(apc.SetParam1(pApc, pThis)); + CHKiRet(apc.ConstructFinalize(pApc, &pThis->apcID)); + +finalize_it: + RETiRet; +} + + /* write memory buffer to a stream object */ static rsRetVal strmWrite(strm_t *pThis, uchar *pBuf, size_t lenBuf) { + DEFVARS_mutexProtection_uncond; DEFiRet; size_t iPartial; @@ -980,6 +1029,11 @@ dbgprintf("strmWrite(%p, '%65.65s', %ld);, disabled %d, sizelim %ld, size %lld\n if(pThis->bDisabled) ABORT_FINALIZE(RS_RET_STREAM_DISABLED); +RUNLOG_VAR("%d", pThis->iFlushInterval); + if(pThis->iFlushInterval != 0) { + BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&pThis->mut); + } + /* check if the to-be-written data is larger than our buffer size */ if(lenBuf >= pThis->sIOBufSize) { /* it is - so we do a direct write, that is most efficient. @@ -1008,7 +1062,17 @@ dbgprintf("strmWrite(%p, '%65.65s', %ld);, disabled %d, sizelim %ld, size %lld\n } } + /* we ignore the outcome of scheduleFlushRequest(), as we will write the data always at + * termination. For Zip mode, it could be fatal if we write after each record. + */ + if(pThis->iFlushInterval != 0) + scheduleFlushRequest(pThis); + finalize_it: + if(pThis->iFlushInterval != 0) { + END_MTX_PROTECTED_OPERATIONS_UNCOND(&pThis->mut); + } + RETiRet; } @@ -1025,6 +1089,7 @@ DEFpropSetMeth(strm, iZipLevel, int) DEFpropSetMeth(strm, bSync, int) DEFpropSetMeth(strm, sIOBufSize, size_t) DEFpropSetMeth(strm, iSizeLimit, off_t) +DEFpropSetMeth(strm, iFlushInterval, int) DEFpropSetMeth(strm, pszSizeLimitCmd, uchar*) static rsRetVal strmSetiMaxFiles(strm_t *pThis, int iNewVal) @@ -1308,6 +1373,7 @@ CODESTARTobjQueryInterface(strm) pIf->SetbSync = strmSetbSync; pIf->SetsIOBufSize = strmSetsIOBufSize; pIf->SetiSizeLimit = strmSetiSizeLimit; + pIf->SetiFlushInterval = strmSetiFlushInterval; pIf->SetpszSizeLimitCmd = strmSetpszSizeLimitCmd; finalize_it: ENDobjQueryInterface(strm) @@ -1319,6 +1385,8 @@ ENDobjQueryInterface(strm) */ BEGINObjClassInit(strm, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ + CHKiRet(objUse(apc, CORE_COMPONENT)); + OBJSetMethodHandler(objMethod_SERIALIZE, strmSerialize); OBJSetMethodHandler(objMethod_SETPROPERTY, strmSetProperty); OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, strmConstructFinalize); diff --git a/runtime/stream.h b/runtime/stream.h index 021c4792..e3ad32b1 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -70,6 +70,7 @@ #include "glbl.h" #include "stream.h" #include "zlibw.h" +#include "apc.h" /* stream types */ typedef enum { @@ -118,6 +119,10 @@ typedef struct strm_s { bool bInRecord; /* if 1, indicates that we are currently writing a not-yet complete record */ int iZipLevel; /* zip level (0..9). If 0, zip is completely disabled */ Bytef *pZipBuf; + /* support for async flush procesing */ + int iFlushInterval; /* flush in which interval - 0, no flushing */ + apc_id_t apcID; /* id of current Apc request (used for cancelling) */ + pthread_mutex_t mut;/* mutex for flush in async mode */ /* support for omfile size-limiting commands, special counters, NOT persisted! */ off_t iSizeLimit; /* file size limit, 0 = no limit */ uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ @@ -127,7 +132,7 @@ typedef struct strm_s { /* interfaces */ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Construct)(strm_t **ppThis); - rsRetVal (*ConstructFinalize)(strm_t __attribute__((unused)) *pThis); + rsRetVal (*ConstructFinalize)(strm_t *pThis); rsRetVal (*Destruct)(strm_t **ppThis); rsRetVal (*SetMaxFileSize)(strm_t *pThis, int64 iMaxFileSize); rsRetVal (*SetFileName)(strm_t *pThis, uchar *pszName, size_t iLenName); @@ -157,9 +162,10 @@ BEGINinterface(strm) /* name must also be changed in ENDinterface macro! */ INTERFACEpropSetMeth(strm, bSync, int); INTERFACEpropSetMeth(strm, sIOBufSize, size_t); INTERFACEpropSetMeth(strm, iSizeLimit, off_t); + INTERFACEpropSetMeth(strm, iFlushInterval, int); INTERFACEpropSetMeth(strm, pszSizeLimitCmd, uchar*); ENDinterface(strm) -#define strmCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define strmCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/tools/omfile.c b/tools/omfile.c index 6d9eb096..675d313e 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -88,6 +88,7 @@ typedef struct s_dynaFileCacheEntry dynaFileCacheEntry; #define IOBUF_DFLT_SIZE 1024 /* default size for io buffers */ +#define FLUSH_INTRVL_DFLT 1 /* default buffer flush interval (in seconds) */ /* globals for default values */ static int iDynaFileCacheSize = 10; /* max cache for dynamic files */ @@ -103,6 +104,7 @@ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathn static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ static bool bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */ static int iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ +static int iFlushInterval = FLUSH_INTRVL_DFLT; /* how often flush the output buffer on inactivity? */ static uchar *pszTplName = NULL; /* name of the default template to use */ /* end globals for default values */ @@ -132,6 +134,7 @@ typedef struct _instanceData { uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ int iZipLevel; /* zip mode to use for this selector */ int iIOBufSize; /* size of associated io buffer */ + int iFlushInterval; /* how fast flush buffer on inactivity? */ bool bFlushOnTXEnd; /* flush write buffers when transaction has ended? */ } instanceData; @@ -400,6 +403,7 @@ prepareFile(instanceData *pData, uchar *newFileName) CHKiRet(strm.SetDir(pData->pStrm, szDirName, ustrlen(szDirName))); CHKiRet(strm.SetiZipLevel(pData->pStrm, pData->iZipLevel)); CHKiRet(strm.SetsIOBufSize(pData->pStrm, (size_t) pData->iIOBufSize)); + CHKiRet(strm.SetiFlushInterval(pData->pStrm, pData->iFlushInterval)); CHKiRet(strm.SettOperationsMode(pData->pStrm, STREAMMODE_WRITE_APPEND)); CHKiRet(strm.SettOpenMode(pData->pStrm, fCreateMode)); CHKiRet(strm.SetbSync(pData->pStrm, pData->bSyncFile)); @@ -678,6 +682,7 @@ CODESTARTparseSelectorAct pData->iZipLevel = iZipLevel; pData->bFlushOnTXEnd = bFlushOnTXEnd; pData->iIOBufSize = iIOBufSize; + pData->iFlushInterval = iFlushInterval; if(pData->bDynamicName == 0) { /* try open and emit error message if not possible. At this stage, we ignore the @@ -712,6 +717,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iZipLevel = 0; bFlushOnTXEnd = 0; iIOBufSize = IOBUF_DFLT_SIZE; + iFlushInterval = FLUSH_INTRVL_DFLT; if(pszTplName != NULL) { free(pszTplName); pszTplName = NULL; @@ -760,6 +766,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(strm, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dynafilecachesize", 0, eCmdHdlrInt, (void*) setDynaFileCacheSize, NULL, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileziplevel", 0, eCmdHdlrInt, NULL, &iZipLevel, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushinterval", 0, eCmdHdlrInt, NULL, &iFlushInterval, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileflushontxend", 0, eCmdHdlrBinary, NULL, &bFlushOnTXEnd, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"omfileiobuffersize", 0, eCmdHdlrSize, NULL, &iIOBufSize, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"dirowner", 0, eCmdHdlrUID, NULL, &dirUID, STD_LOADABLE_MODULE_ID)); diff --git a/tools/syslogd.c b/tools/syslogd.c index 99bf281d..2b29cd7c 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2575,7 +2575,8 @@ mainloop(void) * powertop, for example). In that case, we primarily wait for a signal, * but a once-a-day wakeup should be quite acceptable. -- rgerhards, 2008-06-09 */ - tvSelectTimeout.tv_sec = (bReduceRepeatMsgs == 1) ? TIMERINTVL : 86400 /*1 day*/; + //tvSelectTimeout.tv_sec = (bReduceRepeatMsgs == 1) ? TIMERINTVL : 86400 /*1 day*/; +tvSelectTimeout.tv_sec = 5; // TESTING ONLY!!! TODO: change back!!! tvSelectTimeout.tv_usec = 0; select(1, NULL, NULL, NULL, &tvSelectTimeout); if(bFinished) @@ -2610,49 +2611,11 @@ mainloop(void) bHadHUP = 0; continue; } + execScheduled(); /* handle Apc calls (if any) */ } ENDfunc } -/* If user is not root, prints warnings or even exits - * TODO: check all dynafiles for write permission - * ... but it is probably better to wait here until we have - * a module interface - rgerhards, 2007-07-23 - */ -static void checkPermissions() -{ -#if 0 - /* TODO: this function must either be redone or removed - now with the input modules, - * there is no such simple check we can do. What we can check, however, is if there is - * any input module active and terminate, if not. -- rgerhards, 2007-12-26 - */ - /* we are not root */ - if (geteuid() != 0) - { - fputs("WARNING: Local messages will not be logged! If you want to log them, run rsyslog as root.\n",stderr); -#ifdef SYSLOG_INET - /* udp enabled and port number less than or equal to 1024 */ - if ( AcceptRemote && (atoi(LogPort) <= 1024) ) - fprintf(stderr, "WARNING: Will not listen on UDP port %s. Use port number higher than 1024 or run rsyslog as root!\n", LogPort); - - /* tcp enabled and port number less or equal to 1024 */ - if( bEnableTCP && (atoi(TCPLstnPort) <= 1024) ) - fprintf(stderr, "WARNING: Will not listen on TCP port %s. Use port number higher than 1024 or run rsyslog as root!\n", TCPLstnPort); - - /* Neither explicit high UDP port nor explicit high TCP port. - * It is useless to run anymore */ - if( !(AcceptRemote && (atoi(LogPort) > 1024)) && !( bEnableTCP && (atoi(TCPLstnPort) > 1024)) ) - { -#endif - fprintf(stderr, "ERROR: Nothing to log, no reason to run. Please run rsyslog as root.\n"); - exit(EXIT_FAILURE); -#ifdef SYSLOG_INET - } -#endif - } -#endif -} - /* load build-in modules * very first version begun on 2007-07-23 by rgerhards @@ -3053,7 +3016,6 @@ doGlblProcessInit(void) int i; DEFiRet; - checkPermissions(); thrdInit(); if( !(Debug || NoFork) ) -- cgit v1.2.3 From 4af40abc0a9cdc9f328830b6074f6182f43c171a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 07:15:48 +0200 Subject: cleaned up small nit --- runtime/unicode-helper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/unicode-helper.h b/runtime/unicode-helper.h index 8216e992..7a776f68 100644 --- a/runtime/unicode-helper.h +++ b/runtime/unicode-helper.h @@ -40,7 +40,7 @@ #else static inline uchar* ustrncpy(uchar *psz1, uchar *psz2, size_t len) { - return strncpy((char*) psz1, (char*) psz2, len); + return (uchar*) strncpy((char*) psz1, (char*) psz2, len); } static inline uchar* ustrdup(uchar *psz) -- cgit v1.2.3 From 015d17ca70e81ad998e32cdfeed3cd925fd7dedc Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 08:46:45 +0200 Subject: some performance optimizations - saved gettimeofday() calls in imtcp (and increased reception buffer) - somewhat optimized stringbuf.c - some other optimizations --- plugins/imdiag/imdiag.c | 2 +- plugins/imfile/imfile.c | 2 +- plugins/imklog/imklog.c | 2 +- plugins/imudp/imudp.c | 4 ++-- runtime/msg.c | 57 ++++++++++++++++++++----------------------------- runtime/msg.h | 48 ++++++++++++++++++----------------------- runtime/obj-types.h | 4 ++-- runtime/parser.c | 17 +++++++++++++++ runtime/rsyslog.h | 2 ++ runtime/stringbuf.c | 6 ++---- tcps_sess.c | 54 ++++++++++++++++++++++------------------------ tcpsrv.c | 3 ++- tcpsrv.h | 1 + tools/syslogd.c | 4 ++-- 14 files changed, 102 insertions(+), 104 deletions(-) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index c700cab7..51f319ca 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -207,7 +207,7 @@ doInjectMsg(int iNum) CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); CHKmalloc(pMsg->pszRawMsg = ustrdup(szMsg)); pMsg->iLenRawMsg = ustrlen(szMsg); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imdiag")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imdiag"), sizeof("imdiag")-1); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 92fd30c3..e3882ce5 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -95,7 +95,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); MsgSetUxTradMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index ecb6c100..45933436 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -95,7 +95,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); MsgSetRawMsg(pMsg, (char*)msg); MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index a486b818..57aec9b6 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -6,7 +6,7 @@ * * File begun on 2007-12-21 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -220,7 +220,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); pMsg->iLenRawMsg = lenRcvBuf; - MsgSetInputName(pMsg, UCHAR_CONSTANT("imudp")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("imudp"), sizeof("imudp")-1); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; diff --git a/runtime/msg.c b/runtime/msg.c index dbc3c779..b3c76089 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -104,6 +104,7 @@ static syslogCODE rs_facilitynames[] = /* some forward declarations */ static int getAPPNAMELen(msg_t *pM); +static int getProtocolVersion(msg_t *pM); /* The following functions will support advanced output module * multithreading, once this is implemented. Currently, we @@ -364,7 +365,6 @@ CODESTARTobjDestruct(msg) free(pThis->pszTIMESTAMP_SecFrac); free(pThis->pszTIMESTAMP_MySQL); free(pThis->pszTIMESTAMP_PgSQL); - free(pThis->pszPRI); if(pThis->pCSProgName != NULL) rsCStrDestruct(&pThis->pCSProgName); if(pThis->pCSStrucData != NULL) @@ -670,7 +670,7 @@ void setProtocolVersion(msg_t *pM, int iNewVersion) pM->iProtocolVersion = iNewVersion; } -int getProtocolVersion(msg_t *pM) +static int getProtocolVersion(msg_t *pM) { assert(pM != NULL); return(pM->iProtocolVersion); @@ -689,7 +689,7 @@ int getMSGLen(msg_t *pM) } -char *getRawMsg(msg_t *pM) +static char *getRawMsg(msg_t *pM) { if(pM == NULL) return ""; @@ -724,7 +724,7 @@ char *getMSG(msg_t *pM) /* Get PRI value in text form */ -char *getPRI(msg_t *pM) +static char *getPRI(msg_t *pM) { int pri; BEGINfunc @@ -742,7 +742,7 @@ char *getPRI(msg_t *pM) * report. -- rgerhards, 2008-07-14 */ pri = pM->iFacility * 8 + pM->iSeverity; - if((pM->pszPRI = malloc(5)) == NULL) return ""; + pM->pszPRI = pM->bufPRI; pM->iLenPRI = snprintf((char*)pM->pszPRI, 5, "%d", pri); } MsgUnlock(pM); @@ -753,14 +753,14 @@ char *getPRI(msg_t *pM) /* Get PRI value as integer */ -int getPRIi(msg_t *pM) +static int getPRIi(msg_t *pM) { assert(pM != NULL); return (pM->iFacility << 3) + (pM->iSeverity); } -char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) +static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) { BEGINfunc if(pM == NULL) @@ -838,7 +838,7 @@ char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) return "INVALID eFmt OPTION!"; } -char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) +static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) { BEGINfunc if(pM == NULL) @@ -917,7 +917,7 @@ char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) } -char *getSeverity(msg_t *pM) +static inline char *getSeverity(msg_t *pM) { if(pM == NULL) return ""; @@ -934,7 +934,7 @@ char *getSeverity(msg_t *pM) } -char *getSeverityStr(msg_t *pM) +static inline char *getSeverityStr(msg_t *pM) { syslogCODE *c; int val; @@ -964,7 +964,7 @@ char *getSeverityStr(msg_t *pM) return((char*)pM->pszSeverityStr); } -char *getFacility(msg_t *pM) +static inline char *getFacility(msg_t *pM) { if(pM == NULL) return ""; @@ -983,7 +983,7 @@ char *getFacility(msg_t *pM) return((char*)pM->pszFacility); } -char *getFacilityStr(msg_t *pM) +static inline char *getFacilityStr(msg_t *pM) { syslogCODE *c; int val; @@ -1090,7 +1090,7 @@ finalize_it: /* rgerhards, 2005-11-24 */ -int getPROCIDLen(msg_t *pM) +static inline int getPROCIDLen(msg_t *pM) { assert(pM != NULL); MsgLock(pM); @@ -1135,19 +1135,10 @@ finalize_it: RETiRet; } -/* rgerhards, 2005-11-24 - */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static int getMSGIDLen(msg_t *pM) -{ - return (pM->pCSMSGID == NULL) ? 1 : rsCStrLen(pM->pCSMSGID); -} -#endif - /* rgerhards, 2005-11-24 */ -char *getMSGID(msg_t *pM) +static inline char *getMSGID(msg_t *pM) { return (pM->pCSMSGID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSMSGID); } @@ -1171,8 +1162,7 @@ void MsgAssignTAG(msg_t *pMsg, uchar *pBuf) void MsgSetTAG(msg_t *pMsg, char* pszTAG) { assert(pMsg != NULL); - if(pMsg->pszTAG != NULL) - free(pMsg->pszTAG); + free(pMsg->pszTAG); pMsg->iLenTAG = strlen(pszTAG); if((pMsg->pszTAG = malloc(pMsg->iLenTAG + 1)) != NULL) memcpy(pMsg->pszTAG, pszTAG, pMsg->iLenTAG + 1); @@ -1229,7 +1219,7 @@ static int getTAGLen(msg_t *pM) #endif -char *getTAG(msg_t *pM) +static inline char *getTAG(msg_t *pM) { char *ret; @@ -1272,7 +1262,7 @@ char *getHOSTNAME(msg_t *pM) } -uchar *getInputName(msg_t *pM) +static uchar *getInputName(msg_t *pM) { if(pM == NULL) return (uchar*) ""; @@ -1339,7 +1329,7 @@ static int getStructuredDataLen(msg_t *pM) /* get the "STRUCTURED-DATA" as sz string * rgerhards, 2005-11-24 */ -char *getStructuredData(msg_t *pM) +static inline char *getStructuredData(msg_t *pM) { return (pM->pCSStrucData == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSStrucData); } @@ -1465,14 +1455,13 @@ static int getAPPNAMELen(msg_t *pM) } /* rgerhards 2008-09-10: set pszInputName in msg object + * rgerhards, 2009-06-16 */ -void MsgSetInputName(msg_t *pMsg, uchar* pszInputName) +void MsgSetInputName(msg_t *pMsg, uchar* pszInputName, size_t lenInputName) { assert(pMsg != NULL); - if(pMsg->pszInputName != NULL) - free(pMsg->pszInputName); - - pMsg->iLenInputName = ustrlen(pszInputName); + free(pMsg->pszInputName); + pMsg->iLenInputName = lenInputName; if((pMsg->pszInputName = malloc(pMsg->iLenInputName + 1)) != NULL) { memcpy(pMsg->pszInputName, pszInputName, pMsg->iLenInputName + 1); } @@ -2493,7 +2482,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("pszTAG")) { MsgSetTAG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszInputName")) { - MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pszRcvFromIP")) { MsgSetRcvFromIP(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszRcvFrom")) { diff --git a/runtime/msg.h b/runtime/msg.h index a14f6b15..1689bbbc 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -120,6 +120,9 @@ short bDoLock; /* use the mutex? */ char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ + /* now follow fixed-size buffers to safe some time otherwise used for allocs */ + uchar bufPRI[5]; + }; @@ -137,45 +140,20 @@ short bDoLock; /* use the mutex? */ /* function prototypes */ PROTOTYPEObjClassInit(msg); -char* getProgramName(msg_t*); rsRetVal msgConstruct(msg_t **ppThis); rsRetVal msgConstructWithTime(msg_t **ppThis, struct syslogTime *stTime, time_t ttGenTime); rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); void setProtocolVersion(msg_t *pM, int iNewVersion); -int getProtocolVersion(msg_t *pM); -char *getProtocolVersionString(msg_t *pM); -int getMSGLen(msg_t *pM); -char *getRawMsg(msg_t *pM); -char *getUxTradMsg(msg_t *pM); -char *getMSG(msg_t *pM); -char *getPRI(msg_t *pM); -int getPRIi(msg_t *pM); -char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt); -char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt); -char *getSeverity(msg_t *pM); -char *getSeverityStr(msg_t *pM); -char *getFacility(msg_t *pM); -char *getFacilityStr(msg_t *pM); -void MsgSetInputName(msg_t *pMsg, uchar*); +void MsgSetInputName(msg_t *pMsg, uchar*, size_t); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); -char *getAPPNAME(msg_t *pM); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); -int getPROCIDLen(msg_t *pM); -char *getPROCID(msg_t *pM); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); void MsgSetTAG(msg_t *pMsg, char* pszTAG); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); -char *getTAG(msg_t *pM); -int getHOSTNAMELen(msg_t *pM); -char *getHOSTNAME(msg_t *pM); -uchar *getRcvFrom(msg_t *pM); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); -char *getStructuredData(msg_t *pM); -int getProgramNameLen(msg_t *pM); -char *getProgramName(msg_t *pM); void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); @@ -184,13 +162,29 @@ int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); void MsgSetMSG(msg_t *pMsg, char* pszMSG); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); void moveHOSTNAMEtoTAG(msg_t *pM); -char *getMSGID(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, cstr_t *pCSPropName, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); +/* TODO: remove these five (so far used in action.c) */ +char *getMSG(msg_t *pM); +char *getHOSTNAME(msg_t *pM); +char *getPROCID(msg_t *pM); +char *getAPPNAME(msg_t *pM); +int getMSGLen(msg_t *pM); + +char *getHOSTNAME(msg_t *pM); +int getHOSTNAMELen(msg_t *pM); +char *getProgramName(msg_t *pM); +int getProgramNameLen(msg_t *pM); +uchar *getRcvFrom(msg_t *pM); + +#if 0 +char *getUxTradMsg(msg_t *pM); +#endif + /* The MsgPrepareEnqueue() function is a macro for performance reasons. * It needs one global variable to work. This is acceptable, as it gains * us quite some performance and is fully abstracted using this header file. diff --git a/runtime/obj-types.h b/runtime/obj-types.h index 914c2f2c..78829f94 100644 --- a/runtime/obj-types.h +++ b/runtime/obj-types.h @@ -280,7 +280,7 @@ rsRetVal objName##ClassExit(void) \ * rgerhards, 2008-01-30 */ #define BEGINobjDestruct(OBJ) \ - rsRetVal OBJ##Destruct(OBJ##_t **ppThis) \ + rsRetVal OBJ##Destruct(OBJ##_t __attribute__((unused)) **ppThis) \ { \ DEFiRet; \ int iCancelStateSave; \ @@ -314,7 +314,7 @@ rsRetVal objName##ClassExit(void) \ #define PROTOTYPEObjDebugPrint(obj) rsRetVal obj##DebugPrint(obj##_t *pThis) #define INTERFACEObjDebugPrint(obj) rsRetVal (*DebugPrint)(obj##_t *pThis) #define BEGINobjDebugPrint(obj) \ - rsRetVal obj##DebugPrint(obj##_t *pThis) \ + rsRetVal obj##DebugPrint(obj##_t __attribute__((unused)) *pThis) \ { \ DEFiRet; \ diff --git a/runtime/parser.c b/runtime/parser.c index 212d40f3..64e03094 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -191,6 +191,23 @@ sanitizeMessage(msg_t *pMsg) lenMsg--; } + /* it is much quicker to sweep over the message and see if it actually + * needs sanitation than to do the sanitation in any case. So we first do + * this and terminate when it is not needed - which is expectedly the case + * for the vast majority of messages. -- rgerhards, 2009-06-15 + */ + int bNeedSanitize = 0; + for(iSrc = 0 ; iSrc < lenMsg ; iSrc++) { + if(pszMsg[iSrc] < 32) { + if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { + bNeedSanitize = 1; + break; + } + } + } + if(bNeedSanitize == 0) + FINALIZE; + /* now copy over the message and sanitize it */ /* TODO: can we get cheaper memory alloc? {alloca()?}*/ iMaxLine = glbl.GetMaxLine(); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index ea303a51..32177a9f 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -103,6 +103,8 @@ typedef unsigned int u_int32_t; /* TODO: is this correct? */ typedef int socklen_t; #endif +typedef char bool; /* I intentionally use char, to keep it slim so that many fit into the CPU cache! */ + /* settings for flow control * TODO: is there a better place for them? -- rgerhards, 2008-03-14 */ diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 07256fab..f3d9aa48 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -299,10 +299,8 @@ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) { rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - if(pThis->pBuf != NULL) - free(pThis->pBuf); - if(pThis->pszBuf != NULL) - free(pThis->pszBuf); + free(pThis->pBuf); + free(pThis->pszBuf); if(pszNew == NULL) { pThis->iStrLen = 0; pThis->iBufSize = 0; diff --git a/tcps_sess.c b/tcps_sess.c index 62d51f66..cfee0523 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -97,12 +97,9 @@ CODESTARTobjDestruct(tcps_sess) pThis->pSrv->pOnSessDestruct(&pThis->pUsr); } /* now destruct our own properties */ - if(pThis->fromHost != NULL) - free(pThis->fromHost); - if(pThis->fromHostIP != NULL) - free(pThis->fromHostIP); - if(pThis->pMsg != NULL) - free(pThis->pMsg); + free(pThis->fromHost); + free(pThis->fromHostIP); + free(pThis->pMsg); ENDobjDestruct(tcps_sess) @@ -124,10 +121,7 @@ SetHost(tcps_sess_t *pThis, uchar *pszHost) ISOBJ_TYPE_assert(pThis, tcps_sess); - if(pThis->fromHost != NULL) { - free(pThis->fromHost); - } - + free(pThis->fromHost); pThis->fromHost = pszHost; RETiRet; @@ -144,10 +138,7 @@ SetHostIP(tcps_sess_t *pThis, uchar *pszHostIP) ISOBJ_TYPE_assert(pThis, tcps_sess); - if(pThis->fromHostIP != NULL) { - free(pThis->fromHostIP); - } - + free(pThis->fromHostIP); pThis->fromHostIP = pszHostIP; RETiRet; @@ -226,11 +217,9 @@ SetOnMsgReceive(tcps_sess_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar * rgerhards, 2009-04-23 */ static rsRetVal -defaultDoSubmitMessage(tcps_sess_t *pThis) +defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttGenTime) { msg_t *pMsg; - struct syslogTime stTime; - time_t ttGenTime; DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -240,16 +229,13 @@ defaultDoSubmitMessage(tcps_sess_t *pThis) FINALIZE; } - //TODO: if((iTimeRequery == 0) || (iNbrTimeUsed++ % iTimeRequery) == 0) { - datetime.getCurrTime(&stTime, &ttGenTime); - //} /* we now create our own message object and submit it to the queue */ - CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); + CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); /* first trim the buffer to what we have actually received */ CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); pMsg->iLenRawMsg = pThis->iMsg; - MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName); + MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName, pThis->pLstnInfo->lenInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; @@ -281,6 +267,8 @@ finalize_it: static rsRetVal PrepareClose(tcps_sess_t *pThis) { + struct syslogTime stTime; + time_t ttGenTime; DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -306,7 +294,8 @@ PrepareClose(tcps_sess_t *pThis) * this case. */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); - defaultDoSubmitMessage(pThis); + datetime.getCurrTime(&stTime, &ttGenTime); + defaultDoSubmitMessage(pThis, &stTime, ttGenTime); } finalize_it: @@ -341,7 +330,7 @@ Close(tcps_sess_t *pThis) * rgerhards, 2008-03-14 */ static rsRetVal -processDataRcvd(tcps_sess_t *pThis, char c) +processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -387,7 +376,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); - defaultDoSubmitMessage(pThis); + defaultDoSubmitMessage(pThis, stTime, ttGenTime); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -398,7 +387,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) if(( (c == '\n') || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - defaultDoSubmitMessage(pThis); + defaultDoSubmitMessage(pThis, stTime, ttGenTime); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -415,7 +404,7 @@ processDataRcvd(tcps_sess_t *pThis, char c) pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - defaultDoSubmitMessage(pThis); + defaultDoSubmitMessage(pThis, stTime, ttGenTime); pThis->inputState = eAtStrtFram; } } @@ -436,22 +425,29 @@ processDataRcvd(tcps_sess_t *pThis, char c) * RS_RET_OK, which means the session should be kept open * or anything else, which means it must be closed. * rgerhards, 2008-03-01 + * As a performance optimization, we pick up the timestamp here. Acutally, + * this *is* the *correct* reception step for all the data we received, because + * we have just received a bunch of data! -- rgerhards, 2009-06-16 */ static rsRetVal DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) { - DEFiRet; + struct syslogTime stTime; + time_t ttGenTime; char *pEnd; + DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); assert(pData != NULL); assert(iLen > 0); + datetime.getCurrTime(&stTime, &ttGenTime); + /* We now copy the message to the session buffer. */ pEnd = pData + iLen; /* this is one off, which is intensional */ while(pData < pEnd) { - CHKiRet(processDataRcvd(pThis, *pData++)); + CHKiRet(processDataRcvd(pThis, *pData++, &stTime, ttGenTime)); } finalize_it: diff --git a/tcpsrv.c b/tcpsrv.c index 3516b2e3..11619498 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -105,6 +105,7 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) pEntry->pszPort = pszPort; pEntry->pSrv = pThis; CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); + pEntry->lenInputName = ustrlen(pEntry->pszInputName); /* and add to list */ pEntry->pNext = pThis->pLstnPorts; @@ -510,7 +511,7 @@ Run(tcpsrv_t *pThis) while(nfds && iTCPSess != -1) { CHKiRet(nssel.IsReady(pSel, pThis->pSessions[iTCPSess]->pStrm, NSDSEL_RD, &bIsReady, &nfds)); if(bIsReady) { - char buf[8*1024]; /* reception buffer - may hold a partial or multiple messages */ + char buf[128*1024]; /* reception buffer - may hold a partial or multiple messages */ dbgprintf("netstream %p with new data\n", pThis->pSessions[iTCPSess]->pStrm); /* Receive message */ diff --git a/tcpsrv.h b/tcpsrv.h index 2d174ce0..95537d54 100644 --- a/tcpsrv.h +++ b/tcpsrv.h @@ -37,6 +37,7 @@ typedef enum ETCPsyslogFramingAnomaly { struct tcpLstnPortList_s { uchar *pszPort; /**< the ports the listener shall listen on */ uchar *pszInputName; /**< value to be used as input name */ + size_t lenInputName; /**< length of inputName */ tcpsrv_t *pSrv; /**< pointer to higher-level server instance */ tcpLstnPortList_t *pNext; /**< next port or NULL */ }; diff --git a/tools/syslogd.c b/tools/syslogd.c index b43b7a37..a9f1dbdb 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -639,7 +639,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } if(pszInputName != NULL) - MsgSetInputName(pMsg, pszInputName); + MsgSetInputName(pMsg, pszInputName, ustrlen(pszInputName)); MsgSetFlowControlType(pMsg, flowCtlType); MsgSetRawMsg(pMsg, (char*)msg); @@ -947,7 +947,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) DEFiRet; CHKiRet(msgConstruct(&pMsg)); - MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd")); + MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1); MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); -- cgit v1.2.3 From f7579e68a67364c8040966be57c2eae4c9550ee5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 11:36:05 +0200 Subject: done various optimizations to the stringbuf and its users --- outchannel.c | 20 ++--- parse.c | 16 ++-- runtime/cfsysline.c | 12 +-- runtime/conf.c | 40 +++++----- runtime/ctok.c | 4 +- runtime/datetime.c | 5 +- runtime/msg.c | 30 ++----- runtime/msg.h | 4 +- runtime/nsd_gtls.c | 8 +- runtime/obj.c | 6 +- runtime/parser.c | 19 ++++- runtime/stream.c | 2 +- runtime/stringbuf.c | 221 ++++++++++++++++++---------------------------------- runtime/stringbuf.h | 27 +++---- runtime/vm.c | 2 +- runtime/vmop.c | 2 +- template.c | 54 ++++++------- tools/syslogd.c | 170 ++++++++++++++++++++-------------------- 18 files changed, 282 insertions(+), 360 deletions(-) diff --git a/outchannel.c b/outchannel.c index 5c348b63..4f8abb32 100644 --- a/outchannel.c +++ b/outchannel.c @@ -105,22 +105,22 @@ static rsRetVal get_Field(uchar **pp, uchar **pField) skip_Comma((char**)pp); p = *pp; - CHKiRet(rsCStrConstruct(&pStrB)); + CHKiRet(cstrConstruct(&pStrB)); rsCStrSetAllocIncrement(pStrB, 32); /* copy the field */ while(*p && *p != ' ' && *p != ',') { - CHKiRet(rsCStrAppendChar(pStrB, *p++)); + CHKiRet(cstrAppendChar(pStrB, *p++)); } *pp = p; - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, pField, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, pField, 0)); finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -174,22 +174,22 @@ static inline rsRetVal get_restOfLine(uchar **pp, uchar **pBuf) skip_Comma((char**)pp); p = *pp; - CHKiRet(rsCStrConstruct(&pStrB)); + CHKiRet(cstrConstruct(&pStrB)); rsCStrSetAllocIncrement(pStrB, 32); /* copy the field */ while(*p) { - CHKiRet(rsCStrAppendChar(pStrB, *p++)); + CHKiRet(cstrAppendChar(pStrB, *p++)); } *pp = p; - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, pBuf, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, pBuf, 0)); finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; diff --git a/parse.c b/parse.c index 58458d62..87c67676 100644 --- a/parse.c +++ b/parse.c @@ -268,7 +268,7 @@ rsRetVal parsDelimCStr(rsParsObj *pThis, cstr_t **ppCStr, char cDelim, int bTrim /* We got the string, now take it and see if we need to * remove anything at its end. */ - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); if(bTrimTrailing) { CHKiRet(rsCStrTrimTrailingWhiteSpace(pCStr)); @@ -344,7 +344,7 @@ rsRetVal parsQuotedCStr(rsParsObj *pThis, cstr_t **ppCStr) } /* We got the string, let's finish it... */ - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); /* done! */ *ppCStr = pCStr; @@ -380,7 +380,7 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) assert(pIP != NULL); assert(pBits != NULL); - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); parsSkipWhitespace(pThis); pC = rsCStrGetBufBeg(pThis->pCStr) + pThis->iCurrPos; @@ -390,8 +390,8 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) */ while(pThis->iCurrPos < rsCStrLen(pThis->pCStr) && *pC != '/' && *pC != ',' && !isspace((int)*pC)) { - if((iRet = rsCStrAppendChar(pCStr, *pC)) != RS_RET_OK) { - rsCStrDestruct (&pCStr); + if((iRet = cstrAppendChar(pCStr, *pC)) != RS_RET_OK) { + cstrDestruct (&pCStr); FINALIZE; } ++pThis->iCurrPos; @@ -399,15 +399,15 @@ rsRetVal parsAddrWithBits(rsParsObj *pThis, struct NetAddr **pIP, int *pBits) } /* We got the string, let's finish it... */ - if((iRet = rsCStrFinish(pCStr)) != RS_RET_OK) { - rsCStrDestruct (&pCStr); + if((iRet = cstrFinalize(pCStr)) != RS_RET_OK) { + cstrDestruct(&pCStr); FINALIZE; } /* now we have the string and must check/convert it to * an NetAddr structure. */ - CHKiRet(rsCStrConvSzStrAndDestruct(pCStr, &pszIP, 0)); + CHKiRet(cstrConvSzStrAndDestruct(pCStr, &pszIP, 0)); *pIP = calloc(1, sizeof(struct NetAddr)); diff --git a/runtime/cfsysline.c b/runtime/cfsysline.c index e1e4a6a4..0fb4247d 100644 --- a/runtime/cfsysline.c +++ b/runtime/cfsysline.c @@ -462,7 +462,7 @@ getWord(uchar **pp, cstr_t **ppStrB) ASSERT(*pp != NULL); ASSERT(ppStrB != NULL); - CHKiRet(rsCStrConstruct(ppStrB)); + CHKiRet(cstrConstruct(ppStrB)); skipWhiteSpace(pp); /* skip over any whitespace */ @@ -470,9 +470,9 @@ getWord(uchar **pp, cstr_t **ppStrB) p = *pp; while(*p && !isspace((int) *p)) { - CHKiRet(rsCStrAppendChar(*ppStrB, *p++)); + CHKiRet(cstrAppendChar(*ppStrB, *p++)); } - CHKiRet(rsCStrFinish(*ppStrB)); + CHKiRet(cstrFinalize(*ppStrB)); *pp = p; @@ -506,7 +506,7 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void ASSERT(*pp != NULL); CHKiRet(getWord(pp, &pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &pNewVal, 0)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pNewVal, 0)); pStrB = NULL; /* we got the word, now set it */ @@ -525,7 +525,7 @@ static rsRetVal doGetWord(uchar **pp, rsRetVal (*pSetHdlr)(void*, uchar*), void finalize_it: if(iRet != RS_RET_OK) { if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -548,7 +548,7 @@ doSyslogName(uchar **pp, rsRetVal (*pSetHdlr)(void*, int), void *pVal, syslogNam ASSERT(*pp != NULL); CHKiRet(getWord(pp, &pStrB)); /* get word */ - iNewVal = decodeSyslogName(rsCStrGetSzStr(pStrB), pNameTable); + iNewVal = decodeSyslogName(cstrGetSzStr(pStrB), pNameTable); if(pSetHdlr == NULL) { /* we should set value directly to var */ diff --git a/runtime/conf.c b/runtime/conf.c index 7cdcf5ec..65fc66f1 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -526,17 +526,15 @@ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEn tplName = (uchar*) strdup((char*)dfltTplName); } else { /* template specified, pick it up */ - if(rsCStrConstruct(&pStrB) != RS_RET_OK) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKiRet(cstrConstruct(&pStrB)); /* now copy the string */ while(*p && *p != '#' && !isspace((int) *p)) { - CHKiRet(rsCStrAppendChar(pStrB, *p)); + CHKiRet(cstrAppendChar(pStrB, *p)); ++p; } - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &tplName, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &tplName, 0)); } iRet = OMSRsetEntry(pOMSR, iEntry, tplName, iTplOpts); @@ -925,7 +923,7 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register selector_t *f) */ static rsRetVal cflineProcessHostSelector(uchar **pline) { - rsRetVal iRet; + DEFiRet; ASSERT(pline != NULL); ASSERT(*pline != NULL); @@ -951,21 +949,20 @@ static rsRetVal cflineProcessHostSelector(uchar **pline) dbgprintf("resetting BSD-like hostname filter\n"); eDfltHostnameCmpMode = HN_NO_COMP; if(pDfltHostnameCmp != NULL) { - if((iRet = rsCStrSetSzStr(pDfltHostnameCmp, NULL)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, NULL)); } } else { dbgprintf("setting BSD-like hostname filter to '%s'\n", *pline); if(pDfltHostnameCmp == NULL) { /* create string for parser */ - if((iRet = rsCStrConstructFromszStr(&pDfltHostnameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrConstructFromszStr(&pDfltHostnameCmp, *pline)); } else { /* string objects exists, just update... */ - if((iRet = rsCStrSetSzStr(pDfltHostnameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltHostnameCmp, *pline)); } } - return RS_RET_OK; + +finalize_it: + RETiRet; } @@ -976,7 +973,7 @@ static rsRetVal cflineProcessHostSelector(uchar **pline) */ static rsRetVal cflineProcessTagSelector(uchar **pline) { - rsRetVal iRet; + DEFiRet; ASSERT(pline != NULL); ASSERT(*pline != NULL); @@ -995,21 +992,20 @@ static rsRetVal cflineProcessTagSelector(uchar **pline) if(**pline != '\0' && **pline == '*' && *(*pline+1) == '\0') { dbgprintf("resetting programname filter\n"); if(pDfltProgNameCmp != NULL) { - if((iRet = rsCStrSetSzStr(pDfltProgNameCmp, NULL)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltProgNameCmp, NULL)); } } else { dbgprintf("setting programname filter to '%s'\n", *pline); if(pDfltProgNameCmp == NULL) { /* create string for parser */ - if((iRet = rsCStrConstructFromszStr(&pDfltProgNameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrConstructFromszStr(&pDfltProgNameCmp, *pline)); } else { /* string objects exists, just update... */ - if((iRet = rsCStrSetSzStr(pDfltProgNameCmp, *pline)) != RS_RET_OK) - return(iRet); + CHKiRet(rsCStrSetSzStr(pDfltProgNameCmp, *pline)); } } - return RS_RET_OK; + +finalize_it: + RETiRet; } diff --git a/runtime/ctok.c b/runtime/ctok.c index d2cd8bbd..263e656c 100644 --- a/runtime/ctok.c +++ b/runtime/ctok.c @@ -269,7 +269,7 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) } CHKiRet(ctokUngetCharFromStream(pThis, c)); /* put not processed char back */ - CHKiRet(rsCStrFinish(pstrVal)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); pstrVal = NULL; @@ -319,7 +319,7 @@ ctokGetSimpStr(ctok_t *pThis, ctok_token_t *pToken) } CHKiRet(ctokGetCharFromStream(pThis, &c)); } - CHKiRet(rsCStrFinish(pStrB)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); pstrVal = NULL; diff --git a/runtime/datetime.c b/runtime/datetime.c index 19e61a0a..40ab4e9c 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -152,11 +152,10 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) static int srSLMGParseInt32(uchar** ppsz) { - int i; + register int i; i = 0; - while(isdigit((int) **ppsz)) - { + while(isdigit((int) **ppsz)) { i = i * 10 + **ppsz - '0'; ++(*ppsz); } diff --git a/runtime/msg.c b/runtime/msg.c index b3c76089..55cc48b8 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -437,11 +437,12 @@ msg_t* MsgDup(msg_t* pOld) pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; + memcpy(pNew->bufPRI, pOld->bufPRI, pOld->iLenPRI); + pNew->iLenPRI = pOld->iLenPRI; tmpCOPYSZ(Severity); tmpCOPYSZ(SeverityStr); tmpCOPYSZ(Facility); tmpCOPYSZ(FacilityStr); - tmpCOPYSZ(PRI); tmpCOPYSZ(RawMsg); tmpCOPYSZ(MSG); tmpCOPYSZ(UxTradMsg); @@ -587,7 +588,7 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) } /* OK, finaally we could obtain a PROCID. So let's use it ;) */ - CHKiRet(rsCStrFinish(pM->pCSPROCID)); + CHKiRet(cstrFinalize(pM->pCSPROCID)); finalize_it: RETiRet; @@ -629,7 +630,7 @@ static rsRetVal aquireProgramName(msg_t *pM) ; ++i) { CHKiRet(rsCStrAppendChar(pM->pCSProgName, pM->pszTAG[i])); } - CHKiRet(rsCStrFinish(pM->pCSProgName)); + CHKiRet(cstrFinalize(pM->pCSProgName)); } finalize_it: RETiRet; @@ -724,31 +725,12 @@ char *getMSG(msg_t *pM) /* Get PRI value in text form */ -static char *getPRI(msg_t *pM) +static inline char *getPRI(msg_t *pM) { - int pri; - BEGINfunc - if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszPRI == NULL) { - /* OK, we need to construct it... we use a 5 byte buffer - as of - * RFC 3164, it can't be longer. Should it still be, snprintf will truncate... - * Note that we do not use the LOG_MAKEPRI macro. This macro - * is a simple add of the two values under FreeBSD 7. So we implement - * the logic in our own code. This is a change from a bug - * report. -- rgerhards, 2008-07-14 - */ - pri = pM->iFacility * 8 + pM->iSeverity; - pM->pszPRI = pM->bufPRI; - pM->iLenPRI = snprintf((char*)pM->pszPRI, 5, "%d", pri); - } - MsgUnlock(pM); - - ENDfunc - return (char*)pM->pszPRI; + return (char*)pM->bufPRI; } diff --git a/runtime/msg.h b/runtime/msg.h index 1689bbbc..20360641 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -73,7 +73,8 @@ short bDoLock; /* use the mutex? */ int iLenFacility; /* ... and its length. */ uchar *pszFacilityStr; /* facility name... */ int iLenFacilityStr; /* ... and its length. */ - uchar *pszPRI; /* the PRI as a string */ + //uchar *pszPRI; /* the PRI as a string */ + uchar bufPRI[5]; int iLenPRI; /* and its length */ uchar *pszRawMsg; /* message as it was received on the * wire. This is important in case we @@ -121,7 +122,6 @@ short bDoLock; /* use the mutex? */ char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ /* now follow fixed-size buffers to safe some time otherwise used for allocs */ - uchar bufPRI[5]; }; diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 1a50e2f8..19dc8678 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -335,7 +335,7 @@ gtlsGetCertInfo(nsd_gtls_t *pThis, cstr_t **ppStr) gnutls_x509_crt_deinit(cert); } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; finalize_it: @@ -455,7 +455,7 @@ GenFingerprintStr(uchar *pFingerprint, size_t sizeFingerprint, cstr_t **ppStr) snprintf((char*)buf, sizeof(buf), ":%2.2X", pFingerprint[i]); CHKiRet(rsCStrAppendStrWithLen(pStr, buf, 3)); } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; @@ -723,7 +723,7 @@ gtlsGetCN(nsd_gtls_t *pThis, gnutls_x509_crt *pCert, cstr_t **ppstrCN) } ++i; /* char processed */ } - CHKiRet(rsCStrFinish(pstrCN)); + CHKiRet(cstrFinalize(pstrCN)); /* we got it - we ignore the rest of the DN string (if any). So we may * not detect if it contains more than one CN @@ -884,7 +884,7 @@ gtlsChkPeerName(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) if(!bFoundPositiveMatch) { dbgprintf("invalid peer name, not permitted to talk to it\n"); if(pThis->bReportAuthErr == 1) { - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); errno = 0; errmsg.LogError(0, RS_RET_INVALID_FINGERPRINT, "error: peer name not authorized - " "not permitted to talk to it. Names: %s", diff --git a/runtime/obj.c b/runtime/obj.c index 2a9df9ed..355c0f97 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -447,7 +447,7 @@ objDeserializeEmbedStr(cstr_t **ppStr, strm_t *pStrm) CHKiRet(rsCStrAppendChar(pStr, c)); NEXTC; } - CHKiRet(rsCStrFinish(pStr)); + CHKiRet(cstrFinalize(pStr)); *ppStr = pStr; @@ -515,7 +515,7 @@ static rsRetVal objDeserializeStr(cstr_t **ppCStr, int iLen, strm_t *pStrm) CHKiRet(rsCStrAppendChar(pCStr, c)); NEXTC; } - CHKiRet(rsCStrFinish(pCStr)); + CHKiRet(cstrFinalize(pCStr)); /* check terminator */ if(c != ':') ABORT_FINALIZE(RS_RET_INVALID_DELIMITER); @@ -629,7 +629,7 @@ static rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) CHKiRet(rsCStrAppendChar(pProp->pcsName, c)); NEXTC; } - CHKiRet(rsCStrFinish(pProp->pcsName)); + CHKiRet(cstrFinalize(pProp->pcsName)); /* property type */ CHKiRet(objDeserializeNumber(&i, pStrm)); diff --git a/runtime/parser.c b/runtime/parser.c index 64e03094..13fb51ec 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -205,8 +205,16 @@ sanitizeMessage(msg_t *pMsg) } } } - if(bNeedSanitize == 0) + if(bNeedSanitize == 0) { + /* what a shame - we do not have a \0 byte... + * TODO: think about adding it or otherwise be able to use it... + */ + uchar *pRaw; + CHKmalloc(pRaw = realloc(pMsg->pszRawMsg, pMsg->iLenRawMsg + 1)); + pRaw[pMsg->iLenRawMsg] = '\0'; + pMsg->pszRawMsg = pRaw; FINALIZE; + } /* now copy over the message and sanitize it */ /* TODO: can we get cheaper memory alloc? {alloca()?}*/ @@ -276,6 +284,7 @@ rsRetVal parseMsg(msg_t *pMsg) DEFiRet; uchar *msg; int pri; + int iPriText; CHKiRet(sanitizeMessage(pMsg)); @@ -285,11 +294,19 @@ rsRetVal parseMsg(msg_t *pMsg) /* pull PRI */ pri = DEFUPRI; msg = pMsg->pszRawMsg; + iPriText = 0; if(*msg == '<') { + /* while we process the PRI, we also fill the PRI textual representation + * inside the msg object. This may not be ideal from an OOP point of view, + * but it offers us performance... + */ pri = 0; while(isdigit((int) *++msg)) { + pMsg->bufPRI[iPriText++ % 4] = *msg; /* mod 4 to guard against malformed messages! */ pri = 10 * pri + (*msg - '0'); } + pMsg->bufPRI[iPriText % 4] = '\0'; + pMsg->iLenPRI = iPriText % 4; if(*msg == '>') ++msg; if(pri & ~(LOG_FACMASK|LOG_PRIMASK)) diff --git a/runtime/stream.c b/runtime/stream.c index f1f69cc8..1cff2da6 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -368,7 +368,7 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr) CHKiRet(rsCStrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } - CHKiRet(rsCStrFinish(*ppCStr)); + CHKiRet(cstrFinalize(*ppCStr)); finalize_it: if(iRet != RS_RET_OK && *ppCStr != NULL) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index f3d9aa48..a2d9c599 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -6,8 +6,9 @@ * Please see syslogd.c for license information. * All functions in this "class" start with rsCStr (rsyslog Counted String). * begun 2005-09-07 rgerhards + * did some optimization (read: bugs!) rgerhards, 2009-06-16 * - * Copyright (C) 2007-2008 by Rainer Gerhards and Adiscon GmbH + * Copyright (C) 2007-2009 by Rainer Gerhards and Adiscon GmbH * * This file is part of the rsyslog runtime library. * @@ -40,6 +41,7 @@ #include "regexp.h" #include "obj.h" +uchar* rsCStrGetSzStr(cstr_t *pThis); /* ################################################################# * * private members * @@ -54,15 +56,14 @@ DEFobjCurrIf(regexp) * ################################################################# */ -rsRetVal rsCStrConstruct(cstr_t **ppThis) +rsRetVal cstrConstruct(cstr_t **ppThis) { DEFiRet; cstr_t *pThis; ASSERT(ppThis != NULL); - if((pThis = (cstr_t*) calloc(1, sizeof(cstr_t))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pThis = (cstr_t*) calloc(1, sizeof(cstr_t))); rsSETOBJTYPE(pThis, OIDrsCStr); pThis->pBuf = NULL; @@ -89,7 +90,7 @@ rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz) CHKiRet(rsCStrConstruct(&pThis)); - pThis->iBufSize = pThis->iStrLen = strlen((char*)(char *) sz); + pThis->iBufSize = pThis->iStrLen = strlen((char *) sz); if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -137,24 +138,8 @@ void rsCStrDestruct(cstr_t **ppThis) { cstr_t *pThis = *ppThis; - /* rgerhards 2005-10-19: The free of pBuf was contained in conditional compilation. - * The code was only compiled if STRINGBUF_TRIM_ALLOCSIZE was set to 1. I honestly - * do not know why it was so, I think it was an artifact. Anyhow, I have changed this - * now. Should there any issue occur, this comment hopefully will shed some light - * on what happened. I re-verified, and this function has never before been called - * by anyone. So changing it can have no impact for obvious reasons... - * - * rgerhards, 2008-02-20: I changed the interface to the new calling conventions, where - * the destructor receives a pointer to the object, so that it can set it to NULL. - */ - if(pThis->pBuf != NULL) { - free(pThis->pBuf); - } - - if(pThis->pszBuf != NULL) { - free(pThis->pszBuf); - } - + free(pThis->pBuf); + free(pThis->pszBuf); RSFREEOBJ(pThis); *ppThis = NULL; } @@ -166,12 +151,14 @@ void rsCStrDestruct(cstr_t **ppThis) * allocated. In practice, a bit more is allocated because we envision that * some more characters may be added after these. * rgerhards, 2008-01-07 + * changed to utilized realloc() -- rgerhards, 2009-06-16 */ -static rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) +static rsRetVal +rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) { - DEFiRet; uchar *pNewBuf; size_t iNewSize; + DEFiRet; /* first compute the new size needed */ if(iMinNeeded > pThis->iAllocIncrement) { @@ -187,15 +174,9 @@ static rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) } iNewSize += pThis->iBufSize; /* add current size */ - /* and then allocate and copy over */ /* DEV debugging only: dbgprintf("extending string buffer, old %d, new %d\n", pThis->iBufSize, iNewSize); */ - if((pNewBuf = (uchar*) malloc(iNewSize * sizeof(uchar))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - memcpy(pNewBuf, pThis->pBuf, pThis->iBufSize); + CHKmalloc(pNewBuf = (uchar*) realloc(pThis->pBuf, iNewSize * sizeof(uchar))); pThis->iBufSize = iNewSize; - if(pThis->pBuf != NULL) { - free(pThis->pBuf); - } pThis->pBuf = pNewBuf; finalize_it: @@ -288,6 +269,29 @@ finalize_it: } +/* NEW VARIANT + * Append a character to the current string object. This may only be done until + * cstrFinalize() is called. + * rgerhards, 2009-06-16 + */ +rsRetVal cstrAppendChar(cstr_t *pThis, uchar c) +{ + DEFiRet; + + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + + if(pThis->iStrLen >= pThis->iBufSize) { + CHKiRet(rsCStrExtendBuf(pThis, 1)); /* need more memory! */ + } + + /* ok, when we reach this, we have sufficient memory */ + *(pThis->pBuf + pThis->iStrLen++) = c; + +finalize_it: + RETiRet; +} + + /* Sets the string object to the classigal sz-string provided. * Any previously stored vlaue is discarded. If a NULL pointer * the the new value (pszNew) is provided, an empty string is @@ -388,35 +392,42 @@ uchar* rsCStrGetSzStr(cstr_t *pThis) } +/* NEW VERSION for interface without separate psz buffer! */ +/* Returns the cstr data as a classical C sz string. We use that the + * Finalizer did properly terminate our string (but we may stil be NULL). + * So it is vital that the finalizer is called BEFORe this function here! + * The caller must not free or otherwise manipulate the returned string and must not + * destroy the CStr object as long as the ascii string is used. + * This function may return NULL, if the string is currently NULL. This + * is a feature, not a bug. If you need non-NULL in any case, use + * cstrGetSzStrNoNULL() instead. + * Note that due to the new single-buffer interface this function almost does nothing! + * rgerhards, 2006-09-16 + */ +uchar* cstrGetSzStr(cstr_t *pThis) +{ + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + return(pThis->pBuf); +} + + /* Converts the CStr object to a classical zero-terminated C string, * returns that string and destroys the CStr object. The returned string * MUST be freed by the caller. The function might return NULL if * no memory can be allocated. * - * TODO: - * This function should at some time become special. The base idea is to - * add one extra byte to the end of the regular buffer, so that we can - * convert it to an szString without the need to copy. The extra memory - * footprint is not hefty, but the performance gain is potentially large. - * To get it done now, I am not doing the optimiziation right now. - * rgerhards, 2005-09-07 + * This is the NEW replacement for rsCStrConvSzStrAndDestruct which does + * no longer utilize a special buffer but soley works on pBuf (and also + * assumes that cstrFinalize had been called). * - * rgerhards, 2007-09-04: I have changed the interface of this function. It now - * returns an rsRetVal, so that we can communicate back if we have an error. - * Using the standard method is much better than returning NULL. Secondly, NULL - * was not actually an error - it was in indication if the string was empty. - * This was needed in some parts of the code, in others not. I have now added - * a second parameter to specify what the caller needs. I hope these changes - * will make it less likely that the function is called incorrectly, what - * previously happend quite often and was the cause of a number of program - * aborts. So the parameters are now: + * Parameters are as follows: * pointer to the object, pointer to string-pointer to receive string and * bRetNULL: 0 - must not return NULL on empty string, return "" in that * case, 1 - return NULL instead of an empty string. * PLEASE NOTE: the caller must free the memory returned in ppSz in any case * (except, of course, if it is NULL). */ -rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) +rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) { DEFiRet; uchar* pRetBuf; @@ -427,14 +438,13 @@ rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) if(pThis->pBuf == NULL) { if(bRetNULL == 0) { - if((pRetBuf = malloc(sizeof(uchar))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pRetBuf = malloc(sizeof(uchar))); *pRetBuf = '\0'; } else { pRetBuf = NULL; } } else - pRetBuf = rsCStrGetSzStr(pThis); + pRetBuf = pThis->pBuf; *ppSz = pRetBuf; @@ -443,60 +453,39 @@ finalize_it: * that we can NOT use the rsCStrDestruct function as it would * also free the sz String buffer, which we pass on to the user. */ - if(pThis->pBuf != NULL) - free(pThis->pBuf); RSFREEOBJ(pThis); - RETiRet; } -#if STRINGBUF_TRIM_ALLOCSIZE == 1 - /* Only in this mode, we need to trim the string. To do - * so, we must allocate a new buffer of the exact - * string size, and then copy the old one over. - */ - /* WARNING - * STRINGBUF_TRIM_ALLOCSIZE can, in theory, be used to trim - * memory buffers. This part of the code was inherited from - * liblogging (where it is used in a different context) but - * never put to use in rsyslog. The reason is that it is hardly - * imaginable where the extra performance cost is worth the save - * in memory alloc. Then Anders Blomdel rightfully pointed out that - * the code does not work at all - and nobody even know that it - * probably shouldn't. Rather than removing, I deciced to somewhat - * fix the code, so that this feature may be enabled if somebody - * really has a need for it. Be warned, however, that I NEVER - * tested the fix. So if you intend to use this feature, you must - * do full testing before you rely on it. -- rgerhards, 2008-02-12 - */ -rsRetVal rsCStrFinish(cstr_t __attribute__((unused)) *pThis) +/* Finalize the string object. This must be called after all data is added to it + * but before that data is used. + * rgerhards, 2009-06-16 + */ +rsRetVal +cstrFinalize(cstr_t *pThis) { DEFiRet; - uchar* pBuf; rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + + assert(pThis->bIsFinalized == 0); - if((pBuf = malloc((pThis->iStrLen) * sizeof(uchar))) == NULL) - { /* OK, in this case we use the previous buffer. At least - * we have it ;) - */ - } - else - { /* got the new buffer, so let's use it */ - memcpy(pBuf, pThis->pBuf, pThis->iStrLen); - pThis->pBuf = pBuf; + if(pThis->iStrLen > 0) { + /* terminate string only if one exists */ + CHKiRet(cstrAppendChar(pThis, '\0')); + --pThis->iStrLen; /* do NOT count the \0 byte */ } + pThis->bIsFinalized = 1; +finalize_it: RETiRet; } -#endif /* #if STRINGBUF_TRIM_ALLOCSIZE == 1 */ void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement) { rsCHECKVALIDOBJECT(pThis, OIDrsCStr); assert(iNewIncrement > 0); - pThis->iAllocIncrement = iNewIncrement; } @@ -1025,56 +1014,6 @@ int rsCStrCaseInsensitiveLocateInSzStr(cstr_t *pThis, uchar *sz) } -#if 0 /* read comment below why this is commented out. In short: for future use! */ -/* locate the first occurence of a standard sz string inside a rsCStr object. - * Returns the offset (0-bound) of this first occurrence. If not found, -1 is - * returned. - * rgerhards 2005-09-19 - * WARNING: I accidently created this function (I later noticed I didn't relly - * need it... I will not remove the function, as it probably is useful - * some time later. However, it is not fully tested, so start with testing - * it before you put it to first use). - */ -int rsCStrLocateSzStr(cstr_t *pThis, uchar *sz) -{ - int iLenSz; - int i; - int iMax; - int bFound; - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if(sz == NULL) - return 0; - - iLenSz = strlen((char*)sz); - if(iLenSz == 0) - return 0; - - /* compute the largest index where a match could occur - after all, - * the to-be-located string must be able to be present in the - * searched string (it needs its size ;)). - */ - iMax = pThis->iStrLen - iLenSz; - - bFound = 0; - i = 0; - while(i < iMax && !bFound) { - int iCheck; - uchar *pComp = pThis->pBuf + i; - for(iCheck = 0 ; iCheck < iLenSz ; ++iCheck) - if(*(pComp + iCheck) != *(sz + iCheck)) - break; - if(iCheck == iLenSz) - bFound = 1; /* found! - else it wouldn't be equal */ - else - ++i; /* on to the next try */ - } - - return(bFound ? i : -1); -} -#endif /* end comment out */ - - /* our exit function. TODO: remove once converted to a class * rgerhards, 2008-03-11 */ @@ -1098,11 +1037,5 @@ finalize_it: } -/* - * Local variables: - * c-indent-level: 8 - * c-basic-offset: 8 - * tab-width: 8 - * End: - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 684133bb..d28aee26 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -49,13 +49,15 @@ typedef struct cstr_s size_t iBufSize; /**< current maximum size of the string buffer */ size_t iStrLen; /**< length of the string in characters. */ size_t iAllocIncrement; /**< the amount of bytes the string should be expanded if it needs to */ + bool bIsFinalized; /**< is this object finished and ready for use? (a debug aid, may be removed later TODO 2009-06-16) */ } cstr_t; /** * Construct a rsCStr object. */ -rsRetVal rsCStrConstruct(cstr_t **ppThis); +rsRetVal cstrConstruct(cstr_t **ppThis); +#define rsCStrConstruct(x) cstrConstruct((x)) rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz); rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom); @@ -63,6 +65,7 @@ rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom); * Destruct the string buffer object. */ void rsCStrDestruct(cstr_t **ppThis); +#define cstrDestruct(x) rsCStrDestruct((x)) /** * Append a character to an existing string. If necessary, the @@ -71,6 +74,7 @@ void rsCStrDestruct(cstr_t **ppThis); * \param c Character to append to string. */ rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c); +rsRetVal cstrAppendChar(cstr_t *pThis, uchar c); /** * Truncate "n" number of characters from the end of the @@ -123,10 +127,9 @@ rsRetVal rsCStrAppendInt(cstr_t *pThis, long i); rsRetVal strExit(void); /* TODO: remove once we have a real object interface! */ -uchar* rsCStrGetSzStr(cstr_t *pThis); +uchar* __attribute__((deprecated)) rsCStrGetSzStr(cstr_t *pThis); uchar* rsCStrGetSzStrNoNULL(cstr_t *pThis); rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew); -rsRetVal rsCStrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); int rsCStrCStrCmp(cstr_t *pCS1, cstr_t *pCS2); int rsCStrSzStrCmp(cstr_t *pCS1, uchar *psz, size_t iLenSz); int rsCStrOffsetSzStrCmp(cstr_t *pCS1, size_t iOffset, uchar *psz, size_t iLenSz); @@ -142,6 +145,11 @@ rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); +/* new calling interface */ +rsRetVal cstrFinalize(cstr_t *pThis); +rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); +uchar* cstrGetSzStr(cstr_t *pThis); + /* now come inline-like functions */ #ifdef NDEBUG # define rsCStrLen(x) ((int)((x)->iStrLen)) @@ -149,19 +157,6 @@ rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); int rsCStrLen(cstr_t *pThis); #endif -#if STRINGBUF_TRIM_ALLOCSIZE != 1 -/* This is the normal case (see comment in rsCStrFinish!). In those cases, the function - * simply needs to do nothing, so that we can save us the function call. - * rgerhards, 2008-02-12 - */ -# define rsCStrFinish(pThis) RS_RET_OK -#else - /** - * Finish the string buffer dynamic allocation. - */ - rsRetVal rsCStrFinish(cstr_t *pThis); -#endif - #define rsCStrGetBufBeg(x) ((x)->pBuf) rsRetVal strInit(); diff --git a/runtime/vm.c b/runtime/vm.c index 125b0d21..8cbf9e12 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -563,7 +563,7 @@ rsf_tolower(vmstk_t *pStk, int numOperands) } /* Store result and cleanup */ - CHKiRet(rsCStrFinish(pcstr)); + CHKiRet(cstrFinalize(pcstr)); var.SetString(operand1, pcstr); vmstk.Push(pStk, operand1); finalize_it: diff --git a/runtime/vmop.c b/runtime/vmop.c index 3e001d27..acacfc9e 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -85,7 +85,7 @@ CODESTARTobjDebugPrint(vmop) CHKiRet(var.Obj2Str(pThis->operand.pVar, pStrVar)); } } - CHKiRet(rsCStrFinish(&pStrVar)); + CHKiRet(cstrFinalize(pStrVar)); dbgoprint((obj_t*) pThis, "%.12s\t%s\n", pOpcodeName, rsCStrGetSzStrNoNULL(pStrVar)); if(pThis->opcode != opcode_FUNC_CALL) rsCStrDestruct(&pStrVar); diff --git a/template.c b/template.c index 4a12aa3b..6e34bcd8 100644 --- a/template.c +++ b/template.c @@ -88,7 +88,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) * free the obtained value (if requested). We continue this * loop until we got hold of all values. */ - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); pTpe = pTpl->pEntryRoot; while(pTpe != NULL) { @@ -99,7 +99,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) ) { dbgprintf("error %d during tplToString()\n", iRet); /* it does not make sense to continue now */ - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); FINALIZE; } } else if(pTpe->eEntryType == FIELD) { @@ -119,7 +119,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) CHKiRet_Hdlr(rsCStrAppendStrWithLen(pCStr, (uchar*) pVal, iLenVal)) { dbgprintf("error %d during tplToString()\n", iRet); /* it does not make sense to continue now */ - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); if(bMustBeFreed) free(pVal); FINALIZE; @@ -133,8 +133,8 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) /* we are done with the template, now let's convert the result into a * "real" (usable) string and discard the helper structures. */ - CHKiRet(rsCStrFinish(pCStr)); - CHKiRet(rsCStrConvSzStrAndDestruct(pCStr, &pVal, 0)); + CHKiRet(cstrFinalize(pCStr)); + CHKiRet(cstrConvSzStrAndDestruct(pCStr, &pVal, 0)); finalize_it: *ppSz = (iRet == RS_RET_OK) ? pVal : NULL; @@ -272,21 +272,21 @@ doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeM p = *pp; iLen = *pLen; - CHKiRet(rsCStrConstruct(&pStrB)); + CHKiRet(cstrConstruct(&pStrB)); while(*p) { if(*p == '\'') { - CHKiRet(rsCStrAppendChar(pStrB, (escapeMode == 0) ? '\'' : '\\')); + CHKiRet(cstrAppendChar(pStrB, (escapeMode == 0) ? '\'' : '\\')); iLen++; /* reflect the extra character */ } else if((escapeMode == 1) && (*p == '\\')) { - CHKiRet(rsCStrAppendChar(pStrB, '\\')); + CHKiRet(cstrAppendChar(pStrB, '\\')); iLen++; /* reflect the extra character */ } - CHKiRet(rsCStrAppendChar(pStrB, *p)); + CHKiRet(cstrAppendChar(pStrB, *p)); ++p; } - CHKiRet(rsCStrFinish(pStrB)); - CHKiRet(rsCStrConvSzStrAndDestruct(pStrB, &pszGenerated, 0)); + CHKiRet(cstrFinalize(pStrB)); + CHKiRet(cstrConvSzStrAndDestruct(pStrB, &pszGenerated, 0)); if(*pbMustBeFreed) free(*pp); /* discard previous value */ @@ -299,7 +299,7 @@ finalize_it: if(iRet != RS_RET_OK) { doSQLEmergencyEscape(*pp, escapeMode); if(pStrB != NULL) - rsCStrDestruct(&pStrB); + cstrDestruct(&pStrB); } RETiRet; @@ -376,7 +376,7 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) p = *pp; - if(rsCStrConstruct(&pStrB) != RS_RET_OK) + if(cstrConstruct(&pStrB) != RS_RET_OK) return 1; rsCStrSetAllocIncrement(pStrB, 32); /* process the message and expand escapes @@ -387,22 +387,22 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) switch(*++p) { case '\0': /* the best we can do - it's invalid anyhow... */ - rsCStrAppendChar(pStrB, *p); + cstrAppendChar(pStrB, *p); break; case 'n': - rsCStrAppendChar(pStrB, '\n'); + cstrAppendChar(pStrB, '\n'); ++p; break; case 'r': - rsCStrAppendChar(pStrB, '\r'); + cstrAppendChar(pStrB, '\r'); ++p; break; case '\\': - rsCStrAppendChar(pStrB, '\\'); + cstrAppendChar(pStrB, '\\'); ++p; break; case '%': - rsCStrAppendChar(pStrB, '%'); + cstrAppendChar(pStrB, '%'); ++p; break; case '0': /* numerical escape sequence */ @@ -419,15 +419,15 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) while(*p && isdigit((int)*p)) { i = i * 10 + *p++ - '0'; } - rsCStrAppendChar(pStrB, i); + cstrAppendChar(pStrB, i); break; default: - rsCStrAppendChar(pStrB, *p++); + cstrAppendChar(pStrB, *p++); break; } } else - rsCStrAppendChar(pStrB, *p++); + cstrAppendChar(pStrB, *p++); } if((pTpe = tpeConstruct(pTpl)) == NULL) { @@ -438,14 +438,14 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) return 1; } pTpe->eEntryType = CONSTANT; - rsCStrFinish(pStrB); + cstrFinalize(pStrB); /* We obtain the length from the counted string object * (before we delete it). Later we might take additional * benefit from the counted string object. * 2005-09-09 rgerhards */ pTpe->data.constant.iLenConstant = rsCStrLen(pStrB); - if(rsCStrConvSzStrAndDestruct(pStrB, &pTpe->data.constant.pConstant, 0) != RS_RET_OK) + if(cstrConvSzStrAndDestruct(pStrB, &pTpe->data.constant.pConstant, 0) != RS_RET_OK) return 1; *pp = p; @@ -552,7 +552,7 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) p = (unsigned char*) *pp; - if(rsCStrConstruct(&pStrB) != RS_RET_OK) + if(cstrConstruct(&pStrB) != RS_RET_OK) return 1; if((pTpe = tpeConstruct(pTpl)) == NULL) { @@ -563,13 +563,13 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) pTpe->eEntryType = FIELD; while(*p && *p != '%' && *p != ':') { - rsCStrAppendChar(pStrB, tolower(*p)); + cstrAppendChar(pStrB, tolower(*p)); ++p; /* do NOT do this in tolower()! */ } /* got the name*/ - rsCStrFinish(pStrB); - if(rsCStrConvSzStrAndDestruct(pStrB, &pTpe->data.field.pPropRepl, 0) != RS_RET_OK) + cstrFinalize(pStrB); + if(cstrConvSzStrAndDestruct(pStrB, &pTpe->data.field.pPropRepl, 0) != RS_RET_OK) return 1; /* Check frompos, if it has an R, then topos should be a regex */ diff --git a/tools/syslogd.c b/tools/syslogd.c index a9f1dbdb..c224087d 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -553,7 +553,7 @@ static char **crunch_list(char *list) #if 0 count=0; while (result[count]) - dbgprintf("#%d: %s\n", count, StripDomains[count++]); + DBGPRINTF("#%d: %s\n", count, StripDomains[count++]); #endif return result; } @@ -780,7 +780,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla * rgerhards, 2007-09-14 */ if(*(msg + len - 1) == '\0') { - dbgprintf("dropped NUL at very end of message\n"); + DBGPRINTF("dropped NUL at very end of message\n"); len--; } @@ -790,7 +790,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla * turn on/off this handling. rgerhards, 2007-07-23 */ if(bDropTrailingLF && *(msg + len - 1) == '\n') { - dbgprintf("dropped LF at very end of message (DropTrailingLF is set)\n"); + DBGPRINTF("dropped LF at very end of message (DropTrailingLF is set)\n"); len--; } @@ -815,7 +815,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla iLenDefBuf = iMaxLine; CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iMaxLine + 1))); ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) msg+1, len-1); - dbgprintf("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", + DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", ret, (long) iLenDefBuf, len-1); /* Now check if the uncompression worked. If not, there is not much we can do. In * that case, we log an error message but ignore the message itself. Storing the @@ -861,7 +861,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla * (I couldn't do any more smart things anyway...). * rgerhards, 2007-9-20 */ - dbgprintf("internal error: iMsg > max msg size in printchopped()\n"); + DBGPRINTF("internal error: iMsg > max msg size in printchopped()\n"); } FINALIZE; /* in this case, we are done... nothing left we can do */ } @@ -1025,14 +1025,14 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce } else if(f->eHostnameCmpMode == HN_COMP_MATCH) { if(rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { /* not equal, so we are already done... */ - dbgprintf("hostname filter '+%s' does not match '%s'\n", + DBGPRINTF("hostname filter '+%s' does not match '%s'\n", rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); FINALIZE; } } else { /* must be -hostname */ if(!rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { /* not equal, so we are already done... */ - dbgprintf("hostname filter '-%s' does not match '%s'\n", + DBGPRINTF("hostname filter '-%s' does not match '%s'\n", rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); FINALIZE; } @@ -1053,7 +1053,7 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce if((!bEqv && !bInv) || (bEqv && bInv)) { /* not equal or inverted selection, so we are already done... */ - dbgprintf("programname filter '%s' does not match '%s'\n", + DBGPRINTF("programname filter '%s' does not match '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp), getProgramName(pMsg)); FINALIZE; } @@ -1063,7 +1063,7 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce if(f->f_filter_type == FILTER_PRI) { /* skip messages that are incorrect priority */ - if ( (f->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || \ + if ( (f->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || ((f->f_filterData.f_pmask[pMsg->iFacility] & (1<iSeverity)) == 0) ) bRet = 0; else @@ -1074,7 +1074,7 @@ static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProce CHKiRet(vm.SetMsg(pVM, pMsg)); CHKiRet(vm.ExecProg(pVM, f->f_filterData.f_expr->pVmprg)); CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); - dbgprintf("result of expression evaluation: %lld\n", pResult->val.num); + DBGPRINTF("result of expression evaluation: %lld\n", pResult->val.num); /* VM is destructed on function exit */ bRet = (pResult->val.num) ? 1 : 0; } else { @@ -1166,7 +1166,7 @@ DEFFUNC_llExecFunc(processMsgDoActions) assert(pAction != NULL); if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { - dbgprintf("not calling action because the previous one is not suspended\n"); + DBGPRINTF("not calling action because the previous one is not suspended\n"); ABORT_FINALIZE(RS_RET_OK); } @@ -1399,7 +1399,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); } } else { - dbgprintf("no TIMESTAMP detected!\n"); + DBGPRINTF("no TIMESTAMP detected!\n"); bContParse = 0; } @@ -1561,7 +1561,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) /* indeed, this smells like a TAG, so lets use it for this. We take * the HOSTNAME from the sender system instead. */ - dbgprintf("HOSTNAME contains invalid characters, assuming it to be a TAG.\n"); + DBGPRINTF("HOSTNAME contains invalid characters, assuming it to be a TAG.\n"); moveHOSTNAMEtoTAG(pMsg); MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } @@ -1586,22 +1586,21 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) * the records: the code is currently clean, but we could optimize it! */ if(!bTAGCharDetected) { uchar *pszTAG; - if(rsCStrConstruct(&pStrB) != RS_RET_OK) + if(cstrConstruct(&pStrB) != RS_RET_OK) return 1; rsCStrSetAllocIncrement(pStrB, 33); pWork = pBuf; iCnt = 0; while(*p2parse && *p2parse != ':' && *p2parse != ' ') { - rsCStrAppendChar(pStrB, *p2parse++); + cstrAppendChar(pStrB, *p2parse++); ++iCnt; } if(*p2parse == ':') { ++p2parse; - rsCStrAppendChar(pStrB, ':'); + cstrAppendChar(pStrB, ':'); } - rsCStrFinish(pStrB); - - rsCStrConvSzStrAndDestruct(pStrB, &pszTAG, 1); + cstrFinalize(pStrB); + cstrConvSzStrAndDestruct(pStrB, &pszTAG, 1); if(pszTAG == NULL) { /* rger, 2005-11-10: no TAG found - this implies that what * we have considered to be the HOSTNAME is most probably the @@ -1611,7 +1610,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) * the hostname. This situation is the standard case with * stock BSD syslogd. */ - dbgprintf("No TAG in message, assuming that HOSTNAME is missing.\n"); + DBGPRINTF("No TAG in message, assuming that HOSTNAME is missing.\n"); moveHOSTNAMEtoTAG(pMsg); MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } else { /* we have a TAG, so we can happily set it ;) */ @@ -1627,7 +1626,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) */ if(!(flags & INTERNAL_MSG)) { - dbgprintf("HOSTNAME and TAG not parsed by user configuraton.\n"); + DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n"); MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } } @@ -1688,7 +1687,7 @@ logmsg(msg_t *pMsg, int flags) assert(pMsg != NULL); assert(pMsg->pszUxTradMsg != NULL); msg = (char*) pMsg->pszUxTradMsg; - dbgprintf("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg); + DBGPRINTF("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg); /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have * a traditional syslog message or one formatted according to syslog-protocol. @@ -1696,14 +1695,14 @@ logmsg(msg_t *pMsg, int flags) * -protocol VERSION field for the detection. */ if(msg[0] == '1' && msg[1] == ' ') { - dbgprintf("Message has syslog-protocol format.\n"); + DBGPRINTF("Message has syslog-protocol format.\n"); setProtocolVersion(pMsg, 1); if(parseRFCSyslogMsg(pMsg, flags) == 1) { msgDestruct(&pMsg); return; } } else { /* we have legacy syslog */ - dbgprintf("Message has legacy syslog format.\n"); + DBGPRINTF("Message has legacy syslog format.\n"); setProtocolVersion(pMsg, 0); if(parseLegacySyslogMsg(pMsg, flags) == 1) { msgDestruct(&pMsg); @@ -1753,7 +1752,7 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions) * in an acceptable way. -- rgerhards, 2008-09-16 */ if (pAction->f_prevcount && time(NULL) >= REPEATTIME(pAction)) { - dbgprintf("flush %s: repeated %d times, %d sec.\n", + DBGPRINTF("flush %s: repeated %d times, %d sec.\n", module.GetStateName(pAction->pMod), pAction->f_prevcount, repeatinterval[pAction->f_repeatcount]); actionWriteToAction(pAction); @@ -1789,9 +1788,9 @@ static void debug_switch() if(debugging_on == 0) { debugging_on = 1; - dbgprintf("Switching debugging_on to true\n"); + DBGPRINTF("Switching debugging_on to true\n"); } else { - dbgprintf("Switching debugging_on to false\n"); + DBGPRINTF("Switching debugging_on to false\n"); debugging_on = 0; } @@ -1935,7 +1934,7 @@ void legacyOptsParseTCP(char ch, char *arg) * a minimal delay, but it is much cleaner than the approach of doing everything * inside the signal handler. * rgerhards, 2005-10-26 - * Note: we do not call dbgprintf() as this may cause us to block in case something + * Note: we do not call DBGPRINTF() as this may cause us to block in case something * with the threading is wrong. */ static void doDie(int sig) @@ -1983,7 +1982,7 @@ die(int sig) { char buf[256]; - dbgprintf("exiting on signal %d\n", sig); + DBGPRINTF("exiting on signal %d\n", sig); /* IMPORTANT: we should close the inputs first, and THEN send our termination * message. If we do it the other way around, logmsgInternal() may block on @@ -1998,7 +1997,7 @@ die(int sig) */ /* close the inputs */ - dbgprintf("Terminating input threads...\n"); + DBGPRINTF("Terminating input threads...\n"); thrdTerminateAll(); /* and THEN send the termination log message (see long comment above) */ @@ -2012,17 +2011,17 @@ die(int sig) } /* drain queue (if configured so) and stop main queue worker thread pool */ - dbgprintf("Terminating main queue...\n"); + DBGPRINTF("Terminating main queue...\n"); qqueueDestruct(&pMsgQueue); pMsgQueue = NULL; /* Free ressources and close connections. This includes flushing any remaining * repeated msgs. */ - dbgprintf("Terminating outputs...\n"); + DBGPRINTF("Terminating outputs...\n"); freeSelectors(); - dbgprintf("all primary multi-thread sources have been terminated - now doing aux cleanup...\n"); + DBGPRINTF("all primary multi-thread sources have been terminated - now doing aux cleanup...\n"); /* rger 2005-02-22 * now clean up the in-memory structures. OK, the OS * would also take care of that, but if we do it @@ -2060,7 +2059,7 @@ die(int sig) */ module.UnloadAndDestructAll(eMOD_LINK_ALL); - dbgprintf("Clean shutdown completed, bye\n"); + DBGPRINTF("Clean shutdown completed, bye\n"); /* dbgClassExit MUST be the last one, because it de-inits the debug system */ dbgClassExit(); @@ -2117,7 +2116,7 @@ static rsRetVal setMaxFiles(void __attribute__((unused)) *pVal, int iFiles) iFiles, errStr, (long) maxFiles.rlim_max); ABORT_FINALIZE(RS_RET_ERR_RLIM_NOFILE); } - dbgprintf("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); + DBGPRINTF("Max number of files set to %d [kernel max %ld].\n", iFiles, (long) maxFiles.rlim_max); finalize_it: RETiRet; @@ -2128,7 +2127,7 @@ finalize_it: static rsRetVal setUmask(void __attribute__((unused)) *pVal, int iUmask) { umask(iUmask); - dbgprintf("umask set to 0%3.3o.\n", iUmask); + DBGPRINTF("umask set to 0%3.3o.\n", iUmask); return RS_RET_OK; } @@ -2213,7 +2212,7 @@ static void freeSelectors(void) selector_t *fPrev; if(Files != NULL) { - dbgprintf("Freeing log structures.\n"); + DBGPRINTF("Freeing log structures.\n"); for(f = Files ; f != NULL ; f = f->f_next) { llExecFunc(&f->llActList, freeSelectorsActions, NULL); @@ -2420,7 +2419,7 @@ DEFFUNC_llExecFunc(dbgPrintInitInfoAction) { DEFiRet; iRet = actionDbgPrint((action_t*) pData); - dbgprintf("\n"); + DBGPRINTF("\n"); RETiRet; } @@ -2437,43 +2436,44 @@ static void dbgPrintInitInfo(void) int iSelNbr = 1; int i; - dbgprintf("\nActive selectors:\n"); + DBGPRINTF("\nActive selectors:\n"); for (f = Files; f != NULL ; f = f->f_next) { - dbgprintf("Selector %d:\n", iSelNbr++); + DBGPRINTF("Selector %d:\n", iSelNbr++); if(f->pCSProgNameComp != NULL) - dbgprintf("tag: '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp)); + DBGPRINTF("tag: '%s'\n", rsCStrGetSzStrNoNULL(f->pCSProgNameComp)); if(f->eHostnameCmpMode != HN_NO_COMP) - dbgprintf("hostname: %s '%s'\n", + DBGPRINTF("hostname: %s '%s'\n", f->eHostnameCmpMode == HN_COMP_MATCH ? "only" : "allbut", rsCStrGetSzStrNoNULL(f->pCSHostnameComp)); if(f->f_filter_type == FILTER_PRI) { for (i = 0; i <= LOG_NFACILITIES; i++) - if (f->f_filterData.f_pmask[i] == TABLE_NOPRI) - dbgprintf(" X "); - else - dbgprintf("%2X ", f->f_filterData.f_pmask[i]); + if (f->f_filterData.f_pmask[i] == TABLE_NOPRI) { + DBGPRINTF(" X "); + } else { + DBGPRINTF("%2X ", f->f_filterData.f_pmask[i]); + } } else if(f->f_filter_type == FILTER_EXPR) { - dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); + DBGPRINTF("EXPRESSION-BASED Filter: can currently not be displayed"); } else { - dbgprintf("PROPERTY-BASED Filter:\n"); - dbgprintf("\tProperty.: '%s'\n", + DBGPRINTF("PROPERTY-BASED Filter:\n"); + DBGPRINTF("\tProperty.: '%s'\n", rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName)); - dbgprintf("\tOperation: "); + DBGPRINTF("\tOperation: "); if(f->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("'%s'\n", getFIOPName(f->f_filterData.prop.operation)); - dbgprintf("\tValue....: '%s'\n", + DBGPRINTF("NOT "); + DBGPRINTF("'%s'\n", getFIOPName(f->f_filterData.prop.operation)); + DBGPRINTF("\tValue....: '%s'\n", rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue)); - dbgprintf("\tAction...: "); + DBGPRINTF("\tAction...: "); } - dbgprintf("\nActions:\n"); + DBGPRINTF("\nActions:\n"); llExecFunc(&f->llActList, dbgPrintInitInfoAction, NULL); /* actions */ - dbgprintf("\n"); + DBGPRINTF("\n"); } - dbgprintf("\n"); + DBGPRINTF("\n"); if(bDebugPrintTemplateList) tplPrintList(); if(bDebugPrintModuleList) @@ -2483,24 +2483,24 @@ static void dbgPrintInitInfo(void) if(bDebugPrintCfSysLineHandlerList) dbgPrintCfSysLineHandlers(); - dbgprintf("Messages with malicious PTR DNS Records are %sdropped.\n", + DBGPRINTF("Messages with malicious PTR DNS Records are %sdropped.\n", glbl.GetDropMalPTRMsgs() ? "" : "not "); - dbgprintf("Control characters are %sreplaced upon reception.\n", + DBGPRINTF("Control characters are %sreplaced upon reception.\n", bEscapeCCOnRcv? "" : "not "); if(bEscapeCCOnRcv) - dbgprintf("Control character escape sequence prefix is '%c'.\n", + DBGPRINTF("Control character escape sequence prefix is '%c'.\n", cCCEscapeChar); - dbgprintf("Main queue size %d messages.\n", iMainMsgQueueSize); - dbgprintf("Main queue worker threads: %d, wThread shutdown: %d, Perists every %d updates.\n", + DBGPRINTF("Main queue size %d messages.\n", iMainMsgQueueSize); + DBGPRINTF("Main queue worker threads: %d, wThread shutdown: %d, Perists every %d updates.\n", iMainMsgQueueNumWorkers, iMainMsgQtoWrkShutdown, iMainMsgQPersistUpdCnt); - dbgprintf("Main queue timeouts: shutdown: %d, action completion shutdown: %d, enq: %d\n", + DBGPRINTF("Main queue timeouts: shutdown: %d, action completion shutdown: %d, enq: %d\n", iMainMsgQtoQShutdown, iMainMsgQtoActShutdown, iMainMsgQtoEnq); - dbgprintf("Main queue watermarks: high: %d, low: %d, discard: %d, discard-severity: %d\n", + DBGPRINTF("Main queue watermarks: high: %d, low: %d, discard: %d, discard-severity: %d\n", iMainMsgQHighWtrMark, iMainMsgQLowWtrMark, iMainMsgQDiscardMark, iMainMsgQDiscardSeverity); - dbgprintf("Main queue save on shutdown %d, max disk space allowed %lld\n", + DBGPRINTF("Main queue save on shutdown %d, max disk space allowed %lld\n", bMainMsgQSaveOnShutdown, iMainMsgQueMaxDiskSpace); /* TODO: add iActionRetryCount = 0; @@ -2511,7 +2511,7 @@ static void dbgPrintInitInfo(void) setQPROP(qqueueSetiMinMsgsPerWrkr, "$MainMsgQueueWorkerThreadMinimumMessages", 100); setQPROP(qqueueSetbSaveOnShutdown, "$MainMsgQueueSaveOnShutdown", 1); */ - dbgprintf("Work Directory: '%s'.\n", glbl.GetWorkDir()); + DBGPRINTF("Work Directory: '%s'.\n", glbl.GetWorkDir()); } @@ -2534,7 +2534,7 @@ startInputModules(void) /* activate here */ thrdCreate(pMod->mod.im.runInput, pMod->mod.im.afterRun); } else { - dbgprintf("module %lx will not run, iRet %d\n", (unsigned long) pMod, iRet); + DBGPRINTF("module %lx will not run, iRet %d\n", (unsigned long) pMod, iRet); } pMod = module.GetNxtType(pMod, eMOD_IN); } @@ -2567,11 +2567,11 @@ init(void) pDfltProgNameCmp = NULL; eDfltHostnameCmpMode = HN_NO_COMP; - dbgprintf("rsyslog %s - called init()\n", VERSION); + DBGPRINTF("rsyslog %s - called init()\n", VERSION); /* delete the message queue, which also flushes all messages left over */ if(pMsgQueue != NULL) { - dbgprintf("deleting main message queue\n"); + DBGPRINTF("deleting main message queue\n"); qqueueDestruct(&pMsgQueue); /* delete pThis here! */ pMsgQueue = NULL; } @@ -2582,10 +2582,10 @@ init(void) freeSelectors(); /* Unload all non-static modules */ - dbgprintf("Unloading non-static modules.\n"); + DBGPRINTF("Unloading non-static modules.\n"); module.UnloadAndDestructAll(eMOD_LINK_DYNAMIC_LOADED); - dbgprintf("Clearing templates.\n"); + DBGPRINTF("Clearing templates.\n"); tplDeleteNew(); /* re-setting values to defaults (where applicable) */ @@ -2635,7 +2635,7 @@ init(void) snprintf(cbuf,sizeof(cbuf), "*.*\t%s", szTTYNameBuf); conf.cfline((uchar*)cbuf, &f); } else { - dbgprintf("error %d obtaining controlling terminal, not using that emergency rule\n", errno); + DBGPRINTF("error %d obtaining controlling terminal, not using that emergency rule\n", errno); } selectorAddList(f); } @@ -2741,7 +2741,7 @@ init(void) } bHaveMainQueue = (MainMsgQueType == QUEUETYPE_DIRECT) ? 0 : 1; - dbgprintf("Main processing queue is initialized and running\n"); + DBGPRINTF("Main processing queue is initialized and running\n"); /* the output part and the queue is now ready to run. So it is a good time * to start the inputs. Please note that the net code above should be @@ -2768,7 +2768,7 @@ init(void) sigAct.sa_handler = sighup_handler; sigaction(SIGHUP, &sigAct, NULL); - dbgprintf(" (re)started.\n"); + DBGPRINTF(" (re)started.\n"); finalize_it: RETiRet; @@ -2799,7 +2799,7 @@ selectorAddList(selector_t *f) selectorDestruct(f); } else { /* successfully created an entry */ - dbgprintf("selector line successfully processed\n"); + DBGPRINTF("selector line successfully processed\n"); /* TODO: we should use the linked list class for the selector list, else we need to add globals * ... well nextp could be added temporarily... * Thanks to varmojfekoj for having the idea to just use "Files" to make this @@ -2831,16 +2831,16 @@ static rsRetVal setMainMsgQueType(void __attribute__((unused)) *pVal, uchar *psz if (!strcasecmp((char *) pszType, "fixedarray")) { MainMsgQueType = QUEUETYPE_FIXED_ARRAY; - dbgprintf("main message queue type set to FIXED_ARRAY\n"); + DBGPRINTF("main message queue type set to FIXED_ARRAY\n"); } else if (!strcasecmp((char *) pszType, "linkedlist")) { MainMsgQueType = QUEUETYPE_LINKEDLIST; - dbgprintf("main message queue type set to LINKEDLIST\n"); + DBGPRINTF("main message queue type set to LINKEDLIST\n"); } else if (!strcasecmp((char *) pszType, "disk")) { MainMsgQueType = QUEUETYPE_DISK; - dbgprintf("main message queue type set to DISK\n"); + DBGPRINTF("main message queue type set to DISK\n"); } else if (!strcasecmp((char *) pszType, "direct")) { MainMsgQueType = QUEUETYPE_DIRECT; - dbgprintf("main message queue type set to DIRECT (no queueing at all)\n"); + DBGPRINTF("main message queue type set to DIRECT (no queueing at all)\n"); } else { errmsg.LogError(0, RS_RET_INVALID_PARAMS, "unknown mainmessagequeuetype parameter: %s", (char *) pszType); iRet = RS_RET_INVALID_PARAMS; @@ -3219,7 +3219,7 @@ static rsRetVal mainThread() CHKiRet(init()); if(Debug && debugging_on) { - dbgprintf("Debugging enabled, SIGUSR1 to turn off debugging.\n"); + DBGPRINTF("Debugging enabled, SIGUSR1 to turn off debugging.\n"); } /* Send a signal to the parent so it can terminate. */ @@ -3251,7 +3251,7 @@ static rsRetVal mainThread() * do the init() and then restart things. * rgerhards, 2005-10-24 */ - dbgprintf("initialization completed, transitioning to regular run mode\n"); + DBGPRINTF("initialization completed, transitioning to regular run mode\n"); mainloop(); @@ -3458,7 +3458,7 @@ doGlblProcessInit(void) if( !(Debug || NoFork) ) { - dbgprintf("Checking pidfile.\n"); + DBGPRINTF("Checking pidfile.\n"); if (!check_pid(PidFile)) { memset(&sigAct, 0, sizeof (sigAct)); @@ -3494,7 +3494,7 @@ doGlblProcessInit(void) } /* tuck my process id away */ - dbgprintf("Writing pidfile %s.\n", PidFile); + DBGPRINTF("Writing pidfile %s.\n", PidFile); if (!check_pid(PidFile)) { if (!write_pid(PidFile)) @@ -3640,7 +3640,7 @@ int realMain(int argc, char **argv) if ((argc -= optind)) usage(); - dbgprintf("rsyslogd %s startup, compatibility mode %d, module path '%s'\n", + DBGPRINTF("rsyslogd %s startup, compatibility mode %d, module path '%s'\n", VERSION, iCompatibilityMode, glblModPath == NULL ? "" : (char*)glblModPath); /* we are done with the initial option parsing and processing. Now we init the system. */ @@ -3722,7 +3722,7 @@ int realMain(int argc, char **argv) /* END core initializations - we now come back to carrying out command line options*/ while((iRet = bufOptRemove(&ch, &arg)) == RS_RET_OK) { - dbgprintf("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg); + DBGPRINTF("deque option %c, optarg '%s'\n", ch, (arg == NULL) ? "" : arg); switch((char)ch) { case '4': glbl.SetDefPFFamily(PF_INET); -- cgit v1.2.3 From 74b2b24f508be90d20961304d5e3cce648f3eb7c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 12:31:27 +0200 Subject: removed long-obsoleted property UxTradMsg ... as well as some cleanup (some commented-out code is left to support UxTradMsg again is someone really complains ;)). --- ChangeLog | 1 + doc/property_replacer.html | 4 -- plugins/imfile/imfile.c | 1 - plugins/imklog/imklog.c | 2 - runtime/msg.c | 100 ++++++++++++++++++++++----------------------- runtime/msg.h | 9 ++-- runtime/parser.c | 2 +- tools/syslogd.c | 23 ++++------- 8 files changed, 65 insertions(+), 77 deletions(-) diff --git a/ChangeLog b/ChangeLog index f9e3e32b..beed5663 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ --------------------------------------------------------------------------- Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +- removed long-obsoleted property UxTradMsg - added a generic network stream server (in addition to rather specific syslog tcp server) - added ability for the UDP output action to rebind its send socket after diff --git a/doc/property_replacer.html b/doc/property_replacer.html index a6e9b518..7b604ea0 100644 --- a/doc/property_replacer.html +++ b/doc/property_replacer.html @@ -30,10 +30,6 @@ Currently supported are:

      socket. Should be useful for debugging. -uxtradmsg -will disappear soon - do NOT use! - - hostname hostname from the message diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index e3882ce5..927cb82e 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -96,7 +96,6 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); - MsgSetUxTradMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 45933436..420ebbf1 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -97,8 +97,6 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); MsgSetRawMsg(pMsg, (char*)msg); - MsgSetUxTradMsg(pMsg, (char*)msg); - MsgSetRawMsg(pMsg, (char*)msg); MsgSetMSG(pMsg, (char*)msg); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); diff --git a/runtime/msg.c b/runtime/msg.c index 55cc48b8..8122383a 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -343,7 +343,6 @@ CODESTARTobjDestruct(msg) if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ - free(pThis->pszUxTradMsg); free(pThis->pszRawMsg); free(pThis->pszTAG); free(pThis->pszHOSTNAME); @@ -437,6 +436,10 @@ msg_t* MsgDup(msg_t* pOld) pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + pNew->offAfterPRI = pOld->offAfterPRI; + */ memcpy(pNew->bufPRI, pOld->bufPRI, pOld->iLenPRI); pNew->iLenPRI = pOld->iLenPRI; tmpCOPYSZ(Severity); @@ -445,7 +448,6 @@ msg_t* MsgDup(msg_t* pOld) tmpCOPYSZ(FacilityStr); tmpCOPYSZ(RawMsg); tmpCOPYSZ(MSG); - tmpCOPYSZ(UxTradMsg); tmpCOPYSZ(TAG); tmpCOPYSZ(HOSTNAME); tmpCOPYSZ(RcvFrom); @@ -494,10 +496,13 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializeSCALAR(pStrm, ttGenTime, INT); objSerializeSCALAR(pStrm, tRcvdAt, SYSLOGTIME); objSerializeSCALAR(pStrm, tTIMESTAMP, SYSLOGTIME); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + objSerializeSCALAR(pStrm, offsAfterPRI, SHORT); + */ objSerializePTR(pStrm, pszRawMsg, PSZ); objSerializePTR(pStrm, pszMSG, PSZ); - objSerializePTR(pStrm, pszUxTradMsg, PSZ); objSerializePTR(pStrm, pszTAG, PSZ); objSerializePTR(pStrm, pszHOSTNAME, PSZ); objSerializePTR(pStrm, pszInputName, PSZ); @@ -701,16 +706,17 @@ static char *getRawMsg(msg_t *pM) return (char*)pM->pszRawMsg; } + +/* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 char *getUxTradMsg(msg_t *pM) { if(pM == NULL) return ""; else - if(pM->pszUxTradMsg == NULL) - return ""; - else - return (char*)pM->pszUxTradMsg; + return (char*)pM->pszRawMsg + pM->offAfterPRI; } +*/ char *getMSG(msg_t *pM) { @@ -724,21 +730,30 @@ char *getMSG(msg_t *pM) } +/* Get PRI value as integer */ +static int getPRIi(msg_t *pM) +{ + assert(pM != NULL); + return (pM->iFacility << 3) + (pM->iSeverity); +} + + /* Get PRI value in text form */ static inline char *getPRI(msg_t *pM) { if(pM == NULL) return ""; - return (char*)pM->bufPRI; -} - + /* there are some cases where bufPRI may not contain a valid string, + * and then we need to build it. + */ + MsgLock(pM); + if(pM->bufPRI[0] == '\0') { + snprintf((char*)pM->bufPRI, sizeof(pM->bufPRI), "<%d>", getPRIi(pM)); + } + MsgUnlock(pM); -/* Get PRI value as integer */ -static int getPRIi(msg_t *pM) -{ - assert(pM != NULL); - return (pM->iFacility << 3) + (pM->iSeverity); + return (char*)pM->bufPRI; } @@ -1017,6 +1032,17 @@ MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl) RETiRet; } +/* set offset after which PRI in raw msg starts + * rgerhards, 2009-06-16 + */ +rsRetVal +MsgSetAfterPRIOffs(msg_t *pMsg, short offs) +{ + assert(pMsg != NULL); + pMsg->offAfterPRI = offs; + return RS_RET_OK; +} + /* rgerhards 2004-11-24: set APP-NAME in msg object * TODO: revisit msg locking code! @@ -1520,40 +1546,6 @@ void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME) } -/* Set the UxTradMsg to a caller-provided string. This is thought - * to be a heap buffer that the caller will no longer use. This - * function is a performance optimization over MsgSetUxTradMsg(). - * rgerhards 2004-11-19 - */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static void MsgAssignUxTradMsg(msg_t *pMsg, char *pBuf) -{ - assert(pMsg != NULL); - assert(pBuf != NULL); - pMsg->iLenUxTradMsg = strlen(pBuf); - pMsg->pszUxTradMsg = pBuf; -} -#endif - - -/* rgerhards 2004-11-17: set the traditional Unix message in msg object - */ -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg) -{ - assert(pMsg != NULL); - assert(pszUxTradMsg != NULL); - pMsg->iLenUxTradMsg = strlen(pszUxTradMsg); - if(pMsg->pszUxTradMsg != NULL) - free(pMsg->pszUxTradMsg); - if((pMsg->pszUxTradMsg = malloc(pMsg->iLenUxTradMsg + 1)) != NULL) - memcpy(pMsg->pszUxTradMsg, pszUxTradMsg, pMsg->iLenUxTradMsg + 1); - else - dbgprintf("Could not allocate memory for pszUxTradMsg buffer."); - - return(0); -} - - /* rgerhards 2004-11-09: set MSG in msg object */ void MsgSetMSG(msg_t *pMsg, char* pszMSG) @@ -1732,8 +1724,11 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = getMSG(pMsg); } else if(!strcmp((char*) pName, "rawmsg")) { pRes = getRawMsg(pMsg); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 } else if(!strcmp((char*) pName, "uxtradmsg")) { pRes = getUxTradMsg(pMsg); + */ } else if(!strcmp((char*) pName, "inputname")) { pRes = (char*) getInputName(pMsg); } else if(!strcmp((char*) pName, "fromhost")) { @@ -2457,10 +2452,15 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) pThis->msgFlags = pProp->val.num; } else if(isProp("pszRawMsg")) { MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + } else if(isProp("offAfterPRI")) { + pThis->offAfterPRI = pProp->val.num; + */ } else if(isProp("pszMSG")) { MsgSetMSG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszUxTradMsg")) { - MsgSetUxTradMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + /*IGNORE*/; /* this *was* a property, but does no longer exist */ } else if(isProp("pszTAG")) { MsgSetTAG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszInputName")) { diff --git a/runtime/msg.h b/runtime/msg.h index 20360641..fe9f87fa 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,7 +51,7 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; -short bDoLock; /* use the mutex? */ + bool bDoLock; /* use the mutex? */ pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ @@ -73,13 +73,13 @@ short bDoLock; /* use the mutex? */ int iLenFacility; /* ... and its length. */ uchar *pszFacilityStr; /* facility name... */ int iLenFacilityStr; /* ... and its length. */ - //uchar *pszPRI; /* the PRI as a string */ - uchar bufPRI[5]; + uchar bufPRI[5]; /* PRI as string */ int iLenPRI; /* and its length */ uchar *pszRawMsg; /* message as it was received on the * wire. This is important in case we * need to preserve cryptographic verifiers. */ + short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ int iLenRawMsg; /* length of raw message */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ @@ -158,7 +158,7 @@ void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); +rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSG(msg_t *pMsg, char* pszMSG); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); void moveHOSTNAMEtoTAG(msg_t *pM); @@ -183,6 +183,7 @@ uchar *getRcvFrom(msg_t *pM); #if 0 char *getUxTradMsg(msg_t *pM); +int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); #endif /* The MsgPrepareEnqueue() function is a macro for performance reasons. diff --git a/runtime/parser.c b/runtime/parser.c index 13fb51ec..0b45bfd5 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -314,7 +314,7 @@ rsRetVal parseMsg(msg_t *pMsg) } pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); - MsgSetUxTradMsg(pMsg, (char*) msg); + MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); if(pMsg->bParseHOSTNAME == 0) MsgSetHOSTNAME(pMsg, pMsg->pszRcvFrom); diff --git a/tools/syslogd.c b/tools/syslogd.c index c224087d..2f06d273 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -670,16 +670,9 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) MsgSetHOSTNAME(pMsg, hname); MsgSetRcvFrom(pMsg, hname); + MsgSetAfterPRIOffs(pMsg, p - msg); CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP)); - /* rgerhards 2004-11-19: well, well... we've now seen that we - * have the "hostname problem" also with the traditional Unix - * message. As we like to emulate it, we need to add the hostname - * to it. - */ - if(MsgSetUxTradMsg(pMsg, (char*)p) != 0) - ABORT_FINALIZE(RS_RET_ERR); - logmsg(pMsg, flags); finalize_it: @@ -948,7 +941,6 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) CHKiRet(msgConstruct(&pMsg)); MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1); - MsgSetUxTradMsg(pMsg, (char*)msg); MsgSetRawMsg(pMsg, (char*)msg); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); @@ -1370,8 +1362,8 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) BEGINfunc assert(pMsg != NULL); - assert(pMsg->pszUxTradMsg != NULL); - p2parse = pMsg->pszUxTradMsg; + assert(pMsg->pszRawMsg != NULL); + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ /* do a sanity check on the version and eat it */ assert(p2parse[0] == '1' && p2parse[1] == ' '); @@ -1471,8 +1463,8 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) BEGINfunc assert(pMsg != NULL); - assert(pMsg->pszUxTradMsg != NULL); - p2parse = pMsg->pszUxTradMsg; + assert(pMsg->pszRawMsg != NULL); + p2parse = pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ /* Check to see if msg contains a timestamp. We start by assuming * that the message timestamp is the time of reciption (which we @@ -1685,8 +1677,9 @@ logmsg(msg_t *pMsg, int flags) BEGINfunc assert(pMsg != NULL); - assert(pMsg->pszUxTradMsg != NULL); - msg = (char*) pMsg->pszUxTradMsg; + assert(pMsg->pszRawMsg != NULL); + + msg = (char*) pMsg->pszRawMsg + pMsg->offAfterPRI; /* point to start of text, after PRI */ DBGPRINTF("logmsg: flags %x, from '%s', msg %s\n", flags, getRcvFrom(pMsg), msg); /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have -- cgit v1.2.3 From f45dfe93af0c7f5598a8ea953e6b2b82a5427a7c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 13:00:04 +0200 Subject: minor cleanup --- runtime/rsyslog.h | 1 - tcps_sess.c | 2 -- 2 files changed, 3 deletions(-) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 0fafd700..e81c9eef 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -99,7 +99,6 @@ typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename? typedef long long int64; typedef long long unsigned uint64; typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ -typedef short bool; #ifdef __hpux typedef unsigned int u_int32_t; /* TODO: is this correct? */ diff --git a/tcps_sess.c b/tcps_sess.c index b8ea0878..62882f5b 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -56,8 +56,6 @@ DEFobjCurrIf(datetime) static int iMaxLine; /* maximum size of a single message */ -#define TIME_REQUERY_DFLT 16 // TODO change back! 2 -static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ static int iNbrTimeUsed = 0; /* how often has previous time been used so far? */ -- cgit v1.2.3 From aef1a38fe8c7472362904b2f90c67113b21034ab Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 13:28:18 +0200 Subject: fixing problems that occurred during the last merge --- tcps_sess.c | 2 - tools/syslogd.c | 291 -------------------------------------------------------- 2 files changed, 293 deletions(-) diff --git a/tcps_sess.c b/tcps_sess.c index 62882f5b..f887e702 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -526,7 +526,5 @@ BEGINObjClassInit(tcps_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE c OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, tcps_sessConstructFinalize); ENDObjClassInit(tcps_sess) - - /* vim:set ai: */ diff --git a/tools/syslogd.c b/tools/syslogd.c index fd4bc937..ea8eff7a 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -925,233 +925,6 @@ finalize_it: RETiRet; } -/* This functions looks at the given message and checks if it matches the - * provided filter condition. If so, it returns true, else it returns - * false. This is a helper to logmsg() and meant to drive the decision - * process if a message is to be processed or not. As I expect this - * decision code to grow more complex over time AND logmsg() is already - * a very lengthy function, I thought a separate function is more appropriate. - * 2005-09-19 rgerhards - * 2008-02-25 rgerhards: changed interface, now utilizes iRet, bProcessMsg - * returns is message should be procesed. - */ -static rsRetVal shouldProcessThisMessage(selector_t *f, msg_t *pMsg, int *bProcessMsg) -{ - DEFiRet; - unsigned short pbMustBeFreed; - char *pszPropVal; - int bRet = 0; - vm_t *pVM = NULL; - var_t *pResult = NULL; - - assert(f != NULL); - assert(pMsg != NULL); - - /* we first have a look at the global, BSD-style block filters (for tag - * and host). Only if they match, we evaluate the actual filter. - * rgerhards, 2005-10-18 - */ - if(f->eHostnameCmpMode == HN_NO_COMP) { - /* EMPTY BY INTENSION - we check this value first, because - * it is the one most often used, so this saves us time! - */ - } else if(f->eHostnameCmpMode == HN_COMP_MATCH) { - if(rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, so we are already done... */ - DBGPRINTF("hostname filter '+%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } else { /* must be -hostname */ - if(!rsCStrSzStrCmp(f->pCSHostnameComp, (uchar*) getHOSTNAME(pMsg), getHOSTNAMELen(pMsg))) { - /* not equal, so we are already done... */ - DBGPRINTF("hostname filter '-%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSHostnameComp), getHOSTNAME(pMsg)); - FINALIZE; - } - } - - if(f->pCSProgNameComp != NULL) { - int bInv = 0, bEqv = 0, offset = 0; - if(*(rsCStrGetSzStrNoNULL(f->pCSProgNameComp)) == '-') { - if(*(rsCStrGetSzStrNoNULL(f->pCSProgNameComp) + 1) == '-') - offset = 1; - else { - bInv = 1; - offset = 1; - } - } - if(!rsCStrOffsetSzStrCmp(f->pCSProgNameComp, offset, (uchar*) getProgramName(pMsg), getProgramNameLen(pMsg))) - bEqv = 1; - - if((!bEqv && !bInv) || (bEqv && bInv)) { - /* not equal or inverted selection, so we are already done... */ - DBGPRINTF("programname filter '%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(f->pCSProgNameComp), getProgramName(pMsg)); - FINALIZE; - } - } - - /* done with the BSD-style block filters */ - - if(f->f_filter_type == FILTER_PRI) { - /* skip messages that are incorrect priority */ - if ( (f->f_filterData.f_pmask[pMsg->iFacility] == TABLE_NOPRI) || - ((f->f_filterData.f_pmask[pMsg->iFacility] & (1<iSeverity)) == 0) ) - bRet = 0; - else - bRet = 1; - } else if(f->f_filter_type == FILTER_EXPR) { - CHKiRet(vm.Construct(&pVM)); - CHKiRet(vm.ConstructFinalize(pVM)); - CHKiRet(vm.SetMsg(pVM, pMsg)); - CHKiRet(vm.ExecProg(pVM, f->f_filterData.f_expr->pVmprg)); - CHKiRet(vm.PopBoolFromStack(pVM, &pResult)); - DBGPRINTF("result of expression evaluation: %lld\n", pResult->val.num); - /* VM is destructed on function exit */ - bRet = (pResult->val.num) ? 1 : 0; - } else { - assert(f->f_filter_type == FILTER_PROP); /* assert() just in case... */ - pszPropVal = MsgGetProp(pMsg, NULL, f->f_filterData.prop.pCSPropName, &pbMustBeFreed); - - /* Now do the compares (short list currently ;)) */ - switch(f->f_filterData.prop.operation ) { - case FIOP_CONTAINS: - if(rsCStrLocateInSzStr(f->f_filterData.prop.pCSCompValue, (uchar*) pszPropVal) != -1) - bRet = 1; - break; - case FIOP_ISEQUAL: - if(rsCStrSzStrCmp(f->f_filterData.prop.pCSCompValue, - (uchar*) pszPropVal, strlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_STARTSWITH: - if(rsCStrSzStrStartsWithCStr(f->f_filterData.prop.pCSCompValue, - (uchar*) pszPropVal, strlen(pszPropVal)) == 0) - bRet = 1; /* process message! */ - break; - case FIOP_REGEX: - if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 0, &f->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - case FIOP_EREREGEX: - if(rsCStrSzStrMatchRegex(f->f_filterData.prop.pCSCompValue, - (unsigned char*) pszPropVal, 1, &f->f_filterData.prop.regex_cache) == RS_RET_OK) - bRet = 1; - break; - default: - /* here, it handles NOP (for performance reasons) */ - assert(f->f_filterData.prop.operation == FIOP_NOP); - bRet = 1; /* as good as any other default ;) */ - break; - } - - /* now check if the value must be negated */ - if(f->f_filterData.prop.isNegated) - bRet = (bRet == 1) ? 0 : 1; - - if(Debug) { - dbgprintf("Filter: check for property '%s' (value '%s') ", - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSPropName), - pszPropVal); - if(f->f_filterData.prop.isNegated) - dbgprintf("NOT "); - dbgprintf("%s '%s': %s\n", - getFIOPName(f->f_filterData.prop.operation), - rsCStrGetSzStrNoNULL(f->f_filterData.prop.pCSCompValue), - bRet ? "TRUE" : "FALSE"); - } - - /* cleanup */ - if(pbMustBeFreed) - free(pszPropVal); - } - -finalize_it: - /* destruct in any case, not just on error, but it makes error handling much easier */ - if(pVM != NULL) - vm.Destruct(&pVM); - - if(pResult != NULL) - var.Destruct(&pResult); - - *bProcessMsg = bRet; - RETiRet; -} - - -/* helper to processMsg(), used to call the configured actions. It is - * executed from within llExecFunc() of the action list. - * rgerhards, 2007-08-02 - */ -typedef struct processMsgDoActions_s { - int bPrevWasSuspended; /* was the previous action suspended? */ - msg_t *pMsg; -} processMsgDoActions_t; -DEFFUNC_llExecFunc(processMsgDoActions) -{ - DEFiRet; - rsRetVal iRetMod; /* return value of module - we do not always pass that back */ - action_t *pAction = (action_t*) pData; - processMsgDoActions_t *pDoActData = (processMsgDoActions_t*) pParam; - - assert(pAction != NULL); - - if((pAction->bExecWhenPrevSusp == 1) && (pDoActData->bPrevWasSuspended == 0)) { - DBGPRINTF("not calling action because the previous one is not suspended\n"); - ABORT_FINALIZE(RS_RET_OK); - } - - iRetMod = actionCallAction(pAction, pDoActData->pMsg); - if(iRetMod == RS_RET_DISCARDMSG) { - ABORT_FINALIZE(RS_RET_DISCARDMSG); - } else if(iRetMod == RS_RET_SUSPENDED) { - /* indicate suspension for next module to be called */ - pDoActData->bPrevWasSuspended = 1; - } else { - pDoActData->bPrevWasSuspended = 0; - } - -finalize_it: - RETiRet; -} - - -/* Process (consume) a received message. Calls the actions configured. - * rgerhards, 2005-10-13 - */ -static void -processMsg(msg_t *pMsg) -{ - selector_t *f; - int bContinue; - int bProcessMsg; - processMsgDoActions_t DoActData; - rsRetVal iRet; - - BEGINfunc - assert(pMsg != NULL); - - /* log the message to the particular outputs */ - - bContinue = 1; - for (f = Files; f != NULL && bContinue ; f = f->f_next) { - /* first check the filters... */ - iRet = shouldProcessThisMessage(f, pMsg, &bProcessMsg); - if(!bProcessMsg) { - continue; - } - - /* ok -- from here, we have action-specific code, nothing really selector-specific -- rger 2007-08-01 */ - DoActData.pMsg = pMsg; - DoActData.bPrevWasSuspended = 0; - if(llExecFunc(&f->llActList, processMsgDoActions, (void*)&DoActData) == RS_RET_DISCARDMSG) - bContinue = 0; - } - ENDfunc -} - /* The consumer of dequeued messages. This function is called by the * queue engine on dequeueing of a message. It runs on a SEPARATE @@ -2118,56 +1891,6 @@ static void doDropPrivUid(int iUid) } -/* helper to freeSelectors(), used with llExecFunc() to flush - * pending output. -- rgerhards, 2007-08-02 - * We do not need to lock the action object here as the processing - * queue is already empty and no other threads are running when - * we call this function. -- rgerhards, 2007-12-12 - */ -DEFFUNC_llExecFunc(freeSelectorsActions) -{ - action_t *pAction = (action_t*) pData; - - assert(pAction != NULL); - - /* flush any pending output */ - if(pAction->f_prevcount) { - actionWriteToAction(pAction); - } - - return RS_RET_OK; /* never fails ;) */ -} - - -/* Close all open log files and free selector descriptor array. - */ -static void freeSelectors(void) -{ - selector_t *f; - selector_t *fPrev; - - if(Files != NULL) { - DBGPRINTF("Freeing log structures.\n"); - - for(f = Files ; f != NULL ; f = f->f_next) { - llExecFunc(&f->llActList, freeSelectorsActions, NULL); - } - - /* actions flushed and ready for destruction - so do that... */ - f = Files; - while (f != NULL) { - fPrev = f; - f = f->f_next; - selectorDestruct(fPrev); - } - - /* Reflect the deletion of the selectors linked list. */ - Files = NULL; - bHaveMainQueue = 0; - } -} - - /* helper to generateConfigDAG, to print out all actions via * the llExecFunc() facility. * rgerhards, 2007-08-02 @@ -2349,20 +2072,6 @@ finalize_it: } -/* helper to dbPrintInitInfo, to print out all actions via - * the llExecFunc() facility. - * rgerhards, 2007-08-02 - */ -DEFFUNC_llExecFunc(dbgPrintInitInfoAction) -{ - DEFiRet; - iRet = actionDbgPrint((action_t*) pData); - DBGPRINTF("\n"); - - RETiRet; -} - - /* print debug information as part of init(). This pretty much * outputs the whole config of rsyslogd. I've moved this code * out of init() to clean it somewhat up. -- cgit v1.2.3 From 6f0db63e9b962edf6305860b608500e8c650b71b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 15:13:47 +0200 Subject: milestone: input-side multiSubmit capability ... commit before I try to touch the queue side ;) --- runtime/rsyslog.h | 13 +++++++++++++ runtime/stream.c | 2 +- tcps_sess.c | 34 ++++++++++++++++++++++++++-------- tools/syslogd.c | 26 ++++++++++++++++++++++++-- 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index e81c9eef..ea8a5222 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -127,6 +127,19 @@ typedef enum { } fiop_t; +/* multi-submit support. + * This is done via a simple data structure, which holds the number of elements + * as well as an array of to-be-submitted messages. + * rgerhards, 2009-06-16 + */ +typedef struct multi_submit_s multi_submit_t; +struct multi_submit_s { + short maxElem; /* maximum number of Elements */ + short nElem; /* current number of Elements, points to the next one FREE */ + msg_t **ppMsgs; +}; + + #ifndef _PATH_CONSOLE #define _PATH_CONSOLE "/dev/console" #endif diff --git a/runtime/stream.c b/runtime/stream.c index f13258b5..8cbe0297 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -188,7 +188,7 @@ finalize_it: static rsRetVal doPhysOpen(strm_t *pThis) { - int iFlags; + int iFlags = 0; DEFiRet; ISOBJ_TYPE_assert(pThis, strm); diff --git a/tcps_sess.c b/tcps_sess.c index f887e702..379faeab 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -220,7 +220,7 @@ SetOnMsgReceive(tcps_sess_t *pThis, rsRetVal (*OnMsgReceive)(tcps_sess_t*, uchar * rgerhards, 2009-04-23 */ static rsRetVal -defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttGenTime) +defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttGenTime, multi_submit_t *pMultiSub) { msg_t *pMsg; DEFiRet; @@ -245,7 +245,15 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttG MsgSetRcvFrom(pMsg, pThis->fromHost); MsgSetRuleset(pMsg, pThis->pLstnInfo->pRuleset); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); - CHKiRet(submitMsg(pMsg)); + + if(pMultiSub == NULL) { + CHKiRet(submitMsg(pMsg)); + } else { + pMultiSub->ppMsgs[pMultiSub->nElem++] = pMsg; + if(pMultiSub->nElem == pMultiSub->maxElem) + CHKiRet(multiSubmitMsg(pMultiSub)); + } + finalize_it: /* reset status variables */ @@ -299,7 +307,7 @@ PrepareClose(tcps_sess_t *pThis) */ dbgprintf("Extra data at end of stream in legacy syslog/tcp message - processing\n"); datetime.getCurrTime(&stTime, &ttGenTime); - defaultDoSubmitMessage(pThis, &stTime, ttGenTime); + defaultDoSubmitMessage(pThis, &stTime, ttGenTime, NULL); } finalize_it: @@ -334,7 +342,7 @@ Close(tcps_sess_t *pThis) * rgerhards, 2008-03-14 */ static rsRetVal -processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t ttGenTime) +processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t ttGenTime, multi_submit_t *pMultiSub) { DEFiRet; ISOBJ_TYPE_assert(pThis, tcps_sess); @@ -380,7 +388,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt if(pThis->iMsg >= iMaxLine) { /* emergency, we now need to flush, no matter if we are at end of message or not... */ dbgprintf("error: message received is larger than max msg size, we split it\n"); - defaultDoSubmitMessage(pThis, stTime, ttGenTime); + defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); /* we might think if it is better to ignore the rest of the * message than to treat it as a new one. Maybe this is a good * candidate for a configuration parameter... @@ -391,7 +399,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt if(( (c == '\n') || ((pThis->pSrv->addtlFrameDelim != TCPSRV_NO_ADDTL_DELIMITER) && (c == pThis->pSrv->addtlFrameDelim)) ) && pThis->eFraming == TCP_FRAMING_OCTET_STUFFING) { /* record delimiter? */ - defaultDoSubmitMessage(pThis, stTime, ttGenTime); + defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); pThis->inputState = eAtStrtFram; } else { /* IMPORTANT: here we copy the actual frame content to the message - for BOTH framing modes! @@ -408,7 +416,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt pThis->iOctetsRemain--; if(pThis->iOctetsRemain < 1) { /* we have end of frame! */ - defaultDoSubmitMessage(pThis, stTime, ttGenTime); + defaultDoSubmitMessage(pThis, stTime, ttGenTime, pMultiSub); pThis->inputState = eAtStrtFram; } } @@ -433,9 +441,12 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt * this *is* the *correct* reception step for all the data we received, because * we have just received a bunch of data! -- rgerhards, 2009-06-16 */ +#define NUM_MULTISUB 128 static rsRetVal DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) { + multi_submit_t multiSub; + msg_t *pMsgs[NUM_MULTISUB]; struct syslogTime stTime; time_t ttGenTime; char *pEnd; @@ -446,18 +457,25 @@ DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) assert(iLen > 0); datetime.getCurrTime(&stTime, &ttGenTime); + multiSub.ppMsgs = pMsgs; + multiSub.maxElem = NUM_MULTISUB; + multiSub.nElem = 0; /* We now copy the message to the session buffer. */ pEnd = pData + iLen; /* this is one off, which is intensional */ iNbrTimeUsed = 0; /* full time query */ while(pData < pEnd) { - CHKiRet(processDataRcvd(pThis, *pData++, &stTime, ttGenTime)); + CHKiRet(processDataRcvd(pThis, *pData++, &stTime, ttGenTime, &multiSub)); } + /* submit anything that was not yet submitted */ + CHKiRet(multiSubmitMsg(&multiSub)); + finalize_it: RETiRet; } +#undef NUM_MULTISUB /* queryInterface function diff --git a/tools/syslogd.c b/tools/syslogd.c index ea8eff7a..4d2839a1 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1345,8 +1345,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) } -/* submit a fully created message to the main message queue. The message is - * fully processed and parsed, so no parsing at all happens. This is primarily +/* submit a message to the main message queue. This is primarily * a hook to prevent the need for callers to know about the main message queue * (which may change in the future as we will probably have multiple rule * sets and thus queues...). @@ -1366,6 +1365,29 @@ submitMsg(msg_t *pMsg) } +/* submit multiple messages at once, very similar to submitMsg, just + * for multi_submit_t. + * rgerhards, 2009-06-16 + */ +rsRetVal +multiSubmitMsg(multi_submit_t *pMultiSub) +{ + int i; + DEFiRet; + assert(pMultiSub != NULL); + + for(i = 0 ; i < pMultiSub->nElem ; ++i) { +dbgprintf("multiSubmitMsg, index %d\n", i); + MsgPrepareEnqueue(pMultiSub->ppMsgs[i]); + qqueueEnqObj(pMsgQueue, pMultiSub->ppMsgs[i]->flowCtlType, (void*) pMultiSub->ppMsgs[i]); + } + + pMultiSub->nElem = 0; + + RETiRet; +} + + /* Log a message to the appropriate log files, users, etc. based on * the priority. * rgerhards 2004-11-08: actually, this also decodes all but the PRI part. -- cgit v1.2.3 From c4e18c5bab66537ee9c8fb482edce62c7c39c247 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 15:43:22 +0200 Subject: implemented first version of multi-enqueue support, queue side --- runtime/queue.c | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tcps_sess.c | 2 +- tools/syslogd.c | 2 +- 3 files changed, 128 insertions(+), 2 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index aa8e6c21..dbf59210 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -49,6 +49,7 @@ #include "obj.h" #include "wtp.h" #include "wti.h" +#include "msg.h" #include "atomic.h" #ifdef OS_SOLARIS @@ -2230,6 +2231,131 @@ finalize_it: } +/* enqueue a single data object. This currently is a helper to qqueueMultiEnqObj. + * Note that the queue mutex MUST already be locked when this function is called. + * rgerhards, 2009-06-16 + */ +static inline rsRetVal +doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) +{ + DEFiRet; + struct timespec t; + + /* first check if we need to discard this message (which will cause CHKiRet() to exit) + */ + CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); + + /* then check if we need to add an assistance disk queue */ + if(pThis->bIsDA) + CHKiRet(qqueueChkStrtDA(pThis)); + + /* handle flow control + * There are two different flow control mechanisms: basic and advanced flow control. + * Basic flow control has always been implemented and protects the queue structures + * in that it makes sure no more data is enqueued than the queue is configured to + * support. Enhanced flow control is being added today. There are some sources which + * can easily be stopped, e.g. a file reader. This is the case because it is unlikely + * that blocking those sources will have negative effects (after all, the file is + * continued to be written). Other sources can somewhat be blocked (e.g. the kernel + * log reader or the local log stream reader): in general, nothing is lost if messages + * from these sources are not picked up immediately. HOWEVER, they can not block for + * an extended period of time, as this either causes message loss or - even worse - some + * other bad effects (e.g. unresponsive system in respect to the main system log socket). + * Finally, there are some (few) sources which can not be blocked at all. UDP syslog is + * a prime example. If a UDP message is not received, it is simply lost. So we can't + * do anything against UDP sockets that come in too fast. The core idea of advanced + * flow control is that we take into account the different natures of the sources and + * select flow control mechanisms that fit these needs. This also means, in the end + * result, that non-blockable sources like UDP syslog receive priority in the system. + * It's a side effect, but a good one ;) -- rgerhards, 2008-03-14 + */ + if(flowCtlType == eFLOWCTL_FULL_DELAY) { + while(pThis->iQueueSize >= pThis->iFullDlyMrk) { + dbgoprint((obj_t*) pThis, "enqueueMsg: FullDelay mark reached for full delayable message - blocking.\n"); + pthread_cond_wait(&pThis->belowFullDlyWtrMrk, pThis->mut); /* TODO error check? But what do then? */ + } + } else if(flowCtlType == eFLOWCTL_LIGHT_DELAY) { + if(pThis->iQueueSize >= pThis->iLightDlyMrk) { + dbgoprint((obj_t*) pThis, "enqueueMsg: LightDelay mark reached for light delayable message - blocking a bit.\n"); + timeoutComp(&t, 1000); /* 1000 millisconds = 1 second TODO: make configurable */ + pthread_cond_timedwait(&pThis->belowLightDlyWtrMrk, pThis->mut, &t); /* TODO error check? But what do then? */ + } + } + + /* from our regular flow control settings, we are now ready to enqueue the object. + * However, we now need to do a check if the queue permits to add more data. If that + * is not the case, basic flow control enters the field, which means we wait for + * the queue to become ready or drop the new message. -- rgerhards, 2008-03-14 + */ + while( (pThis->iMaxQueueSize > 0 && pThis->iQueueSize >= pThis->iMaxQueueSize) + || (pThis->qType == QUEUETYPE_DISK && pThis->sizeOnDiskMax != 0 + && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) { + dbgoprint((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); + timeoutComp(&t, pThis->toEnq); + if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) { + dbgoprint((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); + objDestruct(pUsr); + ABORT_FINALIZE(RS_RET_QUEUE_FULL); + } + } + + /* and finally enqueue the message */ + CHKiRet(qqueueAdd(pThis, pUsr)); + qqueueChkPersist(pThis); // TODO: optimize, do in outer function! + +finalize_it: + RETiRet; +} + +/* enqueue multiple user data elements at once. The aim is to provide a faster interface + * for object submission. Uses the multi_submit_t helper object. + * Please note that this function is not cancel-safe and consequently + * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE + * during its execution. If that is not done, race conditions occur if the + * thread is canceled (most important use case is input module termination). + * rgerhards, 2009-06-16 + */ +rsRetVal +qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub) +{ + int iCancelStateSave; + int i; + DEFiRet; + + ISOBJ_TYPE_assert(pThis, qqueue); + assert(pMultiSub != NULL); + + if(pThis->qType != QUEUETYPE_DIRECT) { + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + d_pthread_mutex_lock(pThis->mut); + } + + for(i = 0 ; i < pMultiSub->nElem ; ++i) { +dbgprintf("queueMultiEnq: %d\n", i); + CHKiRet(doEnqSingleObj(pThis, pMultiSub->ppMsgs[i]->flowCtlType, (void*)pMultiSub->ppMsgs[i])); + } + +finalize_it: + if(pThis->qType != QUEUETYPE_DIRECT) { + /* make sure at least one worker is running. */ + qqueueAdviseMaxWorkers(pThis); + /* and release the mutex */ + d_pthread_mutex_unlock(pThis->mut); + pthread_setcancelstate(iCancelStateSave, NULL); + dbgoprint((obj_t*) pThis, "MultiEnqObj advised worker start\n"); + /* the following pthread_yield is experimental, but brought us performance + * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 + * rgerhards, 2008-10-09 + * but this is only true for uniprocessors, so we guard it with an optimize flag -- rgerhards, 2008-10-22 + */ + if(pThis->bOptimizeUniProc) + pthread_yield(); + } + + RETiRet; +} + + /* set queue mode to enqueue only or not * There is one subtle issue: this method may be called during queue * construction or while it is running. In the former case, the queue diff --git a/tcps_sess.c b/tcps_sess.c index 379faeab..9353571c 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -441,7 +441,7 @@ processDataRcvd(tcps_sess_t *pThis, char c, struct syslogTime *stTime, time_t tt * this *is* the *correct* reception step for all the data we received, because * we have just received a bunch of data! -- rgerhards, 2009-06-16 */ -#define NUM_MULTISUB 128 +#define NUM_MULTISUB 1024 static rsRetVal DataRcvd(tcps_sess_t *pThis, char *pData, size_t iLen) { diff --git a/tools/syslogd.c b/tools/syslogd.c index 4d2839a1..e3cd54f7 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1379,9 +1379,9 @@ multiSubmitMsg(multi_submit_t *pMultiSub) for(i = 0 ; i < pMultiSub->nElem ; ++i) { dbgprintf("multiSubmitMsg, index %d\n", i); MsgPrepareEnqueue(pMultiSub->ppMsgs[i]); - qqueueEnqObj(pMsgQueue, pMultiSub->ppMsgs[i]->flowCtlType, (void*) pMultiSub->ppMsgs[i]); } + iRet = qqueueMultiEnqObj(pMsgQueue, pMultiSub); pMultiSub->nElem = 0; RETiRet; -- cgit v1.2.3 From 8a37736287fd4f7e4fd0c0190aabac11dff19af2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 19:18:41 +0200 Subject: some cleanup --- runtime/queue.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index dbf59210..46a3a971 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -63,7 +63,7 @@ DEFobjCurrIf(glbl) DEFobjCurrIf(strm) /* forward-definitions */ -rsRetVal qqueueChkPersist(qqueue_t *pThis); +static rsRetVal qqueueChkPersist(qqueue_t *pThis); static rsRetVal qqueueSetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); static rsRetVal qqueueRateLimiter(qqueue_t *pThis); static int qqueueChkStopWrkrDA(qqueue_t *pThis); @@ -1984,10 +1984,8 @@ finalize_it: * abide to our regular call interface)... * rgerhards, 2008-01-13 */ -rsRetVal qqueueChkPersist(qqueue_t *pThis) +static rsRetVal qqueueChkPersist(qqueue_t *pThis) { - DEFiRet; - ISOBJ_TYPE_assert(pThis, qqueue); if(pThis->iPersistUpdCnt && ++pThis->iUpdsSincePersist >= pThis->iPersistUpdCnt) { @@ -1995,7 +1993,7 @@ rsRetVal qqueueChkPersist(qqueue_t *pThis) pThis->iUpdsSincePersist = 0; } - RETiRet; + return RS_RET_OK; } @@ -2301,7 +2299,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* and finally enqueue the message */ CHKiRet(qqueueAdd(pThis, pUsr)); - qqueueChkPersist(pThis); // TODO: optimize, do in outer function! + qqueueChkPersist(pThis); // TODO: optimize, do in outer function! (but we need parts from v5?) finalize_it: RETiRet; -- cgit v1.2.3 From 1e30f67584ca0770e1e5b88ea75da7f9bc1022df Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 16 Jun 2009 19:32:53 +0200 Subject: added basic plumbing to support message properties separate from message will fill this with live somewhat later, noticed I need to do some stage work first (at least this is useful). --- runtime/Makefile.am | 2 + runtime/prop.c | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++ runtime/prop.h | 46 +++++++++++++++++++ runtime/rsyslog.h | 1 + 4 files changed, 175 insertions(+) create mode 100644 runtime/prop.c create mode 100644 runtime/prop.h diff --git a/runtime/Makefile.am b/runtime/Makefile.am index c2ef7cfa..14abe722 100644 --- a/runtime/Makefile.am +++ b/runtime/Makefile.am @@ -73,6 +73,8 @@ librsyslog_la_SOURCES = \ ruleset.h \ rule.c \ rule.h \ + prop.c \ + prop.h \ cfsysline.c \ cfsysline.h \ \ diff --git a/runtime/prop.c b/runtime/prop.c new file mode 100644 index 00000000..4130d4b7 --- /dev/null +++ b/runtime/prop.c @@ -0,0 +1,126 @@ +/* prop.c - rsyslog's prop object + * + * This object is meant to support message properties that are stored + * seperately from the message. The main intent is to support properties + * that are "constant" during a period of time, so that many messages may + * contain a reference to the same property. It is important, though, that + * properties are destroyed when they are no longer needed. + * + * Please note that this is a performance-critical part of the software and + * as such we may use some methods in here which do not look elegant, but + * which are fast... + * + * Module begun 2009-06-17 by Rainer Gerhards + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ + +#include "config.h" +#include +#include + +#include "rsyslog.h" +#include "obj.h" +#include "prop.h" + +/* static data */ +DEFobjStaticHelpers + + +/* Standard-Constructor + */ +BEGINobjConstruct(prop) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(prop) + + +/* ConstructionFinalizer + * rgerhards, 2008-01-09 + */ +static rsRetVal +propConstructFinalize(prop_t *pThis) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, prop); + +finalize_it: + RETiRet; +} + + +/* destructor for the prop object */ +BEGINobjDestruct(prop) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(prop) +ENDobjDestruct(prop) + + +/* debugprint for the prop object */ +BEGINobjDebugPrint(prop) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDebugPrint(prop) + dbgprintf("prop object %p - no further debug info implemented\n", pThis); +ENDobjDebugPrint(prop) + + +/* queryInterface function + * rgerhards, 2008-02-21 + */ +BEGINobjQueryInterface(prop) +CODESTARTobjQueryInterface(prop) + if(pIf->ifVersion != propCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = propConstruct; + pIf->ConstructFinalize = propConstructFinalize; + pIf->Destruct = propDestruct; + pIf->DebugPrint = propDebugPrint; + +finalize_it: +ENDobjQueryInterface(prop) + + +/* Exit the prop class. + * rgerhards, 2009-04-06 + */ +BEGINObjClassExit(prop, OBJ_IS_CORE_MODULE) /* class, version */ +// objRelease(errmsg, CORE_COMPONENT); +ENDObjClassExit(prop) + + +/* Initialize the prop class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2008-02-19 + */ +BEGINObjClassInit(prop, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ +// CHKiRet(objUse(errmsg, CORE_COMPONENT)); + + /* set our own handlers */ + OBJSetMethodHandler(objMethod_DEBUGPRINT, propDebugPrint); + OBJSetMethodHandler(objMethod_CONSTRUCTION_FINALIZER, propConstructFinalize); +ENDObjClassInit(prop) + +/* vi:set ai: + */ diff --git a/runtime/prop.h b/runtime/prop.h new file mode 100644 index 00000000..7fc466b5 --- /dev/null +++ b/runtime/prop.h @@ -0,0 +1,46 @@ +/* The prop object. + * + * This implements props within rsyslog. + * + * Copyright 2009 Rainer Gerhards and Adiscon GmbH. + * + * This file is part of the rsyslog runtime library. + * + * The rsyslog runtime library is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The rsyslog runtime library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with the rsyslog runtime library. If not, see . + * + * A copy of the GPL can be found in the file "COPYING" in this distribution. + * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. + */ +#ifndef INCLUDED_PROP_H +#define INCLUDED_PROP_H + +/* the prop object */ +struct prop_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ +}; + +/* interfaces */ +BEGINinterface(prop) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(prop); + rsRetVal (*Construct)(prop_t **ppThis); + rsRetVal (*ConstructFinalize)(prop_t __attribute__((unused)) *pThis); + rsRetVal (*Destruct)(prop_t **ppThis); +ENDinterface(prop) +#define propCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ + + +/* prototypes */ +PROTOTYPEObj(prop); + +#endif /* #ifndef INCLUDED_PROP_H */ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index ea8a5222..2e0a4150 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -79,6 +79,7 @@ typedef struct nsdsel_gtls_s nsdsel_gtls_t; typedef obj_t nsd_t; typedef obj_t nsdsel_t; typedef struct msg msg_t; +typedef struct prop_s prop_t; typedef struct interface_s interface_t; typedef struct objInfo_s objInfo_t; typedef enum rsRetVal_ rsRetVal; /**< friendly type for global return value */ -- cgit v1.2.3 From 56e462610db0dc71cfc2e4af17d1eb27bd67fae7 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 17 Jun 2009 12:56:58 +0200 Subject: further optimized message object pri, facility and severity string generation simplified --- dirty.h | 1 + runtime/msg.c | 345 ++++++++++++++++++++++++++++++++++++------------------- runtime/msg.h | 18 +-- runtime/parser.c | 5 - runtime/queue.h | 1 + tools/syslogd.c | 1 - 6 files changed, 234 insertions(+), 137 deletions(-) diff --git a/dirty.h b/dirty.h index 513886b5..8a0cbc80 100644 --- a/dirty.h +++ b/dirty.h @@ -27,6 +27,7 @@ #ifndef DIRTY_H_INCLUDED #define DIRTY_H_INCLUDED 1 +rsRetVal multiSubmitMsg(multi_submit_t *pMultiSub); rsRetVal submitMsg(msg_t *pMsg); rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); diff --git a/runtime/msg.c b/runtime/msg.c index 65041a31..8230a340 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -55,53 +55,212 @@ DEFobjCurrIf(datetime) DEFobjCurrIf(glbl) DEFobjCurrIf(regexp) -static syslogCODE rs_prioritynames[] = - { - { "alert", LOG_ALERT }, - { "crit", LOG_CRIT }, - { "debug", LOG_DEBUG }, - { "emerg", LOG_EMERG }, - { "err", LOG_ERR }, - { "error", LOG_ERR }, /* DEPRECATED */ - { "info", LOG_INFO }, - { "none", INTERNAL_NOPRI }, /* INTERNAL */ - { "notice", LOG_NOTICE }, - { "panic", LOG_EMERG }, /* DEPRECATED */ - { "warn", LOG_WARNING }, /* DEPRECATED */ - { "warning", LOG_WARNING }, - { NULL, -1 } - }; - -#ifndef LOG_AUTHPRIV -# define LOG_AUTHPRIV LOG_AUTH -#endif -static syslogCODE rs_facilitynames[] = - { - { "auth", LOG_AUTH }, - { "authpriv", LOG_AUTHPRIV }, - { "cron", LOG_CRON }, - { "daemon", LOG_DAEMON }, -#if defined(LOG_FTP) - {"ftp", LOG_FTP}, -#endif - { "kern", LOG_KERN }, - { "lpr", LOG_LPR }, - { "mail", LOG_MAIL }, - { "news", LOG_NEWS }, - { "security", LOG_AUTH }, /* DEPRECATED */ - { "syslog", LOG_SYSLOG }, - { "user", LOG_USER }, - { "uucp", LOG_UUCP }, - { "local0", LOG_LOCAL0 }, - { "local1", LOG_LOCAL1 }, - { "local2", LOG_LOCAL2 }, - { "local3", LOG_LOCAL3 }, - { "local4", LOG_LOCAL4 }, - { "local5", LOG_LOCAL5 }, - { "local6", LOG_LOCAL6 }, - { "local7", LOG_LOCAL7 }, - { NULL, -1 } - }; +static struct { + uchar *pszName; + short lenName; +} syslog_pri_names[192] = { + { UCHAR_CONSTANT("0"), 3}, + { UCHAR_CONSTANT("1"), 3}, + { UCHAR_CONSTANT("2"), 3}, + { UCHAR_CONSTANT("3"), 3}, + { UCHAR_CONSTANT("4"), 3}, + { UCHAR_CONSTANT("5"), 3}, + { UCHAR_CONSTANT("6"), 3}, + { UCHAR_CONSTANT("7"), 3}, + { UCHAR_CONSTANT("8"), 3}, + { UCHAR_CONSTANT("9"), 3}, + { UCHAR_CONSTANT("10"), 4}, + { UCHAR_CONSTANT("11"), 4}, + { UCHAR_CONSTANT("12"), 4}, + { UCHAR_CONSTANT("13"), 4}, + { UCHAR_CONSTANT("14"), 4}, + { UCHAR_CONSTANT("15"), 4}, + { UCHAR_CONSTANT("16"), 4}, + { UCHAR_CONSTANT("17"), 4}, + { UCHAR_CONSTANT("18"), 4}, + { UCHAR_CONSTANT("19"), 4}, + { UCHAR_CONSTANT("20"), 4}, + { UCHAR_CONSTANT("21"), 4}, + { UCHAR_CONSTANT("22"), 4}, + { UCHAR_CONSTANT("23"), 4}, + { UCHAR_CONSTANT("24"), 4}, + { UCHAR_CONSTANT("25"), 4}, + { UCHAR_CONSTANT("26"), 4}, + { UCHAR_CONSTANT("27"), 4}, + { UCHAR_CONSTANT("28"), 4}, + { UCHAR_CONSTANT("29"), 4}, + { UCHAR_CONSTANT("30"), 4}, + { UCHAR_CONSTANT("31"), 4}, + { UCHAR_CONSTANT("32"), 4}, + { UCHAR_CONSTANT("33"), 4}, + { UCHAR_CONSTANT("34"), 4}, + { UCHAR_CONSTANT("35"), 4}, + { UCHAR_CONSTANT("36"), 4}, + { UCHAR_CONSTANT("37"), 4}, + { UCHAR_CONSTANT("38"), 4}, + { UCHAR_CONSTANT("39"), 4}, + { UCHAR_CONSTANT("40"), 4}, + { UCHAR_CONSTANT("41"), 4}, + { UCHAR_CONSTANT("42"), 4}, + { UCHAR_CONSTANT("43"), 4}, + { UCHAR_CONSTANT("44"), 4}, + { UCHAR_CONSTANT("45"), 4}, + { UCHAR_CONSTANT("46"), 4}, + { UCHAR_CONSTANT("47"), 4}, + { UCHAR_CONSTANT("48"), 4}, + { UCHAR_CONSTANT("49"), 4}, + { UCHAR_CONSTANT("50"), 4}, + { UCHAR_CONSTANT("51"), 4}, + { UCHAR_CONSTANT("52"), 4}, + { UCHAR_CONSTANT("53"), 4}, + { UCHAR_CONSTANT("54"), 4}, + { UCHAR_CONSTANT("55"), 4}, + { UCHAR_CONSTANT("56"), 4}, + { UCHAR_CONSTANT("57"), 4}, + { UCHAR_CONSTANT("58"), 4}, + { UCHAR_CONSTANT("59"), 4}, + { UCHAR_CONSTANT("60"), 4}, + { UCHAR_CONSTANT("61"), 4}, + { UCHAR_CONSTANT("62"), 4}, + { UCHAR_CONSTANT("63"), 4}, + { UCHAR_CONSTANT("64"), 4}, + { UCHAR_CONSTANT("65"), 4}, + { UCHAR_CONSTANT("66"), 4}, + { UCHAR_CONSTANT("67"), 4}, + { UCHAR_CONSTANT("68"), 4}, + { UCHAR_CONSTANT("69"), 4}, + { UCHAR_CONSTANT("70"), 4}, + { UCHAR_CONSTANT("71"), 4}, + { UCHAR_CONSTANT("72"), 4}, + { UCHAR_CONSTANT("73"), 4}, + { UCHAR_CONSTANT("74"), 4}, + { UCHAR_CONSTANT("75"), 4}, + { UCHAR_CONSTANT("76"), 4}, + { UCHAR_CONSTANT("77"), 4}, + { UCHAR_CONSTANT("78"), 4}, + { UCHAR_CONSTANT("79"), 4}, + { UCHAR_CONSTANT("80"), 4}, + { UCHAR_CONSTANT("81"), 4}, + { UCHAR_CONSTANT("82"), 4}, + { UCHAR_CONSTANT("83"), 4}, + { UCHAR_CONSTANT("84"), 4}, + { UCHAR_CONSTANT("85"), 4}, + { UCHAR_CONSTANT("86"), 4}, + { UCHAR_CONSTANT("87"), 4}, + { UCHAR_CONSTANT("88"), 4}, + { UCHAR_CONSTANT("89"), 4}, + { UCHAR_CONSTANT("90"), 4}, + { UCHAR_CONSTANT("91"), 4}, + { UCHAR_CONSTANT("92"), 4}, + { UCHAR_CONSTANT("93"), 4}, + { UCHAR_CONSTANT("94"), 4}, + { UCHAR_CONSTANT("95"), 4}, + { UCHAR_CONSTANT("96"), 4}, + { UCHAR_CONSTANT("97"), 4}, + { UCHAR_CONSTANT("98"), 4}, + { UCHAR_CONSTANT("99"), 4}, + { UCHAR_CONSTANT("100"), 5}, + { UCHAR_CONSTANT("101"), 5}, + { UCHAR_CONSTANT("102"), 5}, + { UCHAR_CONSTANT("103"), 5}, + { UCHAR_CONSTANT("104"), 5}, + { UCHAR_CONSTANT("105"), 5}, + { UCHAR_CONSTANT("106"), 5}, + { UCHAR_CONSTANT("107"), 5}, + { UCHAR_CONSTANT("108"), 5}, + { UCHAR_CONSTANT("109"), 5}, + { UCHAR_CONSTANT("110"), 5}, + { UCHAR_CONSTANT("111"), 5}, + { UCHAR_CONSTANT("112"), 5}, + { UCHAR_CONSTANT("113"), 5}, + { UCHAR_CONSTANT("114"), 5}, + { UCHAR_CONSTANT("115"), 5}, + { UCHAR_CONSTANT("116"), 5}, + { UCHAR_CONSTANT("117"), 5}, + { UCHAR_CONSTANT("118"), 5}, + { UCHAR_CONSTANT("119"), 5}, + { UCHAR_CONSTANT("120"), 5}, + { UCHAR_CONSTANT("121"), 5}, + { UCHAR_CONSTANT("122"), 5}, + { UCHAR_CONSTANT("123"), 5}, + { UCHAR_CONSTANT("124"), 5}, + { UCHAR_CONSTANT("125"), 5}, + { UCHAR_CONSTANT("126"), 5}, + { UCHAR_CONSTANT("127"), 5}, + { UCHAR_CONSTANT("128"), 5}, + { UCHAR_CONSTANT("129"), 5}, + { UCHAR_CONSTANT("130"), 5}, + { UCHAR_CONSTANT("131"), 5}, + { UCHAR_CONSTANT("132"), 5}, + { UCHAR_CONSTANT("133"), 5}, + { UCHAR_CONSTANT("134"), 5}, + { UCHAR_CONSTANT("135"), 5}, + { UCHAR_CONSTANT("136"), 5}, + { UCHAR_CONSTANT("137"), 5}, + { UCHAR_CONSTANT("138"), 5}, + { UCHAR_CONSTANT("139"), 5}, + { UCHAR_CONSTANT("140"), 5}, + { UCHAR_CONSTANT("141"), 5}, + { UCHAR_CONSTANT("142"), 5}, + { UCHAR_CONSTANT("143"), 5}, + { UCHAR_CONSTANT("144"), 5}, + { UCHAR_CONSTANT("145"), 5}, + { UCHAR_CONSTANT("146"), 5}, + { UCHAR_CONSTANT("147"), 5}, + { UCHAR_CONSTANT("148"), 5}, + { UCHAR_CONSTANT("149"), 5}, + { UCHAR_CONSTANT("150"), 5}, + { UCHAR_CONSTANT("151"), 5}, + { UCHAR_CONSTANT("152"), 5}, + { UCHAR_CONSTANT("153"), 5}, + { UCHAR_CONSTANT("154"), 5}, + { UCHAR_CONSTANT("155"), 5}, + { UCHAR_CONSTANT("156"), 5}, + { UCHAR_CONSTANT("157"), 5}, + { UCHAR_CONSTANT("158"), 5}, + { UCHAR_CONSTANT("159"), 5}, + { UCHAR_CONSTANT("160"), 5}, + { UCHAR_CONSTANT("161"), 5}, + { UCHAR_CONSTANT("162"), 5}, + { UCHAR_CONSTANT("163"), 5}, + { UCHAR_CONSTANT("164"), 5}, + { UCHAR_CONSTANT("165"), 5}, + { UCHAR_CONSTANT("166"), 5}, + { UCHAR_CONSTANT("167"), 5}, + { UCHAR_CONSTANT("168"), 5}, + { UCHAR_CONSTANT("169"), 5}, + { UCHAR_CONSTANT("170"), 5}, + { UCHAR_CONSTANT("171"), 5}, + { UCHAR_CONSTANT("172"), 5}, + { UCHAR_CONSTANT("173"), 5}, + { UCHAR_CONSTANT("174"), 5}, + { UCHAR_CONSTANT("175"), 5}, + { UCHAR_CONSTANT("176"), 5}, + { UCHAR_CONSTANT("177"), 5}, + { UCHAR_CONSTANT("178"), 5}, + { UCHAR_CONSTANT("179"), 5}, + { UCHAR_CONSTANT("180"), 5}, + { UCHAR_CONSTANT("181"), 5}, + { UCHAR_CONSTANT("182"), 5}, + { UCHAR_CONSTANT("183"), 5}, + { UCHAR_CONSTANT("184"), 5}, + { UCHAR_CONSTANT("185"), 5}, + { UCHAR_CONSTANT("186"), 5}, + { UCHAR_CONSTANT("187"), 5}, + { UCHAR_CONSTANT("188"), 5}, + { UCHAR_CONSTANT("189"), 5}, + { UCHAR_CONSTANT("190"), 5}, + { UCHAR_CONSTANT("191"), 5} + }; + +/*syslog facility names (as of RFC5424) */ +static char *syslog_fac_names[24] = { "kern", "user", "mail", "daemon", "auth", "syslog", "lpr", + "news", "uucp", "cron", "authpriv", "ftp", "ntp", "audit", + "alert", "clock", "local0", "local1", "local2", "local3", + "local4", "local5", "local6", "local7" }; + +/* table of severity names (in numerical order)*/ +static char *syslog_severity_names[8] = { "emerg", "alert", "crit", "err", "warning", "notice", "info", "debug" }; /* some forward declarations */ static int getAPPNAMELen(msg_t *pM); @@ -352,9 +511,7 @@ CODESTARTobjDestruct(msg) free(pThis->pszRcvFromIP); free(pThis->pszMSG); free(pThis->pszFacility); - free(pThis->pszFacilityStr); free(pThis->pszSeverity); - free(pThis->pszSeverityStr); free(pThis->pszRcvdAt3164); free(pThis->pszRcvdAt3339); free(pThis->pszRcvdAt_SecFrac); @@ -441,12 +598,8 @@ msg_t* MsgDup(msg_t* pOld) * passed and nobody complained -- rgerhards, 2009-06-16 pNew->offAfterPRI = pOld->offAfterPRI; */ - memcpy(pNew->bufPRI, pOld->bufPRI, pOld->iLenPRI); - pNew->iLenPRI = pOld->iLenPRI; tmpCOPYSZ(Severity); - tmpCOPYSZ(SeverityStr); tmpCOPYSZ(Facility); - tmpCOPYSZ(FacilityStr); tmpCOPYSZ(RawMsg); tmpCOPYSZ(MSG); tmpCOPYSZ(TAG); @@ -734,27 +887,24 @@ char *getMSG(msg_t *pM) /* Get PRI value as integer */ static int getPRIi(msg_t *pM) { - assert(pM != NULL); return (pM->iFacility << 3) + (pM->iSeverity); } -/* Get PRI value in text form */ +/* Get PRI value in text form + */ static inline char *getPRI(msg_t *pM) { + /* PRI is a number in the range 0..191. Thus, we use a simple lookup table to obtain the + * string value. It looks a bit clumpsy here in code ;) + */ + int iPRI; + if(pM == NULL) return ""; - /* there are some cases where bufPRI may not contain a valid string, - * and then we need to build it. - */ - MsgLock(pM); - if(pM->bufPRI[0] == '\0') { - snprintf((char*)pM->bufPRI, sizeof(pM->bufPRI), "<%d>", getPRIi(pM)); - } - MsgUnlock(pM); - - return (char*)pM->bufPRI; + iPRI = getPRIi(pM); + return (iPRI > 191) ? "invld" : (char*)syslog_pri_names[iPRI].pszName; } @@ -934,32 +1084,18 @@ static inline char *getSeverity(msg_t *pM) static inline char *getSeverityStr(msg_t *pM) { - syslogCODE *c; - int val; char *name = NULL; if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszSeverityStr == NULL) { - for(c = rs_prioritynames, val = pM->iSeverity; c->c_name; c++) - if(c->c_val == val) { - name = c->c_name; - break; - } - if(name == NULL) { - /* we use a 2 byte buffer - can only be one digit */ - if((pM->pszSeverityStr = malloc(2)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenSeverityStr = - snprintf((char*)pM->pszSeverityStr, 2, "%d", pM->iSeverity); - } else { - if((pM->pszSeverityStr = (uchar*) strdup(name)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenSeverityStr = strlen((char*)name); - } + if(pM->iSeverity < 0 || pM->iSeverity > 7) { + name = "invld"; + } else { + name = syslog_severity_names[pM->iSeverity]; } - MsgUnlock(pM); - return((char*)pM->pszSeverityStr); + + return name; } static inline char *getFacility(msg_t *pM) @@ -983,35 +1119,18 @@ static inline char *getFacility(msg_t *pM) static inline char *getFacilityStr(msg_t *pM) { - syslogCODE *c; - int val; char *name = NULL; if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszFacilityStr == NULL) { - for(c = rs_facilitynames, val = pM->iFacility << 3; c->c_name; c++) - if(c->c_val == val) { - name = c->c_name; - break; - } - if(name == NULL) { - /* we use a 12 byte buffer - as of - * syslog-protocol, facility can go - * up to 2^32 -1 - */ - if((pM->pszFacilityStr = malloc(12)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenFacilityStr = - snprintf((char*)pM->pszFacilityStr, 12, "%d", val >> 3); - } else { - if((pM->pszFacilityStr = (uchar*)strdup(name)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenFacilityStr = strlen((char*)name); - } - } - MsgUnlock(pM); - return((char*)pM->pszFacilityStr); + if(pM->iFacility < 0 || pM->iFacility > 23) { + name = "invld"; + } else { + name = syslog_fac_names[pM->iFacility]; + } + + return name; } @@ -1599,15 +1718,11 @@ void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg) */ char *textpri(char *pRes, size_t pResLen, int pri) { - syslogCODE *c_pri, *c_fac; - assert(pRes != NULL); assert(pResLen > 0); - for (c_fac = rs_facilitynames; c_fac->c_name && !(c_fac->c_val == LOG_FAC(pri)<<3); c_fac++); - for (c_pri = rs_prioritynames; c_pri->c_name && !(c_pri->c_val == LOG_PRI(pri)); c_pri++); - - snprintf (pRes, pResLen, "%s.%s<%d>", c_fac->c_name, c_pri->c_name, pri); + snprintf(pRes, pResLen, "%s.%s<%d>", syslog_fac_names[LOG_FAC(pri)], + syslog_severity_names[LOG_PRI(pri)], pri); return pRes; } diff --git a/runtime/msg.h b/runtime/msg.h index 74ff9e60..a3058349 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,12 +51,12 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ pthread_mutexattr_t mutAttr; - bool bDoLock; /* use the mutex? */ + bool bDoLock; /* use the mutex? */ + bool bParseHOSTNAME; /* should the hostname be parsed from the message? */ pthread_mutex_t mut; flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ short iRefCount; /* reference counter (0 = unused) */ - short bParseHOSTNAME; /* should the hostname be parsed from the message? */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, * it is available when we have a forwarder (e.g. rfc3195d) using local @@ -66,15 +66,9 @@ struct msg { short iSeverity; /* the severity 0..7 */ uchar *pszSeverity; /* severity as string... */ int iLenSeverity; /* ... and its length. */ - uchar *pszSeverityStr; /* severity name... */ - int iLenSeverityStr; /* ... and its length. */ short iFacility; /* Facility code 0 .. 23*/ uchar *pszFacility; /* Facility as string... */ int iLenFacility; /* ... and its length. */ - uchar *pszFacilityStr; /* facility name... */ - int iLenFacilityStr; /* ... and its length. */ - uchar bufPRI[5]; /* PRI as string */ - int iLenPRI; /* and its length */ uchar *pszRawMsg; /* message as it was received on the * wire. This is important in case we * need to preserve cryptographic verifiers. @@ -83,8 +77,6 @@ struct msg { int iLenRawMsg; /* length of raw message */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ - uchar *pszUxTradMsg; /* the traditional UNIX message */ - int iLenUxTradMsg;/* Length of the traditional UNIX message */ uchar *pszTAG; /* pointer to tag value */ int iLenTAG; /* Length of the TAG part */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ @@ -122,7 +114,6 @@ struct msg { char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ - /* now follow fixed-size buffers to safe some time otherwise used for allocs */ }; @@ -182,11 +173,6 @@ char *getProgramName(msg_t *pM); int getProgramNameLen(msg_t *pM); uchar *getRcvFrom(msg_t *pM); -#if 0 -char *getUxTradMsg(msg_t *pM); -int MsgSetUxTradMsg(msg_t *pMsg, char* pszUxTradMsg); -#endif - /* The MsgPrepareEnqueue() function is a macro for performance reasons. * It needs one global variable to work. This is acceptable, as it gains * us quite some performance and is fully abstracted using this header file. diff --git a/runtime/parser.c b/runtime/parser.c index 0b45bfd5..c13edb8f 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -284,7 +284,6 @@ rsRetVal parseMsg(msg_t *pMsg) DEFiRet; uchar *msg; int pri; - int iPriText; CHKiRet(sanitizeMessage(pMsg)); @@ -294,7 +293,6 @@ rsRetVal parseMsg(msg_t *pMsg) /* pull PRI */ pri = DEFUPRI; msg = pMsg->pszRawMsg; - iPriText = 0; if(*msg == '<') { /* while we process the PRI, we also fill the PRI textual representation * inside the msg object. This may not be ideal from an OOP point of view, @@ -302,11 +300,8 @@ rsRetVal parseMsg(msg_t *pMsg) */ pri = 0; while(isdigit((int) *++msg)) { - pMsg->bufPRI[iPriText++ % 4] = *msg; /* mod 4 to guard against malformed messages! */ pri = 10 * pri + (*msg - '0'); } - pMsg->bufPRI[iPriText % 4] = '\0'; - pMsg->iLenPRI = iPriText % 4; if(*msg == '>') ++msg; if(pri & ~(LOG_FACMASK|LOG_PRIMASK)) diff --git a/runtime/queue.h b/runtime/queue.h index 07f134aa..5bc03254 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -179,6 +179,7 @@ typedef struct queue_s { /* prototypes */ rsRetVal qqueueDestruct(qqueue_t **ppThis); +rsRetVal qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub); rsRetVal qqueueEnqObj(qqueue_t *pThis, flowControl_t flwCtlType, void *pUsr); rsRetVal qqueueStart(qqueue_t *pThis); rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); diff --git a/tools/syslogd.c b/tools/syslogd.c index e3cd54f7..9f70ff26 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1377,7 +1377,6 @@ multiSubmitMsg(multi_submit_t *pMultiSub) assert(pMultiSub != NULL); for(i = 0 ; i < pMultiSub->nElem ; ++i) { -dbgprintf("multiSubmitMsg, index %d\n", i); MsgPrepareEnqueue(pMultiSub->ppMsgs[i]); } -- cgit v1.2.3 From 34b60b389e463e260f880ca3efc591b4db2cd18a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 17 Jun 2009 13:33:57 +0200 Subject: some more optimization in msg object string handling --- runtime/msg.c | 70 ++++++++++++++++++++++------------------------------------- runtime/msg.h | 21 ++++++++---------- 2 files changed, 35 insertions(+), 56 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 8230a340..8e9d2b71 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -262,6 +262,13 @@ static char *syslog_fac_names[24] = { "kern", "user", "mail", "daemon", "auth", /* table of severity names (in numerical order)*/ static char *syslog_severity_names[8] = { "emerg", "alert", "crit", "err", "warning", "notice", "info", "debug" }; +/* numerical values as string - this is the most efficient approach to convert severity + * and facility values to a numerical string... -- rgerhars, 2009-06-17 + */ + +static char *syslog_number_names[24] = { "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", + "15", "16", "17", "18", "19", "20", "21", "22", "23" }; + /* some forward declarations */ static int getAPPNAMELen(msg_t *pM); static int getProtocolVersion(msg_t *pM); @@ -510,15 +517,11 @@ CODESTARTobjDestruct(msg) free(pThis->pszRcvFrom); free(pThis->pszRcvFromIP); free(pThis->pszMSG); - free(pThis->pszFacility); - free(pThis->pszSeverity); free(pThis->pszRcvdAt3164); free(pThis->pszRcvdAt3339); free(pThis->pszRcvdAt_SecFrac); free(pThis->pszRcvdAt_MySQL); free(pThis->pszRcvdAt_PgSQL); - free(pThis->pszTIMESTAMP3164); - free(pThis->pszTIMESTAMP3339); free(pThis->pszTIMESTAMP_SecFrac); free(pThis->pszTIMESTAMP_MySQL); free(pThis->pszTIMESTAMP_PgSQL); @@ -598,8 +601,6 @@ msg_t* MsgDup(msg_t* pOld) * passed and nobody complained -- rgerhards, 2009-06-16 pNew->offAfterPRI = pOld->offAfterPRI; */ - tmpCOPYSZ(Severity); - tmpCOPYSZ(Facility); tmpCOPYSZ(RawMsg); tmpCOPYSZ(MSG); tmpCOPYSZ(TAG); @@ -916,12 +917,10 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) switch(eFmt) { case tplFmtDefault: + case tplFmtRFC3164Date: MsgLock(pM); if(pM->pszTIMESTAMP3164 == NULL) { - if((pM->pszTIMESTAMP3164 = malloc(16)) == NULL) { - MsgUnlock(pM); - return ""; - } + pM->pszTIMESTAMP3164 = pM->pszTimestamp3164; datetime.formatTimestamp3164(&pM->tTIMESTAMP, pM->pszTIMESTAMP3164, 16); } MsgUnlock(pM); @@ -948,24 +947,10 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) } MsgUnlock(pM); return(pM->pszTIMESTAMP_PgSQL); - case tplFmtRFC3164Date: - MsgLock(pM); - if(pM->pszTIMESTAMP3164 == NULL) { - if((pM->pszTIMESTAMP3164 = malloc(16)) == NULL) { - MsgUnlock(pM); - return ""; - } - datetime.formatTimestamp3164(&pM->tTIMESTAMP, pM->pszTIMESTAMP3164, 16); - } - MsgUnlock(pM); - return(pM->pszTIMESTAMP3164); case tplFmtRFC3339Date: MsgLock(pM); if(pM->pszTIMESTAMP3339 == NULL) { - if((pM->pszTIMESTAMP3339 = malloc(33)) == NULL) { - MsgUnlock(pM); - return ""; /* TODO: check this: can it cause a free() of constant memory?) */ - } + pM->pszTIMESTAMP3339 = pM->pszTimestamp3339; datetime.formatTimestamp3339(&pM->tTIMESTAMP, pM->pszTIMESTAMP3339, 33); } MsgUnlock(pM); @@ -1067,18 +1052,18 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) static inline char *getSeverity(msg_t *pM) { + char *name = NULL; + if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszSeverity == NULL) { - /* we use a 2 byte buffer - can only be one digit */ - if((pM->pszSeverity = malloc(2)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenSeverity = - snprintf((char*)pM->pszSeverity, 2, "%d", pM->iSeverity); + if(pM->iSeverity < 0 || pM->iSeverity > 7) { + name = "invld"; + } else { + name = syslog_number_names[pM->iSeverity]; } - MsgUnlock(pM); - return((char*)pM->pszSeverity); + + return name; } @@ -1100,21 +1085,18 @@ static inline char *getSeverityStr(msg_t *pM) static inline char *getFacility(msg_t *pM) { + char *name = NULL; + if(pM == NULL) return ""; - MsgLock(pM); - if(pM->pszFacility == NULL) { - /* we use a 12 byte buffer - as of - * syslog-protocol, facility can go - * up to 2^32 -1 - */ - if((pM->pszFacility = malloc(12)) == NULL) { MsgUnlock(pM) ; return ""; } - pM->iLenFacility = - snprintf((char*)pM->pszFacility, 12, "%d", pM->iFacility); + if(pM->iFacility < 0 || pM->iFacility > 23) { + name = "invld"; + } else { + name = syslog_number_names[pM->iFacility]; } - MsgUnlock(pM); - return((char*)pM->pszFacility); + + return name; } static inline char *getFacilityStr(msg_t *pM) diff --git a/runtime/msg.h b/runtime/msg.h index a3058349..703fdd9f 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -50,12 +50,12 @@ */ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because + once data has entered the queue, this property is no longer needed. */ pthread_mutexattr_t mutAttr; + pthread_mutex_t mut; bool bDoLock; /* use the mutex? */ bool bParseHOSTNAME; /* should the hostname be parsed from the message? */ - pthread_mutex_t mut; - flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because - once data has entered the queue, this property is no longer needed. */ short iRefCount; /* reference counter (0 = unused) */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, @@ -64,17 +64,11 @@ struct msg { * resolve all these issues... rgerhards, 2005-10-06 */ short iSeverity; /* the severity 0..7 */ - uchar *pszSeverity; /* severity as string... */ - int iLenSeverity; /* ... and its length. */ short iFacility; /* Facility code 0 .. 23*/ - uchar *pszFacility; /* Facility as string... */ - int iLenFacility; /* ... and its length. */ - uchar *pszRawMsg; /* message as it was received on the - * wire. This is important in case we - * need to preserve cryptographic verifiers. - */ - short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ + uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we + * need to preserve cryptographic verifiers. */ int iLenRawMsg; /* length of raw message */ + short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ uchar *pszTAG; /* pointer to tag value */ @@ -114,6 +108,9 @@ struct msg { char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ int msgFlags; /* flags associated with this message */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ + /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ + char pszTimestamp3164[16]; + char pszTimestamp3339[33]; }; -- cgit v1.2.3 From d2d54013aebb756169182ed8716b142d27134a70 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 17 Jun 2009 15:22:13 +0200 Subject: going forward in moving string-handling functions to new interface... --- parse.c | 18 ++++++------- runtime/ctok.c | 17 ++++++------ runtime/msg.c | 20 +++++++-------- runtime/nsd_gtls.c | 34 ++++++++++++------------ runtime/obj.c | 16 ++++++------ runtime/prop.c | 4 +-- runtime/stream.c | 6 ++--- runtime/stringbuf.c | 74 +++++++++++++++++++++++++++++++---------------------- runtime/stringbuf.h | 22 +++++++++------- runtime/vm.c | 12 +++++---- runtime/vmop.c | 2 +- tests/nettester.c | 4 ++- tools/syslogd.c | 2 +- 13 files changed, 126 insertions(+), 105 deletions(-) diff --git a/parse.c b/parse.c index 87c67676..5288c8b4 100644 --- a/parse.c +++ b/parse.c @@ -256,7 +256,7 @@ rsRetVal parsDelimCStr(rsParsObj *pThis, cstr_t **ppCStr, char cDelim, int bTrim pC = rsCStrGetBufBeg(pThis->pCStr) + pThis->iCurrPos; while(pThis->iCurrPos < rsCStrLen(pThis->pCStr) && *pC != cDelim) { - CHKiRet(rsCStrAppendChar(pCStr, bConvLower ? tolower(*pC) : *pC)); + CHKiRet(cstrAppendChar(pCStr, bConvLower ? tolower(*pC) : *pC)); ++pThis->iCurrPos; ++pC; } @@ -271,7 +271,7 @@ rsRetVal parsDelimCStr(rsParsObj *pThis, cstr_t **ppCStr, char cDelim, int bTrim CHKiRet(cstrFinalize(pCStr)); if(bTrimTrailing) { - CHKiRet(rsCStrTrimTrailingWhiteSpace(pCStr)); + CHKiRet(cstrTrimTrailingWhiteSpace(pCStr)); } /* done! */ @@ -313,23 +313,23 @@ rsRetVal parsQuotedCStr(rsParsObj *pThis, cstr_t **ppCStr) pC = rsCStrGetBufBeg(pThis->pCStr) + pThis->iCurrPos; /* OK, we most probably can obtain a value... */ - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); - while(pThis->iCurrPos < rsCStrLen(pThis->pCStr)) { + while(pThis->iCurrPos < cstrLen(pThis->pCStr)) { if(*pC == '"') { break; /* we are done! */ } else if(*pC == '\\') { ++pThis->iCurrPos; ++pC; - if(pThis->iCurrPos < rsCStrLen(pThis->pCStr)) { + if(pThis->iCurrPos < cstrLen(pThis->pCStr)) { /* in this case, we copy the escaped character * to the output buffer (but do not rely on this, * we might later introduce other things, like \007! */ - CHKiRet(rsCStrAppendChar(pCStr, *pC)); + CHKiRet(cstrAppendChar(pCStr, *pC)); } } else { /* regular character */ - CHKiRet(rsCStrAppendChar(pCStr, *pC)); + CHKiRet(cstrAppendChar(pCStr, *pC)); } ++pThis->iCurrPos; ++pC; @@ -339,7 +339,7 @@ rsRetVal parsQuotedCStr(rsParsObj *pThis, cstr_t **ppCStr) ++pThis->iCurrPos; /* 'eat' trailing quote */ } else { /* error - improperly quoted string! */ - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); ABORT_FINALIZE(RS_RET_MISSING_TRAIL_QUOTE); } @@ -352,7 +352,7 @@ rsRetVal parsQuotedCStr(rsParsObj *pThis, cstr_t **ppCStr) finalize_it: if(iRet != RS_RET_OK) { if(pCStr != NULL) - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); } RETiRet; diff --git a/runtime/ctok.c b/runtime/ctok.c index 263e656c..6f5f0273 100644 --- a/runtime/ctok.c +++ b/runtime/ctok.c @@ -258,13 +258,13 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) pToken->tok = ctok_MSGVAR; } - CHKiRet(rsCStrConstruct(&pstrVal)); + CHKiRet(cstrConstruct(&pstrVal)); /* this loop is quite simple, a variable name is terminated when a non-supported * character is detected. Note that we currently permit a numerical digit as the * first char, which is not permitted by ABNF. -- rgerhards, 2009-03-10 */ while(isalpha(c) || isdigit(c) || (c == '_') || (c == '-')) { - CHKiRet(rsCStrAppendChar(pstrVal, tolower(c))); + CHKiRet(cstrAppendChar(pstrVal, tolower(c))); CHKiRet(ctokGetCharFromStream(pThis, &c)); } CHKiRet(ctokUngetCharFromStream(pThis, c)); /* put not processed char back */ @@ -277,7 +277,7 @@ ctokGetVar(ctok_t *pThis, ctok_token_t *pToken) finalize_it: if(iRet != RS_RET_OK) { if(pstrVal != NULL) { - rsCStrDestruct(&pstrVal); + cstrDestruct(&pstrVal); } } @@ -301,20 +301,20 @@ ctokGetSimpStr(ctok_t *pThis, ctok_token_t *pToken) pToken->tok = ctok_SIMPSTR; - CHKiRet(rsCStrConstruct(&pstrVal)); + CHKiRet(cstrConstruct(&pstrVal)); CHKiRet(ctokGetCharFromStream(pThis, &c)); /* while we are in escape mode (had a backslash), no sequence * terminates the loop. If outside, it is terminated by a single quote. */ while(bInEsc || c != '\'') { if(bInEsc) { - CHKiRet(rsCStrAppendChar(pstrVal, c)); + CHKiRet(cstrAppendChar(pstrVal, c)); bInEsc = 0; } else { if(c == '\\') { bInEsc = 1; } else { - CHKiRet(rsCStrAppendChar(pstrVal, c)); + CHKiRet(cstrAppendChar(pstrVal, c)); } } CHKiRet(ctokGetCharFromStream(pThis, &c)); @@ -327,7 +327,7 @@ ctokGetSimpStr(ctok_t *pThis, ctok_token_t *pToken) finalize_it: if(iRet != RS_RET_OK) { if(pstrVal != NULL) { - rsCStrDestruct(&pstrVal); + cstrDestruct(&pstrVal); } } @@ -519,8 +519,9 @@ ctokGetToken(ctok_t *pThis, ctok_token_t **ppToken) CHKiRet(ctokUngetCharFromStream(pThis, c)); pToken->tok = ctok_FUNCTION; /* fill function name */ - CHKiRet(rsCStrConstruct(&pstrVal)); + CHKiRet(cstrConstruct(&pstrVal)); CHKiRet(rsCStrSetSzStr(pstrVal, szWord)); + CHKiRet(cstrFinalize(pstrVal)); CHKiRet(var.SetString(pToken->pVar, pstrVal)); } else { /* give up... */ dbgprintf("parser has an invalid word (token) '%s'\n", szWord); diff --git a/runtime/msg.c b/runtime/msg.c index 8e9d2b71..d9ff2e73 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -730,10 +730,10 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) ++i; /* skip '[' */ /* now obtain the PROCID string... */ - CHKiRet(rsCStrConstruct(&pM->pCSPROCID)); + CHKiRet(cstrConstruct(&pM->pCSPROCID)); rsCStrSetAllocIncrement(pM->pCSPROCID, 16); while((i < pM->iLenTAG) && (pM->pszTAG[i] != ']')) { - CHKiRet(rsCStrAppendChar(pM->pCSPROCID, pM->pszTAG[i])); + CHKiRet(cstrAppendChar(pM->pCSPROCID, pM->pszTAG[i])); ++i; } @@ -743,7 +743,7 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) * the buffer and simply return. Note that this is NOT an error * case! */ - rsCStrDestruct(&pM->pCSPROCID); + cstrDestruct(&pM->pCSPROCID); FINALIZE; } @@ -781,14 +781,14 @@ static rsRetVal aquireProgramName(msg_t *pM) /* ok, we do not yet have it. So let's parse the TAG * to obtain it. */ - CHKiRet(rsCStrConstruct(&pM->pCSProgName)); + CHKiRet(cstrConstruct(&pM->pCSProgName)); rsCStrSetAllocIncrement(pM->pCSProgName, 33); for( i = 0 ; (i < pM->iLenTAG) && isprint((int) pM->pszTAG[i]) && (pM->pszTAG[i] != '\0') && (pM->pszTAG[i] != ':') && (pM->pszTAG[i] != '[') && (pM->pszTAG[i] != '/') ; ++i) { - CHKiRet(rsCStrAppendChar(pM->pCSProgName, pM->pszTAG[i])); + CHKiRet(cstrAppendChar(pM->pCSProgName, pM->pszTAG[i])); } CHKiRet(cstrFinalize(pM->pCSProgName)); } @@ -811,8 +811,7 @@ finalize_it: void moveHOSTNAMEtoTAG(msg_t *pM) { assert(pM != NULL); - if(pM->pszTAG != NULL) - free(pM->pszTAG); + free(pM->pszTAG); pM->pszTAG = pM->pszHOSTNAME; pM->iLenTAG = pM->iLenHOSTNAME; pM->pszHOSTNAME = NULL; @@ -1188,11 +1187,11 @@ rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID) ISOBJ_TYPE_assert(pMsg, msg); if(pMsg->pCSPROCID == NULL) { /* we need to obtain the object first */ - CHKiRet(rsCStrConstruct(&pMsg->pCSPROCID)); - rsCStrSetAllocIncrement(pMsg->pCSPROCID, 128); + CHKiRet(cstrConstruct(&pMsg->pCSPROCID)); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSPROCID, (uchar*) pszPROCID); + CHKiRet(cstrFinalize(pMsg->pCSPROCID)); finalize_it: RETiRet; @@ -1221,7 +1220,8 @@ char *getPROCID(msg_t *pM) MsgLock(pM); if(pM->pCSPROCID == NULL) aquirePROCIDFromTAG(pM); - pszRet = (pM->pCSPROCID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSPROCID); + pszRet = (pM->pCSPROCID == NULL) ? "-" : (char*) cstrGetSzStrNoNULL(pM->pCSPROCID); + //pszRet = (pM->pCSPROCID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSPROCID); MsgUnlock(pM); return pszRet; } diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 19dc8678..79ceffb3 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -710,16 +710,16 @@ gtlsGetCN(nsd_gtls_t *pThis, gnutls_x509_crt *pCert, cstr_t **ppstrCN) } /* we found a common name, now extract it */ - CHKiRet(rsCStrConstruct(&pstrCN)); + CHKiRet(cstrConstruct(&pstrCN)); while(szDN[i] != '\0' && szDN[i] != ',') { if(szDN[i] == '\\') { /* hex escapes are not implemented */ ++i; /* escape char processed */ if(szDN[i] == '\0') ABORT_FINALIZE(RS_RET_CERT_INVALID_DN); - CHKiRet(rsCStrAppendChar(pstrCN, szDN[i])); + CHKiRet(cstrAppendChar(pstrCN, szDN[i])); } else { - CHKiRet(rsCStrAppendChar(pstrCN, szDN[i])); + CHKiRet(cstrAppendChar(pstrCN, szDN[i])); } ++i; /* char processed */ } @@ -734,7 +734,7 @@ gtlsGetCN(nsd_gtls_t *pThis, gnutls_x509_crt *pCert, cstr_t **ppstrCN) finalize_it: if(iRet != RS_RET_OK) { if(pstrCN != NULL) - rsCStrDestruct(&pstrCN); + cstrDestruct(&pstrCN); } RETiRet; @@ -761,7 +761,7 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) size = sizeof(fingerprint); CHKgnutls(gnutls_x509_crt_get_fingerprint(*pCert, GNUTLS_DIG_SHA1, fingerprint, &size)); CHKiRet(GenFingerprintStr(fingerprint, size, &pstrFingerprint)); - dbgprintf("peer's certificate SHA1 fingerprint: %s\n", rsCStrGetSzStr(pstrFingerprint)); + dbgprintf("peer's certificate SHA1 fingerprint: %s\n", cstrGetSzStr(pstrFingerprint)); /* now search through the permitted peers to see if we can find a permitted one */ bFoundPositiveMatch = 0; @@ -779,7 +779,7 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) if(pThis->bReportAuthErr == 1) { errno = 0; errmsg.LogError(0, RS_RET_INVALID_FINGERPRINT, "error: peer fingerprint '%s' unknown - we are " - "not permitted to talk to it", rsCStrGetSzStr(pstrFingerprint)); + "not permitted to talk to it", cstrGetSzStr(pstrFingerprint)); pThis->bReportAuthErr = 0; } ABORT_FINALIZE(RS_RET_INVALID_FINGERPRINT); @@ -787,7 +787,7 @@ gtlsChkPeerFingerprint(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) finalize_it: if(pstrFingerprint != NULL) - rsCStrDestruct(&pstrFingerprint); + cstrDestruct(&pstrFingerprint); RETiRet; } @@ -874,10 +874,10 @@ gtlsChkPeerName(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) /* if we did not succeed so far, we try the CN part of the DN... */ CHKiRet(gtlsGetCN(pThis, pCert, &pstrCN)); if(pstrCN != NULL) { /* NULL if there was no CN present */ - dbgprintf("gtls now checking auth for CN '%s'\n", rsCStrGetSzStr(pstrCN)); - snprintf((char*)lnBuf, sizeof(lnBuf), "CN: %s; ", rsCStrGetSzStr(pstrCN)); + dbgprintf("gtls now checking auth for CN '%s'\n", cstrGetSzStr(pstrCN)); + snprintf((char*)lnBuf, sizeof(lnBuf), "CN: %s; ", cstrGetSzStr(pstrCN)); CHKiRet(rsCStrAppendStr(pStr, lnBuf)); - CHKiRet(gtlsChkOnePeerName(pThis, rsCStrGetSzStr(pstrCN), &bFoundPositiveMatch)); + CHKiRet(gtlsChkOnePeerName(pThis, cstrGetSzStr(pstrCN), &bFoundPositiveMatch)); } } @@ -888,7 +888,7 @@ gtlsChkPeerName(nsd_gtls_t *pThis, gnutls_x509_crt *pCert) errno = 0; errmsg.LogError(0, RS_RET_INVALID_FINGERPRINT, "error: peer name not authorized - " "not permitted to talk to it. Names: %s", - rsCStrGetSzStr(pStr)); + cstrGetSzStr(pStr)); pThis->bReportAuthErr = 0; } ABORT_FINALIZE(RS_RET_INVALID_FINGERPRINT); @@ -1010,8 +1010,8 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) errmsg.LogError(0, NO_ERRCODE, "not permitted to talk to peer, certificate invalid: %s", pszErrCause); gtlsGetCertInfo(pThis, &pStr); - errmsg.LogError(0, NO_ERRCODE, "invalid cert info: %s", rsCStrGetSzStr(pStr)); - rsCStrDestruct(&pStr); + errmsg.LogError(0, NO_ERRCODE, "invalid cert info: %s", cstrGetSzStr(pStr)); + cstrDestruct(&pStr); ABORT_FINALIZE(RS_RET_CERT_INVALID); } @@ -1032,8 +1032,8 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) else if(ttCert > ttNow) { errmsg.LogError(0, RS_RET_CERT_NOT_YET_ACTIVE, "not permitted to talk to peer: certificate %d not yet active", i); gtlsGetCertInfo(pThis, &pStr); - errmsg.LogError(0, RS_RET_CERT_NOT_YET_ACTIVE, "invalid cert info: %s", rsCStrGetSzStr(pStr)); - rsCStrDestruct(&pStr); + errmsg.LogError(0, RS_RET_CERT_NOT_YET_ACTIVE, "invalid cert info: %s", cstrGetSzStr(pStr)); + cstrDestruct(&pStr); ABORT_FINALIZE(RS_RET_CERT_NOT_YET_ACTIVE); } @@ -1043,8 +1043,8 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) else if(ttCert < ttNow) { errmsg.LogError(0, RS_RET_CERT_EXPIRED, "not permitted to talk to peer: certificate %d expired", i); gtlsGetCertInfo(pThis, &pStr); - errmsg.LogError(0, RS_RET_CERT_EXPIRED, "invalid cert info: %s", rsCStrGetSzStr(pStr)); - rsCStrDestruct(&pStr); + errmsg.LogError(0, RS_RET_CERT_EXPIRED, "invalid cert info: %s", cstrGetSzStr(pStr)); + cstrDestruct(&pStr); ABORT_FINALIZE(RS_RET_CERT_EXPIRED); } gnutls_x509_crt_deinit(cert); diff --git a/runtime/obj.c b/runtime/obj.c index f2cb447e..6ca05cc4 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -440,11 +440,11 @@ objDeserializeEmbedStr(cstr_t **ppStr, strm_t *pStrm) assert(ppStr != NULL); - CHKiRet(rsCStrConstruct(&pStr)); + CHKiRet(cstrConstruct(&pStr)); NEXTC; while(c != ':') { - CHKiRet(rsCStrAppendChar(pStr, c)); + CHKiRet(cstrAppendChar(pStr, c)); NEXTC; } CHKiRet(cstrFinalize(pStr)); @@ -453,7 +453,7 @@ objDeserializeEmbedStr(cstr_t **ppStr, strm_t *pStrm) finalize_it: if(iRet != RS_RET_OK && pStr != NULL) - rsCStrDestruct(&pStr); + cstrDestruct(&pStr); RETiRet; } @@ -508,11 +508,11 @@ static rsRetVal objDeserializeStr(cstr_t **ppCStr, int iLen, strm_t *pStrm) assert(ppCStr != NULL); assert(iLen >= 0); - CHKiRet(rsCStrConstruct(&pCStr)); + CHKiRet(cstrConstruct(&pCStr)); NEXTC; for(i = 0 ; i < iLen ; ++i) { - CHKiRet(rsCStrAppendChar(pCStr, c)); + CHKiRet(cstrAppendChar(pCStr, c)); NEXTC; } CHKiRet(cstrFinalize(pCStr)); @@ -524,7 +524,7 @@ static rsRetVal objDeserializeStr(cstr_t **ppCStr, int iLen, strm_t *pStrm) finalize_it: if(iRet != RS_RET_OK && pCStr != NULL) - rsCStrDestruct(&pCStr); + cstrDestruct(&pCStr); RETiRet; } @@ -622,11 +622,11 @@ static rsRetVal objDeserializeProperty(var_t *pProp, strm_t *pStrm) } /* get the property name first */ - CHKiRet(rsCStrConstruct(&pProp->pcsName)); + CHKiRet(cstrConstruct(&pProp->pcsName)); NEXTC; while(c != ':') { - CHKiRet(rsCStrAppendChar(pProp->pcsName, c)); + CHKiRet(cstrAppendChar(pProp->pcsName, c)); NEXTC; } CHKiRet(cstrFinalize(pProp->pcsName)); diff --git a/runtime/prop.c b/runtime/prop.c index 4130d4b7..02be315f 100644 --- a/runtime/prop.c +++ b/runtime/prop.c @@ -55,12 +55,10 @@ ENDobjConstruct(prop) * rgerhards, 2008-01-09 */ static rsRetVal -propConstructFinalize(prop_t *pThis) +propConstructFinalize(prop_t __attribute__((unused)) *pThis) { DEFiRet; ISOBJ_TYPE_assert(pThis, prop); - -finalize_it: RETiRet; } diff --git a/runtime/stream.c b/runtime/stream.c index 8cbe0297..f9859617 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -531,19 +531,19 @@ strmReadLine(strm_t *pThis, cstr_t **ppCStr) ASSERT(pThis != NULL); ASSERT(ppCStr != NULL); - CHKiRet(rsCStrConstruct(ppCStr)); + CHKiRet(cstrConstruct(ppCStr)); /* now read the line */ CHKiRet(strmReadChar(pThis, &c)); while(c != '\n') { - CHKiRet(rsCStrAppendChar(*ppCStr, c)); + CHKiRet(cstrAppendChar(*ppCStr, c)); CHKiRet(strmReadChar(pThis, &c)); } CHKiRet(cstrFinalize(*ppCStr)); finalize_it: if(iRet != RS_RET_OK && *ppCStr != NULL) - rsCStrDestruct(ppCStr); + cstrDestruct(ppCStr); RETiRet; } diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index a2d9c599..e7fa8c41 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -1,3 +1,4 @@ +#include /* This is the byte-counted string class for rsyslog. It is a replacement * for classical \0 terminated string functions. We introduce it in * the hope it will make the program more secure, obtain some performance @@ -157,7 +158,7 @@ static rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) { uchar *pNewBuf; - size_t iNewSize; + unsigned short iNewSize; DEFiRet; /* first compute the new size needed */ @@ -224,7 +225,7 @@ rsRetVal rsCStrAppendStr(cstr_t *pThis, uchar* psz) /* append the contents of one cstr_t object to another * rgerhards, 2008-02-25 */ -rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend) +rsRetVal cstrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend) { return rsCStrAppendStrWithLen(pThis, pstrAppend->pBuf, pstrAppend->iStrLen); } @@ -245,32 +246,7 @@ finalize_it: } -rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c) -{ - DEFiRet; - - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if(pThis->iStrLen >= pThis->iBufSize) { - CHKiRet(rsCStrExtendBuf(pThis, 1)); /* need more memory! */ - } - - /* ok, when we reach this, we have sufficient memory */ - *(pThis->pBuf + pThis->iStrLen++) = c; - - /* check if we need to invalidate an sz representation! */ - if(pThis->pszBuf != NULL) { - free(pThis->pszBuf); - pThis->pszBuf = NULL; - } - -finalize_it: - RETiRet; -} - - -/* NEW VARIANT - * Append a character to the current string object. This may only be done until +/* Append a character to the current string object. This may only be done until * cstrFinalize() is called. * rgerhards, 2009-06-16 */ @@ -392,7 +368,24 @@ uchar* rsCStrGetSzStr(cstr_t *pThis) } -/* NEW VERSION for interface without separate psz buffer! */ +/* Converts the CStr object to a classical sz string and returns that. + * Same restrictions as in cstrGetSzStr() applies (see there!). This + * function here guarantees that a valid string is returned, even if + * the CStr object currently holds a NULL pointer string buffer. If so, + * "" is returned. + * rgerhards 2005-10-19 + * WARNING: The returned pointer MUST NOT be freed, as it may be + * obtained from that constant memory pool (in case of NULL!) + */ +uchar* cstrGetSzStrNoNULL(cstr_t *pThis) +{ + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + if(pThis->pBuf == NULL) + return (uchar*) ""; + else + return cstrGetSzStr(pThis); +} + /* Returns the cstr data as a classical C sz string. We use that the * Finalizer did properly terminate our string (but we may stil be NULL). * So it is vital that the finalizer is called BEFORe this function here! @@ -497,7 +490,7 @@ void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement) * This is due to performance reasons. */ #ifndef NDEBUG -int rsCStrLen(cstr_t *pThis) +int cstrLen(cstr_t *pThis) { rsCHECKVALIDOBJECT(pThis, OIDrsCStr); return(pThis->iStrLen); @@ -548,6 +541,27 @@ rsRetVal rsCStrTrimTrailingWhiteSpace(cstr_t *pThis) return RS_RET_OK; } +/* Trim trailing whitespace from a given string + */ +rsRetVal cstrTrimTrailingWhiteSpace(cstr_t *pThis) +{ + register int i; + register uchar *pC; + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + + i = pThis->iStrLen; + pC = pThis->pBuf + i - 1; + while(i > 0 && isspace((int)*pC)) { + --pC; + --i; + } + /* i now is the new string length! */ + pThis->iStrLen = i; + pThis->pBuf[pThis->iStrLen] = '0'; /* we always have this space */ + + return RS_RET_OK; +} + /* compare two string objects - works like strcmp(), but operates * on CStr objects. Please note that this version here is * faster in the majority of cases, simply because it can diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index d28aee26..b4fc3f3c 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -1,5 +1,5 @@ -/*! \file stringbuf.h - * \brief The counted string object +/* stringbuf.h + * The counted string object * * This is the byte-counted string class for rsyslog. It is a replacement * for classical \0 terminated string functions. We introduce it in @@ -11,8 +11,7 @@ * \date 2005-09-07 * Initial version begun. * - * All functions in this "class" start with rsCStr (rsyslog Counted String). - * Copyright 2005 + * Copyright 2005-2009 * Rainer Gerhards and Adiscon GmbH. All Rights Reserved. * * This file is part of the rsyslog runtime library. @@ -48,7 +47,7 @@ typedef struct cstr_s uchar *pszBuf; /**< pointer to the sz version of the string (after it has been created )*/ size_t iBufSize; /**< current maximum size of the string buffer */ size_t iStrLen; /**< length of the string in characters. */ - size_t iAllocIncrement; /**< the amount of bytes the string should be expanded if it needs to */ + unsigned short iAllocIncrement; /**< the amount of bytes the string should be expanded if it needs to */ bool bIsFinalized; /**< is this object finished and ready for use? (a debug aid, may be removed later TODO 2009-06-16) */ } cstr_t; @@ -73,7 +72,6 @@ void rsCStrDestruct(cstr_t **ppThis); * * \param c Character to append to string. */ -rsRetVal rsCStrAppendChar(cstr_t *pThis, uchar c); rsRetVal cstrAppendChar(cstr_t *pThis, uchar c); /** @@ -85,6 +83,7 @@ rsRetVal cstrAppendChar(cstr_t *pThis, uchar c); rsRetVal rsCStrTruncate(cstr_t *pThis, size_t nTrunc); rsRetVal rsCStrTrimTrailingWhiteSpace(cstr_t *pThis); +rsRetVal cstrTrimTrailingWhiteSpace(cstr_t *pThis); /** * Append a string to the buffer. For performance reasons, @@ -143,19 +142,24 @@ rsRetVal rsCStrSzStrMatchRegex(cstr_t *pCS1, uchar *psz, int iType, void *cache) void rsCStrRegexDestruct(void *rc); rsRetVal rsCStrConvertToNumber(cstr_t *pStr, number_t *pNumber); rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); -rsRetVal rsCStrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); + +/* in migration */ +#define rsCStrAppendCStr(pThis, pstrAppend) cstrAppendCStr(pThis, pstrAppend) /* new calling interface */ rsRetVal cstrFinalize(cstr_t *pThis); rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); uchar* cstrGetSzStr(cstr_t *pThis); +uchar* cstrGetSzStrNoNULL(cstr_t *pThis); +rsRetVal cstrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); /* now come inline-like functions */ #ifdef NDEBUG -# define rsCStrLen(x) ((int)((x)->iStrLen)) +# define cstrLen(x) ((int)((x)->iStrLen)) #else - int rsCStrLen(cstr_t *pThis); + int cstrLen(cstr_t *pThis); #endif +#define rsCStrLen(s) cstrLen((s)) #define rsCStrGetBufBeg(x) ((x)->pBuf) diff --git a/runtime/vm.c b/runtime/vm.c index 8cbf9e12..d7cd52d5 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -82,6 +82,7 @@ rsfrAddFunction(uchar *szName, prsf_t rsf) /* unique name, so add to head of list */ CHKmalloc(pEntry = calloc(1, sizeof(rsf_entry_t))); CHKiRet(rsCStrConstructFromszStr(&pEntry->pName, szName)); + CHKiRet(cstrFinalize(pEntry->pName)); pEntry->rsf = rsf; pEntry->pNext = funcRegRoot; funcRegRoot = pEntry; @@ -167,7 +168,7 @@ rsfrRemoveAll(void) while(pEntry != NULL) { pEntryDel = pEntry; pEntry = pEntry->pNext; - rsCStrDestruct(&pEntryDel->pName); + cstrDestruct(&pEntryDel->pName); free(pEntryDel); } funcRegRoot = NULL; @@ -405,6 +406,7 @@ CODESTARTop(STRADD) vmstk.PopString(pThis->pStk, &operand1); CHKiRet(rsCStrAppendCStr(operand1->val.pStr, operand2->val.pStr)); + CHKiRet(cstrFinalize(operand1->val.pStr)); /* we have a result, so let's push it */ vmstk.Push(pThis->pStk, operand1); @@ -554,12 +556,12 @@ rsf_tolower(vmstk_t *pStk, int numOperands) ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); /* pop args and do operaton */ - CHKiRet(rsCStrConstruct(&pcstr)); + CHKiRet(cstrConstruct(&pcstr)); vmstk.PopString(pStk, &operand1); - pSrc = rsCStrGetSzStr(operand1->val.pStr); - iStrlen = strlen((char*)pSrc); + pSrc = cstrGetSzStr(operand1->val.pStr); + iStrlen = strlen((char*)pSrc); // TODO: use count from string! while(iStrlen--) { - CHKiRet(rsCStrAppendChar(pcstr, tolower(*pSrc++))); + CHKiRet(cstrAppendChar(pcstr, tolower(*pSrc++))); } /* Store result and cleanup */ diff --git a/runtime/vmop.c b/runtime/vmop.c index acacfc9e..ea627220 100644 --- a/runtime/vmop.c +++ b/runtime/vmop.c @@ -125,7 +125,7 @@ Obj2Str(vmop_t *pThis, cstr_t *pstrPrg) if(pThis->operand.pVar != NULL) CHKiRet(var.Obj2Str(pThis->operand.pVar, pstrPrg)); } - CHKiRet(rsCStrAppendChar(pstrPrg, '\n')); + CHKiRet(cstrAppendChar(pstrPrg, '\n')); finalize_it: RETiRet; diff --git a/tests/nettester.c b/tests/nettester.c index 566f553b..b7b3f892 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -321,7 +321,9 @@ doTests(int fd, char *files) if(ret == 0) { if(verbose) printf("successfully completed\n"); } else { - if(verbose) printf("failed!\n"); + if(!verbose) + printf("test '%s' ", testFile); + printf("failed!\n"); ++iFailed; } } diff --git a/tools/syslogd.c b/tools/syslogd.c index 9f70ff26..cfecfd83 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1051,7 +1051,7 @@ static int parseRFCStructuredData(uchar **pp2parse, uchar *pResult) return 0; } -/* parse a RFC-formatted syslog message. This function returns +/* parse a RFC5424-formatted syslog message. This function returns * 0 if processing of the message shall continue and 1 if something * went wrong and this messe should be ignored. This function has been * implemented in the effort to support syslog-protocol. Please note that -- cgit v1.2.3 From 8628312396b1535c41124e499d292f4d1e77d955 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 13:22:21 +0200 Subject: cleaned up/optimized raw message handling in msg object --- action.c | 2 +- plugins/imdiag/imdiag.c | 3 +- plugins/imfile/imfile.c | 2 +- plugins/imklog/imklog.c | 2 +- plugins/imudp/imudp.c | 5 +--- runtime/msg.c | 43 ++++++++++++++++++++------- runtime/msg.h | 7 ++++- runtime/parser.c | 79 ++++++++++++++++--------------------------------- tcps_sess.c | 6 ++-- tests/nettester.c | 2 +- tests/parsertest.sh | 6 ++-- tools/syslogd.c | 4 +-- 12 files changed, 77 insertions(+), 84 deletions(-) diff --git a/action.c b/action.c index 8bdb6dec..08cdd6fd 100644 --- a/action.c +++ b/action.c @@ -675,7 +675,7 @@ actionWriteToAction(action_t *pAction) datetime.getCurrTime(&(pMsg->tRcvdAt), &(pMsg->ttGenTime)); memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); MsgSetMSG(pMsg, (char*)szRepMsg); - MsgSetRawMsg(pMsg, (char*)szRepMsg); + MsgSetRawMsgWOSize(pMsg, (char*)szRepMsg); pMsgSave = pAction->f_pMsg; /* save message pointer for later restoration */ pAction->f_pMsg = pMsg; /* use the new msg (pointer will be restored below) */ diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 51f319ca..bfb4a2e5 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -205,8 +205,7 @@ doInjectMsg(int iNum) datetime.getCurrTime(&stTime, &ttGenTime); /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); - CHKmalloc(pMsg->pszRawMsg = ustrdup(szMsg)); - pMsg->iLenRawMsg = ustrlen(szMsg); + MsgSetRawMsg(pMsg, (char*) szMsg, ustrlen(szMsg)); MsgSetInputName(pMsg, UCHAR_CONSTANT("imdiag"), sizeof("imdiag")-1); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 86270e2d..bdd222c4 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -97,7 +97,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); - MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine)); + MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine), cstrLen(cstrLine)); MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetTAG(pMsg, (char*)pInfo->pszTag); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 420ebbf1..24f15510 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -96,7 +96,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); - MsgSetRawMsg(pMsg, (char*)msg); + MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetMSG(pMsg, (char*)msg); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 97e66e8e..2340aac4 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -241,10 +241,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, } /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); - /* first trim the buffer to what we have actually received */ - CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar)* lenRcvBuf)); - memcpy(pMsg->pszRawMsg, pRcvBuf, lenRcvBuf); - pMsg->iLenRawMsg = lenRcvBuf; + MsgSetRawMsg(pMsg, (char*)pRcvBuf, lenRcvBuf); MsgSetInputName(pMsg, UCHAR_CONSTANT("imudp"), sizeof("imudp")-1); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; diff --git a/runtime/msg.c b/runtime/msg.c index d9ff2e73..be62e52f 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -510,7 +510,8 @@ CODESTARTobjDestruct(msg) if(currRefCount == 0) { /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ - free(pThis->pszRawMsg); + if(pThis->pszRawMsg != pThis->szRawMsg) + free(pThis->pszRawMsg); free(pThis->pszTAG); free(pThis->pszHOSTNAME); free(pThis->pszInputName); @@ -876,11 +877,12 @@ char *getMSG(msg_t *pM) { if(pM == NULL) return ""; - else + else { if(pM->pszMSG == NULL) return ""; else return (char*)pM->pszMSG; + } } @@ -1674,19 +1676,38 @@ void MsgSetMSG(msg_t *pMsg, char* pszMSG) dbgprintf("MsgSetMSG could not allocate memory for pszMSG buffer."); } -/* rgerhards 2004-11-11: set RawMsg in msg object +/* set raw message in message object. Size of message is provided. + * rgerhards, 2009-06-16 */ -void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg) +void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg) { assert(pMsg != NULL); - if(pMsg->pszRawMsg != NULL) + if(pMsg->pszRawMsg != pMsg->szRawMsg) free(pMsg->pszRawMsg); - pMsg->iLenRawMsg = strlen(pszRawMsg); - if((pMsg->pszRawMsg = (uchar*) malloc(pMsg->iLenRawMsg + 1)) != NULL) - memcpy(pMsg->pszRawMsg, pszRawMsg, pMsg->iLenRawMsg + 1); - else - dbgprintf("Could not allocate memory for pszRawMsg buffer."); + pMsg->iLenRawMsg = lenMsg; + if(pMsg->iLenRawMsg < CONF_RAWMSG_BUFSIZE) { + /* small enough: use fixed buffer (faster!) */ + pMsg->pszRawMsg = pMsg->szRawMsg; + } else if((pMsg->pszRawMsg = (uchar*) malloc(pMsg->iLenRawMsg + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pMsg->pszRawMsg = pMsg->szRawMsg; + pMsg->iLenRawMsg = CONF_RAWMSG_BUFSIZE - 1; + } + + memcpy(pMsg->pszRawMsg, pszRawMsg, pMsg->iLenRawMsg); + pMsg->pszRawMsg[pMsg->iLenRawMsg] = '\0'; /* this also works with truncation! */ +} + + +/* set raw message in message object. Size of message is not provided. This + * function should only be used when it is unavoidable (and over time we should + * try to remove it altogether). + * rgerhards, 2009-06-16 + */ +void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg) +{ + MsgSetRawMsg(pMsg, pszRawMsg, strlen(pszRawMsg)); } @@ -2558,7 +2579,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("msgFlags")) { pThis->msgFlags = pProp->val.num; } else if(isProp("pszRawMsg")) { - MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr)); /* enable this, if someone actually uses UxTradMsg, delete after some time has * passed and nobody complained -- rgerhards, 2009-06-16 } else if(isProp("offAfterPRI")) { diff --git a/runtime/msg.h b/runtime/msg.h index 703fdd9f..f2701780 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -28,6 +28,9 @@ #ifndef MSG_H_INCLUDED #define MSG_H_INCLUDED 1 +/* some configuration constants */ +#define CONF_RAWMSG_BUFSIZE 101 + #include #include "obj.h" #include "syslogd-types.h" @@ -109,6 +112,7 @@ struct msg { int msgFlags; /* flags associated with this message */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ + uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ char pszTimestamp3164[16]; char pszTimestamp3339[33]; }; @@ -149,7 +153,8 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSG(msg_t *pMsg, char* pszMSG); -void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg); +void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); +void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); void moveHOSTNAMEtoTAG(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, cstr_t *pCSPropName, unsigned short *pbMustBeFreed); diff --git a/runtime/parser.c b/runtime/parser.c index c13edb8f..75cde343 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -114,10 +114,8 @@ static inline rsRetVal uncompressMessage(msg_t *pMsg) "Message ignored.", ret); FINALIZE; /* unconditional exit, nothing left to do... */ } - free(pMsg->pszRawMsg); - pMsg->pszRawMsg = deflateBuf; - pMsg->iLenRawMsg = iLenDefBuf; - deflateBuf = NULL; /* logically "freed" - caller is now responsible */ + MsgSetRawMsg(pMsg, (char*)deflateBuf, iLenDefBuf); + free(deflateBuf); } finalize_it: if(deflateBuf != NULL) @@ -165,6 +163,8 @@ sanitizeMessage(msg_t *pMsg) size_t iSrc; size_t iDst; size_t iMaxLine; + size_t maxDest; + uchar szSanBuf[32*1024]; /* buffer used for sanitizing a string */ assert(pMsg != NULL); @@ -205,70 +205,43 @@ sanitizeMessage(msg_t *pMsg) } } } - if(bNeedSanitize == 0) { - /* what a shame - we do not have a \0 byte... - * TODO: think about adding it or otherwise be able to use it... - */ - uchar *pRaw; - CHKmalloc(pRaw = realloc(pMsg->pszRawMsg, pMsg->iLenRawMsg + 1)); - pRaw[pMsg->iLenRawMsg] = '\0'; - pMsg->pszRawMsg = pRaw; + + if(!bNeedSanitize) FINALIZE; - } /* now copy over the message and sanitize it */ - /* TODO: can we get cheaper memory alloc? {alloca()?}*/ iMaxLine = glbl.GetMaxLine(); - CHKmalloc(pDst = malloc(sizeof(uchar) * (iMaxLine + 1))); + maxDest = lenMsg * 4; /* message can grow at most four-fold */ + if(maxDest > iMaxLine) + maxDest = iMaxLine; /* but not more than the max size! */ + if(maxDest < sizeof(szSanBuf)) + pDst = szSanBuf; + else + CHKmalloc(pDst = malloc(sizeof(uchar) * (iMaxLine + 1))); iSrc = iDst = 0; - while(iSrc < lenMsg && iDst < iMaxLine) { + while(iSrc < lenMsg && iDst < maxDest - 3) { /* leave some space if last char must be escaped */ if(pszMsg[iSrc] == '\0') { /* guard against \0 characters... */ - /* changed to the sequence (somewhat) proposed in - * draft-ietf-syslog-protocol-19. rgerhards, 2006-11-30 - */ - if(iDst + 3 < iMaxLine) { /* do we have space? */ - pDst[iDst++] = cCCEscapeChar; - pDst[iDst++] = '0'; - pDst[iDst++] = '0'; - pDst[iDst++] = '0'; - } /* if we do not have space, we simply ignore the '\0'... */ - /* log an error? Very questionable... rgerhards, 2006-11-30 */ - /* decided: we do not log an error, it won't help... rger, 2007-06-21 */ - } else if(bEscapeCCOnRcv && iscntrl((int) pszMsg[iSrc])) { + } else if(iscntrl((int) pszMsg[iSrc])) { + if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { /* we are configured to escape control characters. Please note * that this most probably break non-western character sets like * Japanese, Korean or Chinese. rgerhards, 2007-07-17 - * Note: sysklogd logs octal values only for DEL and CCs above 127. - * For others, it logs ^n where n is the control char converted to an - * alphabet character. We like consistency and thus escape it to octal - * in all cases. If someone complains, we may change the mode. At least - * we known now what's going on. - * rgerhards, 2007-07-17 */ - if(iDst + 3 < iMaxLine) { /* do we have space? */ - pDst[iDst++] = cCCEscapeChar; - pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); - pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); - pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); - } /* again, if we do not have space, we ignore the char - see comment at '\0' */ + pDst[iDst++] = cCCEscapeChar; + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); + } } else { pDst[iDst++] = pszMsg[iSrc]; } ++iSrc; } - pDst[iDst] = '\0'; /* space *is* reserved for this! */ - - /* we have a sanitized string. Let's save it now */ - free(pMsg->pszRawMsg); - if((pMsg->pszRawMsg = malloc((iDst+1) * sizeof(uchar))) == NULL) { - /* when we get no new buffer, we use what we already have ;) */ - pMsg->pszRawMsg = pDst; - } else { - /* trim buffer */ - memcpy(pMsg->pszRawMsg, pDst, iDst+1); - free(pDst); /* too big! */ - pMsg->iLenRawMsg = iDst; - } + + MsgSetRawMsg(pMsg, (char*)pDst, iDst); /* save sanitized string */ + + if(pDst != szSanBuf) + free(pDst); finalize_it: RETiRet; diff --git a/tcps_sess.c b/tcps_sess.c index 9353571c..e0bec949 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -234,10 +234,8 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttG /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); - /* first trim the buffer to what we have actually received */ - CHKmalloc(pMsg->pszRawMsg = malloc(sizeof(uchar) * pThis->iMsg)); - memcpy(pMsg->pszRawMsg, pThis->pMsg, pThis->iMsg); - pMsg->iLenRawMsg = pThis->iMsg; +dbgprintf("defaultDoSubmit, iMsg %d\n", pThis->iMsg); + MsgSetRawMsg(pMsg, (char*)pThis->pMsg, pThis->iMsg); MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName, pThis->pLstnInfo->lenInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; diff --git a/tests/nettester.c b/tests/nettester.c index b7b3f892..e5e6278c 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -191,7 +191,7 @@ int openPipe(char *configFile, pid_t *pid, int *pfd) char *newenviron[] = { NULL }; /* debug aide... char *newenviron[] = { "RSYSLOG_DEBUG=debug nostdout", - "RSYSLOG_DEBUGLOG=tmp", NULL }; + "RSYSLOG_DEBUGLOG=log", NULL }; */ diff --git a/tests/parsertest.sh b/tests/parsertest.sh index afdb9469..8e04b95e 100755 --- a/tests/parsertest.sh +++ b/tests/parsertest.sh @@ -1,13 +1,13 @@ echo test parsertest via udp $srcdir/killrsyslog.sh # kill rsyslogd if it runs for some reason -./nettester -tparse1 -iudp +echo test parsertest via tcp +./nettester -tparse1 -itcp if [ "$?" -ne "0" ]; then exit 1 fi -echo test parsertest via tcp -./nettester -tparse1 -itcp +./nettester -tparse1 -iudp if [ "$?" -ne "0" ]; then exit 1 fi diff --git a/tools/syslogd.c b/tools/syslogd.c index 034111f5..91918d96 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -582,7 +582,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f if(pszInputName != NULL) MsgSetInputName(pMsg, pszInputName, ustrlen(pszInputName)); MsgSetFlowControlType(pMsg, flowCtlType); - MsgSetRawMsg(pMsg, (char*)msg); + MsgSetRawMsgWOSize(pMsg, (char*)msg); /* test for special codes */ pri = DEFUPRI; @@ -882,7 +882,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) CHKiRet(msgConstruct(&pMsg)); MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1); - MsgSetRawMsg(pMsg, (char*)msg); + MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1")); -- cgit v1.2.3 From 2f86678c577ee469852ffae35123c4a90b12d214 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 14:30:21 +0200 Subject: optimized TAG handling in msg object --- plugins/imfile/imfile.c | 4 +- plugins/imklog/imklog.c | 2 +- runtime/msg.c | 117 +++++++++++++++++++++++++++++------------------- runtime/msg.h | 8 +++- tools/syslogd.c | 6 +-- 5 files changed, 85 insertions(+), 52 deletions(-) diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index bdd222c4..0dabc757 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -61,6 +61,7 @@ DEFobjCurrIf(strm) typedef struct fileInfo_s { uchar *pszFileName; uchar *pszTag; + size_t lenTag; uchar *pszStateFile; /* file in which state between runs is to be stored */ int iFacility; int iSeverity; @@ -100,7 +101,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine), cstrLen(cstrLine)); MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); - MsgSetTAG(pMsg, (char*)pInfo->pszTag); + MsgSetTAG(pMsg, pInfo->pszTag, pInfo->lenTag); pMsg->iFacility = LOG_FAC(pInfo->iFacility); pMsg->iSeverity = LOG_PRI(pInfo->iSeverity); pMsg->bParseHOSTNAME = 0; @@ -471,6 +472,7 @@ static rsRetVal addMonitor(void __attribute__((unused)) *pVal, uchar *pNewVal) ABORT_FINALIZE(RS_RET_CONFIG_ERROR); } else { pThis->pszTag = (uchar*) strdup((char*) pszFileTag); + pThis->lenTag = ustrlen(pThis->pszTag); } if(pszStateFile == NULL) { diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 24f15510..bf60ebe5 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -101,7 +101,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); - MsgSetTAG(pMsg, (char*)pszTag); + MsgSetTAG(pMsg, pszTag, ustrlen(pszTag)); pMsg->iFacility = LOG_FAC(iFacility); pMsg->iSeverity = LOG_PRI(iSeverity); pMsg->bParseHOSTNAME = 0; diff --git a/runtime/msg.c b/runtime/msg.c index be62e52f..2df04b2a 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -497,6 +497,16 @@ finalize_it: } +/* some free handlers for (slightly) complicated cases... All of them may be called + * with an empty element. + */ +static inline void freeTAG(msg_t *pThis) +{ + if(pThis->iLenTAG >= CONF_TAG_BUFSIZE) + free(pThis->TAG.pszTAG); +} + + BEGINobjDestruct(msg) /* be sure to specify the object type also in END and CODESTART macros! */ int currRefCount; CODESTARTobjDestruct(msg) @@ -512,7 +522,7 @@ CODESTARTobjDestruct(msg) /* DEV Debugging Only! dbgprintf("msgDestruct\t0x%lx, RefCount now 0, doing DESTROY\n", (unsigned long)pThis); */ if(pThis->pszRawMsg != pThis->szRawMsg) free(pThis->pszRawMsg); - free(pThis->pszTAG); + freeTAG(pThis); free(pThis->pszHOSTNAME); free(pThis->pszInputName); free(pThis->pszRcvFrom); @@ -602,9 +612,19 @@ msg_t* MsgDup(msg_t* pOld) * passed and nobody complained -- rgerhards, 2009-06-16 pNew->offAfterPRI = pOld->offAfterPRI; */ + if(pOld->iLenTAG > 0) { + if(pOld->iLenTAG < CONF_TAG_BUFSIZE) { + memcpy(pNew->TAG.szBuf, pOld->TAG.szBuf, pOld->iLenTAG); + } else { + if((pNew->TAG.pszTAG = srUtilStrDup(pOld->TAG.pszTAG, pOld->iLenTAG)) == NULL) { + msgDestruct(&pNew); + return NULL; + } + pNew->iLenTAG = pOld->iLenTAG; + } + } tmpCOPYSZ(RawMsg); tmpCOPYSZ(MSG); - tmpCOPYSZ(TAG); tmpCOPYSZ(HOSTNAME); tmpCOPYSZ(RcvFrom); @@ -659,7 +679,10 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializePTR(pStrm, pszRawMsg, PSZ); objSerializePTR(pStrm, pszMSG, PSZ); - objSerializePTR(pStrm, pszTAG, PSZ); + + CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszTAG"), PROPTYPE_PSZ, (void*) + ((pThis->iLenTAG < CONF_TAG_BUFSIZE) ? pThis->TAG.szBuf : pThis->TAG.pszTAG))); + objSerializePTR(pStrm, pszHOSTNAME, PSZ); objSerializePTR(pStrm, pszInputName, PSZ); objSerializePTR(pStrm, pszRcvFrom, PSZ); @@ -712,18 +735,22 @@ msg_t *MsgAddRef(msg_t *pM) static rsRetVal aquirePROCIDFromTAG(msg_t *pM) { register int i; + uchar *pszTag; DEFiRet; assert(pM != NULL); + if(pM->pCSPROCID != NULL) return RS_RET_OK; /* we are already done ;) */ if(getProtocolVersion(pM) != 0) return RS_RET_OK; /* we can only emulate if we have legacy format */ + pszTag = (uchar*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); + /* find first '['... */ i = 0; - while((i < pM->iLenTAG) && (pM->pszTAG[i] != '[')) + while((i < pM->iLenTAG) && (pszTag[i] != '[')) ++i; if(!(i < pM->iLenTAG)) return RS_RET_OK; /* no [, so can not emulate... */ @@ -733,8 +760,8 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) /* now obtain the PROCID string... */ CHKiRet(cstrConstruct(&pM->pCSPROCID)); rsCStrSetAllocIncrement(pM->pCSPROCID, 16); - while((i < pM->iLenTAG) && (pM->pszTAG[i] != ']')) { - CHKiRet(cstrAppendChar(pM->pCSPROCID, pM->pszTAG[i])); + while((i < pM->iLenTAG) && (pszTag[i] != ']')) { + CHKiRet(cstrAppendChar(pM->pCSPROCID, pszTag[i])); ++i; } @@ -774,22 +801,24 @@ finalize_it: */ static rsRetVal aquireProgramName(msg_t *pM) { - DEFiRet; register int i; + uchar *pszTag; + DEFiRet; assert(pM != NULL); if(pM->pCSProgName == NULL) { /* ok, we do not yet have it. So let's parse the TAG * to obtain it. */ + pszTag = (uchar*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); CHKiRet(cstrConstruct(&pM->pCSProgName)); rsCStrSetAllocIncrement(pM->pCSProgName, 33); for( i = 0 - ; (i < pM->iLenTAG) && isprint((int) pM->pszTAG[i]) - && (pM->pszTAG[i] != '\0') && (pM->pszTAG[i] != ':') - && (pM->pszTAG[i] != '[') && (pM->pszTAG[i] != '/') + ; (i < pM->iLenTAG) && isprint((int) pszTag[i]) + && (pszTag[i] != '\0') && (pszTag[i] != ':') + && (pszTag[i] != '[') && (pszTag[i] != '/') ; ++i) { - CHKiRet(cstrAppendChar(pM->pCSProgName, pM->pszTAG[i])); + CHKiRet(cstrAppendChar(pM->pCSProgName, pszTag[i])); } CHKiRet(cstrFinalize(pM->pCSProgName)); } @@ -808,13 +837,16 @@ finalize_it: * This is especially important as this can be a very common case, e.g. * when BSD syslog is acting as a sender. * rgerhards, 2005-11-10. + * NOTE ********* 2009-06-18 / rgerhards ************* + * This function is being obsoleted by the new handling. I keep it for + * a while, and for oversize tags it is somewhat less optimal than in previous + * versions. This should only happen very seldom. */ void moveHOSTNAMEtoTAG(msg_t *pM) { assert(pM != NULL); - free(pM->pszTAG); - pM->pszTAG = pM->pszHOSTNAME; - pM->iLenTAG = pM->iLenHOSTNAME; + MsgSetTAG(pM, pM->pszHOSTNAME, pM->iLenHOSTNAME); + free(pM->pszHOSTNAME); pM->pszHOSTNAME = NULL; pM->iLenHOSTNAME = 0; } @@ -1264,8 +1296,8 @@ static inline char *getMSGID(msg_t *pM) void MsgAssignTAG(msg_t *pMsg, uchar *pBuf) { assert(pMsg != NULL); - pMsg->iLenTAG = (pBuf == NULL) ? 0 : strlen((char*)pBuf); - pMsg->pszTAG = (uchar*) pBuf; + MsgSetTAG(pMsg, pBuf, ustrlen(pBuf)); + free(pBuf); } @@ -1278,17 +1310,28 @@ void MsgSetRuleset(msg_t *pMsg, ruleset_t *pRuleset) } -/* rgerhards 2004-11-16: set TAG in msg object +/* set TAG in msg object + * (rewritten 2009-06-18 rgerhards) */ -void MsgSetTAG(msg_t *pMsg, char* pszTAG) +void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf) { + uchar *pBuf; assert(pMsg != NULL); - free(pMsg->pszTAG); - pMsg->iLenTAG = strlen(pszTAG); - if((pMsg->pszTAG = malloc(pMsg->iLenTAG + 1)) != NULL) - memcpy(pMsg->pszTAG, pszTAG, pMsg->iLenTAG + 1); - else - dbgprintf("Could not allocate memory in MsgSetTAG()\n"); + + freeTAG(pMsg); + + pMsg->iLenTAG = lenBuf; + if(pMsg->iLenTAG < CONF_RAWMSG_BUFSIZE) { + /* small enough: use fixed buffer (faster!) */ + pBuf = pMsg->TAG.szBuf; + } else if((pBuf = (uchar*) malloc(pMsg->iLenTAG + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pBuf = pMsg->TAG.szBuf; + pMsg->iLenTAG = CONF_RAWMSG_BUFSIZE - 1; + } + + memcpy(pBuf, pszBuf, pMsg->iLenTAG); + pBuf[pMsg->iLenTAG] = '\0'; /* this also works with truncation! */ } @@ -1305,13 +1348,13 @@ static void tryEmulateTAG(msg_t *pM) uchar *pBuf; assert(pM != NULL); - if(pM->pszTAG != NULL) + if(pM->iLenTAG > 0) return; /* done, no need to emulate */ if(getProtocolVersion(pM) == 1) { if(!strcmp(getPROCID(pM), "-")) { /* no process ID, use APP-NAME only */ - MsgSetTAG(pM, getAPPNAME(pM)); + MsgSetTAG(pM, (uchar*) getAPPNAME(pM), getAPPNAMELen(pM)); } else { /* now we can try to emulate */ iTAGLen = getAPPNAMELen(pM) + getPROCIDLen(pM) + 3; @@ -1324,22 +1367,6 @@ static void tryEmulateTAG(msg_t *pM) } -#if 0 /* This method is currently not called, be we like to preserve it */ -static int getTAGLen(msg_t *pM) -{ - if(pM == NULL) - return 0; - else { - tryEmulateTAG(pM); - if(pM->pszTAG == NULL) - return 0; - else - return pM->iLenTAG; - } -} -#endif - - static inline char *getTAG(msg_t *pM) { char *ret; @@ -1349,10 +1376,10 @@ static inline char *getTAG(msg_t *pM) else { MsgLock(pM); tryEmulateTAG(pM); - if(pM->pszTAG == NULL) + if(pM->iLenTAG == 0) ret = ""; else - ret = (char*) pM->pszTAG; + ret = (char*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); MsgUnlock(pM); } return(ret); @@ -2590,7 +2617,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("pszUxTradMsg")) { /*IGNORE*/; /* this *was* a property, but does no longer exist */ } else if(isProp("pszTAG")) { - MsgSetTAG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetTAG(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr)); } else if(isProp("pszInputName")) { MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pszRcvFromIP")) { diff --git a/runtime/msg.h b/runtime/msg.h index f2701780..13c2c4ea 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -30,6 +30,7 @@ /* some configuration constants */ #define CONF_RAWMSG_BUFSIZE 101 +#define CONF_TAG_BUFSIZE 33 /* RFC says 32 chars (+ \0), but in practice we see longer ones... */ #include #include "obj.h" @@ -74,7 +75,6 @@ struct msg { short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ - uchar *pszTAG; /* pointer to tag value */ int iLenTAG; /* Length of the TAG part */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ int iLenHOSTNAME; /* Length of HOSTNAME */ @@ -113,6 +113,10 @@ struct msg { ruleset_t *pRuleset; /* ruleset to be used for processing this message */ /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ + union { + uchar *pszTAG; /* pointer to tag value */ + uchar szBuf[CONF_TAG_BUFSIZE]; + } TAG; char pszTimestamp3164[16]; char pszTimestamp3339[33]; }; @@ -143,7 +147,7 @@ rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); -void MsgSetTAG(msg_t *pMsg, char* pszTAG); +void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf); void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); diff --git a/tools/syslogd.c b/tools/syslogd.c index 91918d96..8e804d8f 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -890,11 +890,11 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) * adjust the tag. -- rgerhards, 2008-06-27 */ if(iErr == NO_ERRCODE) { - MsgSetTAG(pMsg, "rsyslogd:"); + MsgSetTAG(pMsg, UCHAR_CONSTANT("rsyslogd:"), sizeof("rsyslogd:") - 1); } else { - snprintf((char*)pszTag, sizeof(pszTag), "rsyslogd%d:", iErr); + size_t len = snprintf((char*)pszTag, sizeof(pszTag), "rsyslogd%d:", iErr); pszTag[32] = '\0'; /* just to make sure... */ - MsgSetTAG(pMsg, (char*)pszTag); + MsgSetTAG(pMsg, pszTag, len); } pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); -- cgit v1.2.3 From f18c0ffb9a6de737d2b86b3df164ead22ac5ef58 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 15:57:16 +0200 Subject: some optimization in the property replacer --- runtime/msg.c | 88 ++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 2df04b2a..39f2370c 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -876,11 +876,6 @@ char *getProtocolVersionString(msg_t *pM) return(pM->iProtocolVersion ? "1" : "0"); } -int getMSGLen(msg_t *pM) -{ - return((pM == NULL) ? 0 : pM->iLenMSG); -} - static char *getRawMsg(msg_t *pM) { @@ -905,6 +900,12 @@ char *getUxTradMsg(msg_t *pM) } */ + +int getMSGLen(msg_t *pM) +{ + return((pM == NULL) ? 0 : pM->iLenMSG); +} + char *getMSG(msg_t *pM) { if(pM == NULL) @@ -1851,6 +1852,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, { uchar *pName; char *pRes; /* result pointer */ + int bufLen = -1; /* length of string or -1, if not known */ char *pBufStart; char *pBuf; int iLen; @@ -1877,6 +1879,14 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, * property names. These come after || in the ifs below. */ if(!strcmp((char*) pName, "msg")) { pRes = getMSG(pMsg); + bufLen = getMSGLen(pMsg); + } else if(!strcmp((char*) pName, "timestamp") + || !strcmp((char*) pName, "timereported")) { + pRes = getTimeReported(pMsg, pTpe->data.field.eDateFormat); + } else if(!strcmp((char*) pName, "hostname") || !strcmp((char*) pName, "source")) { + pRes = getHOSTNAME(pMsg); + } else if(!strcmp((char*) pName, "syslogtag")) { + pRes = getTAG(pMsg); } else if(!strcmp((char*) pName, "rawmsg")) { pRes = getRawMsg(pMsg); /* enable this, if someone actually uses UxTradMsg, delete after some time has @@ -1890,10 +1900,6 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = (char*) getRcvFrom(pMsg); } else if(!strcmp((char*) pName, "fromhost-ip")) { pRes = (char*) getRcvFromIP(pMsg); - } else if(!strcmp((char*) pName, "source") || !strcmp((char*) pName, "hostname")) { - pRes = getHOSTNAME(pMsg); - } else if(!strcmp((char*) pName, "syslogtag")) { - pRes = getTAG(pMsg); } else if(!strcmp((char*) pName, "pri")) { pRes = getPRI(pMsg); } else if(!strcmp((char*) pName, "pri-text")) { @@ -1917,9 +1923,6 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = getSeverityStr(pMsg); } else if(!strcmp((char*) pName, "timegenerated")) { pRes = getTimeGenerated(pMsg, pTpe->data.field.eDateFormat); - } else if(!strcmp((char*) pName, "timereported") - || !strcmp((char*) pName, "timestamp")) { - pRes = getTimeReported(pMsg, pTpe->data.field.eDateFormat); } else if(!strcmp((char*) pName, "programname")) { pRes = getProgramName(pMsg); } else if(!strcmp((char*) pName, "protocol-version")) { @@ -2042,6 +2045,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } /* now copy */ memcpy(pBuf, pFld, iLen); + bufLen = iLen; pBuf[iLen] = '\0'; /* terminate it */ if(*pbMustBeFreed == 1) free(pRes); @@ -2086,6 +2090,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } } /* OK, we are at the begin - now let's copy... */ + bufLen = iLen; while(*pSb && iLen) { *pBuf++ = *pSb; ++pSb; @@ -2181,6 +2186,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* Lets copy the matched substring to the buffer */ memcpy(pB, pRes + iOffs + pmatch[pTpe->data.field.iSubMatchToUse].rm_so, iLenBuf); + bufLen = iLenBuf - 1; pB[iLenBuf] = '\0';/* terminate string, did not happen before */ if (*pbMustBeFreed == 1) @@ -2206,28 +2212,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* now check if we need to do our "SP if first char is non-space" hack logic */ if(*pRes && pTpe->data.field.options.bSPIffNo1stSP) { - char *pB; - uchar cFirst = *pRes; - /* here, we always destruct the buffer and return a new one */ - pB = (char *) malloc(2 * sizeof(char)); - if(pB == NULL) { - if(*pbMustBeFreed == 1) - free(pRes); - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; - } - pRes = pB; - *pbMustBeFreed = 1; - - if(cFirst == ' ') { - /* if we have a SP, we must return an empty string */ - *pRes = '\0'; /* empty */ - } else { - /* if it is no SP, we need to return one */ - *pRes = ' '; - *(pRes+1) = '\0'; - } + uchar cFirst = *pRes; /* save first char */ + if(*pbMustBeFreed == 1) + free(pRes); + pRes = (cFirst == ' ') ? "" : " "; + bufLen = (cFirst == ' ') ? 0 : 1; + *pbMustBeFreed = 0; } if(*pRes) { @@ -2236,11 +2227,12 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ if(pTpe->data.field.eCaseConv != tplCaseConvNo) { /* we need to obtain a private copy */ - int iBufLen = strlen(pRes); + if(bufLen == -1) + bufLen = strlen(pRes); char *pBStart; char *pB; char *pSrc; - pBStart = pB = malloc((iBufLen + 1) * sizeof(char)); + pBStart = pB = malloc((bufLen + 1) * sizeof(char)); if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2302,6 +2294,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pDstStart; + bufLen = iLenBuf; *pbMustBeFreed = 1; } } else if(pTpe->data.field.options.bSpaceCC) { @@ -2320,7 +2313,9 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pDst = ' '; } } else { - pDst = pDstStart = malloc(strlen(pRes) + 1); + if(bufLen == -1) + bufLen = strlen(pRes); + pDst = pDstStart = malloc(bufLen + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2381,6 +2376,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pBStart; + bufLen = -1; *pbMustBeFreed = 1; } } @@ -2420,6 +2416,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pDstStart; + bufLen = -1; /* TODO: can we do better? */ *pbMustBeFreed = 1; } } else { @@ -2438,7 +2435,9 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pDst++ = '_'; } } else { - pDst = pDstStart = malloc(strlen(pRes) + 1); + if(bufLen == -1) + bufLen = strlen(pRes); + pDst = pDstStart = malloc(bufLen + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2476,16 +2475,20 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = "_"; + bufLen = 1; *pbMustBeFreed = 0; } } /* Now drop last LF if present (pls note that this must not be done - * if bEscapeCC was set! + * if bEscapeCC was set)! */ if(pTpe->data.field.options.bDropLastLF && !pTpe->data.field.options.bEscapeCC) { - int iLn = strlen(pRes); + int iLn; char *pB; + if(bufLen == -1) + bufLen = strlen(pRes); + iLn = bufLen; if(iLn > 0 && *(pRes + iLn - 1) == '\n') { /* we have a LF! */ /* check if we need to obtain a private copy */ @@ -2501,6 +2504,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pbMustBeFreed = 1; } *(pRes + iLn - 1) = '\0'; /* drop LF ;) */ + --bufLen; } } @@ -2511,10 +2515,13 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, */ if(pTpe->data.field.options.bCSV) { /* we need to obtain a private copy, as we need to at least add the double quotes */ - int iBufLen = strlen(pRes); + int iBufLen; char *pBStart; char *pDst; char *pSrc; + if(bufLen == -1) + bufLen = strlen(pRes); + iBufLen = bufLen; /* the malloc may be optimized, we currently use the worst case... */ pBStart = pDst = malloc((2 * iBufLen + 3) * sizeof(char)); if(pDst == NULL) { @@ -2535,6 +2542,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, if(*pbMustBeFreed == 1) free(pRes); pRes = pBStart; + bufLen = -1; *pbMustBeFreed = 1; } -- cgit v1.2.3 From df9012f755a305ef48037f10fcc9413406894e66 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 16:20:06 +0200 Subject: slight optimization of template generation --- runtime/msg.c | 10 ++++++++-- runtime/msg.h | 2 +- runtime/rule.c | 4 +++- template.c | 6 +++--- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 39f2370c..67aaf250 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1848,7 +1848,8 @@ static uchar *getNOW(eNOWType eNow) * rgerhards 2005-09-15 */ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - cstr_t *pCSPropName, unsigned short *pbMustBeFreed) + cstr_t *pCSPropName, size_t *pPropLen, + unsigned short *pbMustBeFreed) { uchar *pName; char *pRes; /* result pointer */ @@ -2546,6 +2547,10 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pbMustBeFreed = 1; } + if(bufLen == -1) + bufLen = strlen(pRes); + *pPropLen = bufLen; + /*dbgprintf("MsgGetProp(\"%s\"): \"%s\"\n", pName, pRes); only for verbose debug logging */ return(pRes); } @@ -2561,6 +2566,7 @@ msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar) { DEFiRet; var_t *pVar; + size_t propLen; uchar *pszProp = NULL; cstr_t *pstrProp; unsigned short bMustBeFreed = 0; @@ -2574,7 +2580,7 @@ msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar) CHKiRet(var.ConstructFinalize(pVar)); /* always call MsgGetProp() without a template specifier */ - pszProp = (uchar*) MsgGetProp(pThis, NULL, pstrPropName, &bMustBeFreed); + pszProp = (uchar*) MsgGetProp(pThis, NULL, pstrPropName, &propLen, &bMustBeFreed); /* now create a string object out of it and hand that over to the var */ CHKiRet(rsCStrConstructFromszStr(&pstrProp, pszProp)); diff --git a/runtime/msg.h b/runtime/msg.h index 13c2c4ea..48c1090e 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -161,7 +161,7 @@ void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); void moveHOSTNAMEtoTAG(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - cstr_t *pCSPropName, unsigned short *pbMustBeFreed); + cstr_t *pCSPropName, size_t *pPropLen, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); diff --git a/runtime/rule.c b/runtime/rule.c index f17c524e..12494543 100644 --- a/runtime/rule.c +++ b/runtime/rule.c @@ -106,6 +106,7 @@ shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) unsigned short pbMustBeFreed; char *pszPropVal; int bRet = 0; + size_t propLen; vm_t *pVM = NULL; var_t *pResult = NULL; @@ -177,7 +178,8 @@ shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) bRet = (pResult->val.num) ? 1 : 0; } else { assert(pRule->f_filter_type == FILTER_PROP); /* assert() just in case... */ - pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.pCSPropName, &pbMustBeFreed); + pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.pCSPropName, &propLen, &pbMustBeFreed); + // TODO: optimize, we now have the length of the property! /* Now do the compares (short list currently ;)) */ switch(pRule->f_filterData.prop.operation ) { diff --git a/template.c b/template.c index 6e34bcd8..aacd4dbd 100644 --- a/template.c +++ b/template.c @@ -103,8 +103,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) FINALIZE; } } else if(pTpe->eEntryType == FIELD) { - pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &bMustBeFreed); - iLenVal = strlen((char*) pVal); + pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &iLenVal, &bMustBeFreed); /* we now need to check if we should use SQL option. In this case, * we must go over the generated string and escape '\'' characters. * rgerhards, 2005-09-22: the option values below look somewhat misplaced, @@ -158,6 +157,7 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) struct templateEntry *pTpe; uchar **pArr; int iArr; + size_t propLen; unsigned short bMustBeFreed; uchar *pVal; @@ -178,7 +178,7 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) if(pTpe->eEntryType == CONSTANT) { CHKmalloc(pArr[iArr] = (uchar*)strdup((char*) pTpe->data.constant.pConstant)); } else if(pTpe->eEntryType == FIELD) { - pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &bMustBeFreed); + pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &propLen, &bMustBeFreed); if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */ pArr[iArr] = pVal; /* ... so we can use it! */ } else { -- cgit v1.2.3 From 2de4964affabc1ccf61bc72426a468fc871a54d0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 16:51:40 +0200 Subject: optimized handling of MSG part in msg object WARNING: currently, message repeation processing is disabled, must be reenabled (but prefer to do some other tests first) --- action.c | 3 ++- plugins/imfile/imfile.c | 2 +- plugins/imklog/imklog.c | 2 +- runtime/msg.c | 30 +++++++++++++++++++++--------- runtime/msg.h | 5 +++-- tools/syslogd.c | 7 +++++-- 6 files changed, 33 insertions(+), 16 deletions(-) diff --git a/action.c b/action.c index 08cdd6fd..4ab5b81f 100644 --- a/action.c +++ b/action.c @@ -674,7 +674,8 @@ actionWriteToAction(action_t *pAction) */ datetime.getCurrTime(&(pMsg->tRcvdAt), &(pMsg->ttGenTime)); memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); - MsgSetMSG(pMsg, (char*)szRepMsg); +#pragma warn "need fix msg repeationg handling" + //MsgSetMSG(pMsg, (char*)szRepMsg); MsgSetRawMsgWOSize(pMsg, (char*)szRepMsg); pMsgSave = pAction->f_pMsg; /* save message pointer for later restoration */ diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 0dabc757..0f5f49dc 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -99,7 +99,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine), cstrLen(cstrLine)); - MsgSetMSG(pMsg, (char*)rsCStrGetSzStr(cstrLine)); + MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); MsgSetTAG(pMsg, pInfo->pszTag, pInfo->lenTag); pMsg->iFacility = LOG_FAC(pInfo->iFacility); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index bf60ebe5..67fdc8a3 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -97,7 +97,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); MsgSetRawMsgWOSize(pMsg, (char*)msg); - MsgSetMSG(pMsg, (char*)msg); + MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); diff --git a/runtime/msg.c b/runtime/msg.c index 67aaf250..507d041e 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -438,6 +438,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->iRefCount = 1; pM->iSeverity = -1; pM->iFacility = -1; + pM->offMSG = -1; objConstructSetObjInfo(pM); /* DEV debugging only! dbgprintf("msgConstruct\t0x%x, ref 1\n", (int)pM);*/ @@ -527,7 +528,6 @@ CODESTARTobjDestruct(msg) free(pThis->pszInputName); free(pThis->pszRcvFrom); free(pThis->pszRcvFromIP); - free(pThis->pszMSG); free(pThis->pszRcvdAt3164); free(pThis->pszRcvdAt3339); free(pThis->pszRcvdAt_SecFrac); @@ -608,6 +608,7 @@ msg_t* MsgDup(msg_t* pOld) pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; + pNew->offMSG = pOld->offMSG; /* enable this, if someone actually uses UxTradMsg, delete after some time has * passed and nobody complained -- rgerhards, 2009-06-16 pNew->offAfterPRI = pOld->offAfterPRI; @@ -624,7 +625,6 @@ msg_t* MsgDup(msg_t* pOld) } } tmpCOPYSZ(RawMsg); - tmpCOPYSZ(MSG); tmpCOPYSZ(HOSTNAME); tmpCOPYSZ(RcvFrom); @@ -668,6 +668,7 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializeSCALAR(pStrm, iProtocolVersion, SHORT); objSerializeSCALAR(pStrm, iSeverity, SHORT); objSerializeSCALAR(pStrm, iFacility, SHORT); + objSerializeSCALAR(pStrm, offMSG, SHORT); objSerializeSCALAR(pStrm, msgFlags, INT); objSerializeSCALAR(pStrm, ttGenTime, INT); objSerializeSCALAR(pStrm, tRcvdAt, SYSLOGTIME); @@ -677,12 +678,10 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializeSCALAR(pStrm, offsAfterPRI, SHORT); */ - objSerializePTR(pStrm, pszRawMsg, PSZ); - objSerializePTR(pStrm, pszMSG, PSZ); - CHKiRet(obj.SerializeProp(pStrm, UCHAR_CONSTANT("pszTAG"), PROPTYPE_PSZ, (void*) ((pThis->iLenTAG < CONF_TAG_BUFSIZE) ? pThis->TAG.szBuf : pThis->TAG.pszTAG))); + objSerializePTR(pStrm, pszRawMsg, PSZ); objSerializePTR(pStrm, pszHOSTNAME, PSZ); objSerializePTR(pStrm, pszInputName, PSZ); objSerializePTR(pStrm, pszRcvFrom, PSZ); @@ -911,10 +910,10 @@ char *getMSG(msg_t *pM) if(pM == NULL) return ""; else { - if(pM->pszMSG == NULL) + if(pM->offMSG == -1) return ""; else - return (char*)pM->pszMSG; + return (char*)(pM->pszRawMsg + pM->offMSG); } } @@ -1687,6 +1686,16 @@ void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME) } +/* rgerhards 2004-11-09: set MSG in msg object + */ +void MsgSetMSGoffs(msg_t *pMsg, short offs) +{ + assert(pMsg != NULL); + + pMsg->iLenMSG = ustrlen(pMsg->pszRawMsg + offs); + pMsg->offMSG = offs; +} +#if 0 /* rgerhards 2004-11-09: set MSG in msg object */ void MsgSetMSG(msg_t *pMsg, char* pszMSG) @@ -1703,6 +1712,7 @@ void MsgSetMSG(msg_t *pMsg, char* pszMSG) else dbgprintf("MsgSetMSG could not allocate memory for pszMSG buffer."); } +#endif /* set raw message in message object. Size of message is provided. * rgerhards, 2009-06-16 @@ -2619,6 +2629,8 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) pThis->iFacility = pProp->val.num; } else if(isProp("msgFlags")) { pThis->msgFlags = pProp->val.num; + } else if(isProp("offMSG")) { + pThis->offMSG = pProp->val.num; } else if(isProp("pszRawMsg")) { MsgSetRawMsg(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr)); /* enable this, if someone actually uses UxTradMsg, delete after some time has @@ -2626,8 +2638,6 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("offAfterPRI")) { pThis->offAfterPRI = pProp->val.num; */ - } else if(isProp("pszMSG")) { - MsgSetMSG(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszUxTradMsg")) { /*IGNORE*/; /* this *was* a property, but does no longer exist */ } else if(isProp("pszTAG")) { @@ -2654,6 +2664,8 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) memcpy(&pThis->tRcvdAt, &pProp->val.vSyslogTime, sizeof(struct syslogTime)); } else if(isProp("tTIMESTAMP")) { memcpy(&pThis->tTIMESTAMP, &pProp->val.vSyslogTime, sizeof(struct syslogTime)); + } else if(isProp("pszMSG")) { + dbgprintf("no longer supported property pszMSG silently ignored\n"); } RETiRet; diff --git a/runtime/msg.h b/runtime/msg.h index 48c1090e..4cdb0762 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -73,7 +73,8 @@ struct msg { * need to preserve cryptographic verifiers. */ int iLenRawMsg; /* length of raw message */ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ - uchar *pszMSG; /* the MSG part itself */ + short offMSG; /* offset at which the MSG part starts in pszRawMsg */ + //uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ @@ -156,7 +157,7 @@ rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); -void MsgSetMSG(msg_t *pMsg, char* pszMSG); +void MsgSetMSGoffs(msg_t *pMsg, short offs); void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); void moveHOSTNAMEtoTAG(msg_t *pM); diff --git a/tools/syslogd.c b/tools/syslogd.c index 8e804d8f..11f4ebd9 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1146,7 +1146,8 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) } /* MSG */ - MsgSetMSG(pMsg, (char*)p2parse); + MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); + //MsgSetMSG(pMsg, (char*)p2parse); free(pBuf); ENDfunc @@ -1339,7 +1340,9 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) } /* The rest is the actual MSG */ - MsgSetMSG(pMsg, (char*)p2parse); +dbgprintf("XXX: msg set msg offset %d, str: '%s', prev '%s'\n", p2parse - pMsg->pszRawMsg, pMsg->pszRawMsg + (p2parse - pMsg->pszRawMsg), p2parse); +// MsgSetMSG(pMsg, (char*)p2parse); + MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); ENDfunc return 0; /* all ok */ -- cgit v1.2.3 From f33dd51c802a8d49839aa73fb9167d8bc31ea912 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 18 Jun 2009 17:48:11 +0200 Subject: fixed abort condition with oversize tags this was a regression I introduced this afternoon --- runtime/msg.c | 33 +++++++++++---------------------- runtime/msg.h | 1 - template.c | 1 + tests/Makefile.am | 1 + tests/testsuites/oversizeTag-1.parse1 | 3 +++ 5 files changed, 16 insertions(+), 23 deletions(-) create mode 100644 tests/testsuites/oversizeTag-1.parse1 diff --git a/runtime/msg.c b/runtime/msg.c index 507d041e..4b7a0ad4 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1321,13 +1321,17 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf) freeTAG(pMsg); pMsg->iLenTAG = lenBuf; - if(pMsg->iLenTAG < CONF_RAWMSG_BUFSIZE) { + if(pMsg->iLenTAG < CONF_TAG_BUFSIZE) { /* small enough: use fixed buffer (faster!) */ pBuf = pMsg->TAG.szBuf; - } else if((pBuf = (uchar*) malloc(pMsg->iLenTAG + 1)) == NULL) { - /* truncate message, better than completely loosing it... */ - pBuf = pMsg->TAG.szBuf; - pMsg->iLenTAG = CONF_RAWMSG_BUFSIZE - 1; + } else { + if((pBuf = (uchar*) malloc(pMsg->iLenTAG + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pBuf = pMsg->TAG.szBuf; + pMsg->iLenTAG = CONF_TAG_BUFSIZE - 1; + } else { + pMsg->TAG.pszTAG = pBuf; + } } memcpy(pBuf, pszBuf, pMsg->iLenTAG); @@ -1695,24 +1699,7 @@ void MsgSetMSGoffs(msg_t *pMsg, short offs) pMsg->iLenMSG = ustrlen(pMsg->pszRawMsg + offs); pMsg->offMSG = offs; } -#if 0 -/* rgerhards 2004-11-09: set MSG in msg object - */ -void MsgSetMSG(msg_t *pMsg, char* pszMSG) -{ - assert(pMsg != NULL); - assert(pszMSG != NULL); - if(pMsg->pszMSG != NULL) - free(pMsg->pszMSG); - - pMsg->iLenMSG = strlen(pszMSG); - if((pMsg->pszMSG = (uchar*) malloc(pMsg->iLenMSG + 1)) != NULL) - memcpy(pMsg->pszMSG, pszMSG, pMsg->iLenMSG + 1); - else - dbgprintf("MsgSetMSG could not allocate memory for pszMSG buffer."); -} -#endif /* set raw message in message object. Size of message is provided. * rgerhards, 2009-06-16 @@ -1869,6 +1856,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, int iLen; short iOffs; + BEGINfunc #ifdef FEATURE_REGEXP /* Variables necessary for regular expression matching */ size_t nmatch = 10; @@ -2562,6 +2550,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, *pPropLen = bufLen; /*dbgprintf("MsgGetProp(\"%s\"): \"%s\"\n", pName, pRes); only for verbose debug logging */ + ENDfunc return(pRes); } diff --git a/runtime/msg.h b/runtime/msg.h index 4cdb0762..ec18b29d 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -74,7 +74,6 @@ struct msg { int iLenRawMsg; /* length of raw message */ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ short offMSG; /* offset at which the MSG part starts in pszRawMsg */ - //uchar *pszMSG; /* the MSG part itself */ int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ diff --git a/template.c b/template.c index aacd4dbd..93d0539c 100644 --- a/template.c +++ b/template.c @@ -571,6 +571,7 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) cstrFinalize(pStrB); if(cstrConvSzStrAndDestruct(pStrB, &pTpe->data.field.pPropRepl, 0) != RS_RET_OK) return 1; +// TODO: another optimization: map name to integer id OPT /* Check frompos, if it has an R, then topos should be a regex */ if(*p == ':') { diff --git a/tests/Makefile.am b/tests/Makefile.am index dbaf85f0..0800f667 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -42,6 +42,7 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.parse1 \ testsuites/2.parse1 \ testsuites/3.parse1 \ + testsuites/oversizeTag-1.parse1 \ testsuites/date1.parse1 \ testsuites/date2.parse1 \ testsuites/date3.parse1 \ diff --git a/tests/testsuites/oversizeTag-1.parse1 b/tests/testsuites/oversizeTag-1.parse1 new file mode 100644 index 00000000..56510c63 --- /dev/null +++ b/tests/testsuites/oversizeTag-1.parse1 @@ -0,0 +1,3 @@ +<38>Mar 27 19:06:53 source_server 0123456780123456780123456780123456789: MSG part +38,auth,info,Mar 27 19:06:53,source_server,0123456780123456780123456780123456789,0123456780123456780123456780123456789:, MSG part +# yet another real-life sample where we had some issues with -- cgit v1.2.3 From dc603a3dbf0bc0654830e7830a07816c23c85ce5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 09:58:33 +0200 Subject: optimized rfc3339 timestamp string generation --- runtime/datetime.c | 103 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 73 insertions(+), 30 deletions(-) diff --git a/runtime/datetime.c b/runtime/datetime.c index 40ab4e9c..eac55083 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -616,6 +616,18 @@ int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf, size_t iLenBuf) } +static char *const_number_str[100] = + { "00", "01", "02", "00", "04", "05", "06", "07", "08", "09", + "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", + "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", + "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", + "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", + "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", + "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", + "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", + "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", + "90", "91", "92", "93", "94", "95", "96", "97", "98", "99", + }; /** * Format a syslogTimestamp to a RFC3339 timestamp string (as * specified in syslog-protocol). @@ -626,46 +638,77 @@ int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf, size_t iLenBuf) */ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) { - int iRet; - char szTZ[7]; /* buffer for TZ information */ + int iBuf; + int power; + int secfrac; + short digit; + /* the following table of ten powers saves us some computation */ + static const int tenPowers[6] = { 1, 10, 100, 1000, 10000, 100000 }; + + BEGINfunc assert(ts != NULL); assert(pBuf != NULL); - if(iLenBuf < 20) + if(iLenBuf < 33) return(0); /* we NEED at least 20 bytes */ - /* do TZ information first, this is easier to take care of "Z" zone in rfc3339 */ + /* start with fixed parts */ + /* year yyyy */ + pBuf[0] = (ts->year / 1000) % 10 + '0'; + pBuf[1] = (ts->year / 100) % 10 + '0'; + pBuf[2] = (ts->year / 10) % 10 + '0'; + pBuf[3] = ts->year % 10 + '0'; + pBuf[4] = '-'; + /* month */ + pBuf[5] = (ts->month / 10) % 10 + '0'; + pBuf[6] = ts->month % 10 + '0'; + pBuf[7] = '-'; + /* day */ + pBuf[8] = (ts->day / 10) % 10 + '0'; + pBuf[9] = ts->day % 10 + '0'; + pBuf[10] = 'T'; + /* hour */ + pBuf[11] = (ts->hour / 10) % 10 + '0'; + pBuf[12] = ts->hour % 10 + '0'; + pBuf[13] = '-'; + /* minute */ + pBuf[14] = (ts->minute / 10) % 10 + '0'; + pBuf[15] = ts->minute % 10 + '0'; + pBuf[16] = '-'; + /* second */ + pBuf[17] = (ts->second / 10) % 10 + '0'; + pBuf[18] = ts->second % 10 + '0'; + + iBuf = 19; /* points to next free entry, now it becomes dynamic! */ + + if(ts->secfracPrecision > 0) { + pBuf[iBuf++] = '.'; + power = tenPowers[(ts->secfracPrecision - 1) % 6]; + secfrac = ts->secfrac; + while(power > 0) { + digit = secfrac / power; + secfrac -= digit * power; + power /= 10; + pBuf[iBuf++] = digit + '0'; + } + } + if(ts->OffsetMode == 'Z') { - szTZ[0] = 'Z'; - szTZ[1] = '\0'; + pBuf[iBuf++] = 'Z'; } else { - snprintf(szTZ, sizeof(szTZ) / sizeof(char), "%c%2.2d:%2.2d", - ts->OffsetMode, ts->OffsetHour, ts->OffsetMinute); + pBuf[iBuf++] = ts->OffsetMode; + pBuf[iBuf++] = (ts->OffsetHour / 10) % 10 + '0'; + pBuf[iBuf++] = ts->OffsetHour % 10 + '0'; + pBuf[iBuf++] = ':'; + pBuf[iBuf++] = (ts->OffsetMinute / 10) % 10 + '0'; + pBuf[iBuf++] = ts->OffsetMinute % 10 + '0'; } - if(ts->secfracPrecision > 0) - { /* we now need to include fractional seconds. While doing so, we must look at - * the precision specified. For example, if we have millisec precision (3 digits), a - * secFrac value of 12 is not equivalent to ".12" but ".012". Obviously, this - * is a huge difference ;). To avoid this, we first create a format string with - * the specific precision and *then* use that format string to do the actual - * formating (mmmmhhh... kind of self-modifying code... ;)). - */ - char szFmtStr[64]; - /* be careful: there is ONE actual %d in the format string below ;) */ - snprintf(szFmtStr, sizeof(szFmtStr), - "%%04d-%%02d-%%02dT%%02d:%%02d:%%02d.%%0%dd%%s", - ts->secfracPrecision); - iRet = snprintf(pBuf, iLenBuf, szFmtStr, ts->year, ts->month, ts->day, - ts->hour, ts->minute, ts->second, ts->secfrac, szTZ); - } - else - iRet = snprintf(pBuf, iLenBuf, - "%4.4d-%2.2d-%2.2dT%2.2d:%2.2d:%2.2d%s", - ts->year, ts->month, ts->day, - ts->hour, ts->minute, ts->second, szTZ); - return(iRet); + pBuf[iBuf] = '\0'; + + ENDfunc + return iBuf; } /** -- cgit v1.2.3 From 77c992e2155420702460e835ce2d561cf2d10fcb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 11:30:57 +0200 Subject: fully optimized datetime module and enhanced test suite tests for the various timestamp formats have been added --- ChangeLog | 4 + runtime/datetime.c | 166 ++++++++++++++++++++------------------ tests/Makefile.am | 22 ++++- tests/diag.sh | 6 ++ tests/nettester.c | 70 +++++++++------- tests/testsuites/1.retry.conf | 2 + tests/testsuites/Apr.ts3164 | 3 + tests/testsuites/Aug.ts3164 | 3 + tests/testsuites/Dec.ts3164 | 3 + tests/testsuites/Feb.ts3164 | 3 + tests/testsuites/Jan.ts3164 | 3 + tests/testsuites/Jul.ts3164 | 3 + tests/testsuites/Jun.ts3164 | 3 + tests/testsuites/Mar.ts3164 | 3 + tests/testsuites/May.ts3164 | 3 + tests/testsuites/Nov.ts3164 | 3 + tests/testsuites/Oct.ts3164 | 3 + tests/testsuites/Sep.ts3164 | 3 + tests/testsuites/master.subsecond | 8 ++ tests/testsuites/master.ts3339 | 22 +++++ tests/testsuites/master.tsmysql | 2 + tests/testsuites/master.tspgsql | 2 + tests/testsuites/mon1digit.ts3164 | 3 + tests/testsuites/mon2digit.ts3164 | 3 + tests/testsuites/parse1udp.conf | 9 +++ tests/testsuites/subsecond.conf | 8 ++ tests/testsuites/ts3164.conf | 8 ++ tests/testsuites/ts3339.conf | 8 ++ tests/testsuites/tsmysql.conf | 8 ++ tests/testsuites/tspgsql.conf | 8 ++ tests/timestamp.sh | 13 +++ 31 files changed, 298 insertions(+), 110 deletions(-) create mode 100644 tests/testsuites/1.retry.conf create mode 100644 tests/testsuites/Apr.ts3164 create mode 100644 tests/testsuites/Aug.ts3164 create mode 100644 tests/testsuites/Dec.ts3164 create mode 100644 tests/testsuites/Feb.ts3164 create mode 100644 tests/testsuites/Jan.ts3164 create mode 100644 tests/testsuites/Jul.ts3164 create mode 100644 tests/testsuites/Jun.ts3164 create mode 100644 tests/testsuites/Mar.ts3164 create mode 100644 tests/testsuites/May.ts3164 create mode 100644 tests/testsuites/Nov.ts3164 create mode 100644 tests/testsuites/Oct.ts3164 create mode 100644 tests/testsuites/Sep.ts3164 create mode 100644 tests/testsuites/master.subsecond create mode 100644 tests/testsuites/master.ts3339 create mode 100644 tests/testsuites/master.tsmysql create mode 100644 tests/testsuites/master.tspgsql create mode 100644 tests/testsuites/mon1digit.ts3164 create mode 100644 tests/testsuites/mon2digit.ts3164 create mode 100644 tests/testsuites/parse1udp.conf create mode 100644 tests/testsuites/subsecond.conf create mode 100644 tests/testsuites/ts3164.conf create mode 100644 tests/testsuites/ts3339.conf create mode 100644 tests/testsuites/tsmysql.conf create mode 100644 tests/testsuites/tspgsql.conf create mode 100755 tests/timestamp.sh diff --git a/ChangeLog b/ChangeLog index 330fd56c..36a819a3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,9 @@ --------------------------------------------------------------------------- Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +- greatly reduced memory requirements of msg object + to around half of the previous demand. This means that more messages can + be stored in core! Due to fewer cache misses, this also means some + performance improvement. - improved config error messages: now contain a copy of the config line that (most likely) caused the error - removed long-obsoleted property UxTradMsg diff --git a/runtime/datetime.c b/runtime/datetime.c index eac55083..114bd27f 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -49,6 +49,8 @@ DEFobjStaticHelpers DEFobjCurrIf(errmsg) +/* the following table of ten powers saves us some computation */ +static const int tenPowers[6] = { 1, 10, 100, 1000, 10000, 100000 }; /* ------------------------------ methods ------------------------------ */ @@ -544,7 +546,7 @@ finalize_it: * returns the size of the timestamp written in bytes (without * the string terminator). If 0 is returend, an error occured. */ -int formatTimestampToMySQL(struct syslogTime *ts, char* pDst, size_t iLenDst) +int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf, size_t iLenDst) { /* currently we do not consider localtime/utc. This may later be * added. If so, I recommend using a property replacer option @@ -553,28 +555,60 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pDst, size_t iLenDst) * rgerhards, 2007-06-26 */ assert(ts != NULL); - assert(pDst != NULL); + assert(pBuf != NULL); - if (iLenDst < 15) /* we need at least 14 bytes - 14 digits for timestamp + '\n' */ + if (iLenDst < 15) /* we need at least 14 bytes */ return(0); - return(snprintf(pDst, iLenDst, "%4.4d%2.2d%2.2d%2.2d%2.2d%2.2d", - ts->year, ts->month, ts->day, ts->hour, ts->minute, ts->second)); + pBuf[0] = (ts->year / 1000) % 10 + '0'; + pBuf[1] = (ts->year / 100) % 10 + '0'; + pBuf[2] = (ts->year / 10) % 10 + '0'; + pBuf[3] = ts->year % 10 + '0'; + pBuf[4] = (ts->month / 10) % 10 + '0'; + pBuf[5] = ts->month % 10 + '0'; + pBuf[6] = (ts->day / 10) % 10 + '0'; + pBuf[7] = ts->day % 10 + '0'; + pBuf[8] = (ts->hour / 10) % 10 + '0'; + pBuf[9] = ts->hour % 10 + '0'; + pBuf[10] = (ts->minute / 10) % 10 + '0'; + pBuf[11] = ts->minute % 10 + '0'; + pBuf[12] = (ts->second / 10) % 10 + '0'; + pBuf[13] = ts->second % 10 + '0'; + pBuf[14] = '\0'; + return 15; } -int formatTimestampToPgSQL(struct syslogTime *ts, char *pDst, size_t iLenDst) +int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf, size_t iLenDst) { /* see note in formatTimestampToMySQL, applies here as well */ assert(ts != NULL); - assert(pDst != NULL); + assert(pBuf != NULL); - if (iLenDst < 21) /* we need 20 bytes + '\n' */ - return(0); + if (iLenDst < 20) /* we need 20 bytes */ + return(0); - return(snprintf(pDst, iLenDst, "%4.4d-%2.2d-%2.2d %2.2d:%2.2d:%2.2d", - ts->year, ts->month, ts->day, ts->hour, ts->minute, ts->second)); + pBuf[0] = (ts->year / 1000) % 10 + '0'; + pBuf[1] = (ts->year / 100) % 10 + '0'; + pBuf[2] = (ts->year / 10) % 10 + '0'; + pBuf[3] = ts->year % 10 + '0'; + pBuf[4] = '-'; + pBuf[5] = (ts->month / 10) % 10 + '0'; + pBuf[6] = ts->month % 10 + '0'; + pBuf[7] = '-'; + pBuf[8] = (ts->day / 10) % 10 + '0'; + pBuf[9] = ts->day % 10 + '0'; + pBuf[10] = ' '; + pBuf[11] = (ts->hour / 10) % 10 + '0'; + pBuf[12] = ts->hour % 10 + '0'; + pBuf[13] = ':'; + pBuf[14] = (ts->minute / 10) % 10 + '0'; + pBuf[15] = ts->minute % 10 + '0'; + pBuf[16] = ':'; + pBuf[17] = (ts->second / 10) % 10 + '0'; + pBuf[18] = ts->second % 10 + '0'; + pBuf[19] = '\0'; + return 19; } @@ -589,45 +623,35 @@ int formatTimestampToPgSQL(struct syslogTime *ts, char *pDst, size_t iLenDst) */ int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf, size_t iLenBuf) { - int lenRet; - char szFmtStr[64]; + int iBuf; + int power; + int secfrac; + short digit; assert(ts != NULL); assert(pBuf != NULL); assert(iLenBuf >= 10); + iBuf = 0; if(ts->secfracPrecision > 0) - { /* We must look at - * the precision specified. For example, if we have millisec precision (3 digits), a - * secFrac value of 12 is not equivalent to ".12" but ".012". Obviously, this - * is a huge difference ;). To avoid this, we first create a format string with - * the specific precision and *then* use that format string to do the actual formating. - */ - /* be careful: there is ONE actual %d in the format string below ;) */ - snprintf(szFmtStr, sizeof(szFmtStr), "%%0%dd", ts->secfracPrecision); - lenRet = snprintf(pBuf, iLenBuf, szFmtStr, ts->secfrac); + { + power = tenPowers[(ts->secfracPrecision - 1) % 6]; + secfrac = ts->secfrac; + while(power > 0) { + digit = secfrac / power; + secfrac -= digit * power; + power /= 10; + pBuf[iBuf++] = digit + '0'; + } } else { - pBuf[0] = '0'; - pBuf[1] = '\0'; - lenRet = 1; + pBuf[iBuf++] = '0'; } + pBuf[iBuf] = '\0'; - return(lenRet); + return iBuf; } -static char *const_number_str[100] = - { "00", "01", "02", "00", "04", "05", "06", "07", "08", "09", - "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", - "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", - "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", - "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", - "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", - "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", - "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", - "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", - "90", "91", "92", "93", "94", "95", "96", "97", "98", "99", - }; /** * Format a syslogTimestamp to a RFC3339 timestamp string (as * specified in syslog-protocol). @@ -643,15 +667,10 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) int secfrac; short digit; - /* the following table of ten powers saves us some computation */ - static const int tenPowers[6] = { 1, 10, 100, 1000, 10000, 100000 }; - BEGINfunc assert(ts != NULL); assert(pBuf != NULL); - - if(iLenBuf < 33) - return(0); /* we NEED at least 20 bytes */ + assert(iLenBuf < 33); /* start with fixed parts */ /* year yyyy */ @@ -671,11 +690,11 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) /* hour */ pBuf[11] = (ts->hour / 10) % 10 + '0'; pBuf[12] = ts->hour % 10 + '0'; - pBuf[13] = '-'; + pBuf[13] = ':'; /* minute */ pBuf[14] = (ts->minute / 10) % 10 + '0'; pBuf[15] = ts->minute % 10 + '0'; - pBuf[16] = '-'; + pBuf[16] = ':'; /* second */ pBuf[17] = (ts->second / 10) % 10 + '0'; pBuf[18] = ts->second % 10 + '0'; @@ -720,44 +739,36 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) */ int formatTimestamp3164(struct syslogTime *ts, char* pBuf, size_t iLenBuf) { - static char* monthNames[13] = {"ERR", "Jan", "Feb", "Mar", - "Apr", "May", "Jun", "Jul", - "Aug", "Sep", "Oct", "Nov", "Dec"}; + static char* monthNames[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; + int iDay; assert(ts != NULL); assert(pBuf != NULL); if(iLenBuf < 16) return(0); /* we NEED 16 bytes */ - return(snprintf(pBuf, iLenBuf, "%s %2d %2.2d:%2.2d:%2.2d", - monthNames[ts->month], ts->day, ts->hour, - ts->minute, ts->second - )); -} -/** - * Format a syslogTimestamp to a text format. - * The caller must provide the timestamp as well as a character - * buffer that will receive the resulting string. The function - * returns the size of the timestamp written in bytes (without - * the string termnator). If 0 is returend, an error occured. - */ -#if 0 /* This method is currently not called, be we like to preserve it */ -static int formatTimestamp(struct syslogTime *ts, char* pBuf, size_t iLenBuf) -{ - assert(ts != NULL); - assert(pBuf != NULL); - - if(ts->timeType == 1) { - return(formatTimestamp3164(ts, pBuf, iLenBuf)); - } + pBuf[0] = monthNames[(ts->month - 1)% 12][0]; + pBuf[1] = monthNames[(ts->month - 1) % 12][1]; + pBuf[2] = monthNames[(ts->month - 1) % 12][2]; + pBuf[3] = ' '; + iDay = (ts->day / 10) % 10; /* we need to write a space if the first digit is 0 */ + pBuf[4] = iDay ? iDay + '0' : ' '; + pBuf[5] = ts->day % 10 + '0'; + pBuf[6] = ' '; + pBuf[7] = (ts->hour / 10) % 10 + '0'; + pBuf[8] = ts->hour % 10 + '0'; + pBuf[9] = ':'; + pBuf[10] = (ts->minute / 10) % 10 + '0'; + pBuf[11] = ts->minute % 10 + '0'; + pBuf[12] = ':'; + pBuf[13] = (ts->second / 10) % 10 + '0'; + pBuf[14] = ts->second % 10 + '0'; + pBuf[15] = '\0'; + return 16; /* traditional: number of bytes written */ +} - if(ts->timeType == 2) { - return(formatTimestamp3339(ts, pBuf, iLenBuf)); - } - return(0); -} -#endif /* queryInterface function * rgerhards, 2008-03-05 */ @@ -791,7 +802,6 @@ ENDobjQueryInterface(datetime) BEGINAbstractObjClassInit(datetime, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); - ENDObjClassInit(datetime) /* vi:set ai: diff --git a/tests/Makefile.am b/tests/Makefile.am index 0800f667..955ad405 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -10,7 +10,11 @@ TESTS = $(TESTRUNS) cfg.sh \ queue-persist.sh if ENABLE_OMSTDOUT -TESTS += omod-if-array.sh parsertest.sh inputname.sh fieldtest.sh +TESTS += omod-if-array.sh \ + parsertest.sh \ + timestamp.sh \ + inputname.sh \ + fieldtest.sh endif endif # if ENABLE_TESTBENCH @@ -37,6 +41,22 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ DevNull.cfgtest \ err1.rstest \ NoExistFile.cfgtest \ + timestamp.sh \ + testsuites/ts3164.conf \ + testsuites/mon1digit.ts3164 \ + testsuites/mon2digit.ts3164 \ + testsuites/Jan.ts3164 \ + testsuites/Feb.ts3164 \ + testsuites/Mar.ts3164 \ + testsuites/Apr.ts3164 \ + testsuites/May.ts3164 \ + testsuites/Jun.ts3164 \ + testsuites/Jul.ts3164 \ + testsuites/Aug.ts3164 \ + testsuites/Sep.ts3164 \ + testsuites/Oct.ts3164 \ + testsuites/Nov.ts3164 \ + testsuites/Dec.ts3164 \ testsuites/parse1.conf \ testsuites/field1.conf \ testsuites/1.parse1 \ diff --git a/tests/diag.sh b/tests/diag.sh index 1fa8f62a..2a9d0ee3 100755 --- a/tests/diag.sh +++ b/tests/diag.sh @@ -82,5 +82,11 @@ case $1 in exit 1 fi ;; + 'nettester') # perform nettester-based tests + ./nettester -t$2 -i$3 + if [ "$?" -ne "0" ]; then + exit 1 + fi + ;; *) echo "invalid argument" $1 esac diff --git a/tests/nettester.c b/tests/nettester.c index e5e6278c..b8816b7c 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -61,6 +61,9 @@ static int iPort = 12514; /* port which shall be used for sending data */ static char* pszCustomConf = NULL; /* custom config file, use -c conf to specify */ static int verbose = 0; /* verbose output? -v option */ +/* these two are quick hacks... */ +int iFailed = 0; +int iTests = 0; /* provide user-friednly name of input mode */ @@ -247,38 +250,47 @@ processTestFile(int fd, char *pszFileName) /* skip comments at start of file */ - getline(&testdata, &lenLn, fp); while(!feof(fp)) { - if(*testdata == '#') - getline(&testdata, &lenLn, fp); - else - break; /* first non-comment */ - } + getline(&testdata, &lenLn, fp); + while(!feof(fp)) { + if(*testdata == '#') + getline(&testdata, &lenLn, fp); + else + break; /* first non-comment */ + } + /* this is not perfect, but works ;) */ + if(feof(fp)) + break; - testdata[strlen(testdata)-1] = '\0'; /* remove \n */ - /* now we have the test data to send (we could use function pointers here...) */ - if(inputMode == inputUDP) { - if(udpSend(testdata, strlen(testdata)) != 0) - return(2); - } else { - if(tcpSend(testdata, strlen(testdata)) != 0) - return(2); - } + ++iTests; /* increment test count, we now do one! */ - /* next line is expected output - * we do not care about EOF here, this will lead to a failure and thus - * draw enough attention. -- rgerhards, 2009-03-31 - */ - getline(&expected, &lenLn, fp); - expected[strlen(expected)-1] = '\0'; /* remove \n */ + testdata[strlen(testdata)-1] = '\0'; /* remove \n */ + /* now we have the test data to send (we could use function pointers here...) */ + if(inputMode == inputUDP) { + if(udpSend(testdata, strlen(testdata)) != 0) + return(2); + } else { + if(tcpSend(testdata, strlen(testdata)) != 0) + return(2); + } + + /* next line is expected output + * we do not care about EOF here, this will lead to a failure and thus + * draw enough attention. -- rgerhards, 2009-03-31 + */ + getline(&expected, &lenLn, fp); + expected[strlen(expected)-1] = '\0'; /* remove \n */ + + /* pull response from server and then check if it meets our expectation */ + readLine(fd, buf); + if(strcmp(expected, buf)) { + ++iFailed; + printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", + expected, buf); + ret = 1; + } - /* pull response from server and then check if it meets our expectation */ - readLine(fd, buf); - if(strcmp(expected, buf)) { - printf("\nExpected Response:\n'%s'\nActual Response:\n'%s'\n", - expected, buf); - ret = 1; } free(testdata); @@ -297,8 +309,6 @@ processTestFile(int fd, char *pszFileName) int doTests(int fd, char *files) { - int iFailed = 0; - int iTests = 0; int ret; char *testFile; glob_t testFiles; @@ -313,7 +323,6 @@ doTests(int fd, char *files) if(stat((char*) testFile, &fileInfo) != 0) continue; /* continue with the next file if we can't stat() the file */ - ++iTests; /* all regular files are run through the test logic. Symlinks don't work. */ if(S_ISREG(fileInfo.st_mode)) { /* config file */ if(verbose) printf("processing test case '%s' ... ", testFile); @@ -324,7 +333,6 @@ doTests(int fd, char *files) if(!verbose) printf("test '%s' ", testFile); printf("failed!\n"); - ++iFailed; } } } diff --git a/tests/testsuites/1.retry.conf b/tests/testsuites/1.retry.conf new file mode 100644 index 00000000..c464b19c --- /dev/null +++ b/tests/testsuites/1.retry.conf @@ -0,0 +1,2 @@ +<167>Mar 6 16:57:54 172.20.245.8 %PIX-7-710005: UDP request discarded from SERVER1/2741 to test_app:255.255.255.255/61601 +167,Mar 6 16:57:54,172.20.245.8,%PIX-7-710005,%PIX-7-710005:, diff --git a/tests/testsuites/Apr.ts3164 b/tests/testsuites/Apr.ts3164 new file mode 100644 index 00000000..3134f224 --- /dev/null +++ b/tests/testsuites/Apr.ts3164 @@ -0,0 +1,3 @@ +<167>Apr 6 16:57:54 172.20.245.8 TAG: MSG +Apr 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Aug.ts3164 b/tests/testsuites/Aug.ts3164 new file mode 100644 index 00000000..d9a721eb --- /dev/null +++ b/tests/testsuites/Aug.ts3164 @@ -0,0 +1,3 @@ +<167>Aug 6 16:57:54 172.20.245.8 TAG: MSG +Aug 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Dec.ts3164 b/tests/testsuites/Dec.ts3164 new file mode 100644 index 00000000..080ba401 --- /dev/null +++ b/tests/testsuites/Dec.ts3164 @@ -0,0 +1,3 @@ +<167>Dec 6 16:57:54 172.20.245.8 TAG: MSG +Dec 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Feb.ts3164 b/tests/testsuites/Feb.ts3164 new file mode 100644 index 00000000..d1eaaa33 --- /dev/null +++ b/tests/testsuites/Feb.ts3164 @@ -0,0 +1,3 @@ +<167>Feb 6 16:57:54 172.20.245.8 TAG: MSG +Feb 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Jan.ts3164 b/tests/testsuites/Jan.ts3164 new file mode 100644 index 00000000..0cb1c8e2 --- /dev/null +++ b/tests/testsuites/Jan.ts3164 @@ -0,0 +1,3 @@ +<167>Jan 6 16:57:54 172.20.245.8 TAG: MSG +Jan 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Jul.ts3164 b/tests/testsuites/Jul.ts3164 new file mode 100644 index 00000000..562e1ec4 --- /dev/null +++ b/tests/testsuites/Jul.ts3164 @@ -0,0 +1,3 @@ +<167>Jul 6 16:57:54 172.20.245.8 TAG: MSG +Jul 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Jun.ts3164 b/tests/testsuites/Jun.ts3164 new file mode 100644 index 00000000..ede27e0e --- /dev/null +++ b/tests/testsuites/Jun.ts3164 @@ -0,0 +1,3 @@ +<167>Jun 6 16:57:54 172.20.245.8 TAG: MSG +Jun 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Mar.ts3164 b/tests/testsuites/Mar.ts3164 new file mode 100644 index 00000000..55dd5bc2 --- /dev/null +++ b/tests/testsuites/Mar.ts3164 @@ -0,0 +1,3 @@ +<167>Mar 6 16:57:54 172.20.245.8 TAG: MSG +Mar 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/May.ts3164 b/tests/testsuites/May.ts3164 new file mode 100644 index 00000000..72a5a301 --- /dev/null +++ b/tests/testsuites/May.ts3164 @@ -0,0 +1,3 @@ +<167>May 6 16:57:54 172.20.245.8 TAG: MSG +May 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Nov.ts3164 b/tests/testsuites/Nov.ts3164 new file mode 100644 index 00000000..e8f00e01 --- /dev/null +++ b/tests/testsuites/Nov.ts3164 @@ -0,0 +1,3 @@ +<167>Nov 6 16:57:54 172.20.245.8 TAG: MSG +Nov 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Oct.ts3164 b/tests/testsuites/Oct.ts3164 new file mode 100644 index 00000000..01423fef --- /dev/null +++ b/tests/testsuites/Oct.ts3164 @@ -0,0 +1,3 @@ +<167>Oct 6 16:57:54 172.20.245.8 TAG: MSG +Oct 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/Sep.ts3164 b/tests/testsuites/Sep.ts3164 new file mode 100644 index 00000000..6c9e48e0 --- /dev/null +++ b/tests/testsuites/Sep.ts3164 @@ -0,0 +1,3 @@ +<167>Sep 6 16:57:54 172.20.245.8 TAG: MSG +Sep 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/master.subsecond b/tests/testsuites/master.subsecond new file mode 100644 index 00000000..ee924877 --- /dev/null +++ b/tests/testsuites/master.subsecond @@ -0,0 +1,8 @@ +<34>1 2003-01-23T12:34:56.003Z mymachine.example.com su - ID47 - MSG +003 +# full precision +<34>1 2003-01-23T12:34:56.123456Z mymachine.example.com su - ID47 - MSG +123456 +# without +<34>1 2003-01-23T12:34:56Z mymachine.example.com su - ID47 - MSG +0 diff --git a/tests/testsuites/master.ts3339 b/tests/testsuites/master.ts3339 new file mode 100644 index 00000000..b4dd5f39 --- /dev/null +++ b/tests/testsuites/master.ts3339 @@ -0,0 +1,22 @@ +<34>1 2003-11-11T22:14:15.003Z mymachine.example.com su - ID47 - MSG +2003-11-11T22:14:15.003Z +# next test +<34>1 2003-01-11T22:14:15.003Z mymachine.example.com su - ID47 - MSG +2003-01-11T22:14:15.003Z +# next test +<34>1 2003-11-01T22:04:15.003Z mymachine.example.com su - ID47 - MSG +2003-11-01T22:04:15.003Z +# next test +<34>1 2003-11-11T02:14:15.003Z mymachine.example.com su - ID47 - MSG +2003-11-11T02:14:15.003Z +# next test +<34>1 2003-11-11T22:04:05.003Z mymachine.example.com su - ID47 - MSG +2003-11-11T22:04:05.003Z +# next test +<34>1 2003-11-11T22:04:05.003+02:00 mymachine.example.com su - ID47 - MSG +2003-11-11T22:04:05.003+02:00 +# next test +<34>1 2003-11-11T22:04:05.003+01:30 mymachine.example.com su - ID47 - MSG +2003-11-11T22:04:05.003+01:30 +<34>1 2003-11-11T22:04:05.123456+01:30 mymachine.example.com su - ID47 - MSG +2003-11-11T22:04:05.123456+01:30 diff --git a/tests/testsuites/master.tsmysql b/tests/testsuites/master.tsmysql new file mode 100644 index 00000000..dc6d85be --- /dev/null +++ b/tests/testsuites/master.tsmysql @@ -0,0 +1,2 @@ +<34>1 2003-01-23T12:34:56.003Z mymachine.example.com su - ID47 - MSG +20030123123456 diff --git a/tests/testsuites/master.tspgsql b/tests/testsuites/master.tspgsql new file mode 100644 index 00000000..d7ac19ff --- /dev/null +++ b/tests/testsuites/master.tspgsql @@ -0,0 +1,2 @@ +<34>1 2003-01-23T12:34:56.003Z mymachine.example.com su - ID47 - MSG +2003-01-23 12:34:56 diff --git a/tests/testsuites/mon1digit.ts3164 b/tests/testsuites/mon1digit.ts3164 new file mode 100644 index 00000000..0cb1c8e2 --- /dev/null +++ b/tests/testsuites/mon1digit.ts3164 @@ -0,0 +1,3 @@ +<167>Jan 6 16:57:54 172.20.245.8 TAG: MSG +Jan 6 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/mon2digit.ts3164 b/tests/testsuites/mon2digit.ts3164 new file mode 100644 index 00000000..9606961c --- /dev/null +++ b/tests/testsuites/mon2digit.ts3164 @@ -0,0 +1,3 @@ +<167>Jan 16 16:57:54 172.20.245.8 TAG: MSG +Jan 16 16:57:54 +#Only the first two lines are important, you may place anything behind them! diff --git a/tests/testsuites/parse1udp.conf b/tests/testsuites/parse1udp.conf new file mode 100644 index 00000000..0fb7d16d --- /dev/null +++ b/tests/testsuites/parse1udp.conf @@ -0,0 +1,9 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$ModLoad ../plugins/imudp/.libs/imudp +$UDPServerRun 12514 + +$ErrorMessagesToStderr off + +# use a special format that we can easily parse in expect +$template expect,"%PRI%,%syslogfacility-text%,%syslogseverity-text%,%timestamp%,%hostname%,%programname%,%syslogtag%,%msg%\n" +*.* :omstdout:;expect diff --git a/tests/testsuites/subsecond.conf b/tests/testsuites/subsecond.conf new file mode 100644 index 00000000..58c26cc7 --- /dev/null +++ b/tests/testsuites/subsecond.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-subseconds%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/ts3164.conf b/tests/testsuites/ts3164.conf new file mode 100644 index 00000000..7aa6a8ef --- /dev/null +++ b/tests/testsuites/ts3164.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-rfc3164%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/ts3339.conf b/tests/testsuites/ts3339.conf new file mode 100644 index 00000000..df8f23ac --- /dev/null +++ b/tests/testsuites/ts3339.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-rfc3339%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/tsmysql.conf b/tests/testsuites/tsmysql.conf new file mode 100644 index 00000000..f97d4b0a --- /dev/null +++ b/tests/testsuites/tsmysql.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-mysql%\n" +*.* :omstdout:;fmt diff --git a/tests/testsuites/tspgsql.conf b/tests/testsuites/tspgsql.conf new file mode 100644 index 00000000..eb18c091 --- /dev/null +++ b/tests/testsuites/tspgsql.conf @@ -0,0 +1,8 @@ +$ModLoad ../plugins/omstdout/.libs/omstdout +$IncludeConfig nettest.input.conf # This picks the to be tested input from the test driver! + +$ErrorMessagesToStderr off + +# use a special format +$template fmt,"%timestamp:::date-pgsql%\n" +*.* :omstdout:;fmt diff --git a/tests/timestamp.sh b/tests/timestamp.sh new file mode 100755 index 00000000..7699a4af --- /dev/null +++ b/tests/timestamp.sh @@ -0,0 +1,13 @@ +echo various timestamp tests +source $srcdir/diag.sh init +source $srcdir/diag.sh nettester ts3164 udp +source $srcdir/diag.sh nettester ts3164 tcp +source $srcdir/diag.sh nettester ts3339 udp +source $srcdir/diag.sh nettester ts3339 tcp +source $srcdir/diag.sh nettester tsmysql udp +source $srcdir/diag.sh nettester tsmysql tcp +source $srcdir/diag.sh nettester tspgsql udp +source $srcdir/diag.sh nettester tspgsql tcp +source $srcdir/diag.sh nettester subsecond udp +source $srcdir/diag.sh nettester subsecond tcp +source $srcdir/diag.sh init -- cgit v1.2.3 From 11cb1e008bfb932a159cc746c5d435ae9518ed19 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 11:45:21 +0200 Subject: some cleanup --- runtime/datetime.c | 17 ++++++----------- tests/Makefile.am | 8 ++++++++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/runtime/datetime.c b/runtime/datetime.c index 114bd27f..30f397ff 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -556,9 +556,7 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf, size_t iLenDst) */ assert(ts != NULL); assert(pBuf != NULL); - - if (iLenDst < 15) /* we need at least 14 bytes */ - return(0); + assert(iLenDst < 15); pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; @@ -581,12 +579,10 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf, size_t iLenDst) int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf, size_t iLenDst) { - /* see note in formatTimestampToMySQL, applies here as well */ - assert(ts != NULL); - assert(pBuf != NULL); - - if (iLenDst < 20) /* we need 20 bytes */ - return(0); + /* see note in formatTimestampToMySQL, applies here as well */ + assert(ts != NULL); + assert(pBuf != NULL); + assert(iLenDst < 20): pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; @@ -745,8 +741,7 @@ int formatTimestamp3164(struct syslogTime *ts, char* pBuf, size_t iLenBuf) assert(ts != NULL); assert(pBuf != NULL); - if(iLenBuf < 16) - return(0); /* we NEED 16 bytes */ + assert(iLenBuf < 16); pBuf[0] = monthNames[(ts->month - 1)% 12][0]; pBuf[1] = monthNames[(ts->month - 1) % 12][1]; diff --git a/tests/Makefile.am b/tests/Makefile.am index 955ad405..4d38f875 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -57,6 +57,14 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/Oct.ts3164 \ testsuites/Nov.ts3164 \ testsuites/Dec.ts3164 \ + testsuites/ts3339.conf \ + testsuites/master.ts3339 \ + testsuites/tsmysql.conf \ + testsuites/master.tsmysql \ + testsuites/tspgsql.conf \ + testsuites/master.tspgsql \ + testsuites/subsecond.conf \ + testsuites/master.subsecond \ testsuites/parse1.conf \ testsuites/field1.conf \ testsuites/1.parse1 \ -- cgit v1.2.3 From dd53709083f3671ac711d5d2cb03bfd9c80a7cf3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 11:58:30 +0200 Subject: typo fix --- runtime/datetime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/datetime.c b/runtime/datetime.c index 30f397ff..b5514e7c 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -582,7 +582,7 @@ int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf, size_t iLenDst) /* see note in formatTimestampToMySQL, applies here as well */ assert(ts != NULL); assert(pBuf != NULL); - assert(iLenDst < 20): + assert(iLenDst < 20); pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; -- cgit v1.2.3 From 7a695d171436fe249770e8256ae48cd4ed86fd30 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 12:03:56 +0200 Subject: removed uniprocessor optimization ... as it was not even optimal on uniprocessors any longer ;) I keep the config directive in, maybe we can utilize it again at some later point in time (questionable). --- runtime/queue.c | 15 --------------- runtime/queue.h | 13 ++++++------- runtime/wti.c | 5 ----- runtime/wti.h | 3 +-- runtime/wtp.c | 9 --------- runtime/wtp.h | 3 +-- tools/syslogd.c | 2 -- 7 files changed, 8 insertions(+), 42 deletions(-) diff --git a/runtime/queue.c b/runtime/queue.c index 46a3a971..1e3b761f 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1295,7 +1295,6 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread /* we have an object, so let's fill the properties */ objConstructSetObjInfo(pThis); - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); if((pThis->pszSpoolDir = (uchar*) strdup((char*)glbl.GetWorkDir())) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); @@ -2216,13 +2215,6 @@ finalize_it: d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); dbgoprint((obj_t*) pThis, "EnqueueMsg advised worker start\n"); - /* the following pthread_yield is experimental, but brought us performance - * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 - * rgerhards, 2008-10-09 - * but this is only true for uniprocessors, so we guard it with an optimize flag -- rgerhards, 2008-10-22 - */ - if(pThis->bOptimizeUniProc) - pthread_yield(); } RETiRet; @@ -2341,13 +2333,6 @@ finalize_it: d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); dbgoprint((obj_t*) pThis, "MultiEnqObj advised worker start\n"); - /* the following pthread_yield is experimental, but brought us performance - * benefit. For details, please see http://kb.monitorware.com/post14216.html#p14216 - * rgerhards, 2008-10-09 - * but this is only true for uniprocessors, so we guard it with an optimize flag -- rgerhards, 2008-10-22 - */ - if(pThis->bOptimizeUniProc) - pthread_yield(); } RETiRet; diff --git a/runtime/queue.h b/runtime/queue.h index 5bc03254..1d82d8d9 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -58,11 +58,10 @@ typedef struct qWrkThrd_s { typedef struct queue_s { BEGINobjInstance; queueType_t qType; - int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ - int bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ - int bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ - int bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ - int bQueueInDestruction;/* 1 if queue is in destruction process, 0 otherwise */ + bool bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ + bool bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ + bool bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ + bool bQueueInDestruction;/* 1 if queue is in destruction process, 0 otherwise */ int iQueueSize; /* Current number of elements in the queue */ int iMaxQueueSize; /* how large can the queue grow? */ int iNumWorkerThreads;/* number of worker threads to use */ @@ -73,14 +72,14 @@ typedef struct queue_s { void *pUsr; /* a global, user-supplied pointer. Is passed back to consumer. */ int iUpdsSincePersist;/* nbr of queue updates since the last persist call */ int iPersistUpdCnt; /* persits queue info after this nbr of updates - 0 -> persist only on shutdown */ - int bSyncQueueFiles;/* if working with files, sync them after each write? */ + bool bSyncQueueFiles;/* if working with files, sync them after each write? */ int iHighWtrMrk; /* high water mark for disk-assisted memory queues */ int iLowWtrMrk; /* low water mark for disk-assisted memory queues */ int iDiscardMrk; /* if the queue is above this mark, low-severity messages are discarded */ int iFullDlyMrk; /* if the queue is above this mark, FULL_DELAYable message are put on hold */ int iLightDlyMrk; /* if the queue is above this mark, LIGHT_DELAYable message are put on hold */ int iDiscardSeverity;/* messages of this severity above are discarded on too-full queue */ - int bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */ + bool bNeedDelQIF; /* does the QIF file need to be deleted when queue becomes empty? */ int toQShutdown; /* timeout for regular queue shutdown in ms */ int toActShutdown; /* timeout for long-running action shutdown in ms */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ diff --git a/runtime/wti.c b/runtime/wti.c index 18767ea1..bfe8f863 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -208,7 +208,6 @@ ENDobjDestruct(wti) /* Standard-Constructor for the wti object */ BEGINobjConstruct(wti) /* be sure to specify the object type also in END macro! */ - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); pthread_cond_init(&pThis->condExitDone, NULL); pthread_mutex_init(&pThis->mut, NULL); ENDobjConstruct(wti) @@ -377,10 +376,6 @@ wtiWorker(wti_t *pThis) /* process any pending thread requests */ wtpProcessThrdChanges(pWtp); pthread_testcancel(); /* see big comment in function header */ -# if !defined(__hpux) /* pthread_yield is missing there! */ - if(pThis->bOptimizeUniProc) - pthread_yield(); /* see big comment in function header */ -# endif /* if we have a rate-limiter set for this worker pool, let's call it. Please * keep in mind that the rate-limiter may hold us for an extended period diff --git a/runtime/wti.h b/runtime/wti.h index 6b60b833..72653b15 100644 --- a/runtime/wti.h +++ b/runtime/wti.h @@ -31,14 +31,13 @@ /* the worker thread instance class */ typedef struct wti_s { BEGINobjInstance; - int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ pthread_t thrdID; /* thread ID */ qWrkCmd_t tCurrCmd; /* current command to be carried out by worker */ obj_t *pUsrp; /* pointer to an object meaningful for current user pointer (e.g. queue pUsr data elemt) */ wtp_t *pWtp; /* my worker thread pool (important if only the work thread instance is passed! */ pthread_cond_t condExitDone; /* signaled when the thread exit is done (once per thread existance) */ pthread_mutex_t mut; - int bShutdownRqtd; /* shutdown for this thread requested? 0 - no , 1 - yes */ + bool bShutdownRqtd; /* shutdown for this thread requested? 0 - no , 1 - yes */ uchar *pszDbgHdr; /* header string for debug messages */ } wti_t; diff --git a/runtime/wtp.c b/runtime/wtp.c index df39daa3..611b3f25 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -82,7 +82,6 @@ static rsRetVal NotImplementedDummy() { return RS_RET_OK; } /* Standard-Constructor for the wtp object */ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! */ - pThis->bOptimizeUniProc = glbl.GetOptimizeUniProc(); pthread_mutex_init(&pThis->mut, NULL); pthread_mutex_init(&pThis->mutThrdShutdwn, NULL); pthread_cond_init(&pThis->condThrdTrm, NULL); @@ -509,14 +508,6 @@ wtpStartWrkr(wtp_t *pThis, int bLockMutex) dbgprintf("%s: started with state %d, num workers now %d\n", wtpGetDbgHdr(pThis), iState, pThis->iCurNumWrkThrd); - /* we try to give the starting worker a little boost. It won't help much as we still - * hold the queue's mutex, but at least it has a chance to start on a single-CPU system. - */ -# if !defined(__hpux) /* pthread_yield is missing there! */ - if(pThis->bOptimizeUniProc) - pthread_yield(); -# endif - /* indicate we just started a worker and would like to see it running */ wtpSetInactivityGuard(pThis, 1, MUTEX_ALREADY_LOCKED); diff --git a/runtime/wtp.h b/runtime/wtp.h index b9cb07c5..1ce171cc 100644 --- a/runtime/wtp.h +++ b/runtime/wtp.h @@ -52,13 +52,12 @@ typedef enum { /* the worker thread pool (wtp) object */ typedef struct wtp_s { BEGINobjInstance; - int bOptimizeUniProc; /* cache for the equally-named global setting, pulled at time of queue creation */ wtpState_t wtpState; int iNumWorkerThreads;/* number of worker threads to use */ int iCurNumWrkThrd;/* current number of active worker threads */ struct wti_s **pWrkr;/* array with control structure for the worker thread(s) associated with this wtp */ int toWrkShutdown; /* timeout for idle workers in ms, -1 means indefinite (0 is immediate) */ - int bInactivityGuard;/* prevents inactivity due to race condition */ + bool bInactivityGuard;/* prevents inactivity due to race condition */ rsRetVal (*pConsumer)(void *); /* user-supplied consumer function for dewtpd messages */ /* synchronization variables */ pthread_mutex_t mutThrdShutdwn; /* mutex to guard thread shutdown processing */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 11f4ebd9..16f08c21 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1340,8 +1340,6 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) } /* The rest is the actual MSG */ -dbgprintf("XXX: msg set msg offset %d, str: '%s', prev '%s'\n", p2parse - pMsg->pszRawMsg, pMsg->pszRawMsg + (p2parse - pMsg->pszRawMsg), p2parse); -// MsgSetMSG(pMsg, (char*)p2parse); MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); ENDfunc -- cgit v1.2.3 From 7b3dad877e9faf53609a9f0986925a157f9f6e9b Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 12:10:38 +0200 Subject: removed pthread_testcancel() as it is no longer necessary we usually stay long enough inside the actions, so there should be no problem with reaching a cancellation point. Actually, if we really need to cancel, the thread is in an output action (otherwise it would have willingly terminated). --- runtime/wti.c | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/runtime/wti.c b/runtime/wti.c index bfe8f863..f20812f8 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -327,27 +327,6 @@ wtiWorkerCancelCleanup(void *arg) /* generic worker thread framework - * - * Some special comments below, so that they do not clutter the main function code: - * - * On the use of pthread_testcancel(): - * Now make sure we can get canceled - it is not specified if pthread_setcancelstate() is - * a cancellation point in itself. As we run most of the time without cancel enabled, I fear - * we may never get cancelled if we do not create a cancellation point ourselfs. - * - * On the use of pthread_yield(): - * We yield to give the other threads a chance to obtain the mutex. If we do not - * do that, this thread may very well aquire the mutex again before another thread - * has even a chance to run. The reason is that mutex operations are free to be - * implemented in the quickest possible way (and they typically are!). That is, the - * mutex lock/unlock most probably just does an atomic memory swap and does not necessarily - * schedule other threads waiting on the same mutex. That can lead to the same thread - * aquiring the mutex ever and ever again while all others are starving for it. We - * have exactly seen this behaviour when we deliberately introduced a long-running - * test action which basically did a sleep. I understand that with real actions the - * likelihood of this starvation condition is very low - but it could still happen - * and would be very hard to debug. The yield() is a sure fix, its performance overhead - * should be well accepted given the above facts. -- rgerhards, 2008-01-10 */ #pragma GCC diagnostic ignored "-Wempty-body" rsRetVal @@ -375,7 +354,6 @@ wtiWorker(wti_t *pThis) while(1) { /* loop will be broken below - need to do mutex locks */ /* process any pending thread requests */ wtpProcessThrdChanges(pWtp); - pthread_testcancel(); /* see big comment in function header */ /* if we have a rate-limiter set for this worker pool, let's call it. Please * keep in mind that the rate-limiter may hold us for an extended period -- cgit v1.2.3 From cbad28915a847d9ad461d8099e90ad7903d537a3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 12:39:06 +0200 Subject: optimized action.c a bit --- action.c | 34 +++++++++++++++------------------- action.h | 11 ++++++----- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/action.c b/action.c index 4ab5b81f..22755e86 100644 --- a/action.c +++ b/action.c @@ -195,6 +195,7 @@ rsRetVal actionDestruct(action_t *pThis) pthread_mutex_destroy(&pThis->mutActExec); d_free(pThis->pszName); d_free(pThis->ppTpl); + d_free(pThis->ppMsgs); d_free(pThis); RETiRet; @@ -432,23 +433,17 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) int iSleepPeriod; int bCallAction; int iCancelStateSave; - uchar **ppMsgs; /* array of message pointers for doAction */ ASSERT(pAction != NULL); - /* create the array for doAction() message pointers */ - if((ppMsgs = calloc(pAction->iNumTpls, sizeof(uchar *))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - /* here we must loop to process all requested strings */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { switch(pAction->eParamPassing) { case ACT_STRING_PASSING: - CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(ppMsgs[i]))); + CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(pAction->ppMsgs[i]))); break; case ACT_ARRAY_PASSING: - CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(ppMsgs[i]))); + CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(pAction->ppMsgs[i]))); break; default:assert(0); /* software bug if this happens! */ } @@ -487,7 +482,7 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) if(bCallAction) { /* call configured action */ - iRet = pAction->pMod->mod.om.doAction(ppMsgs, pMsg->msgFlags, pAction->pModData); + iRet = pAction->pMod->mod.om.doAction(pAction->ppMsgs, pMsg->msgFlags, pAction->pModData); if(iRet == RS_RET_SUSPENDED) { dbgprintf("Action requested to be suspended, done that.\n"); actionSuspend(pAction, getActNow(pAction)); @@ -506,25 +501,27 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) finalize_it: /* cleanup */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { - if(ppMsgs[i] != NULL) { + if(pAction->ppMsgs[i] != NULL) { switch(pAction->eParamPassing) { case ACT_ARRAY_PASSING: iArr = 0; - while(((char **)ppMsgs[i])[iArr] != NULL) - d_free(((char **)ppMsgs[i])[iArr++]); - d_free(ppMsgs[i]); + while(((char **)pAction->ppMsgs[i])[iArr] != NULL) { + d_free(((char **)pAction->ppMsgs[i])[iArr++]); + ((char **)pAction->ppMsgs[i])[iArr++] = NULL; + } + d_free(pAction->ppMsgs[i]); + pAction->ppMsgs[i] = NULL; break; case ACT_STRING_PASSING: - d_free(ppMsgs[i]); + d_free(pAction->ppMsgs[i]); break; default: assert(0); } } } - d_free(ppMsgs); - msgDestruct(&pMsg); /* we are now finished with the message */ + msgDestruct(&pMsg); /* we are now finished with the message */ RETiRet; } #pragma GCC diagnostic warning "-Wempty-body" @@ -906,9 +903,8 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques */ if(pAction->iNumTpls > 0) { /* we first need to create the template pointer array */ - if((pAction->ppTpl = calloc(pAction->iNumTpls, sizeof(struct template *))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc(pAction->ppTpl = (struct template *)calloc(pAction->iNumTpls, sizeof(struct template *))); + CHKmalloc(pAction->ppMsgs = (uchar**) malloc(pAction->iNumTpls * sizeof(uchar *))); } for(i = 0 ; i < pAction->iNumTpls ; ++i) { diff --git a/action.h b/action.h index 2a1487a5..a6c02a8b 100644 --- a/action.h +++ b/action.h @@ -43,10 +43,10 @@ struct action_s { time_t tActNow; /* the current time for an action execution. Initially set to -1 and populated on an as-needed basis. This is a performance optimization. */ time_t tLastExec; /* time this action was last executed */ - int bExecWhenPrevSusp;/* execute only when previous action is suspended? */ + bool bExecWhenPrevSusp;/* execute only when previous action is suspended? */ int iSecsExecOnceInterval; /* if non-zero, minimum seconds to wait until action is executed again */ short bEnabled; /* is the related action enabled (1) or disabled (0)? */ - short bSuspended; /* is the related action temporarily suspended? */ + bool bSuspended; /* is the related action temporarily suspended? */ time_t ttResumeRtry; /* when is it time to retry the resume? */ int iResumeInterval;/* resume interval for this action */ int iResumeRetryCount;/* how often shall we retry a suspended action? (-1 --> eternal) */ @@ -57,7 +57,7 @@ struct action_s { time_t tLastOccur; /* time last occurence was seen (for timing them out) */ struct modInfo_s *pMod;/* pointer to output module handling this selector */ void *pModData; /* pointer to module data - content is module-specific */ - short bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ + bool bRepMsgHasMsg; /* "message repeated..." has msg fragment in it (0-no, 1-yes) */ short f_ReduceRepeated;/* reduce repeated lines 0 - no, 1 - yes */ int f_prevcount; /* repetition cnt of prevline */ int f_repeatcount; /* number of "repeated" msgs */ @@ -66,14 +66,15 @@ struct action_s { int iNumTpls; /* number of array entries for template element below */ struct template **ppTpl;/* array of template to use - strings must be passed to doAction * in this order. */ - struct msg* f_pMsg; /* pointer to the message (this will replace the other vars with msg + msg_t *f_pMsg; /* pointer to the message (this will replace the other vars with msg * content later). This is preserved after the message has been * processed - it is also used to detect duplicates. */ qqueue_t *pQueue; /* action queue */ SYNC_OBJ_TOOL; /* required for mutex support */ - uchar *pszName; /* action name (for documentation) */ pthread_mutex_t mutActExec; /* mutex to guard actual execution of doAction for single-threaded modules */ + uchar *pszName; /* action name (for documentation) */ + uchar **ppMsgs; /* pointer to action-calling parameters (kept in structure to save alloc() time!) */ }; typedef struct action_s action_t; -- cgit v1.2.3 From 953d015f440224321796105464294006838b5302 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 14:25:27 +0200 Subject: switched default to keep in line with traditional syslogd behaviour --- doc/rsyslog_conf_global.html | 7 +++++-- tools/omfile.c | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/rsyslog_conf_global.html b/doc/rsyslog_conf_global.html index d58bcac0..332b6282 100644 --- a/doc/rsyslog_conf_global.html +++ b/doc/rsyslog_conf_global.html @@ -196,13 +196,16 @@ supported in order to be compliant to the upcoming new syslog RFC series. of the output file. The higher the number, the better the compression, but also the more CPU is required for zipping.
    • $OMFileIOBufferSize <size_nbr>, default 4k, size of the buffer used to writing output data. The larger the buffer, the potentially better performance is. The default of 4k is quite conservative, it is useful to go up to 64k, and 128K if you used gzip compression (then, even higher sizes may make sense)
      • -
    • $OMFileFlushOnTXEnd <[on/off]>, default off, by default, omfile +
    • $OMFileFlushOnTXEnd <[on/off]>, default on. Omfile has the +capability to writes output using a buffered writer. Disk writes are only done when the buffer is full. So if an error happens during that write, data is potentially lost. In cases where this is unacceptable, set $OMFileFlushOnTXEnd to on. Then, data is written at the end of each transaction (for pre-v5 this means after each log message) and the usual error recovery thus can handle write errors without data loss. Note that this option -severely reduces the effect of zip compression.
      • +severely reduces the effect of zip compression and should be switched to off +for that use case. Note that the default -off- is primarily an aid to preserve +the traditional syslogd behaviour.       +generic syslog application design
    • description of rsyslog modules
    • rsyslogd man page (heavily outdated)
    • $RepeatedMsgContainsOriginalMsg [on/off] - "last message repeated n times" messages, if generated, have a different format that contains the message that is being repeated. Note that only the first "n" characters are included, with n to be at least 80 characters, most diff --git a/tools/omfile.c b/tools/omfile.c index 675d313e..b62d9c57 100644 --- a/tools/omfile.c +++ b/tools/omfile.c @@ -102,7 +102,7 @@ static uid_t dirGID; /* GID to be used for newly created directories */ static int bCreateDirs; /* auto-create directories for dynaFiles: 0 - no, 1 - yes */ static int bEnableSync = 0;/* enable syncing of files (no dash in front of pathname in conf): 0 - no, 1 - yes */ static int iZipLevel = 0; /* zip compression mode (0..9 as usual) */ -static bool bFlushOnTXEnd = 0;/* flush write buffers when transaction has ended? */ +static bool bFlushOnTXEnd = 1;/* flush write buffers when transaction has ended? */ static int iIOBufSize = IOBUF_DFLT_SIZE; /* size of an io buffer */ static int iFlushInterval = FLUSH_INTRVL_DFLT; /* how often flush the output buffer on inactivity? */ static uchar *pszTplName = NULL; /* name of the default template to use */ @@ -715,7 +715,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a bCreateDirs = 1; bEnableSync = 0; iZipLevel = 0; - bFlushOnTXEnd = 0; + bFlushOnTXEnd = 1; iIOBufSize = IOBUF_DFLT_SIZE; iFlushInterval = FLUSH_INTRVL_DFLT; if(pszTplName != NULL) { -- cgit v1.2.3 From 3abf567d2b57014381eda49018a0e2c21fa1b853 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 19 Jun 2009 16:07:17 +0200 Subject: optimized template string generation --- action.c | 103 ++++++++++++++++++++++++++++++++++++---------------- action.h | 1 + runtime/datetime.c | 8 ++-- runtime/stringbuf.c | 3 -- runtime/stringbuf.h | 2 +- template.c | 97 +++++++++++++++++++++++-------------------------- template.h | 2 +- 7 files changed, 125 insertions(+), 91 deletions(-) diff --git a/action.c b/action.c index 22755e86..d214e808 100644 --- a/action.c +++ b/action.c @@ -178,6 +178,7 @@ actionResetQueueParams(void) */ rsRetVal actionDestruct(action_t *pThis) { + int i; DEFiRet; ASSERT(pThis != NULL); @@ -195,7 +196,33 @@ rsRetVal actionDestruct(action_t *pThis) pthread_mutex_destroy(&pThis->mutActExec); d_free(pThis->pszName); d_free(pThis->ppTpl); + + /* message ptr cleanup */ + for(i = 0 ; i < pThis->iNumTpls ; ++i) { + if(pThis->ppMsgs[i] != NULL) { + switch(pThis->eParamPassing) { + case ACT_ARRAY_PASSING: +#if 0 /* later! */ + iArr = 0; + while(((char **)pThis->ppMsgs[i])[iArr] != NULL) { + d_free(((char **)pThis->ppMsgs[i])[iArr++]); + ((char **)pThis->ppMsgs[i])[iArr++] = NULL; + } + d_free(pThis->ppMsgs[i]); + pThis->ppMsgs[i] = NULL; +#endif + break; + case ACT_STRING_PASSING: + d_free(pThis->ppMsgs[i]); + break; + default: + assert(0); + } + } + } d_free(pThis->ppMsgs); + d_free(pThis->lenMsgs); + d_free(pThis); RETiRet; @@ -440,7 +467,7 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) for(i = 0 ; i < pAction->iNumTpls ; ++i) { switch(pAction->eParamPassing) { case ACT_STRING_PASSING: - CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(pAction->ppMsgs[i]))); + CHKiRet(tplToString(pAction->ppTpl[i], pMsg, &(pAction->ppMsgs[i]), &(pAction->lenMsgs[i]))); break; case ACT_ARRAY_PASSING: CHKiRet(tplToArray(pAction->ppTpl[i], pMsg, (uchar***) &(pAction->ppMsgs[i]))); @@ -513,7 +540,6 @@ finalize_it: pAction->ppMsgs[i] = NULL; break; case ACT_STRING_PASSING: - d_free(pAction->ppMsgs[i]); break; default: assert(0); @@ -727,32 +753,13 @@ finalize_it: } -/* call the configured action. Does all necessary housekeeping. - * rgerhards, 2007-08-01 - * FYI: currently, this function is only called from the queue - * consumer. So we (conceptually) run detached from the input - * threads (which also means we may run much later than when the - * message was generated). +/* helper to actonCallAction, mostly needed because of this damn + * pthread_cleanup_push() POSIX macro... */ -#pragma GCC diagnostic ignored "-Wempty-body" -rsRetVal -actionCallAction(action_t *pAction, msg_t *pMsg) +static rsRetVal +doActionCallAction(action_t *pAction, msg_t *pMsg) { DEFiRet; - int iCancelStateSave; - - ISOBJ_TYPE_assert(pMsg, msg); - ASSERT(pAction != NULL); - - /* Make sure nodbody else modifies/uses this action object. Right now, this - * is important because of "message repeated n times" processing and potentially - * multiple worker threads. -- rgerhards, 2007-12-11 - */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - LockObj(pAction); - pthread_cleanup_push(mutexCancelCleanup, pAction->Sync_mut); - pthread_setcancelstate(iCancelStateSave, NULL); - /* first, we need to check if this is a disabled * entry. If so, we must not further process it. * rgerhards 2005-09-26 @@ -813,10 +820,43 @@ actionCallAction(action_t *pAction, msg_t *pMsg) } finalize_it: - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - UnlockObj(pAction); - pthread_cleanup_pop(0); /* remove mutex cleanup handler */ - pthread_setcancelstate(iCancelStateSave, NULL); + RETiRet; +} + +/* call the configured action. Does all necessary housekeeping. + * rgerhards, 2007-08-01 + * FYI: currently, this function is only called from the queue + * consumer. So we (conceptually) run detached from the input + * threads (which also means we may run much later than when the + * message was generated). + */ +#pragma GCC diagnostic ignored "-Wempty-body" +rsRetVal +actionCallAction(action_t *pAction, msg_t *pMsg) +{ + DEFiRet; + int iCancelStateSave; + + ISOBJ_TYPE_assert(pMsg, msg); + ASSERT(pAction != NULL); + + /* We need to lock the mutex only for repeated line processing. + * rgerhards, 2009-06-19 + */ + if(pAction->f_ReduceRepeated == 1) { + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + LockObj(pAction); + pthread_cleanup_push(mutexCancelCleanup, pAction->Sync_mut); + pthread_setcancelstate(iCancelStateSave, NULL); + iRet = doActionCallAction(pAction, pMsg); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + UnlockObj(pAction); + pthread_cleanup_pop(0); /* remove mutex cleanup handler */ + pthread_setcancelstate(iCancelStateSave, NULL); + } else { + iRet = doActionCallAction(pAction, pMsg); + } + RETiRet; } #pragma GCC diagnostic warning "-Wempty-body" @@ -903,8 +943,9 @@ addAction(action_t **ppAction, modInfo_t *pMod, void *pModData, omodStringReques */ if(pAction->iNumTpls > 0) { /* we first need to create the template pointer array */ - CHKmalloc(pAction->ppTpl = (struct template *)calloc(pAction->iNumTpls, sizeof(struct template *))); - CHKmalloc(pAction->ppMsgs = (uchar**) malloc(pAction->iNumTpls * sizeof(uchar *))); + CHKmalloc(pAction->ppTpl = (struct template **)calloc(pAction->iNumTpls, sizeof(struct template *))); + CHKmalloc(pAction->ppMsgs = (uchar**) calloc(pAction->iNumTpls, sizeof(uchar *))); + CHKmalloc(pAction->lenMsgs = (size_t*) calloc(pAction->iNumTpls, sizeof(size_t))); } for(i = 0 ; i < pAction->iNumTpls ; ++i) { diff --git a/action.h b/action.h index a6c02a8b..579a1215 100644 --- a/action.h +++ b/action.h @@ -75,6 +75,7 @@ struct action_s { pthread_mutex_t mutActExec; /* mutex to guard actual execution of doAction for single-threaded modules */ uchar *pszName; /* action name (for documentation) */ uchar **ppMsgs; /* pointer to action-calling parameters (kept in structure to save alloc() time!) */ + size_t *lenMsgs; /* length of message in ppMsgs */ }; typedef struct action_s action_t; diff --git a/runtime/datetime.c b/runtime/datetime.c index b5514e7c..8f4bfa10 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -556,7 +556,7 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf, size_t iLenDst) */ assert(ts != NULL); assert(pBuf != NULL); - assert(iLenDst < 15); + assert(iLenDst >= 15); pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; @@ -582,7 +582,7 @@ int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf, size_t iLenDst) /* see note in formatTimestampToMySQL, applies here as well */ assert(ts != NULL); assert(pBuf != NULL); - assert(iLenDst < 20); + assert(iLenDst >= 20); pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; @@ -666,7 +666,7 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) BEGINfunc assert(ts != NULL); assert(pBuf != NULL); - assert(iLenBuf < 33); + assert(iLenBuf >= 33); /* start with fixed parts */ /* year yyyy */ @@ -741,7 +741,7 @@ int formatTimestamp3164(struct syslogTime *ts, char* pBuf, size_t iLenBuf) assert(ts != NULL); assert(pBuf != NULL); - assert(iLenBuf < 16); + assert(iLenBuf >= 16); pBuf[0] = monthNames[(ts->month - 1)% 12][0]; pBuf[1] = monthNames[(ts->month - 1) % 12][1]; diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index e7fa8c41..d3ddf33a 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -461,14 +461,11 @@ cstrFinalize(cstr_t *pThis) DEFiRet; rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - assert(pThis->bIsFinalized == 0); - if(pThis->iStrLen > 0) { /* terminate string only if one exists */ CHKiRet(cstrAppendChar(pThis, '\0')); --pThis->iStrLen; /* do NOT count the \0 byte */ } - pThis->bIsFinalized = 1; finalize_it: RETiRet; diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index b4fc3f3c..9d2e7865 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -48,7 +48,7 @@ typedef struct cstr_s size_t iBufSize; /**< current maximum size of the string buffer */ size_t iStrLen; /**< length of the string in characters. */ unsigned short iAllocIncrement; /**< the amount of bytes the string should be expanded if it needs to */ - bool bIsFinalized; /**< is this object finished and ready for use? (a debug aid, may be removed later TODO 2009-06-16) */ + bool bIsForeignBuf; /**< is pBuf a buffer provided by someone else? */ } cstr_t; diff --git a/template.c b/template.c index 93d0539c..43a54d98 100644 --- a/template.c +++ b/template.c @@ -49,59 +49,61 @@ static struct template *tplLastStatic = NULL; /* last static element of the temp -/* This functions converts a template into a string. It should - * actually be in template.c, but this requires larger re-structuring - * of the code (because all the property-access functions are static - * to this module). I have placed it next to the iov*() functions, as - * it is somewhat similiar in what it does. - * - * The function takes a pointer to a template and a pointer to a msg object. - * It the creates a string based on the template definition. A pointer - * to that string is returned to the caller. The caller MUST FREE that - * pointer when it is no longer needed. If the function fails, NULL - * is returned. - * If memory allocation fails in this function, we silently return - * NULL. The reason is that we can not do anything against it. And - * if we raise an alert, the memory situation might become even - * worse. So we prefer to let the caller deal with it. - * rgerhards, 2007-07-03 +/* helper to tplToString, extends buffer */ +#define ALLOC_INC 128 +static inline rsRetVal ExtendBuf(uchar **pBuf, size_t *pLenBuf, size_t iMinSize) +{ + uchar *pNewBuf; + size_t iNewSize; + DEFiRet; + + iNewSize = (iMinSize / ALLOC_INC + 1) * ALLOC_INC; + CHKmalloc(pNewBuf = (uchar*) realloc(*pBuf, iNewSize)); + *pBuf = pNewBuf; + *pLenBuf = iNewSize; +dbgprintf("extend buf to at least %ld, done %ld\n", iMinSize, iNewSize); + +finalize_it: + RETiRet; +} + + +/* This functions converts a template into a string. * - * rgerhards, 2007-09-05: I changed the interface to use the standard iRet - * "calling sequence". This greatly eases complexity when it comes to handling - * errors in called modules (plus, it is much nicer). + * The function takes a pointer to a template and a pointer to a msg object + * as well as a pointer to an output buffer and its size. Note that the output + * buffer pointer may be NULL, size 0, in which case a new one is allocated. + * The outpub buffer is grown as required. It is the caller's duty to free the + * buffer when it is done. Note that it is advisable to reuse memory, as this + * offers big performance improvements. + * rewritten 2009-06-19 rgerhards */ -rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) +rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t *pLenBuf) { DEFiRet; struct templateEntry *pTpe; - cstr_t *pCStr; + int iBuf; unsigned short bMustBeFreed; uchar *pVal; size_t iLenVal; assert(pTpl != NULL); assert(pMsg != NULL); - assert(ppSz != NULL); + assert(ppBuf != NULL); + assert(pLenBuf != NULL); /* loop through the template. We obtain one value * and copy it over to our dynamic string buffer. Then, we * free the obtained value (if requested). We continue this * loop until we got hold of all values. */ - CHKiRet(cstrConstruct(&pCStr)); - pTpe = pTpl->pEntryRoot; + iBuf = 0; while(pTpe != NULL) { if(pTpe->eEntryType == CONSTANT) { - CHKiRet_Hdlr(rsCStrAppendStrWithLen(pCStr, - (uchar *) pTpe->data.constant.pConstant, - pTpe->data.constant.iLenConstant) - ) { - dbgprintf("error %d during tplToString()\n", iRet); - /* it does not make sense to continue now */ - cstrDestruct(&pCStr); - FINALIZE; - } + pVal = (uchar*) pTpe->data.constant.pConstant; + iLenVal = pTpe->data.constant.iLenConstant; + bMustBeFreed = 0; } else if(pTpe->eEntryType == FIELD) { pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &iLenVal, &bMustBeFreed); /* we now need to check if we should use SQL option. In this case, @@ -114,30 +116,23 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz) doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 1); else if(pTpl->optFormatForSQL == 2) doSQLEscape(&pVal, &iLenVal, &bMustBeFreed, 0); - /* value extracted, so lets copy */ - CHKiRet_Hdlr(rsCStrAppendStrWithLen(pCStr, (uchar*) pVal, iLenVal)) { - dbgprintf("error %d during tplToString()\n", iRet); - /* it does not make sense to continue now */ - cstrDestruct(&pCStr); - if(bMustBeFreed) - free(pVal); - FINALIZE; - } - if(bMustBeFreed) - free(pVal); } + /* got source, now copy over */ + if(iBuf + iLenVal + 1 >= *pLenBuf) /* we reserve one char for the final \0! */ + CHKiRet(ExtendBuf(ppBuf, pLenBuf, iBuf + iLenVal + 1)); + + memcpy(*ppBuf + iBuf, pVal, iLenVal); + iBuf += iLenVal; + + if(bMustBeFreed) + free(pVal); + pTpe = pTpe->pNext; } - /* we are done with the template, now let's convert the result into a - * "real" (usable) string and discard the helper structures. - */ - CHKiRet(cstrFinalize(pCStr)); - CHKiRet(cstrConvSzStrAndDestruct(pCStr, &pVal, 0)); + (*ppBuf)[iBuf] = '\0'; /* space was reserved above (see copy) */ finalize_it: - *ppSz = (iRet == RS_RET_OK) ? pVal : NULL; - RETiRet; } diff --git a/template.h b/template.h index 9d794f66..73ed0f84 100644 --- a/template.h +++ b/template.h @@ -127,7 +127,7 @@ void tplLastStaticInit(struct template *tpl); * rgerhards, 2007-08-06 */ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr); -rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz); +rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar** ppSz, size_t *); rsRetVal doSQLEscape(uchar **pp, size_t *pLen, unsigned short *pbMustBeFreed, int escapeMode); rsRetVal templateInit(); -- cgit v1.2.3 From 5defa14fed704cabe2074f4bdbb6d389d6dee7cf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 22 Jun 2009 15:32:01 +0200 Subject: some post-merge cleanup --- action.c | 7 ++----- action.h | 2 -- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/action.c b/action.c index 8c41b638..5d591a5d 100644 --- a/action.c +++ b/action.c @@ -608,9 +608,8 @@ rsRetVal actionDbgPrint(action_t *pThis) /* prepare the calling parameters for doAction() * rgerhards, 2009-05-07 */ -static rsRetVal prepareDoActionParams(action_t *pAction, msg_t *pMsg, uchar ***pppMsgs) +static rsRetVal prepareDoActionParams(action_t *pAction, msg_t *pMsg) { - uchar **ppMsgs = *pppMsgs; int i; DEFiRet; @@ -630,7 +629,6 @@ static rsRetVal prepareDoActionParams(action_t *pAction, msg_t *pMsg, uchar ***p } finalize_it: - *pppMsgs = ppMsgs; RETiRet; } @@ -683,14 +681,13 @@ static rsRetVal cleanupDoActionParams(action_t *pAction) rsRetVal actionCallDoAction(action_t *pThis, msg_t *pMsg) { - uchar **ppMsgs; /* array of message pointers for doAction */ DEFiRet; ASSERT(pThis != NULL); ISOBJ_TYPE_assert(pMsg, msg); DBGPRINTF("entering actionCalldoAction(), state: %s\n", getActStateName(pThis)); - CHKiRet(prepareDoActionParams(pThis, pMsg, &ppMsgs)); + CHKiRet(prepareDoActionParams(pThis, pMsg)); pThis->bHadAutoCommit = 0; iRet = pThis->pMod->mod.om.doAction(pThis->ppMsgs, pMsg->msgFlags, pThis->pModData); diff --git a/action.h b/action.h index 6155ee36..a29936e5 100644 --- a/action.h +++ b/action.h @@ -56,8 +56,6 @@ struct action_s { int iSecsExecOnceInterval; /* if non-zero, minimum seconds to wait until action is executed again */ action_state_t eState; /* current state of action */ int bHadAutoCommit; /* did an auto-commit happen during doAction()? */ - //short bEnabled; /* is the related action enabled (1) or disabled (0)? */ - //short bSuspended; /* is the related action temporarily suspended? */ time_t ttResumeRtry; /* when is it time to retry the resume? */ int iResumeInterval;/* resume interval for this action */ int iResumeRetryCount;/* how often shall we retry a suspended action? (-1 --> eternal) */ -- cgit v1.2.3 From 236bdb5cfaf56d9192a9911f140ba3da95b390b6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 22 Jun 2009 16:06:29 +0200 Subject: bugfix: huge memory leak in queue engine (made rsyslogd unusable in production). Occured if at least one queue was in direct mode (the default for action queues). --- ChangeLog | 3 +++ runtime/queue.c | 1 + 2 files changed, 4 insertions(+) diff --git a/ChangeLog b/ChangeLog index 70d58419..f2d6aac6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +- bugfix: huge memory leak in queue engine (made rsyslogd unusable in + production). Occured if at least one queue was in direct mode + (the default for action queues) --------------------------------------------------------------------------- Version 5.1.0 [DEVEL] (rgerhards), 2009-05-29 diff --git a/runtime/queue.c b/runtime/queue.c index e18ce3d7..9462f865 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1089,6 +1089,7 @@ static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) singleBatch.nElem = 1; /* there always is only one in direct mode */ singleBatch.pElem = &batchObj; iRet = pThis->pConsumer(pThis->pUsr, &singleBatch); + objDestruct(pUsr); RETiRet; } -- cgit v1.2.3 From 8962c6b011f70bb7033d58641c2e4f3a73e7801a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 22 Jun 2009 17:35:16 +0200 Subject: reduced memory footprint / "memory leak" Testing has shown that at least the glibc malloc() subsystem returns memory to the OS far too late in our case. So we need to help it a bit, by calling malloc_trim(), which will tell the alloc subsystem to consolidate and return to the OS. --- configure.ac | 2 +- runtime/msg.c | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 4a040e6a..13d922a3 100644 --- a/configure.ac +++ b/configure.ac @@ -98,7 +98,7 @@ AC_TYPE_SIGNAL AC_FUNC_STAT AC_FUNC_STRERROR_R AC_FUNC_VPRINTF -AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r epoll_wait getline]) +AC_CHECK_FUNCS([flock basename alarm clock_gettime gethostbyname gethostname gettimeofday localtime_r memset mkdir regcomp select setid socket strcasecmp strchr strdup strerror strndup strnlen strrchr strstr strtol strtoul uname ttyname_r epoll_wait getline malloc_trim]) # Check for MAXHOSTNAMELEN AC_MSG_CHECKING(for MAXHOSTNAMELEN) diff --git a/runtime/msg.c b/runtime/msg.c index 4b7a0ad4..e47f104c 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "rsyslog.h" #include "srUtils.h" #include "stringbuf.h" @@ -550,6 +551,23 @@ CODESTARTobjDestruct(msg) MsgUnlock(pThis); # endif funcDeleteMutex(pThis); + /* now we need to do our own optimization. Testing has shown that at least the glibc + * malloc() subsystem returns memory to the OS far too late in our case. So we need + * to help it a bit, by calling malloc_trim(), which will tell the alloc subsystem + * to consolidate and return to the OS. We keep 128K for our use, as a safeguard + * to too-frequent reallocs. But more importantly, we call this hook only every + * 100,000 messages (which is an approximation, as we do not work with atomic + * operations on the counter. --- rgerhards, 2009-06-22. + */ +# if HAVE_MALLOC_TRIM + { /* standard C requires a new block for a new variable definition! */ + static unsigned iTrimCtr = 1; + if(iTrimCtr ++ % 100000 == 0) { + iTrimCtr = 1; + malloc_trim(128*1024); + } + } +# endif } else { MsgUnlock(pThis); pThis = NULL; /* tell framework not to destructing the object! */ -- cgit v1.2.3 From 221dc8a3224dcb59a7dd3158716a8d24cee71618 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 22 Jun 2009 18:19:10 +0200 Subject: some more optimizations of the msg_t object (minor) --- runtime/datetime.c | 5 ++--- runtime/datetime.h | 2 +- runtime/msg.c | 37 +++++++++++++++++-------------------- runtime/msg.h | 41 ++++++++++++++++++++--------------------- 4 files changed, 40 insertions(+), 45 deletions(-) diff --git a/runtime/datetime.c b/runtime/datetime.c index 8f4bfa10..c0e145af 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -614,10 +614,10 @@ int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf, size_t iLenDst) * buffer that will receive the resulting string. The function * returns the size of the timestamp written in bytes (without * the string terminator). If 0 is returend, an error occured. - * The buffer must be at least 10 bytes large. + * The buffer must be at least 7 bytes large. * rgerhards, 2008-06-06 */ -int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf, size_t iLenBuf) +int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf) { int iBuf; int power; @@ -626,7 +626,6 @@ int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf, size_t iLenBuf) assert(ts != NULL); assert(pBuf != NULL); - assert(iLenBuf >= 10); iBuf = 0; if(ts->secfracPrecision > 0) diff --git a/runtime/datetime.h b/runtime/datetime.h index efb0a0af..1ea99aba 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -42,7 +42,7 @@ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ int (*formatTimestampToPgSQL)(struct syslogTime *ts, char *pDst, size_t iLenDst); int (*formatTimestamp3339)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); int (*formatTimestamp3164)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); - int (*formatTimestampSecFrac)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); + int (*formatTimestampSecFrac)(struct syslogTime *ts, char* pBuf); ENDinterface(datetime) #define datetimeCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ /* interface changes: diff --git a/runtime/msg.c b/runtime/msg.c index e47f104c..2ff63d9c 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -335,14 +335,15 @@ static void MsgLockingDummy(msg_t __attribute__((unused)) *pMsg) static void MsgPrepareEnqueueLockingCase(msg_t *pThis) { int iErr; + pthread_mutexattr_t mutAttr; BEGINfunc assert(pThis != NULL); - iErr = pthread_mutexattr_init(&pThis->mutAttr); + iErr = pthread_mutexattr_init(&mutAttr); if(iErr != 0) { dbgprintf("error initializing mutex attribute in %s:%d, trying to continue\n", __FILE__, __LINE__); } - iErr = pthread_mutexattr_settype(&pThis->mutAttr, PTHREAD_MUTEX_RECURSIVE); + iErr = pthread_mutexattr_settype(&mutAttr, PTHREAD_MUTEX_RECURSIVE); if(iErr != 0) { dbgprintf("ERROR setting mutex attribute to recursive in %s:%d, trying to continue " "but we will probably either abort or hang soon\n", @@ -352,14 +353,14 @@ static void MsgPrepareEnqueueLockingCase(msg_t *pThis) * down. We should do that over time. -- rgerhards, 2008-07-14 */ } - pthread_mutex_init(&pThis->mut, &pThis->mutAttr); + pthread_mutex_init(&pThis->mut, &mutAttr); /* we do no longer need the attribute. According to the * POSIX spec, we can destroy it without affecting the * initialized mutex (that used the attribute). * rgerhards, 2008-07-14 */ - pthread_mutexattr_destroy(&pThis->mutAttr); + pthread_mutexattr_destroy(&mutAttr); pThis->bDoLock = 1; ENDfunc } @@ -531,10 +532,8 @@ CODESTARTobjDestruct(msg) free(pThis->pszRcvFromIP); free(pThis->pszRcvdAt3164); free(pThis->pszRcvdAt3339); - free(pThis->pszRcvdAt_SecFrac); free(pThis->pszRcvdAt_MySQL); free(pThis->pszRcvdAt_PgSQL); - free(pThis->pszTIMESTAMP_SecFrac); free(pThis->pszTIMESTAMP_MySQL); free(pThis->pszTIMESTAMP_PgSQL); if(pThis->pCSProgName != NULL) @@ -1007,15 +1006,14 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return(pM->pszTIMESTAMP3339); case tplFmtSecFrac: - MsgLock(pM); - if(pM->pszTIMESTAMP_SecFrac == NULL) { - if((pM->pszTIMESTAMP_SecFrac = malloc(10)) == NULL) { - MsgUnlock(pM); - return ""; /* TODO: check this: can it cause a free() of constant memory?) */ + if(pM->pszTIMESTAMP_SecFrac[0] == '\0') { + MsgLock(pM); + /* re-check, may have changed while we did not hold lock */ + if(pM->pszTIMESTAMP_SecFrac[0] == '\0') { + datetime.formatTimestampSecFrac(&pM->tTIMESTAMP, pM->pszTIMESTAMP_SecFrac); } - datetime.formatTimestampSecFrac(&pM->tTIMESTAMP, pM->pszTIMESTAMP_SecFrac, 10); + MsgUnlock(pM); } - MsgUnlock(pM); return(pM->pszTIMESTAMP_SecFrac); } ENDfunc @@ -1085,15 +1083,14 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return(pM->pszRcvdAt3339); case tplFmtSecFrac: - MsgLock(pM); - if(pM->pszRcvdAt_SecFrac == NULL) { - if((pM->pszRcvdAt_SecFrac = malloc(10)) == NULL) { - MsgUnlock(pM); - return ""; /* TODO: check this: can it cause a free() of constant memory?) */ + if(pM->pszRcvdAt_SecFrac[0] == '\0') { + MsgLock(pM); + /* re-check, may have changed while we did not hold lock */ + if(pM->pszRcvdAt_SecFrac[0] == '\0') { + datetime.formatTimestampSecFrac(&pM->tRcvdAt, pM->pszRcvdAt_SecFrac); } - datetime.formatTimestampSecFrac(&pM->tRcvdAt, pM->pszRcvdAt_SecFrac, 10); + MsgUnlock(pM); } - MsgUnlock(pM); return(pM->pszRcvdAt_SecFrac); } ENDfunc diff --git a/runtime/msg.h b/runtime/msg.h index ec18b29d..34983704 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -56,7 +56,6 @@ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ flowControl_t flowCtlType; /**< type of flow control we can apply, for enqueueing, needs not to be persisted because once data has entered the queue, this property is no longer needed. */ - pthread_mutexattr_t mutAttr; pthread_mutex_t mut; bool bDoLock; /* use the mutex? */ bool bParseHOSTNAME; /* should the hostname be parsed from the message? */ @@ -69,27 +68,37 @@ struct msg { */ short iSeverity; /* the severity 0..7 */ short iFacility; /* Facility code 0 .. 23*/ - uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we - * need to preserve cryptographic verifiers. */ - int iLenRawMsg; /* length of raw message */ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ short offMSG; /* offset at which the MSG part starts in pszRawMsg */ + short iLenInputName; /* Length of pszInputName */ + short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */ + int msgFlags; /* flags associated with this message */ + int iLenRawMsg; /* length of raw message */ int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ - uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ int iLenHOSTNAME; /* Length of HOSTNAME */ - uchar *pszRcvFrom; /* System message was received from */ int iLenRcvFrom; /* Length of pszRcvFrom */ - uchar *pszRcvFromIP; /* IP of system message was received from */ int iLenRcvFromIP; /* Length of pszRcvFromIP */ + uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we + * need to preserve cryptographic verifiers. */ + uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ + uchar *pszRcvFrom; /* System message was received from */ + uchar *pszRcvFromIP; /* IP of system message was received from */ uchar *pszInputName; /* name of the input module that submitted this message */ - int iLenInputName; /* Length of pszInputName */ - short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */ + char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ + char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ + char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ + char *pszRcvdAt_PgSQL; /* rcvdAt as PgSQL formatted string (always 21 characters) */ + char *pszTIMESTAMP3164; /* TIMESTAMP as RFC3164 formatted string (always 15 charcters) */ + char *pszTIMESTAMP3339; /* TIMESTAMP as RFC3339 formatted string (32 charcters at most) */ + char *pszTIMESTAMP_MySQL;/* TIMESTAMP as MySQL formatted string (always 14 charcters) */ + char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ cstr_t *pCSProgName; /* the (BSD) program name */ cstr_t *pCSStrucData; /* STRUCTURED-DATA */ cstr_t *pCSAPPNAME; /* APP-NAME */ cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ + ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp is used at later processing stages (namely in the output arena). Thanks to @@ -98,19 +107,7 @@ struct msg { enough to reliable, but I prefer to leave the subtle things to the OS, where it obviously is solved in way or another...). */ struct syslogTime tRcvdAt;/* time the message entered this program */ - char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ - char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ - char *pszRcvdAt_SecFrac;/* time just as fractional seconds (6 charcters) */ - char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ - char *pszRcvdAt_PgSQL; /* rcvdAt as PgSQL formatted string (always 21 characters) */ struct syslogTime tTIMESTAMP;/* (parsed) value of the timestamp */ - char *pszTIMESTAMP3164; /* TIMESTAMP as RFC3164 formatted string (always 15 charcters) */ - char *pszTIMESTAMP3339; /* TIMESTAMP as RFC3339 formatted string (32 charcters at most) */ - char *pszTIMESTAMP_MySQL;/* TIMESTAMP as MySQL formatted string (always 14 charcters) */ - char *pszTIMESTAMP_PgSQL;/* TIMESTAMP as PgSQL formatted string (always 21 characters) */ - char *pszTIMESTAMP_SecFrac;/* TIMESTAMP fractional seconds (always 6 characters) */ - int msgFlags; /* flags associated with this message */ - ruleset_t *pRuleset; /* ruleset to be used for processing this message */ /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ union { @@ -119,6 +116,8 @@ struct msg { } TAG; char pszTimestamp3164[16]; char pszTimestamp3339[33]; + char pszTIMESTAMP_SecFrac[7]; /* Note: a pointer is 64 bits/8 char, so this is actually fewer than a pointer! */ + char pszRcvdAt_SecFrac[7]; /* same as above. Both are fractional seconds for their respective timestamp */ }; -- cgit v1.2.3 From d61bcbbd0e99a8ffadfe1e0c5347dee9e04a0b1d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 22 Jun 2009 18:32:21 +0200 Subject: adapted (and improved) input batching to v5 engine --- runtime/queue.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/queue.c b/runtime/queue.c index 9462f865..1ae386e7 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2635,7 +2635,6 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* and finally enqueue the message */ CHKiRet(qqueueAdd(pThis, pUsr)); - qqueueChkPersist(pThis, 1); // TODO: optimize, do in outer function! (but we need parts from v5?) finalize_it: RETiRet; @@ -2669,6 +2668,8 @@ dbgprintf("queueMultiEnq: %d\n", i); CHKiRet(doEnqSingleObj(pThis, pMultiSub->ppMsgs[i]->flowCtlType, (void*)pMultiSub->ppMsgs[i])); } + qqueueChkPersist(pThis, pMultiSub->nElem); + finalize_it: if(pThis->qType != QUEUETYPE_DIRECT) { /* make sure at least one worker is running. */ -- cgit v1.2.3 From ce5869f7c41c8db943d8cbe804b69af40d43e2e6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 22 Jun 2009 18:52:30 +0200 Subject: optimized processing of TAG message field --- runtime/msg.c | 27 ++++++--------------------- runtime/msg.h | 1 - runtime/rsyslog.h | 1 + tools/syslogd.c | 26 ++++++++++---------------- 4 files changed, 17 insertions(+), 38 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 2ff63d9c..1a864648 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1303,19 +1303,6 @@ static inline char *getMSGID(msg_t *pM) } -/* Set the TAG to a caller-provided string. This is thought - * to be a heap buffer that the caller will no longer use. This - * function is a performance optimization over MsgSetTAG(). - * rgerhards 2004-11-19 - */ -void MsgAssignTAG(msg_t *pMsg, uchar *pBuf) -{ - assert(pMsg != NULL); - MsgSetTAG(pMsg, pBuf, ustrlen(pBuf)); - free(pBuf); -} - - /* rgerhards 2009-06-12: set associated ruleset */ void MsgSetRuleset(msg_t *pMsg, ruleset_t *pRuleset) @@ -1361,10 +1348,10 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf) * if there is a TAG and, if not, if it can emulate it. * rgerhards, 2005-11-24 */ -static void tryEmulateTAG(msg_t *pM) +static inline void tryEmulateTAG(msg_t *pM) { - int iTAGLen; - uchar *pBuf; + size_t lenTAG; + uchar bufTAG[CONF_TAG_MAXSIZE]; assert(pM != NULL); if(pM->iLenTAG > 0) @@ -1376,11 +1363,9 @@ static void tryEmulateTAG(msg_t *pM) MsgSetTAG(pM, (uchar*) getAPPNAME(pM), getAPPNAMELen(pM)); } else { /* now we can try to emulate */ - iTAGLen = getAPPNAMELen(pM) + getPROCIDLen(pM) + 3; - if((pBuf = malloc(iTAGLen * sizeof(char))) == NULL) - return; /* nothing we can do */ - snprintf((char*)pBuf, iTAGLen, "%s[%s]", getAPPNAME(pM), getPROCID(pM)); - MsgAssignTAG(pM, pBuf); + lenTAG = snprintf((char*)bufTAG, CONF_TAG_MAXSIZE, "%s[%s]", getAPPNAME(pM), getPROCID(pM)); + bufTAG[32] = '\0'; /* just to make sure... */ + MsgSetTAG(pM, bufTAG, lenTAG); } } } diff --git a/runtime/msg.h b/runtime/msg.h index 34983704..0f9bd95e 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -145,7 +145,6 @@ void MsgSetInputName(msg_t *pMsg, uchar*, size_t); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); -void MsgAssignTAG(msg_t *pMsg, uchar *pBuf); void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf); void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 2e0a4150..692b1327 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -30,6 +30,7 @@ * # Config Settings # * * ############################################################# */ #define RS_STRINGBUF_ALLOC_INCREMENT 128 +#define CONF_TAG_MAXSIZE 512 /* a value that is deemed far too large for any valid TAG */ /* ############################################################# * * # End Config Settings # * diff --git a/tools/syslogd.c b/tools/syslogd.c index 16f08c21..ed826fc1 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1173,9 +1173,9 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) uchar *p2parse; char *pBuf; char *pWork; - cstr_t *pStrB; - int iCnt; int bTAGCharDetected; + int i; /* general index for parsing */ + uchar bufParseTAG[CONF_TAG_MAXSIZE]; BEGINfunc assert(pMsg != NULL); @@ -1293,23 +1293,16 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) * is the max size ;) we need to shuffle the code again... Just for * the records: the code is currently clean, but we could optimize it! */ if(!bTAGCharDetected) { - uchar *pszTAG; - if(cstrConstruct(&pStrB) != RS_RET_OK) - return 1; - rsCStrSetAllocIncrement(pStrB, 33); - pWork = pBuf; - iCnt = 0; - while(*p2parse && *p2parse != ':' && *p2parse != ' ') { - cstrAppendChar(pStrB, *p2parse++); - ++iCnt; + i = 0; + while(*p2parse && *p2parse != ':' && *p2parse != ' ' && i < CONF_TAG_MAXSIZE) { + bufParseTAG[i++] = *p2parse++; } if(*p2parse == ':') { ++p2parse; - cstrAppendChar(pStrB, ':'); + bufParseTAG[i++] = ':'; } - cstrFinalize(pStrB); - cstrConvSzStrAndDestruct(pStrB, &pszTAG, 1); - if(pszTAG == NULL) + + if(i == 0) { /* rger, 2005-11-10: no TAG found - this implies that what * we have considered to be the HOSTNAME is most probably the * TAG. We consider it so probable, that we now adjust it @@ -1322,7 +1315,8 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) moveHOSTNAMEtoTAG(pMsg); MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } else { /* we have a TAG, so we can happily set it ;) */ - MsgAssignTAG(pMsg, pszTAG); + bufParseTAG[i] = '\0'; /* terminate string */ + MsgSetTAG(pMsg, bufParseTAG, i); } } else { /* we have no TAG, so we ... */ -- cgit v1.2.3 From 5310ccee3159c298cfd6e3af47c50d8b302894ec Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 22 Jun 2009 19:22:24 +0200 Subject: preparing for v4-stable --- ChangeLog | 2 +- configure.ac | 2 +- doc/manual.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 697a6a3b..1da8a4ce 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.1.8 [BETA] (rgerhards), 2009-04-?? +Version 4.2.0 [v4-stable] (rgerhards), 2009-06-23 - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs - bugfix: compile problems in im3195 diff --git a/configure.ac b/configure.ac index 4da2a4ff..3e24c62f 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([rsyslog],[4.1.7],[rsyslog@lists.adiscon.com]) +AC_INIT([rsyslog],[4.2.0],[rsyslog@lists.adiscon.com]) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR([ChangeLog]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/manual.html b/doc/manual.html index 5b9c4b1e..a1657c05 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

      Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

      -

      This documentation is for version 4.1.7 (devel branch) of rsyslog. +

      This documentation is for version 4.2.0 (v4-stable) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

      If you like rsyslog, you might -- cgit v1.2.3 From 2d548d56c2cb5c4018994a9e3bae306bdf35f38f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 22 Jun 2009 19:45:53 +0200 Subject: finishing touches for 4.2.0 --- ChangeLog | 1 + 1 file changed, 1 insertion(+) diff --git a/ChangeLog b/ChangeLog index 1da8a4ce..e59acf00 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,7 @@ Version 4.2.0 [v4-stable] (rgerhards), 2009-06-23 - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs +- imported all patches from 3.22.1 as of today (see below) - bugfix: compile problems in im3195 --------------------------------------------------------------------------- Version 4.1.7 [BETA] (rgerhards), 2009-04-22 -- cgit v1.2.3 From d798f5b66f4ae699f352b6c40abd07495eff8f94 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 11:22:07 +0200 Subject: some more optimizations - done malloc() instead of calloc() for msg_t, as we have large space which needs not be initialized - shrunk syslogTime structure in the hope to get better cache and write performance (non-aligned data should not hurt much here) --- runtime/msg.c | 51 +++++++++++++++++++++++++++++++++++++++++++++---- runtime/msg.h | 5 +++++ runtime/obj.h | 2 +- runtime/rsyslog.h | 1 + runtime/syslogd-types.h | 30 +++++++++++++++-------------- 5 files changed, 70 insertions(+), 19 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 1a864648..6ad0ee4f 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -425,6 +425,11 @@ rsRetVal MsgEnableThreadSafety(void) * itself but rather uses a user-supplied value. This enables the caller * to do some tricks to save processing time (done, for example, in the * udp input). + * NOTE: this constructor does NOT call calloc(), as we have many bytes + * inside the structure which do not need to be cleared. bzero() will + * heavily thrash the cache, so we do the init manually (which also + * is the right thing to do with pointers, as they are not neccessarily + * a binary 0 on all machines [but today almost always...]). * rgerhards, 2008-10-06 */ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) @@ -433,15 +438,53 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) msg_t *pM; assert(ppThis != NULL); - if((pM = calloc(1, sizeof(msg_t))) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pM = malloc(sizeof(msg_t))); + objConstructSetObjInfo(pM); /* intialize object helper entities */ - /* initialize members that are non-zero */ + /* initialize members in ORDER they appear in structure (think "cache line"!) */ + pM->flowCtlType = 0; + pM->bDoLock = 0; + pM->bParseHOSTNAME = 0; pM->iRefCount = 1; pM->iSeverity = -1; pM->iFacility = -1; + pM->offAfterPRI = 0; pM->offMSG = -1; - objConstructSetObjInfo(pM); + pM->iLenInputName = 0; + pM->iProtocolVersion = 0; + pM->msgFlags = 0; + pM->iLenRawMsg = 0; + pM->iLenMSG = 0; + pM->iLenTAG = 0; + pM->iLenHOSTNAME = 0; + pM->iLenRcvFrom = 0; + pM->iLenRcvFromIP = 0; + pM->pszRawMsg = NULL; + pM->pszHOSTNAME = NULL; + pM->pszRcvFrom = NULL; + pM->pszRcvFromIP = NULL; + pM->pszInputName = NULL; + pM->pszRcvdAt3164 = NULL; + pM->pszRcvdAt3339 = NULL; + pM->pszRcvdAt_MySQL = NULL; + pM->pszRcvdAt_PgSQL = NULL; + pM->pszTIMESTAMP3164 = NULL; + pM->pszTIMESTAMP3339 = NULL; + pM->pszTIMESTAMP_MySQL = NULL; + pM->pszTIMESTAMP_PgSQL = NULL; + pM->pCSProgName = NULL; + pM->pCSStrucData = NULL; + pM->pCSAPPNAME = NULL; + pM->pCSPROCID = NULL; + pM->pCSMSGID = NULL; + pM->pRuleset = NULL; + memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt)); + memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP)); + pM->TAG.pszTAG = NULL; + pM->pszTimestamp3164[0] = '\0'; + pM->pszTimestamp3339[0] = '\0'; + pM->pszTIMESTAMP_SecFrac[0] = '\0'; + pM->pszRcvdAt_SecFrac[0] = '\0'; /* DEV debugging only! dbgprintf("msgConstruct\t0x%x, ref 1\n", (int)pM);*/ diff --git a/runtime/msg.h b/runtime/msg.h index 0f9bd95e..9113882a 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -51,6 +51,11 @@ * will be decremented. If it is 1, however, the object is actually * destroyed. To make this work, it is vital that MsgAddRef() is * called each time a "copy" is stored somewhere. + * + * WARNING: this structure is not calloc()ed, so be careful when + * adding new fields. You need to initialize them in + * msgBaseConstruct(). That function header comment also describes + * why this is the case. */ struct msg { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ diff --git a/runtime/obj.h b/runtime/obj.h index 3973a16e..50868cc9 100644 --- a/runtime/obj.h +++ b/runtime/obj.h @@ -77,8 +77,8 @@ /* the next macro MUST be called in Constructors: */ #ifndef NDEBUG /* this means if debug... */ # define objConstructSetObjInfo(pThis) \ - ASSERT(((obj_t*) (pThis))->pObjInfo == NULL); \ ((obj_t*) (pThis))->pObjInfo = pObjInfoOBJ; \ + ((obj_t*) (pThis))->pszName = NULL; \ ((obj_t*) (pThis))->iObjCooCKiE = 0xBADEFEE #else # define objConstructSetObjInfo(pThis) ((obj_t*) (pThis))->pObjInfo = pObjInfoOBJ diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 692b1327..ac068d5e 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -101,6 +101,7 @@ typedef struct strmLstnPortList_s strmLstnPortList_t; // TODO: rename? typedef long long int64; typedef long long unsigned uint64; typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ +typedef char intTiny; /* 0..127! */ #ifdef __hpux typedef unsigned int u_int32_t; /* TODO: is this correct? */ diff --git a/runtime/syslogd-types.h b/runtime/syslogd-types.h index be0dfdd8..83b38f28 100644 --- a/runtime/syslogd-types.h +++ b/runtime/syslogd-types.h @@ -78,23 +78,25 @@ typedef enum _EHostnameCmpMode EHostnameCmpMode; /* rgerhards 2004-11-11: the following structure represents * a time as it is used in syslog. + * rgerhards, 2009-06-23: packed structure for better cache performance + * (but left ultimate decision about packing to compiler) */ struct syslogTime { - int timeType; /* 0 - unitinialized , 1 - RFC 3164, 2 - syslog-protocol */ - int year; - int month; - int day; - int hour; /* 24 hour clock */ - int minute; - int second; - int secfrac; /* fractional seconds (must be 32 bit!) */ - int secfracPrecision; + intTiny timeType; /* 0 - unitinialized , 1 - RFC 3164, 2 - syslog-protocol */ + intTiny month; + intTiny day; + intTiny hour; /* 24 hour clock */ + intTiny minute; + intTiny second; + intTiny secfracPrecision; + intTiny OffsetMinute; /* UTC offset in minutes */ + intTiny OffsetHour; /* UTC offset in hours + * full UTC offset minutes = OffsetHours*60 + OffsetMinute. Then use + * OffsetMode to know the direction. + */ char OffsetMode; /* UTC offset + or - */ - char OffsetHour; /* UTC offset in hours */ - int OffsetMinute; /* UTC offset in minutes */ - /* full UTC offset minutes = OffsetHours*60 + OffsetMinute. Then use - * OffsetMode to know the direction. - */ + short year; + int secfrac; /* fractional seconds (must be 32 bit!) */ }; typedef struct syslogTime syslogTime_t; -- cgit v1.2.3 From 6181156b1c42825bac892d3e1284dcb2d4fcf5d3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 11:33:11 +0200 Subject: fix: previous patch aborted in release mode --- runtime/obj.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/runtime/obj.h b/runtime/obj.h index 50868cc9..419d29cc 100644 --- a/runtime/obj.h +++ b/runtime/obj.h @@ -81,7 +81,9 @@ ((obj_t*) (pThis))->pszName = NULL; \ ((obj_t*) (pThis))->iObjCooCKiE = 0xBADEFEE #else -# define objConstructSetObjInfo(pThis) ((obj_t*) (pThis))->pObjInfo = pObjInfoOBJ +# define objConstructSetObjInfo(pThis) \ + ((obj_t*) (pThis))->pObjInfo = pObjInfoOBJ; \ + ((obj_t*) (pThis))->pszName = NULL #endif #define objDestruct(pThis) (((obj_t*) (pThis))->pObjInfo->objMethods[objMethod_DESTRUCT])(&pThis) #define objSerialize(pThis) (((obj_t*) (pThis))->pObjInfo->objMethods[objMethod_SERIALIZE]) -- cgit v1.2.3 From b50d13a6a97c0b6fa14807775ae0edf52ef015fb Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 14:50:03 +0200 Subject: restored repeated message reduction processing --- action.c | 22 +++++++---------- runtime/msg.c | 70 ++++++++++++++++++++++++++++++++++++++++++------------- runtime/msg.h | 5 +--- runtime/rsyslog.h | 3 +++ runtime/ruleset.c | 7 ++---- tools/syslogd.c | 1 + 6 files changed, 69 insertions(+), 39 deletions(-) diff --git a/action.c b/action.c index d214e808..2040d6bd 100644 --- a/action.c +++ b/action.c @@ -4,7 +4,7 @@ * * File begun on 2007-08-06 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -239,10 +239,7 @@ rsRetVal actionConstruct(action_t **ppThis) ASSERT(ppThis != NULL); - if((pThis = (action_t*) calloc(1, sizeof(action_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } - + CHKmalloc(pThis = (action_t*) calloc(1, sizeof(action_t))); pThis->iResumeInterval = glbliActionResumeInterval; pThis->iResumeRetryCount = glbliActionResumeRetryCount; pThis->tLastOccur = time(NULL); /* done once per action on startup only */ @@ -658,7 +655,7 @@ actionWriteToAction(action_t *pAction) if(pAction->iNbrNoExec < pAction->iExecEveryNthOccur - 1) { ++pAction->iNbrNoExec; dbgprintf("action %p passed %d times to execution - less than neded - discarding\n", - pAction, pAction->iNbrNoExec); + pAction, pAction->iNbrNoExec); FINALIZE; } else { pAction->iNbrNoExec = 0; /* we execute the action now, so the number of no execs is down to */ @@ -675,6 +672,7 @@ actionWriteToAction(action_t *pAction) */ if(pAction->f_prevcount > 1) { msg_t *pMsg; + size_t lenRepMsg; uchar szRepMsg[1024]; if((pMsg = MsgDup(pAction->f_pMsg)) == NULL) { @@ -684,23 +682,19 @@ actionWriteToAction(action_t *pAction) } if(pAction->bRepMsgHasMsg == 0) { /* old format repeat message? */ - snprintf((char*)szRepMsg, sizeof(szRepMsg), "last message repeated %d times", + lenRepMsg = snprintf((char*)szRepMsg, sizeof(szRepMsg), " last message repeated %d times", pAction->f_prevcount); } else { - snprintf((char*)szRepMsg, sizeof(szRepMsg), "message repeated %d times: [%.800s]", + lenRepMsg = snprintf((char*)szRepMsg, sizeof(szRepMsg), " message repeated %d times: [%.800s]", pAction->f_prevcount, getMSG(pAction->f_pMsg)); } - /* We now need to update the other message properties. - * ... RAWMSG is a problem ... Please note that digital + /* We now need to update the other message properties. Please note that digital * signatures inside the message are also invalidated. */ datetime.getCurrTime(&(pMsg->tRcvdAt), &(pMsg->ttGenTime)); memcpy(&pMsg->tTIMESTAMP, &pMsg->tRcvdAt, sizeof(struct syslogTime)); -#pragma warn "need fix msg repeationg handling" - //MsgSetMSG(pMsg, (char*)szRepMsg); - MsgSetRawMsgWOSize(pMsg, (char*)szRepMsg); - + MsgReplaceMSG(pMsg, szRepMsg, lenRepMsg); pMsgSave = pAction->f_pMsg; /* save message pointer for later restoration */ pAction->f_pMsg = pMsg; /* use the new msg (pointer will be restored below) */ } diff --git a/runtime/msg.c b/runtime/msg.c index 6ad0ee4f..8c306aca 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1733,38 +1733,76 @@ void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME) } -/* rgerhards 2004-11-09: set MSG in msg object +/* set the offset of the MSG part into the raw msg buffer */ void MsgSetMSGoffs(msg_t *pMsg, short offs) { - assert(pMsg != NULL); - - pMsg->iLenMSG = ustrlen(pMsg->pszRawMsg + offs); + ISOBJ_TYPE_assert(pMsg, msg); + pMsg->iLenMSG = pMsg->iLenRawMsg - offs; pMsg->offMSG = offs; } +/* replace the MSG part of a message. The update actually takes place inside + * rawmsg. + * There are two cases: either the new message will be larger than the new msg + * or it will be less than or equal. If it is less than or equal, we can utilize + * the previous message buffer. If it is larger, we can utilize the msg_t-included + * message buffer if it fits in there. If this is not the case, we need to alloc + * a new, larger, chunk and copy over the data to it. Note that this function is + * (hopefully) relatively seldom being called, so some performance impact is + * uncritical. In any case, pszMSG is copied, so if it was dynamically allocated, + * the caller is responsible for freeing it. + * rgerhards, 2009-06-23 + */ +rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG) +{ + int lenNew; + uchar *bufNew; + DEFiRet; + ISOBJ_TYPE_assert(pThis, msg); + assert(pszMSG != NULL); + + lenNew = pThis->iLenRawMsg + lenMSG - pThis->iLenMSG; + if(lenMSG > pThis->iLenMSG && lenNew >= CONF_RAWMSG_BUFSIZE) { + /* we have lost and need to alloc a new buffer ;) */ + CHKmalloc(bufNew = malloc(lenNew + 1)); + memcpy(bufNew, pThis->pszRawMsg, pThis->offMSG); + free(pThis->pszRawMsg); + pThis->pszRawMsg = bufNew; + } + + memcpy(pThis->pszRawMsg + pThis->offMSG, pszMSG, lenMSG); + pThis->pszRawMsg[lenNew] = '\0'; /* this also works with truncation! */ + pThis->iLenRawMsg = lenNew; + pThis->iLenMSG = lenMSG; + +finalize_it: + RETiRet; +} + + /* set raw message in message object. Size of message is provided. * rgerhards, 2009-06-16 */ -void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg) +void MsgSetRawMsg(msg_t *pThis, char* pszRawMsg, size_t lenMsg) { - assert(pMsg != NULL); - if(pMsg->pszRawMsg != pMsg->szRawMsg) - free(pMsg->pszRawMsg); + assert(pThis != NULL); + if(pThis->pszRawMsg != pThis->szRawMsg) + free(pThis->pszRawMsg); - pMsg->iLenRawMsg = lenMsg; - if(pMsg->iLenRawMsg < CONF_RAWMSG_BUFSIZE) { + pThis->iLenRawMsg = lenMsg; + if(pThis->iLenRawMsg < CONF_RAWMSG_BUFSIZE) { /* small enough: use fixed buffer (faster!) */ - pMsg->pszRawMsg = pMsg->szRawMsg; - } else if((pMsg->pszRawMsg = (uchar*) malloc(pMsg->iLenRawMsg + 1)) == NULL) { + pThis->pszRawMsg = pThis->szRawMsg; + } else if((pThis->pszRawMsg = (uchar*) malloc(pThis->iLenRawMsg + 1)) == NULL) { /* truncate message, better than completely loosing it... */ - pMsg->pszRawMsg = pMsg->szRawMsg; - pMsg->iLenRawMsg = CONF_RAWMSG_BUFSIZE - 1; + pThis->pszRawMsg = pThis->szRawMsg; + pThis->iLenRawMsg = CONF_RAWMSG_BUFSIZE - 1; } - memcpy(pMsg->pszRawMsg, pszRawMsg, pMsg->iLenRawMsg); - pMsg->pszRawMsg[pMsg->iLenRawMsg] = '\0'; /* this also works with truncation! */ + memcpy(pThis->pszRawMsg, pszRawMsg, pThis->iLenRawMsg); + pThis->pszRawMsg[pThis->iLenRawMsg] = '\0'; /* this also works with truncation! */ } diff --git a/runtime/msg.h b/runtime/msg.h index 9113882a..c38cb788 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -28,10 +28,6 @@ #ifndef MSG_H_INCLUDED #define MSG_H_INCLUDED 1 -/* some configuration constants */ -#define CONF_RAWMSG_BUFSIZE 101 -#define CONF_TAG_BUFSIZE 33 /* RFC says 32 chars (+ \0), but in practice we see longer ones... */ - #include #include "obj.h" #include "syslogd-types.h" @@ -162,6 +158,7 @@ rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSGoffs(msg_t *pMsg, short offs); void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); +rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG); void moveHOSTNAMEtoTAG(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, cstr_t *pCSPropName, size_t *pPropLen, unsigned short *pbMustBeFreed); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index ac068d5e..793df782 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -31,6 +31,9 @@ * ############################################################# */ #define RS_STRINGBUF_ALLOC_INCREMENT 128 #define CONF_TAG_MAXSIZE 512 /* a value that is deemed far too large for any valid TAG */ +#define CONF_RAWMSG_BUFSIZE 101 +#define CONF_TAG_BUFSIZE 33 /* RFC says 32 chars (+ \0), but in practice we see longer ones... */ + /* ############################################################# * * # End Config Settings # * diff --git a/runtime/ruleset.c b/runtime/ruleset.c index 93d40e24..d98b4217 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -84,7 +84,6 @@ DEFFUNC_llExecFunc(doIterateRulesetActions) iRet = rule.IterateAllActions(pRule, pMyParam->pFunc, pMyParam->pParam); RETiRet; } -#if 0 /* iterate over all actions of THIS rule set. */ static rsRetVal @@ -96,7 +95,7 @@ iterateRulesetAllActions(ruleset_t *pThis, rsRetVal (*pFunc)(void*, void*), void params.pFunc = pFunc; params.pParam = pParam; - CHKiRet(llExecFunc(&llRulesets, doIterateRulesetActions, ¶ms)); + CHKiRet(llExecFunc(&(pThis->llRules), doIterateRulesetActions, ¶ms)); finalize_it: RETiRet; @@ -112,7 +111,6 @@ DEFFUNC_llExecFunc(doIterateAllActions) iRet = iterateRulesetAllActions(pThis, pMyParam->pFunc, pMyParam->pParam); RETiRet; } -#endif /* iterate over ALL actions present in the WHOLE system. * this is often needed, for example when HUP processing * must be done or a shutdown is pending. @@ -126,8 +124,7 @@ iterateAllActions(rsRetVal (*pFunc)(void*, void*), void* pParam) params.pFunc = pFunc; params.pParam = pParam; - //CHKiRet(llExecFunc(&llRulesets, doIterateAllActions, ¶ms)); - CHKiRet(llExecFunc(&llRulesets, doIterateRulesetActions, ¶ms)); + CHKiRet(llExecFunc(&llRulesets, doIterateAllActions, ¶ms)); finalize_it: RETiRet; diff --git a/tools/syslogd.c b/tools/syslogd.c index ed826fc1..178ad455 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1469,6 +1469,7 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions) assert(pAction != NULL); BEGINfunc +RUNLOG_VAR("%p", pAction); LockObj(pAction); /* TODO: time() performance: the call below could be moved to * the beginn of the llExec(). This makes it slightly less correct, but -- cgit v1.2.3 From b1f2e53921b7ab19c363a63de47a0e7a35866052 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 15:17:55 +0200 Subject: prevented unneccessary apc calls --- runtime/apc.c | 7 +++++-- runtime/stream.c | 15 ++++++++++----- runtime/stream.h | 5 +++-- tools/syslogd.c | 3 +-- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/runtime/apc.c b/runtime/apc.c index b0b5f298..5919191d 100644 --- a/runtime/apc.c +++ b/runtime/apc.c @@ -257,8 +257,11 @@ execScheduled(void) END_MTX_PROTECTED_OPERATIONS_UNCOND(&listMutex); CHKiRet(iRet); - DBGPRINTF("running apc scheduler - we have %s to execute\n", - pExecList == NULL ? "nothing" : "something"); + if(pExecList != NULL) { + DBGPRINTF("running apc scheduler - we have %s to execute\n", + pExecList == NULL ? "nothing" : "something"); + } + for(pCurr = pExecList ; pCurr != NULL ; pCurr = pNext) { dbgprintf("executing apc list entry %p, apc %p\n", pCurr, pCurr->pApc); pNext = pCurr->pNext; diff --git a/runtime/stream.c b/runtime/stream.c index f9859617..00c726d9 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -74,6 +74,7 @@ flushApc(void *param1, void __attribute__((unused)) *param2) BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(&pThis->mut); strmFlush(pThis); + pThis->apcRequested = 0; END_MTX_PROTECTED_OPERATIONS_UNCOND(&pThis->mut); } @@ -1001,12 +1002,16 @@ scheduleFlushRequest(strm_t *pThis) apc_t *pApc; DEFiRet; - CHKiRet(apc.CancelApc(pThis->apcID)); + if(!pThis->apcRequested) { + /* we do an request only if none is yet pending */ + pThis->apcRequested = 1; + // TODO: find similar thing later CHKiRet(apc.CancelApc(pThis->apcID)); dbgprintf("XXX: requesting to add apc!\n"); - CHKiRet(apc.Construct(&pApc)); - CHKiRet(apc.SetProcedure(pApc, (void (*)(void*, void*))flushApc)); - CHKiRet(apc.SetParam1(pApc, pThis)); - CHKiRet(apc.ConstructFinalize(pApc, &pThis->apcID)); + CHKiRet(apc.Construct(&pApc)); + CHKiRet(apc.SetProcedure(pApc, (void (*)(void*, void*))flushApc)); + CHKiRet(apc.SetParam1(pApc, pThis)); + CHKiRet(apc.ConstructFinalize(pApc, &pThis->apcID)); + } finalize_it: RETiRet; diff --git a/runtime/stream.h b/runtime/stream.h index e3ad32b1..ac003c7b 100644 --- a/runtime/stream.h +++ b/runtime/stream.h @@ -104,8 +104,8 @@ typedef struct strm_s { int64 iCurrOffs;/* current offset */ int64 *pUsrWCntr; /* NULL or a user-provided counter that receives the nbr of bytes written since the last CntrSet() */ /* dynamic properties, valid only during file open, not to be persistet */ - int bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */ - int bSync; /* sync this file after every write? */ + bool bDisabled; /* should file no longer be written to? (currently set only if omfile file size limit fails) */ + bool bSync; /* sync this file after every write? */ size_t sIOBufSize;/* size of IO buffer */ uchar *pszDir; /* Directory */ int lenDir; @@ -123,6 +123,7 @@ typedef struct strm_s { int iFlushInterval; /* flush in which interval - 0, no flushing */ apc_id_t apcID; /* id of current Apc request (used for cancelling) */ pthread_mutex_t mut;/* mutex for flush in async mode */ + int apcRequested; /* is an apc Requested? */ /* support for omfile size-limiting commands, special counters, NOT persisted! */ off_t iSizeLimit; /* file size limit, 0 = no limit */ uchar *pszSizeLimitCmd; /* command to carry out when size limit is reached */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 178ad455..385fef1c 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1469,7 +1469,6 @@ DEFFUNC_llExecFunc(flushRptdMsgsActions) assert(pAction != NULL); BEGINfunc -RUNLOG_VAR("%p", pAction); LockObj(pAction); /* TODO: time() performance: the call below could be moved to * the beginn of the llExec(). This makes it slightly less correct, but @@ -2586,7 +2585,7 @@ mainloop(void) * but a once-a-day wakeup should be quite acceptable. -- rgerhards, 2008-06-09 */ //tvSelectTimeout.tv_sec = (bReduceRepeatMsgs == 1) ? TIMERINTVL : 86400 /*1 day*/; -tvSelectTimeout.tv_sec = 5; // TESTING ONLY!!! TODO: change back!!! + tvSelectTimeout.tv_sec = TIMERINTVL; /* TODO: change this back to the above code when we have a better solution for apc */ tvSelectTimeout.tv_usec = 0; select(1, NULL, NULL, NULL, &tvSelectTimeout); if(bFinished) -- cgit v1.2.3 From 86e37f70fe0e9de0e00362990c73536843c8fef3 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 16:14:19 +0200 Subject: more strict parsing of the hostname in rfc3164 mode ... hopefully removes false positives (but may cause some trouble with hostname parsing). For details, see this bug tracker: http://bugzilla.adiscon.com/show_bug.cgi?id=126 This patch is not optimal for v4 - another one will follow. The spirit of this commit is to enable easier backporting if someone is interested in doing so. --- ChangeLog | 8 +++- runtime/msg.h | 2 +- runtime/rsyslog.h | 12 ++++-- tools/syslogd.c | 114 ++++++++++++++++++++---------------------------------- 4 files changed, 57 insertions(+), 79 deletions(-) diff --git a/ChangeLog b/ChangeLog index 36a819a3..46235ff7 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +Version 4.5.0 [DEVEL] (rgerhards), 2009-??-?? - greatly reduced memory requirements of msg object to around half of the previous demand. This means that more messages can be stored in core! Due to fewer cache misses, this also means some @@ -22,6 +22,10 @@ Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? - added capability to fsync() queue disk files for enhanced reliability (also add's speed, because you do no longer need to run the whole file system in sync mode) +- more strict parsing of the hostname in rfc3164 mode, hopefully + removes false positives (but may cause some trouble with hostname + parsing). For details, see this bug tracker: + http://bugzilla.adiscon.com/show_bug.cgi?id=126 - added configuration commands (see doc for explanations) * $OMFileZipLevel * $OMFileIOBufferSize @@ -29,6 +33,8 @@ Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? * $MainMsgQueueSyncQueueFiles * $ActionQueueSyncQueueFiles --------------------------------------------------------------------------- +Version 4.3.2 [beta] (rgerhards), 2009-??-?? +--------------------------------------------------------------------------- Version 4.3.1 [DEVEL] (rgerhards), 2009-05-25 - added capability to run multiple tcp listeners (on different ports) - performance enhancement: imtcp calls parser no longer on input thread diff --git a/runtime/msg.h b/runtime/msg.h index c38cb788..d70b939a 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -152,7 +152,7 @@ rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); -void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); +//void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSGoffs(msg_t *pMsg, short offs); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 793df782..4fa5100e 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -29,10 +29,14 @@ /* ############################################################# * * # Config Settings # * * ############################################################# */ -#define RS_STRINGBUF_ALLOC_INCREMENT 128 -#define CONF_TAG_MAXSIZE 512 /* a value that is deemed far too large for any valid TAG */ -#define CONF_RAWMSG_BUFSIZE 101 -#define CONF_TAG_BUFSIZE 33 /* RFC says 32 chars (+ \0), but in practice we see longer ones... */ +#define RS_STRINGBUF_ALLOC_INCREMENT 128 +/* MAXSIZE are absolute maxima, while BUFSIZE are just values after which + * processing is more time-intense. + */ +#define CONF_TAG_MAXSIZE 512 /* a value that is deemed far too large for any valid TAG */ +#define CONF_TAG_HOSTNAME 512 /* a value that is deemed far too large for any valid HOSTNAME */ +#define CONF_RAWMSG_BUFSIZE 101 +#define CONF_TAG_BUFSIZE 33 /* RFC says 32 chars (+ \0), but in practice we see longer ones... */ /* ############################################################# * diff --git a/tools/syslogd.c b/tools/syslogd.c index 385fef1c..2f28cde0 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -1176,6 +1176,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) int bTAGCharDetected; int i; /* general index for parsing */ uchar bufParseTAG[CONF_TAG_MAXSIZE]; + uchar bufParseHOSTNAME[CONF_TAG_HOSTNAME]; BEGINfunc assert(pMsg != NULL); @@ -1228,50 +1229,27 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) * If I find them, I set a simple flag but continue. After parsing, I check the flag. * If it was set, then we most probably do not have a hostname but a TAG. Thus, I change * the fields. I think this logic shall work with any type of syslog message. + * rgerhards, 2009-06-23: and I now have extended this logic to every character + * that is not a valid hostname. */ bTAGCharDetected = 0; if(flags & PARSE_HOSTNAME) { - /* TODO: quick and dirty memory allocation */ - /* the memory allocated is far too much in most cases. But on the plus side, - * it is quite fast... - rgerhards, 2007-09-20 - */ - if((pBuf = malloc(sizeof(char)* (ustrlen(p2parse) +1))) == NULL) - return 1; - pWork = pBuf; - /* this is the actual parsing loop */ - while(*p2parse && *p2parse != ' ' && *p2parse != ':') { - if(*p2parse == '[' || *p2parse == ']' || *p2parse == '/') - bTAGCharDetected = 1; - *pWork++ = *p2parse++; + i = 0; + while((isalnum(p2parse[i]) || p2parse[i] == '.' || p2parse[i] == '.' + || p2parse[i] == '_') && i < CONF_TAG_MAXSIZE) { + bufParseHOSTNAME[i] = p2parse[i]; + ++i; + } + + if(i > 0 && p2parse[i] == ' ' && isalnum(p2parse[i-1])) { + /* we got a hostname! */ + p2parse += i + 1; /* "eat" it (including SP delimiter) */ + bufParseHOSTNAME[i] = '\0'; + MsgSetHOSTNAME(pMsg, bufParseHOSTNAME); + //MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i); + } else { + MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } - /* we need to handle ':' seperately, because it terminates the - * TAG - so we also need to terminate the parser here! - * rgerhards, 2007-09-10 *p2parse points to a valid address here in - * any case. We can reach this point only if we are at end of string, - * or we have a ':' or ' '. What the if below does is check if we are - * not at end of string and, if so, advance the parse pointer. If we - * are already at end of string, *p2parse is equal to '\0', neither if - * will be true and the parse pointer remain as is. This is perfectly - * well. - */ - if(*p2parse == ':') { - bTAGCharDetected = 1; - /* We will move hostname to tag, so preserve ':' (otherwise we - * will needlessly change the message format) */ - *pWork++ = *p2parse++; - } else if(*p2parse == ' ') - ++p2parse; - *pWork = '\0'; - MsgAssignHOSTNAME(pMsg, pBuf); - } - /* check if we seem to have a TAG */ - if(bTAGCharDetected) { - /* indeed, this smells like a TAG, so lets use it for this. We take - * the HOSTNAME from the sender system instead. - */ - DBGPRINTF("HOSTNAME contains invalid characters, assuming it to be a TAG.\n"); - moveHOSTNAMEtoTAG(pMsg); - MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } /* now parse TAG - that should be present in message from all sources. @@ -1287,40 +1265,30 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) * in RFC3164...). We now receive the full size, but will modify the * outputs so that only 32 characters max are used by default. */ - /* The following code in general is quick & dirty - I need to get - * it going for a test, rgerhards 2004-11-16 */ - /* lol.. we tried to solve it, just to remind ourselfs that 32 octets - * is the max size ;) we need to shuffle the code again... Just for - * the records: the code is currently clean, but we could optimize it! */ - if(!bTAGCharDetected) { - i = 0; - while(*p2parse && *p2parse != ':' && *p2parse != ' ' && i < CONF_TAG_MAXSIZE) { - bufParseTAG[i++] = *p2parse++; - } - if(*p2parse == ':') { - ++p2parse; - bufParseTAG[i++] = ':'; - } + i = 0; + while(*p2parse && *p2parse != ':' && *p2parse != ' ' && i < CONF_TAG_MAXSIZE) { + bufParseTAG[i++] = *p2parse++; + } + if(*p2parse == ':') { + ++p2parse; + bufParseTAG[i++] = ':'; + } - if(i == 0) - { /* rger, 2005-11-10: no TAG found - this implies that what - * we have considered to be the HOSTNAME is most probably the - * TAG. We consider it so probable, that we now adjust it - * that way. So we pick up the previously set hostname, assign - * it to tag and use the sender system (from IP stack) as - * the hostname. This situation is the standard case with - * stock BSD syslogd. - */ - DBGPRINTF("No TAG in message, assuming that HOSTNAME is missing.\n"); - moveHOSTNAMEtoTAG(pMsg); - MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); - } else { /* we have a TAG, so we can happily set it ;) */ - bufParseTAG[i] = '\0'; /* terminate string */ - MsgSetTAG(pMsg, bufParseTAG, i); - } - } else { - /* we have no TAG, so we ... */ - /*DO NOTHING*/; + if(i == 0) + { /* rger, 2005-11-10: no TAG found - this implies that what + * we have considered to be the HOSTNAME is most probably the + * TAG. We consider it so probable, that we now adjust it + * that way. So we pick up the previously set hostname, assign + * it to tag and use the sender system (from IP stack) as + * the hostname. This situation is the standard case with + * stock BSD syslogd. + */ + DBGPRINTF("No TAG in message, assuming that HOSTNAME is missing.\n"); + moveHOSTNAMEtoTAG(pMsg); + MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); + } else { /* we have a TAG, so we can happily set it ;) */ + bufParseTAG[i] = '\0'; /* terminate string */ + MsgSetTAG(pMsg, bufParseTAG, i); } } else { /* we enter this code area when the user has instructed rsyslog NOT -- cgit v1.2.3 From 662ad3e4bf8dbd317d18aa1afcbf3e8b9e424506 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 16:32:29 +0200 Subject: optimized hostname processing --- plugins/imfile/imfile.c | 2 +- plugins/imklog/imklog.c | 2 +- runtime/msg.c | 10 +++++----- runtime/msg.h | 5 ++--- runtime/parser.c | 3 --- tools/syslogd.c | 47 +++++++++++------------------------------------ 6 files changed, 20 insertions(+), 49 deletions(-) diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 0f5f49dc..631d02ff 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -100,7 +100,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine), cstrLen(cstrLine)); MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ - MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); MsgSetTAG(pMsg, pInfo->pszTag, pInfo->lenTag); pMsg->iFacility = LOG_FAC(pInfo->iFacility); pMsg->iSeverity = LOG_PRI(pInfo->iSeverity); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 67fdc8a3..f63d60ac 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -100,7 +100,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); - MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); MsgSetTAG(pMsg, pszTag, ustrlen(pszTag)); pMsg->iFacility = LOG_FAC(iFacility); pMsg->iSeverity = LOG_PRI(iSeverity); diff --git a/runtime/msg.c b/runtime/msg.c index 8c306aca..6cf41322 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1439,7 +1439,7 @@ int getHOSTNAMELen(msg_t *pM) return 0; else if(pM->pszHOSTNAME == NULL) - return 0; + return pM->iLenRcvFrom; else return pM->iLenHOSTNAME; } @@ -1451,7 +1451,7 @@ char *getHOSTNAME(msg_t *pM) return ""; else if(pM->pszHOSTNAME == NULL) - return ""; + return (char*) pM->pszRcvFrom; else return (char*) pM->pszHOSTNAME; } @@ -1720,12 +1720,12 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) * we need it. The rest of the code already knows how to handle an * unset HOSTNAME. */ -void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME) +void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME, int lenHOSTNAME) { assert(pMsg != NULL); free(pMsg->pszHOSTNAME); - pMsg->iLenHOSTNAME = ustrlen(pszHOSTNAME); + pMsg->iLenHOSTNAME = lenHOSTNAME; if((pMsg->pszHOSTNAME = malloc(pMsg->iLenHOSTNAME + 1)) != NULL) memcpy(pMsg->pszHOSTNAME, pszHOSTNAME, pMsg->iLenHOSTNAME + 1); else @@ -2719,7 +2719,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("pszRcvFrom")) { MsgSetRcvFrom(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszHOSTNAME")) { - MsgSetHOSTNAME(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetHOSTNAME(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pCSStrucData")) { MsgSetStructuredData(pThis, (char*) rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pCSAPPNAME")) { diff --git a/runtime/msg.h b/runtime/msg.h index d70b939a..fd4d650b 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -3,7 +3,7 @@ * * File begun on 2007-07-13 by RGerhards (extracted from syslogd.c) * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -152,8 +152,7 @@ rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); -//void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf); -void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME); +void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME, int lenHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSGoffs(msg_t *pMsg, short offs); void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); diff --git a/runtime/parser.c b/runtime/parser.c index 75cde343..0a27c982 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -284,9 +284,6 @@ rsRetVal parseMsg(msg_t *pMsg) pMsg->iSeverity = LOG_PRI(pri); MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); - if(pMsg->bParseHOSTNAME == 0) - MsgSetHOSTNAME(pMsg, pMsg->pszRcvFrom); - /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have * a traditional syslog message or one formatted according to syslog-protocol. * We need to apply different parsers depending on that. We use the diff --git a/tools/syslogd.c b/tools/syslogd.c index 2f28cde0..faa793fb 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -609,7 +609,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f * being the local host). rgerhards 2004-11-16 */ if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) - MsgSetHOSTNAME(pMsg, hname); + MsgSetHOSTNAME(pMsg, hname, strlen(hname)); MsgSetRcvFrom(pMsg, hname); MsgSetAfterPRIOffs(pMsg, p - msg); CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP)); @@ -883,7 +883,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) CHKiRet(msgConstruct(&pMsg)); MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1); MsgSetRawMsgWOSize(pMsg, (char*)msg); - MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName()); + MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1")); /* check if we have an error code associated and, if so, @@ -1113,12 +1113,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) /* HOSTNAME */ if(bContParse) { parseRFCField(&p2parse, pBuf); - MsgSetHOSTNAME(pMsg, pBuf); - } else { - /* we can not parse, so we get the system we - * received the data from. - */ - MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); + MsgSetHOSTNAME(pMsg, pBuf, strlen(pBuf)); } /* APP-NAME */ @@ -1147,7 +1142,6 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) /* MSG */ MsgSetMSGoffs(pMsg, p2parse - pMsg->pszRawMsg); - //MsgSetMSG(pMsg, (char*)p2parse); free(pBuf); ENDfunc @@ -1171,8 +1165,6 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) int parseLegacySyslogMsg(msg_t *pMsg, int flags) { uchar *p2parse; - char *pBuf; - char *pWork; int bTAGCharDetected; int i; /* general index for parsing */ uchar bufParseTAG[CONF_TAG_MAXSIZE]; @@ -1198,8 +1190,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) if(datetime.ParseTIMESTAMP3164(&(pMsg->tTIMESTAMP), &p2parse) == RS_RET_OK) { /* indeed, we got it! */ /* we are done - parse pointer is moved by ParseTIMESTAMP3164 */; - } else { - /* parse pointer needs to be restored, as we moved it off-by-one + } else {/* parse pointer needs to be restored, as we moved it off-by-one * for this try. */ --p2parse; @@ -1245,10 +1236,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) /* we got a hostname! */ p2parse += i + 1; /* "eat" it (including SP delimiter) */ bufParseHOSTNAME[i] = '\0'; - MsgSetHOSTNAME(pMsg, bufParseHOSTNAME); - //MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i); - } else { - MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); + MsgSetHOSTNAME(pMsg, bufParseHOSTNAME, i); } } @@ -1274,30 +1262,17 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) bufParseTAG[i++] = ':'; } - if(i == 0) - { /* rger, 2005-11-10: no TAG found - this implies that what - * we have considered to be the HOSTNAME is most probably the - * TAG. We consider it so probable, that we now adjust it - * that way. So we pick up the previously set hostname, assign - * it to tag and use the sender system (from IP stack) as - * the hostname. This situation is the standard case with - * stock BSD syslogd. - */ - DBGPRINTF("No TAG in message, assuming that HOSTNAME is missing.\n"); - moveHOSTNAMEtoTAG(pMsg); - MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); - } else { /* we have a TAG, so we can happily set it ;) */ - bufParseTAG[i] = '\0'; /* terminate string */ - MsgSetTAG(pMsg, bufParseTAG, i); - } + /* no TAG can only be detected if the message immediatly ends, in which case an empty TAG + * is considered OK. So we do not need to check for empty TAG. -- rgerhards, 2009-06-23 + */ + bufParseTAG[i] = '\0'; /* terminate string */ + MsgSetTAG(pMsg, bufParseTAG, i); } else { /* we enter this code area when the user has instructed rsyslog NOT * to parse HOSTNAME and TAG - rgerhards, 2006-03-13 */ - if(!(flags & INTERNAL_MSG)) - { + if(!(flags & INTERNAL_MSG)) { DBGPRINTF("HOSTNAME and TAG not parsed by user configuraton.\n"); - MsgSetHOSTNAME(pMsg, getRcvFrom(pMsg)); } } -- cgit v1.2.3 From b2fa740b9ab5fb9e85309b3307f3fca21f625ab1 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 17:14:42 +0200 Subject: optimized TAG handling --- runtime/msg.c | 87 ++++++++++++++++++++++-------------------------------- runtime/msg.h | 2 +- runtime/rsyslog.h | 6 ++-- tests/diskqueue.sh | 2 +- tools/syslogd.c | 7 ++--- 5 files changed, 44 insertions(+), 60 deletions(-) diff --git a/runtime/msg.c b/runtime/msg.c index 6cf41322..1a882597 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -551,6 +551,11 @@ static inline void freeTAG(msg_t *pThis) if(pThis->iLenTAG >= CONF_TAG_BUFSIZE) free(pThis->TAG.pszTAG); } +static inline void freeHOSTNAME(msg_t *pThis) +{ + if(pThis->iLenHOSTNAME >= CONF_HOSTNAME_BUFSIZE) + free(pThis->pszHOSTNAME); +} BEGINobjDestruct(msg) /* be sure to specify the object type also in END and CODESTART macros! */ @@ -569,7 +574,7 @@ CODESTARTobjDestruct(msg) if(pThis->pszRawMsg != pThis->szRawMsg) free(pThis->pszRawMsg); freeTAG(pThis); - free(pThis->pszHOSTNAME); + freeHOSTNAME(pThis); free(pThis->pszInputName); free(pThis->pszRcvFrom); free(pThis->pszRcvFromIP); @@ -669,7 +674,7 @@ msg_t* MsgDup(msg_t* pOld) pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; pNew->offMSG = pOld->offMSG; - /* enable this, if someone actually uses UxTradMsg, delete after some time has + /* enable this, if someone actually uses UxTradMsg, delete after some time has * passed and nobody complained -- rgerhards, 2009-06-16 pNew->offAfterPRI = pOld->offAfterPRI; */ @@ -684,8 +689,18 @@ msg_t* MsgDup(msg_t* pOld) pNew->iLenTAG = pOld->iLenTAG; } } - tmpCOPYSZ(RawMsg); - tmpCOPYSZ(HOSTNAME); + if(pOld->iLenRawMsg < CONF_RAWMSG_BUFSIZE) { + memcpy(pNew->szRawMsg, pOld->szRawMsg, pOld->iLenRawMsg + 1); + pNew->pszRawMsg = pNew->szRawMsg; + } else { + tmpCOPYSZ(RawMsg); + } + if(pOld->iLenHOSTNAME < CONF_HOSTNAME_BUFSIZE) { + memcpy(pNew->szHOSTNAME, pOld->szHOSTNAME, pOld->iLenHOSTNAME + 1); + pNew->pszHOSTNAME = pNew->szHOSTNAME; + } else { + tmpCOPYSZ(HOSTNAME); + } tmpCOPYSZ(RcvFrom); tmpCOPYCSTR(ProgName); @@ -886,30 +901,6 @@ finalize_it: } -/* This function moves the HOSTNAME inside the message object to the - * TAG. It is a specialised function used to handle the condition when - * a message without HOSTNAME is being processed. The missing HOSTNAME - * is only detected at a later stage, during TAG processing, so that - * we already had set the HOSTNAME property and now need to move it to - * the TAG. Of course, we could do this via a couple of get/set methods, - * but it is far more efficient to do it via this specialised method. - * This is especially important as this can be a very common case, e.g. - * when BSD syslog is acting as a sender. - * rgerhards, 2005-11-10. - * NOTE ********* 2009-06-18 / rgerhards ************* - * This function is being obsoleted by the new handling. I keep it for - * a while, and for oversize tags it is somewhat less optimal than in previous - * versions. This should only happen very seldom. - */ -void moveHOSTNAMEtoTAG(msg_t *pM) -{ - assert(pM != NULL); - MsgSetTAG(pM, pM->pszHOSTNAME, pM->iLenHOSTNAME); - free(pM->pszHOSTNAME); - pM->pszHOSTNAME = NULL; - pM->iLenHOSTNAME = 0; -} - /* Access methods - dumb & easy, not a comment for each ;) */ void setProtocolVersion(msg_t *pM, int iNewVersion) @@ -1694,22 +1685,6 @@ finalize_it: } -/* Set the HOSTNAME to a caller-provided string. This is thought - * to be a heap buffer that the caller will no longer use. This - * function is a performance optimization over MsgSetHOSTNAME(). - * rgerhards 2004-11-19 - */ -void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) -{ - assert(pMsg != NULL); - assert(pBuf != NULL); - if(pMsg->pszHOSTNAME != NULL) - free(pMsg->pszHOSTNAME); - pMsg->iLenHOSTNAME = strlen(pBuf); - pMsg->pszHOSTNAME = (uchar*) pBuf; -} - - /* rgerhards 2004-11-09: set HOSTNAME in msg object * rgerhards, 2007-06-21: * Does not return anything. If an error occurs, the hostname is @@ -1720,16 +1695,24 @@ void MsgAssignHOSTNAME(msg_t *pMsg, char *pBuf) * we need it. The rest of the code already knows how to handle an * unset HOSTNAME. */ -void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME, int lenHOSTNAME) +void MsgSetHOSTNAME(msg_t *pThis, uchar* pszHOSTNAME, int lenHOSTNAME) { - assert(pMsg != NULL); - free(pMsg->pszHOSTNAME); + assert(pThis != NULL); - pMsg->iLenHOSTNAME = lenHOSTNAME; - if((pMsg->pszHOSTNAME = malloc(pMsg->iLenHOSTNAME + 1)) != NULL) - memcpy(pMsg->pszHOSTNAME, pszHOSTNAME, pMsg->iLenHOSTNAME + 1); - else - DBGPRINTF("Could not allocate memory in MsgSetHOSTNAME()\n"); + freeHOSTNAME(pThis); + + pThis->iLenHOSTNAME = lenHOSTNAME; + if(pThis->iLenHOSTNAME < CONF_HOSTNAME_BUFSIZE) { + /* small enough: use fixed buffer (faster!) */ + pThis->pszHOSTNAME = pThis->szHOSTNAME; + } else if((pThis->pszHOSTNAME = (uchar*) malloc(pThis->iLenHOSTNAME + 1)) == NULL) { + /* truncate message, better than completely loosing it... */ + pThis->pszHOSTNAME = pThis->szHOSTNAME; + pThis->iLenHOSTNAME = CONF_HOSTNAME_BUFSIZE - 1; + } + + memcpy(pThis->pszHOSTNAME, pszHOSTNAME, pThis->iLenHOSTNAME); + pThis->pszHOSTNAME[pThis->iLenHOSTNAME] = '\0'; /* this also works with truncation! */ } diff --git a/runtime/msg.h b/runtime/msg.h index fd4d650b..4bfc1e3f 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -111,6 +111,7 @@ struct msg { struct syslogTime tTIMESTAMP;/* (parsed) value of the timestamp */ /* some fixed-size buffers to save malloc()/free() for frequently used fields (from the default templates) */ uchar szRawMsg[CONF_RAWMSG_BUFSIZE]; /* most messages are small, and these are stored here (without malloc/free!) */ + uchar szHOSTNAME[CONF_HOSTNAME_BUFSIZE]; union { uchar *pszTAG; /* pointer to tag value */ uchar szBuf[CONF_TAG_BUFSIZE]; @@ -158,7 +159,6 @@ void MsgSetMSGoffs(msg_t *pMsg, short offs); void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG); -void moveHOSTNAMEtoTAG(msg_t *pM); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, cstr_t *pCSPropName, size_t *pPropLen, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 4fa5100e..913ab300 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -31,12 +31,14 @@ * ############################################################# */ #define RS_STRINGBUF_ALLOC_INCREMENT 128 /* MAXSIZE are absolute maxima, while BUFSIZE are just values after which - * processing is more time-intense. + * processing is more time-intense. The BUFSIZE params currently add their + * value to the fixed size of the message object. */ #define CONF_TAG_MAXSIZE 512 /* a value that is deemed far too large for any valid TAG */ #define CONF_TAG_HOSTNAME 512 /* a value that is deemed far too large for any valid HOSTNAME */ #define CONF_RAWMSG_BUFSIZE 101 -#define CONF_TAG_BUFSIZE 33 /* RFC says 32 chars (+ \0), but in practice we see longer ones... */ +#define CONF_TAG_BUFSIZE 32 +#define CONF_HOSTNAME_BUFSIZE 32 /* ############################################################# * diff --git a/tests/diskqueue.sh b/tests/diskqueue.sh index 668a1224..d2e75cbd 100755 --- a/tests/diskqueue.sh +++ b/tests/diskqueue.sh @@ -5,7 +5,7 @@ # added 2009-04-17 by Rgerhards # This file is part of the rsyslog project, released under GPLv3 # uncomment for debugging support: -echo testing queue disk-only mode +echo diskqueue.sh: testing queue disk-only mode source $srcdir/diag.sh init source $srcdir/diag.sh startup diskqueue.conf # 20000 messages should be enough - the disk test is slow enough ;) diff --git a/tools/syslogd.c b/tools/syslogd.c index faa793fb..6f264e5e 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -609,7 +609,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f * being the local host). rgerhards 2004-11-16 */ if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) - MsgSetHOSTNAME(pMsg, hname, strlen(hname)); + MsgSetHOSTNAME(pMsg, hname, ustrlen(hname)); MsgSetRcvFrom(pMsg, hname); MsgSetAfterPRIOffs(pMsg, p - msg); CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP)); @@ -1113,7 +1113,7 @@ int parseRFCSyslogMsg(msg_t *pMsg, int flags) /* HOSTNAME */ if(bContParse) { parseRFCField(&p2parse, pBuf); - MsgSetHOSTNAME(pMsg, pBuf, strlen(pBuf)); + MsgSetHOSTNAME(pMsg, pBuf, ustrlen(pBuf)); } /* APP-NAME */ @@ -1267,8 +1267,7 @@ int parseLegacySyslogMsg(msg_t *pMsg, int flags) */ bufParseTAG[i] = '\0'; /* terminate string */ MsgSetTAG(pMsg, bufParseTAG, i); - } else { - /* we enter this code area when the user has instructed rsyslog NOT + } else {/* we enter this code area when the user has instructed rsyslog NOT * to parse HOSTNAME and TAG - rgerhards, 2006-03-13 */ if(!(flags & INTERNAL_MSG)) { -- cgit v1.2.3 From c53ca3a23429e750aeb6ab1c3600ae8ecb3c8ac6 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 23 Jun 2009 18:23:02 +0200 Subject: fixed a race condition: generated msg string store must be mutex protected the string area that is used to build the string being passed to the output module is now part of the action structure. As such, access to it must also be guarded by the action mutex (an even more optimal solution may be to store it in thread-local storage, but there always must remain some room for improvement ;)). --- action.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/action.c b/action.c index 2040d6bd..01bbfd13 100644 --- a/action.c +++ b/action.c @@ -460,6 +460,16 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) ASSERT(pAction != NULL); + /* We now must guard the output module against execution by multiple threads. The + * plugin interface specifies that output modules must not be thread-safe (except + * if they notify us they are - functionality not yet implemented...). + * rgerhards, 2008-01-30 + */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + d_pthread_mutex_lock(&pAction->mutActExec); + pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); + pthread_setcancelstate(iCancelStateSave, NULL); + /* here we must loop to process all requested strings */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { switch(pAction->eParamPassing) { @@ -472,16 +482,8 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) default:assert(0); /* software bug if this happens! */ } } + iRetries = 0; - /* We now must guard the output module against execution by multiple threads. The - * plugin interface specifies that output modules must not be thread-safe (except - * if they notify us they are - functionality not yet implemented...). - * rgerhards, 2008-01-30 - */ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - d_pthread_mutex_lock(&pAction->mutActExec); - pthread_cleanup_push(mutexCancelCleanup, &pAction->mutActExec); - pthread_setcancelstate(iCancelStateSave, NULL); do { /* on first invocation, this if should never be true. We just put it at the top * of the loop so that processing (and code) is simplified. This code is actually @@ -520,8 +522,6 @@ actionCallDoAction(action_t *pAction, msg_t *pMsg) pAction->bEnabled = 0; /* that's it... */ } - pthread_cleanup_pop(1); /* unlock mutex */ - finalize_it: /* cleanup */ for(i = 0 ; i < pAction->iNumTpls ; ++i) { @@ -544,6 +544,8 @@ finalize_it: } } + pthread_cleanup_pop(1); /* unlock mutex */ + msgDestruct(&pMsg); /* we are now finished with the message */ RETiRet; } -- cgit v1.2.3 From a48918cc2783f4351b246c97cd9669abb9d0de47 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 24 Jun 2009 10:09:57 +0200 Subject: minor: prepped ChangeLog for next beta --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 59d9e194..ff1baba9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,5 @@ --------------------------------------------------------------------------- -Version 4.3.2 [DEVEL] (rgerhards), 2009-??-?? +Version 4.3.2 [beta] (rgerhards), 2009-06-24 - removed long-obsoleted property UxTradMsg - added a generic network stream server (in addition to rather specific syslog tcp server) -- cgit v1.2.3 From 675d46f5b59f64e378968baa5e0dec6810090287 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 24 Jun 2009 10:29:57 +0200 Subject: cleanup (removed left-over from backport) --- tests/Makefile.am | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index a95139f2..88c3137e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -70,8 +70,6 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ testsuites/1.inputname_imtcp_12516 \ omod-if-array.sh \ diag.sh \ - daqueue-persist.sh \ - daqueue-persist-drvr.sh \ testsuites/diag-common.conf \ queue-persist.sh \ queue-persist-drvr.sh \ -- cgit v1.2.3 From 63daa0385cfe4c2999a32c62ff90e9b6e3c1f44e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 24 Jun 2009 10:45:03 +0200 Subject: updated project status --- ChangeLog | 2 ++ doc/manual.html | 2 +- doc/status.html | 27 +++++++++++++++++---------- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/ChangeLog b/ChangeLog index 01dfdd85..b0e104c4 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,5 @@ +--------------------------------------------------------------------------- +Version 5.1.1 [DEVEL] (rgerhards), 2009-06-?? - bugfix: huge memory leak in queue engine (made rsyslogd unusable in production). Occured if at least one queue was in direct mode (the default for action queues) diff --git a/doc/manual.html b/doc/manual.html index 81180f39..1c0e6775 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -19,7 +19,7 @@ rsyslog support available directly from the source!

      Please visit the rsyslog sponsor's page to honor the project sponsors or become one yourself! We are very grateful for any help towards the project goals.

      -

      This documentation is for version 5.1.0 (devel branch) of rsyslog. +

      This documentation is for version 5.1.1 (devel branch) of rsyslog. Visit the rsyslog status page to obtain current version information and project status.

      If you like rsyslog, you might diff --git a/doc/status.html b/doc/status.html index 4e8f1a5f..272b49bc 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,23 +2,30 @@ rsyslog status page

      rsyslog status page

      -

      This page reflects the status as of 2009-05-25.

      +

      This page reflects the status as of 2009-06-24.

      Current Releases

      -

      development: 4.3.1 [2009-05-25] - -change log - -download +

      v5 development: 5.1.0 [2009-05-29] - +change log - +download -
      beta: 3.21.11 [2009-04-03] - -change log - -download

      +
      v4 development: currently no version other than beta -

      v3 stable: 3.22.0 [2009-04-21] - change log - +
      beta: 4.3.2 [2009-06-24] - +change log - +download

      + +

      v4 stable: 4.2.0 [2009-06-23] - +change log - +download + +
      v3 stable: 3.22.0 [2009-04-21] - +change log - download -
      v2 stable: 2.0.7 [2009-04-14] - change log - +
      v2 stable: 2.0.7 [2009-04-14] - change log - download -
      v0 and v1 are deprecated and no longer supported. If you absolutely do not like to +
      v0 to v2 are deprecated and no longer supported. If you absolutely do not like to upgrade, you may consider purchasing a commercial rsyslog support package. Just let us point out that it is really not a good idea to still run a v0 version. -- cgit v1.2.3 From 539f33c6a064b8c28cbb318e856dc50c213fd844 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 24 Jun 2009 11:40:53 +0200 Subject: bugfix: abort when using multiple mainMsgQueue worker threads this bug was introduced by a recent change which was a bit too agressive in avoiding locking. We can probably do better than with this patch, but I think I'll move that into the v5 engine. --- action.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/action.c b/action.c index 01bbfd13..d1d2b78d 100644 --- a/action.c +++ b/action.c @@ -836,22 +836,15 @@ actionCallAction(action_t *pAction, msg_t *pMsg) ISOBJ_TYPE_assert(pMsg, msg); ASSERT(pAction != NULL); - /* We need to lock the mutex only for repeated line processing. - * rgerhards, 2009-06-19 - */ - if(pAction->f_ReduceRepeated == 1) { - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - LockObj(pAction); - pthread_cleanup_push(mutexCancelCleanup, pAction->Sync_mut); - pthread_setcancelstate(iCancelStateSave, NULL); - iRet = doActionCallAction(pAction, pMsg); - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); - UnlockObj(pAction); - pthread_cleanup_pop(0); /* remove mutex cleanup handler */ - pthread_setcancelstate(iCancelStateSave, NULL); - } else { - iRet = doActionCallAction(pAction, pMsg); - } + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + LockObj(pAction); + pthread_cleanup_push(mutexCancelCleanup, pAction->Sync_mut); + pthread_setcancelstate(iCancelStateSave, NULL); + iRet = doActionCallAction(pAction, pMsg); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + UnlockObj(pAction); + pthread_cleanup_pop(0); /* remove mutex cleanup handler */ + pthread_setcancelstate(iCancelStateSave, NULL); RETiRet; } -- cgit v1.2.3 From 11172b62b0f0312fef6a4a0abca982a2a6301649 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 24 Jun 2009 15:05:13 +0200 Subject: quick and dirty fix for one race condition It is intentionally quick & dirty, as I would like to do some better patch, if possible. For that, I probably need the commented-out code, thus no delete. --- action.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/action.c b/action.c index 09dd7df4..d43028b8 100644 --- a/action.c +++ b/action.c @@ -1240,7 +1240,7 @@ actionCallAction(action_t *pAction, msg_t *pMsg) /* We need to lock the mutex only for repeated line processing. * rgerhards, 2009-06-19 */ - if(pAction->f_ReduceRepeated == 1) { + //if(pAction->f_ReduceRepeated == 1) { pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); LockObj(pAction); pthread_cleanup_push(mutexCancelCleanup, pAction->Sync_mut); @@ -1250,9 +1250,9 @@ actionCallAction(action_t *pAction, msg_t *pMsg) UnlockObj(pAction); pthread_cleanup_pop(0); /* remove mutex cleanup handler */ pthread_setcancelstate(iCancelStateSave, NULL); - } else { - iRet = doActionCallAction(pAction, pMsg); - } + //} else { + //iRet = doActionCallAction(pAction, pMsg); + //} RETiRet; } -- cgit v1.2.3 From c60cbd99df4710545587659be6344392e99745ff Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 24 Jun 2009 17:00:10 +0200 Subject: bugfix: mutex was sometimes released when not being held --- runtime/wti.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/runtime/wti.c b/runtime/wti.c index 8b39764e..e43c6cce 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -366,11 +366,12 @@ doIdleProcessing(wti_t *pThis, wtp_t *pWtp, int *pbInactivityTOOccured) rsRetVal wtiWorker(wti_t *pThis) { - DEFVARS_mutexProtection; + DEFVARS_mutexProtection_uncond; wtp_t *pWtp; /* our worker thread pool */ int bInactivityTOOccured = 0; rsRetVal localRet; rsRetVal terminateRet; + bool bMutexIsLocked; DEFiRet; ISOBJ_TYPE_assert(pThis, wti); @@ -380,7 +381,6 @@ wtiWorker(wti_t *pThis) dbgSetThrdName(pThis->pszDbgHdr); pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); - // TODO: if we have a problem, enable again! pThis->batch.nElemDeq = 0; /* re-init dequeue count */ BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); pWtp->pfOnWorkerStartup(pWtp->pUsr); END_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); @@ -396,6 +396,7 @@ wtiWorker(wti_t *pThis) wtpSetInactivityGuard(pThis->pWtp, 0, LOCK_MUTEX); /* must be set before usr mutex is locked! */ BEGIN_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); + bMutexIsLocked = TRUE; /* first check if we are in shutdown process (but evaluate a bit later) */ terminateRet = wtpChkStopWrkr(pWtp, LOCK_MUTEX, MUTEX_ALREADY_LOCKED); @@ -410,6 +411,7 @@ wtiWorker(wti_t *pThis) /* try to execute and process whatever we have */ localRet = pWtp->pfDoWork(pWtp->pUsr, pThis, iCancelStateSave); /* This function must and does RELEASE the MUTEX! */ + bMutexIsLocked = FALSE; if(localRet == RS_RET_IDLE) { if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE) { @@ -429,8 +431,10 @@ wtiWorker(wti_t *pThis) bInactivityTOOccured = 0; /* reset for next run */ } - /* if we exit the loop, the mutex is locked and must be unlocked */ - END_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); + /* if we exit the loop, the mutex may be locked and, if so, must be unlocked */ + if(bMutexIsLocked) { + END_MTX_PROTECTED_OPERATIONS_UNCOND(pWtp->pmutUsr); + } /* indicate termination */ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); -- cgit v1.2.3 From 46024834449840dabf399dda196c9dd11cf78ace Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Jun 2009 11:06:42 +0200 Subject: added a few atomic operations mostly to get thread debugger errors clean (plus, of course, it makes things more deterministic) --- runtime/atomic.h | 4 ++++ runtime/queue.c | 17 ++++++++--------- runtime/wti.c | 24 ++++++++++++++---------- runtime/wtp.c | 3 ++- 4 files changed, 28 insertions(+), 20 deletions(-) diff --git a/runtime/atomic.h b/runtime/atomic.h index fdf64214..4cb832f2 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -46,6 +46,10 @@ # define ATOMIC_DEC_AND_FETCH(data) __sync_sub_and_fetch(&(data), 1) # define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff)) # define ATOMIC_STORE_1_TO_32BIT(data) __sync_lock_test_and_set(&(data), 1) +# define ATOMIC_STORE_0_TO_INT(data) __sync_fetch_and_and(&(data), 0) +# define ATOMIC_STORE_1_TO_INT(data) __sync_fetch_and_or(&(data), 1) +# define ATOMIC_STORE_INT_TO_INT(data, val) __sync_fetch_and_or(&(data), (val)) +# define ATOMIC_CAS(data, oldVal, newVal) __sync_bool_compare_and_swap(&(data), (oldVal), (newVal)); #else /* note that we gained parctical proof that theoretical problems DO occur * if we do not properly address them. See this blog post for details: diff --git a/runtime/queue.c b/runtime/queue.c index 1ae386e7..5102b0df 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -1649,6 +1649,10 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz nDeleted = pWti->batch.nElemDeq; DeleteProcessedBatch(pThis, &pWti->batch); +//int iii = pthread_mutex_trylock(pThis->mut); +//char errStr[1024]; +//rs_strerror_r(iii, errStr, sizeof(errStr)); +//dbgprintf("DequeueConsumableElemnts mutex locked: %d (16 is EBUSY = OK): %s\n", iii, errStr); nDequeued = nDiscarded = 0; while((iQueueSize = getLogicalQueueSize(pThis)) > 0 && nDequeued < pThis->iDeqBatchSize) { //dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); @@ -2473,15 +2477,6 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) ISOBJ_TYPE_assert(pThis, qqueue); - /* first check if we need to discard this message (which will cause CHKiRet() to exit) - * rgerhards, 2008-10-07: It is OK to do this outside of mutex protection. The queue size - * and bRunsDA parameters may not reflect the correct settings here, but they are - * "good enough" in the sense that they can be used to drive the decision. Valgrind's - * threading tools may point this access to be an error, but this is done - * intentional. I do not see this causes problems to us. - */ - CHKiRet(qqueueChkDiscardMsg(pThis, getPhysicalQueueSize(pThis), pThis->bRunsDA, pUsr)); - /* Please note that this function is not cancel-safe and consequently * sets the calling thread's cancelibility state to PTHREAD_CANCEL_DISABLE * during its execution. If that is not done, race conditions occur if the @@ -2493,6 +2488,10 @@ qqueueEnqObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) d_pthread_mutex_lock(pThis->mut); } + /* first check if we need to discard this message (which will cause CHKiRet() to exit) + */ + CHKiRet(qqueueChkDiscardMsg(pThis, getPhysicalQueueSize(pThis), pThis->bRunsDA, pUsr)); + /* then check if we need to add an assistance disk queue */ if(pThis->bIsDA) CHKiRet(ChkStrtDA(pThis)); diff --git a/runtime/wti.c b/runtime/wti.c index e43c6cce..c9fc4879 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -51,6 +51,7 @@ #include "wti.h" #include "obj.h" #include "glbl.h" +#include "atomic.h" /* static data */ DEFobjStaticHelpers @@ -106,6 +107,7 @@ rsRetVal wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex) { DEFiRet; + qWrkCmd_t tCurrCmd; DEFVARS_mutexProtection; ISOBJ_TYPE_assert(pThis, wti); @@ -113,13 +115,14 @@ wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex) BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); + tCurrCmd = pThis->tCurrCmd; /* all worker states must be followed sequentially, only termination can be set in any state */ - if( (bActiveOnly && (pThis->tCurrCmd < eWRKTHRD_RUN_CREATED)) - || (pThis->tCurrCmd > tCmd && !(tCmd == eWRKTHRD_TERMINATING || tCmd == eWRKTHRD_STOPPED))) { - dbgprintf("%s: command %d can not be accepted in current %d processing state - ignored\n", - wtiGetDbgHdr(pThis), tCmd, pThis->tCurrCmd); + if( (bActiveOnly && (tCurrCmd < eWRKTHRD_RUN_CREATED)) + || (tCurrCmd > tCmd && !(tCmd == eWRKTHRD_TERMINATING || tCmd == eWRKTHRD_STOPPED))) { + DBGPRINTF("%s: command %d can not be accepted in current %d processing state - ignored\n", + wtiGetDbgHdr(pThis), tCmd, tCurrCmd); } else { - dbgprintf("%s: receiving command %d\n", wtiGetDbgHdr(pThis), tCmd); + DBGPRINTF("%s: receiving command %d\n", wtiGetDbgHdr(pThis), tCmd); /* we could replace this with a simple if, but we leave the switch in in case we need * to add something at a later stage. -- rgerhards, 2008-09-30 */ @@ -143,7 +146,8 @@ wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex) /* DO NOTHING */ break; } - pThis->tCurrCmd = tCmd; /* apply the new state */ + /* better do a CAS? */ + ATOMIC_STORE_INT_TO_INT(pThis->tCurrCmd, tCmd); /* apply the new state */ } END_MTX_PROTECTED_OPERATIONS(&pThis->mut); @@ -169,7 +173,7 @@ wtiCancelThrd(wti_t *pThis) dbgoprint((obj_t*) pThis, "canceling worker thread\n"); pthread_cancel(pThis->thrdID); wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - pThis->pWtp->bThrdStateChanged = 1; /* indicate change, so harverster will be called */ + ATOMIC_STORE_1_TO_INT(pThis->pWtp->bThrdStateChanged); /* indicate change, so harverster will be called */ } d_pthread_mutex_unlock(&pThis->mut); @@ -324,7 +328,7 @@ wtiWorkerCancelCleanup(void *arg) d_pthread_mutex_lock(&pWtp->mut); wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); /* TODO: sync access? I currently think it is NOT needed -- rgerhards, 2008-01-28 */ - pWtp->bThrdStateChanged = 1; /* indicate change, so harverster will be called */ + ATOMIC_STORE_1_TO_INT(pWtp->bThrdStateChanged); /* indicate change, so harverster will be called */ d_pthread_mutex_unlock(&pWtp->mut); pthread_setcancelstate(iCancelStateSave, NULL); @@ -409,8 +413,8 @@ wtiWorker(wti_t *pThis) } /* try to execute and process whatever we have */ - localRet = pWtp->pfDoWork(pWtp->pUsr, pThis, iCancelStateSave); /* This function must and does RELEASE the MUTEX! */ + localRet = pWtp->pfDoWork(pWtp->pUsr, pThis, iCancelStateSave); bMutexIsLocked = FALSE; if(localRet == RS_RET_IDLE) { @@ -445,7 +449,7 @@ RUNLOG_STR("XXX: Worker shutdown"); pWtp->pfOnWorkerShutdown(pWtp->pUsr); wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - pWtp->bThrdStateChanged = 1; /* indicate change, so harverster will be called */ + ATOMIC_STORE_1_TO_INT(pWtp->bThrdStateChanged); /* indicate change, so harverster will be called */ d_pthread_mutex_unlock(&pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); diff --git a/runtime/wtp.c b/runtime/wtp.c index 267555cd..f5769a72 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -39,6 +39,7 @@ #include #include #include +#include /// TODO: check on solaris if this is any longer needed - I don't think so - rgerhards, 2009-09-20 //#ifdef OS_SOLARIS @@ -217,7 +218,7 @@ wtpProcessThrdChanges(wtp_t *pThis) */ do { /* reset the change marker */ - pThis->bThrdStateChanged = 0; + ATOMIC_STORE_0_TO_INT(pThis->bThrdStateChanged); /* go through all threads */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { wtiProcessThrdChanges(pThis->pWrkr[i], LOCK_MUTEX); -- cgit v1.2.3 From d12b9e0c67cc72c9b1631bf2a5611d383e7ad69d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Jun 2009 12:03:14 +0200 Subject: some memory accesses are now explicitely atomic ... as far as I think this mostly is to keep the thread debuggers happy --- ChangeLog | 1 + runtime/atomic.h | 5 +++++ runtime/wti.c | 26 +++++++++++++++++--------- runtime/wtp.c | 3 ++- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6cbfcf70..e7de1c4d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -23,6 +23,7 @@ Version 4.5.0 [DEVEL] (rgerhards), 2009-??-?? * $OMFileFlushOnTXEnd * $MainMsgQueueSyncQueueFiles * $ActionQueueSyncQueueFiles +- done some memory accesses explicitely atomic --------------------------------------------------------------------------- Version 4.3.2 [beta] (rgerhards), 2009-06-24 - removed long-obsoleted property UxTradMsg diff --git a/runtime/atomic.h b/runtime/atomic.h index fdf64214..f5ae9357 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -46,6 +46,11 @@ # define ATOMIC_DEC_AND_FETCH(data) __sync_sub_and_fetch(&(data), 1) # define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff)) # define ATOMIC_STORE_1_TO_32BIT(data) __sync_lock_test_and_set(&(data), 1) +# define ATOMIC_STORE_0_TO_INT(data) __sync_fetch_and_and(&(data), 0) +# define ATOMIC_STORE_1_TO_INT(data) __sync_fetch_and_or(&(data), 1) +# define ATOMIC_STORE_INT_TO_INT(data, val) __sync_fetch_and_or(&(data), (val)) +# define ATOMIC_CAS(data, oldVal, newVal) __sync_bool_compare_and_swap(&(data), (oldVal), (newVal)); +# define ATOMIC_CAS_VAL(data, oldVal, newVal) __sync_val_compare_and_swap(&(data), (oldVal), (newVal)); #else /* note that we gained parctical proof that theoretical problems DO occur * if we do not properly address them. See this blog post for details: diff --git a/runtime/wti.c b/runtime/wti.c index f20812f8..9de7c365 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -51,6 +51,7 @@ #include "wti.h" #include "obj.h" #include "glbl.h" +#include "atomic.h" /* static data */ DEFobjStaticHelpers @@ -106,6 +107,7 @@ rsRetVal wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex) { DEFiRet; + qWrkCmd_t tCurrCmd; DEFVARS_mutexProtection; ISOBJ_TYPE_assert(pThis, wti); @@ -113,13 +115,14 @@ wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex) BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, bLockMutex); + tCurrCmd = pThis->tCurrCmd; /* all worker states must be followed sequentially, only termination can be set in any state */ - if( (bActiveOnly && (pThis->tCurrCmd < eWRKTHRD_RUN_CREATED)) - || (pThis->tCurrCmd > tCmd && !(tCmd == eWRKTHRD_TERMINATING || tCmd == eWRKTHRD_STOPPED))) { - dbgprintf("%s: command %d can not be accepted in current %d processing state - ignored\n", - wtiGetDbgHdr(pThis), tCmd, pThis->tCurrCmd); + if( (bActiveOnly && (tCurrCmd < eWRKTHRD_RUN_CREATED)) + || (tCurrCmd > tCmd && !(tCmd == eWRKTHRD_TERMINATING || tCmd == eWRKTHRD_STOPPED))) { + DBGPRINTF("%s: command %d can not be accepted in current %d processing state - ignored\n", + wtiGetDbgHdr(pThis), tCmd, tCurrCmd); } else { - dbgprintf("%s: receiving command %d\n", wtiGetDbgHdr(pThis), tCmd); + DBGPRINTF("%s: receiving command %d\n", wtiGetDbgHdr(pThis), tCmd); /* we could replace this with a simple if, but we leave the switch in in case we need * to add something at a later stage. -- rgerhards, 2008-09-30 */ @@ -143,7 +146,12 @@ wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex) /* DO NOTHING */ break; } - pThis->tCurrCmd = tCmd; /* apply the new state */ + /* apply the new state */ + unsigned val = ATOMIC_CAS_VAL(pThis->tCurrCmd, tCurrCmd, tCmd); + if(val != tCurrCmd) { + DBGPRINTF("wtiSetState PROBLEM, tCurrCmd %d overwritten with %d, wanted to set %d\n", tCurrCmd, val, tCmd); + } + } END_MTX_PROTECTED_OPERATIONS(&pThis->mut); @@ -169,7 +177,7 @@ wtiCancelThrd(wti_t *pThis) dbgoprint((obj_t*) pThis, "canceling worker thread\n"); pthread_cancel(pThis->thrdID); wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - pThis->pWtp->bThrdStateChanged = 1; /* indicate change, so harverster will be called */ + ATOMIC_STORE_1_TO_INT(pThis->pWtp->bThrdStateChanged); /* indicate change, so harverster will be called */ } d_pthread_mutex_unlock(&pThis->mut); @@ -318,7 +326,7 @@ wtiWorkerCancelCleanup(void *arg) d_pthread_mutex_lock(&pWtp->mut); wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); /* TODO: sync access? I currently think it is NOT needed -- rgerhards, 2008-01-28 */ - pWtp->bThrdStateChanged = 1; /* indicate change, so harverster will be called */ + ATOMIC_STORE_1_TO_INT(pWtp->bThrdStateChanged); /* indicate change, so harverster will be called */ d_pthread_mutex_unlock(&pWtp->mut); pthread_setcancelstate(iCancelStateSave, NULL); @@ -405,7 +413,7 @@ wtiWorker(wti_t *pThis) pWtp->pfOnWorkerShutdown(pWtp->pUsr); wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); - pWtp->bThrdStateChanged = 1; /* indicate change, so harverster will be called */ + ATOMIC_STORE_1_TO_INT(pWtp->bThrdStateChanged); /* indicate change, so harverster will be called */ d_pthread_mutex_unlock(&pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); diff --git a/runtime/wtp.c b/runtime/wtp.c index 611b3f25..218a5db6 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -39,6 +39,7 @@ #include #include #include +#include #ifdef OS_SOLARIS # include @@ -214,7 +215,7 @@ wtpProcessThrdChanges(wtp_t *pThis) */ do { /* reset the change marker */ - pThis->bThrdStateChanged = 0; + ATOMIC_STORE_0_TO_INT(pThis->bThrdStateChanged); /* go through all threads */ for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { wtiProcessThrdChanges(pThis->pWrkr[i], LOCK_MUTEX); -- cgit v1.2.3 From d116f30a877c65b4f23dbb92601251402b0f957e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Jun 2009 12:53:00 +0200 Subject: improvements/fixes in queue termination timeout handling - bugfix: subtle (and usually irrelevant) issue in timout processing timeout could be one second too early if nanoseconds wrapped - set a more sensible timeout for shutdow, now 1.5 seconds to complete processing (this also removes those cases where the shutdown message was not written because the termination happened before it) --- ChangeLog | 6 ++++++ runtime/srutils.c | 4 +++- runtime/wti.c | 8 ++++++-- tools/syslogd.c | 4 ++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index ca1ea10a..0cef1a0d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -3,6 +3,12 @@ Version 5.1.1 [DEVEL] (rgerhards), 2009-06-?? - bugfix: huge memory leak in queue engine (made rsyslogd unusable in production). Occured if at least one queue was in direct mode (the default for action queues) +- imported many performance optimizations from v4-devel (4.5.0) +- bugfix: subtle (and usually irrelevant) issue in timout processing + timeout could be one second too early if nanoseconds wrapped +- set a more sensible timeout for shutdow, now 1.5 seconds to complete + processing (this also removes those cases where the shutdown message + was not written because the termination happened before it) --------------------------------------------------------------------------- Version 5.1.0 [DEVEL] (rgerhards), 2009-05-29 diff --git a/runtime/srutils.c b/runtime/srutils.c index 5407531f..c403b312 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -366,6 +366,7 @@ int getNumberDigits(long lNum) /* compute an absolute time timeout suitable for calls to pthread_cond_timedwait() + * iTimeout is in milliseconds * rgerhards, 2008-01-14 */ rsRetVal @@ -375,11 +376,12 @@ timeoutComp(struct timespec *pt, long iTimeout) assert(pt != NULL); /* compute timeout */ clock_gettime(CLOCK_REALTIME, pt); + pt->tv_sec += iTimeout / 1000; pt->tv_nsec += (iTimeout % 1000) * 1000000; /* think INTEGER arithmetic! */ if(pt->tv_nsec > 999999999) { /* overrun? */ pt->tv_nsec -= 1000000000; + ++pt->tv_sec; } - pt->tv_sec += iTimeout / 1000; ENDfunc return RS_RET_OK; /* so far, this is static... */ } diff --git a/runtime/wti.c b/runtime/wti.c index 9c137f57..917b456b 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -147,10 +147,12 @@ wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex) break; } /* apply the new state */ +dbgprintf("worker terminator will write stateval %d\n", tCmd); unsigned val = ATOMIC_CAS_VAL(pThis->tCurrCmd, tCurrCmd, tCmd); if(val != tCurrCmd) { DBGPRINTF("wtiSetState PROBLEM, tCurrCmd %d overwritten with %d, wanted to set %d\n", tCurrCmd, val, tCmd); } +//dbgprintf("worker terminator has written stateval %d\n", tCmd); } END_MTX_PROTECTED_OPERATIONS(&pThis->mut); @@ -158,7 +160,7 @@ wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex) } -/* Cancel the thread. If the thread is already cancelled or termination, +/* Cancel the thread. If the thread is already cancelled or terminated, * we do not again cancel it. But it is save and legal to call wtiCancelThrd() in * such situations. * rgerhards, 2008-02-26 @@ -172,8 +174,10 @@ wtiCancelThrd(wti_t *pThis) d_pthread_mutex_lock(&pThis->mut); + wtiProcessThrdChanges(pThis, MUTEX_ALREADY_LOCKED); /* process state change, so that we have current state vars */ + if(pThis->tCurrCmd >= eWRKTHRD_TERMINATING) { - dbgoprint((obj_t*) pThis, "canceling worker thread\n"); + dbgoprint((obj_t*) pThis, "canceling worker thread, curr stat %d\n", pThis->tCurrCmd); pthread_cancel(pThis->thrdID); wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); ATOMIC_STORE_1_TO_INT(pThis->pWtp->bThrdStateChanged); /* indicate change, so harverster will be called */ diff --git a/tools/syslogd.c b/tools/syslogd.c index ace08e58..6a3fa6c9 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -302,7 +302,7 @@ static uchar *pszMainMsgQFName = NULL; /* prefix for the main message queue f static int64 iMainMsgQueMaxFileSize = 1024*1024; static int iMainMsgQPersistUpdCnt = 0; /* persist queue info every n updates */ static int bMainMsgQSyncQeueFiles = 0; /* sync queue files on every write? */ -static int iMainMsgQtoQShutdown = 0; /* queue shutdown */ +static int iMainMsgQtoQShutdown = 1500; /* queue shutdown (ms) */ static int iMainMsgQtoActShutdown = 1000; /* action shutdown (in phase 2) */ static int iMainMsgQtoEnq = 2000; /* timeout for queue enque */ static int iMainMsgQtoWrkShutdown = 60000; /* timeout for worker thread shutdown */ @@ -367,7 +367,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iMainMsgQueueNumWorkers = 1; iMainMsgQPersistUpdCnt = 0; bMainMsgQSyncQeueFiles = 0; - iMainMsgQtoQShutdown = 0; + iMainMsgQtoQShutdown = 1500; iMainMsgQtoActShutdown = 1000; iMainMsgQtoEnq = 2000; iMainMsgQtoWrkShutdown = 60000; -- cgit v1.2.3 From 5af5b1e42868d00dfbc5fa47028768d0a03f8e32 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Jun 2009 12:59:13 +0200 Subject: improvements/fixes in queue termination timeout handling - bugfix: subtle (and usually irrelevant) issue in timout processing timeout could be one second too early if nanoseconds wrapped - set a more sensible timeout for shutdow, now 1.5 seconds to complete processing (this also removes those cases where the shutdown message was not written because the termination happened before it) --- ChangeLog | 5 +++++ runtime/srutils.c | 4 +++- runtime/wti.c | 6 ++++-- tools/syslogd.c | 4 ++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index e7de1c4d..d7c6e408 100644 --- a/ChangeLog +++ b/ChangeLog @@ -24,6 +24,11 @@ Version 4.5.0 [DEVEL] (rgerhards), 2009-??-?? * $MainMsgQueueSyncQueueFiles * $ActionQueueSyncQueueFiles - done some memory accesses explicitely atomic +- bugfix: subtle (and usually irrelevant) issue in timout processing + timeout could be one second too early if nanoseconds wrapped +- set a more sensible timeout for shutdow, now 1.5 seconds to complete + processing (this also removes those cases where the shutdown message + was not written because the termination happened before it) --------------------------------------------------------------------------- Version 4.3.2 [beta] (rgerhards), 2009-06-24 - removed long-obsoleted property UxTradMsg diff --git a/runtime/srutils.c b/runtime/srutils.c index 5407531f..c403b312 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -366,6 +366,7 @@ int getNumberDigits(long lNum) /* compute an absolute time timeout suitable for calls to pthread_cond_timedwait() + * iTimeout is in milliseconds * rgerhards, 2008-01-14 */ rsRetVal @@ -375,11 +376,12 @@ timeoutComp(struct timespec *pt, long iTimeout) assert(pt != NULL); /* compute timeout */ clock_gettime(CLOCK_REALTIME, pt); + pt->tv_sec += iTimeout / 1000; pt->tv_nsec += (iTimeout % 1000) * 1000000; /* think INTEGER arithmetic! */ if(pt->tv_nsec > 999999999) { /* overrun? */ pt->tv_nsec -= 1000000000; + ++pt->tv_sec; } - pt->tv_sec += iTimeout / 1000; ENDfunc return RS_RET_OK; /* so far, this is static... */ } diff --git a/runtime/wti.c b/runtime/wti.c index 9de7c365..156d8116 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -159,7 +159,7 @@ wtiSetState(wti_t *pThis, qWrkCmd_t tCmd, int bActiveOnly, int bLockMutex) } -/* Cancel the thread. If the thread is already cancelled or termination, +/* Cancel the thread. If the thread is already cancelled or terminated, * we do not again cancel it. But it is save and legal to call wtiCancelThrd() in * such situations. * rgerhards, 2008-02-26 @@ -173,8 +173,10 @@ wtiCancelThrd(wti_t *pThis) d_pthread_mutex_lock(&pThis->mut); + wtiProcessThrdChanges(pThis, MUTEX_ALREADY_LOCKED); /* process state change, so that we have current state vars */ + if(pThis->tCurrCmd >= eWRKTHRD_TERMINATING) { - dbgoprint((obj_t*) pThis, "canceling worker thread\n"); + dbgoprint((obj_t*) pThis, "canceling worker thread, curr stat %d\n", pThis->tCurrCmd); pthread_cancel(pThis->thrdID); wtiSetState(pThis, eWRKTHRD_TERMINATING, 0, MUTEX_ALREADY_LOCKED); ATOMIC_STORE_1_TO_INT(pThis->pWtp->bThrdStateChanged); /* indicate change, so harverster will be called */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 6f264e5e..0fdaf5ba 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -299,7 +299,7 @@ static uchar *pszMainMsgQFName = NULL; /* prefix for the main message queue f static int64 iMainMsgQueMaxFileSize = 1024*1024; static int iMainMsgQPersistUpdCnt = 0; /* persist queue info every n updates */ static int bMainMsgQSyncQeueFiles = 0; /* sync queue files on every write? */ -static int iMainMsgQtoQShutdown = 0; /* queue shutdown */ +static int iMainMsgQtoQShutdown = 1500; /* queue shutdown (ms) */ static int iMainMsgQtoActShutdown = 1000; /* action shutdown (in phase 2) */ static int iMainMsgQtoEnq = 2000; /* timeout for queue enque */ static int iMainMsgQtoWrkShutdown = 60000; /* timeout for worker thread shutdown */ @@ -363,7 +363,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a iMainMsgQueueNumWorkers = 1; iMainMsgQPersistUpdCnt = 0; bMainMsgQSyncQeueFiles = 0; - iMainMsgQtoQShutdown = 0; + iMainMsgQtoQShutdown = 1500; iMainMsgQtoActShutdown = 1000; iMainMsgQtoEnq = 2000; iMainMsgQtoWrkShutdown = 60000; -- cgit v1.2.3 From 4818b0081d3a265a87f9f646d79f2a2ffbcda819 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Jun 2009 15:21:29 +0200 Subject: bugfix: subtle synchronization issue This may have caused a segfault under strange circumstances (but if we just run long enough with a high enough message volume, even the strangest circumstances will occur...) --- runtime/atomic.h | 3 +++ runtime/msg.c | 8 +++++--- runtime/queue.c | 13 ++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/runtime/atomic.h b/runtime/atomic.h index f5ae9357..f0733f09 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -41,7 +41,10 @@ * They simply came in too late. -- rgerhards, 2008-04-02 */ #ifdef HAVE_ATOMIC_BUILTINS +# define ATOMIC_SUB(data, val) __sync_fetch_and_sub(&(data), val) +# define ATOMIC_ADD(data, val) __sync_fetch_and_add(&(data), val) # define ATOMIC_INC(data) ((void) __sync_fetch_and_add(&(data), 1)) +# define ATOMIC_INC_AND_FETCH(data) __sync_fetch_and_add(&(data), 1) # define ATOMIC_DEC(data) ((void) __sync_sub_and_fetch(&(data), 1)) # define ATOMIC_DEC_AND_FETCH(data) __sync_sub_and_fetch(&(data), 1) # define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff)) diff --git a/runtime/msg.c b/runtime/msg.c index 8aab5317..a1a4714f 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -607,10 +607,12 @@ CODESTARTobjDestruct(msg) * operations on the counter. --- rgerhards, 2009-06-22. */ # if HAVE_MALLOC_TRIM - { /* standard C requires a new block for a new variable definition! */ + { /* standard C requires a new block for a new variable definition! + * To simplify matters, we use modulo arithmetic and live with the fact + * that we trim too often when the counter wraps. + */ static unsigned iTrimCtr = 1; - if(iTrimCtr ++ % 100000 == 0) { - iTrimCtr = 1; + if(ATOMIC_INC_AND_FETCH(iTrimCtr) % 100000 == 0) { malloc_trim(128*1024); } } diff --git a/runtime/queue.c b/runtime/queue.c index 5102b0df..7438fbaa 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -212,7 +212,7 @@ static inline void queueDrain(qqueue_t *pThis) BEGINfunc dbgoprint((obj_t*) pThis, "queue (type %d) will lose %d messages, destroying...\n", pThis->qType, pThis->iQueueSize); /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ - while(pThis->iQueueSize-- > 0) { + while(ATOMIC_DEC_AND_FETCH(pThis->iQueueSize) > 0) { pThis->qDeq(pThis, &pUsr); if(pUsr != NULL) { objDestruct(pUsr); @@ -1547,15 +1547,14 @@ DoDeleteBatchFromQStore(qqueue_t *pThis, int nElem) ISOBJ_TYPE_assert(pThis, qqueue); -//dbgprintf("pre delete batch from store, new sizes: log %d, phys %d, nElem %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis), nElem); /* now send delete request to storage driver */ for(i = 0 ; i < nElem ; ++i) { pThis->qDel(pThis); } /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ - pThis->iQueueSize -= nElem; - pThis->nLogDeq -= nElem; + ATOMIC_SUB(pThis->iQueueSize, nElem); + ATOMIC_SUB(pThis->nLogDeq, nElem); dbgprintf("delete batch from store, new sizes: log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); ++pThis->deqIDDel; /* one more batch dequeued */ @@ -1649,13 +1648,9 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz nDeleted = pWti->batch.nElemDeq; DeleteProcessedBatch(pThis, &pWti->batch); -//int iii = pthread_mutex_trylock(pThis->mut); -//char errStr[1024]; -//rs_strerror_r(iii, errStr, sizeof(errStr)); -//dbgprintf("DequeueConsumableElemnts mutex locked: %d (16 is EBUSY = OK): %s\n", iii, errStr); nDequeued = nDiscarded = 0; while((iQueueSize = getLogicalQueueSize(pThis)) > 0 && nDequeued < pThis->iDeqBatchSize) { -//dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); +dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); CHKiRet(qqueueDeq(pThis, &pUsr)); /* check if we should discard this element */ -- cgit v1.2.3 From b82f9ae304b8cdfada55d7dce93daf50cf7f8578 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Jun 2009 15:29:09 +0200 Subject: bugfix: subtle potential issue during queue shutdown ... this one could cause trouble, but I really don't think it caused any actual harm. --- runtime/queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/queue.c b/runtime/queue.c index 1e3b761f..13e7007a 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -110,7 +110,7 @@ static inline void queueDrain(qqueue_t *pThis) ASSERT(pThis != NULL); /* iQueueSize is not decremented by qDel(), so we need to do it ourselves */ - while(pThis->iQueueSize-- > 0) { + while(ATOMIC_DEC_AND_FETCH(pThis->iQueueSize) > 0) { pThis->qDel(pThis, &pUsr); if(pUsr != NULL) { objDestruct(pUsr); -- cgit v1.2.3 From 46435e2c3b2ca31aca641aba290173142fae540c Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Jun 2009 15:33:49 +0200 Subject: cleanup (removed now-unused parameters) --- runtime/datetime.c | 8 ++++---- runtime/datetime.h | 8 ++++---- runtime/msg.c | 18 +++++++++--------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/runtime/datetime.c b/runtime/datetime.c index c0e145af..e0f3f5fa 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -546,7 +546,7 @@ finalize_it: * returns the size of the timestamp written in bytes (without * the string terminator). If 0 is returend, an error occured. */ -int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf, size_t iLenDst) +int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf) { /* currently we do not consider localtime/utc. This may later be * added. If so, I recommend using a property replacer option @@ -577,7 +577,7 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf, size_t iLenDst) } -int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf, size_t iLenDst) +int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf) { /* see note in formatTimestampToMySQL, applies here as well */ assert(ts != NULL); @@ -655,7 +655,7 @@ int formatTimestampSecFrac(struct syslogTime *ts, char* pBuf) * returns the size of the timestamp written in bytes (without * the string terminator). If 0 is returend, an error occured. */ -int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) +int formatTimestamp3339(struct syslogTime *ts, char* pBuf) { int iBuf; int power; @@ -732,7 +732,7 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf, size_t iLenBuf) * returns the size of the timestamp written in bytes (without * the string termnator). If 0 is returend, an error occured. */ -int formatTimestamp3164(struct syslogTime *ts, char* pBuf, size_t iLenBuf) +int formatTimestamp3164(struct syslogTime *ts, char* pBuf) { static char* monthNames[12] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; diff --git a/runtime/datetime.h b/runtime/datetime.h index 1ea99aba..79a86d05 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -38,10 +38,10 @@ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ void (*getCurrTime)(struct syslogTime *t, time_t *ttSeconds); rsRetVal (*ParseTIMESTAMP3339)(struct syslogTime *pTime, uchar** ppszTS); rsRetVal (*ParseTIMESTAMP3164)(struct syslogTime *pTime, uchar** pszTS); - int (*formatTimestampToMySQL)(struct syslogTime *ts, char* pDst, size_t iLenDst); - int (*formatTimestampToPgSQL)(struct syslogTime *ts, char *pDst, size_t iLenDst); - int (*formatTimestamp3339)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); - int (*formatTimestamp3164)(struct syslogTime *ts, char* pBuf, size_t iLenBuf); + int (*formatTimestampToMySQL)(struct syslogTime *ts, char* pDst); + int (*formatTimestampToPgSQL)(struct syslogTime *ts, char *pDst); + int (*formatTimestamp3339)(struct syslogTime *ts, char* pBuf); + int (*formatTimestamp3164)(struct syslogTime *ts, char* pBuf); int (*formatTimestampSecFrac)(struct syslogTime *ts, char* pBuf); ENDinterface(datetime) #define datetimeCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ diff --git a/runtime/msg.c b/runtime/msg.c index 8aab5317..cbdfff78 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1007,7 +1007,7 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) MsgLock(pM); if(pM->pszTIMESTAMP3164 == NULL) { pM->pszTIMESTAMP3164 = pM->pszTimestamp3164; - datetime.formatTimestamp3164(&pM->tTIMESTAMP, pM->pszTIMESTAMP3164, 16); + datetime.formatTimestamp3164(&pM->tTIMESTAMP, pM->pszTIMESTAMP3164); } MsgUnlock(pM); return(pM->pszTIMESTAMP3164); @@ -1018,7 +1018,7 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return ""; } - datetime.formatTimestampToMySQL(&pM->tTIMESTAMP, pM->pszTIMESTAMP_MySQL, 15); + datetime.formatTimestampToMySQL(&pM->tTIMESTAMP, pM->pszTIMESTAMP_MySQL); } MsgUnlock(pM); return(pM->pszTIMESTAMP_MySQL); @@ -1029,7 +1029,7 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return ""; } - datetime.formatTimestampToPgSQL(&pM->tTIMESTAMP, pM->pszTIMESTAMP_PgSQL, 21); + datetime.formatTimestampToPgSQL(&pM->tTIMESTAMP, pM->pszTIMESTAMP_PgSQL); } MsgUnlock(pM); return(pM->pszTIMESTAMP_PgSQL); @@ -1037,7 +1037,7 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) MsgLock(pM); if(pM->pszTIMESTAMP3339 == NULL) { pM->pszTIMESTAMP3339 = pM->pszTimestamp3339; - datetime.formatTimestamp3339(&pM->tTIMESTAMP, pM->pszTIMESTAMP3339, 33); + datetime.formatTimestamp3339(&pM->tTIMESTAMP, pM->pszTIMESTAMP3339); } MsgUnlock(pM); return(pM->pszTIMESTAMP3339); @@ -1070,7 +1070,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return ""; } - datetime.formatTimestamp3164(&pM->tRcvdAt, pM->pszRcvdAt3164, 16); + datetime.formatTimestamp3164(&pM->tRcvdAt, pM->pszRcvdAt3164); } MsgUnlock(pM); return(pM->pszRcvdAt3164); @@ -1081,7 +1081,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return ""; } - datetime.formatTimestampToMySQL(&pM->tRcvdAt, pM->pszRcvdAt_MySQL, 15); + datetime.formatTimestampToMySQL(&pM->tRcvdAt, pM->pszRcvdAt_MySQL); } MsgUnlock(pM); return(pM->pszRcvdAt_MySQL); @@ -1092,7 +1092,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return ""; } - datetime.formatTimestampToPgSQL(&pM->tRcvdAt, pM->pszRcvdAt_PgSQL, 21); + datetime.formatTimestampToPgSQL(&pM->tRcvdAt, pM->pszRcvdAt_PgSQL); } MsgUnlock(pM); return(pM->pszRcvdAt_PgSQL); @@ -1103,7 +1103,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return ""; } - datetime.formatTimestamp3164(&pM->tRcvdAt, pM->pszRcvdAt3164, 16); + datetime.formatTimestamp3164(&pM->tRcvdAt, pM->pszRcvdAt3164); } MsgUnlock(pM); return(pM->pszRcvdAt3164); @@ -1114,7 +1114,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) MsgUnlock(pM); return ""; } - datetime.formatTimestamp3339(&pM->tRcvdAt, pM->pszRcvdAt3339, 33); + datetime.formatTimestamp3339(&pM->tRcvdAt, pM->pszRcvdAt3339); } MsgUnlock(pM); return(pM->pszRcvdAt3339); -- cgit v1.2.3 From 1f79c785975261e4158c9b85f6451d5bd00b2495 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Jun 2009 16:03:28 +0200 Subject: named queue worker threads ... but I don't see the name anywhere...? --- runtime/wtp.c | 6 ++++++ tools/syslogd.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/runtime/wtp.c b/runtime/wtp.c index 218a5db6..02662cde 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -40,6 +40,7 @@ #include #include #include +#include #ifdef OS_SOLARIS # include @@ -433,6 +434,11 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in sigfillset(&sigSet); pthread_sigmask(SIG_BLOCK, &sigSet, NULL); + /* set thread name - we ignore if the call fails, has no harsh consequences... */ + if(prctl(PR_SET_NAME, wtpGetDbgHdr(pThis), 0, 0, 0) != 0) { + DBGPRINTF("prctl failed, not setting thread name for '%s'\n", wtpGetDbgHdr(pThis)); + } + BEGIN_MTX_PROTECTED_OPERATIONS(&pThis->mut, LOCK_MUTEX); /* do some late initialization */ diff --git a/tools/syslogd.c b/tools/syslogd.c index 0fdaf5ba..8bc4939f 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2272,7 +2272,7 @@ init(void) exit(1); } /* name our main queue object (it's not fatal if it fails...) */ - obj.SetName((obj_t*) pMsgQueue, (uchar*) "main queue"); + obj.SetName((obj_t*) pMsgQueue, (uchar*) "main Q"); /* ... set some properties ... */ # define setQPROP(func, directive, data) \ -- cgit v1.2.3 From 12dc91a157cf2375596920c1f4f7c361ad717103 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Jun 2009 17:09:51 +0200 Subject: backported "clean" increment of memory trim counter (not protected by mutex) --- runtime/atomic.h | 1 + runtime/msg.c | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/runtime/atomic.h b/runtime/atomic.h index f5ae9357..d5aaf56b 100644 --- a/runtime/atomic.h +++ b/runtime/atomic.h @@ -42,6 +42,7 @@ */ #ifdef HAVE_ATOMIC_BUILTINS # define ATOMIC_INC(data) ((void) __sync_fetch_and_add(&(data), 1)) +# define ATOMIC_INC_AND_FETCH(data) __sync_fetch_and_add(&(data), 1) # define ATOMIC_DEC(data) ((void) __sync_sub_and_fetch(&(data), 1)) # define ATOMIC_DEC_AND_FETCH(data) __sync_sub_and_fetch(&(data), 1) # define ATOMIC_FETCH_32BIT(data) ((unsigned) __sync_fetch_and_and(&(data), 0xffffffff)) diff --git a/runtime/msg.c b/runtime/msg.c index cbdfff78..75933d68 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -607,10 +607,12 @@ CODESTARTobjDestruct(msg) * operations on the counter. --- rgerhards, 2009-06-22. */ # if HAVE_MALLOC_TRIM - { /* standard C requires a new block for a new variable definition! */ + { /* standard C requires a new block for a new variable definition! + * To simplify matters, we use modulo arithmetic and live with the fact + * that we trim too often when the counter wraps. + */ static unsigned iTrimCtr = 1; - if(iTrimCtr ++ % 100000 == 0) { - iTrimCtr = 1; + if(ATOMIC_INC_AND_FETCH(iTrimCtr) % 100000 == 0) { malloc_trim(128*1024); } } -- cgit v1.2.3 From 07a7152ea0ec499a481942e9003079efffd7bb8f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Jun 2009 18:23:03 +0200 Subject: bugfix: msg_t mutex was sometimes initialized twice --- runtime/parser.c | 1 - 1 file changed, 1 deletion(-) diff --git a/runtime/parser.c b/runtime/parser.c index 0a27c982..d4ca7673 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -307,7 +307,6 @@ rsRetVal parseMsg(msg_t *pMsg) /* finalize message object */ pMsg->msgFlags &= ~NEEDS_PARSING; /* this message is now parsed */ - MsgPrepareEnqueue(pMsg); /* "historical" name - preparese for multi-threading */ finalize_it: RETiRet; -- cgit v1.2.3 From 464dcf2339634a2cda0244011abe7ed8f6ed2fe5 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 25 Jun 2009 18:44:48 +0200 Subject: some more stringbuffer optimization --- outchannel.c | 2 -- runtime/msg.c | 8 ++------ runtime/stringbuf.c | 45 ++++++--------------------------------------- runtime/stringbuf.h | 45 ++++++++++++++++++++++----------------------- template.c | 1 - 5 files changed, 30 insertions(+), 71 deletions(-) diff --git a/outchannel.c b/outchannel.c index 4f8abb32..74c18218 100644 --- a/outchannel.c +++ b/outchannel.c @@ -106,7 +106,6 @@ static rsRetVal get_Field(uchar **pp, uchar **pField) p = *pp; CHKiRet(cstrConstruct(&pStrB)); - rsCStrSetAllocIncrement(pStrB, 32); /* copy the field */ while(*p && *p != ' ' && *p != ',') { @@ -175,7 +174,6 @@ static inline rsRetVal get_restOfLine(uchar **pp, uchar **pBuf) p = *pp; CHKiRet(cstrConstruct(&pStrB)); - rsCStrSetAllocIncrement(pStrB, 32); /* copy the field */ while(*p) { diff --git a/runtime/msg.c b/runtime/msg.c index 75933d68..fa820893 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -353,7 +353,8 @@ static void MsgPrepareEnqueueLockingCase(msg_t *pThis) * down. We should do that over time. -- rgerhards, 2008-07-14 */ } - pthread_mutex_init(&pThis->mut, &mutAttr); +// pthread_mutex_init(&pThis->mut, &mutAttr); +pthread_mutex_init(&pThis->mut, NULL); /* we do no longer need the attribute. According to the * POSIX spec, we can destroy it without affecting the @@ -837,7 +838,6 @@ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) /* now obtain the PROCID string... */ CHKiRet(cstrConstruct(&pM->pCSPROCID)); - rsCStrSetAllocIncrement(pM->pCSPROCID, 16); while((i < pM->iLenTAG) && (pszTag[i] != ']')) { CHKiRet(cstrAppendChar(pM->pCSPROCID, pszTag[i])); ++i; @@ -890,7 +890,6 @@ static rsRetVal aquireProgramName(msg_t *pM) */ pszTag = (uchar*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); CHKiRet(cstrConstruct(&pM->pCSProgName)); - rsCStrSetAllocIncrement(pM->pCSProgName, 33); for( i = 0 ; (i < pM->iLenTAG) && isprint((int) pszTag[i]) && (pszTag[i] != '\0') && (pszTag[i] != ':') @@ -1242,7 +1241,6 @@ rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME) if(pMsg->pCSAPPNAME == NULL) { /* we need to obtain the object first */ CHKiRet(rsCStrConstruct(&pMsg->pCSAPPNAME)); - rsCStrSetAllocIncrement(pMsg->pCSAPPNAME, 128); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSAPPNAME, (uchar*) pszAPPNAME); @@ -1323,7 +1321,6 @@ rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID) if(pMsg->pCSMSGID == NULL) { /* we need to obtain the object first */ CHKiRet(rsCStrConstruct(&pMsg->pCSMSGID)); - rsCStrSetAllocIncrement(pMsg->pCSMSGID, 128); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSMSGID, (uchar*) pszMSGID); @@ -1496,7 +1493,6 @@ rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData) if(pMsg->pCSStrucData == NULL) { /* we need to obtain the object first */ CHKiRet(rsCStrConstruct(&pMsg->pCSStrucData)); - rsCStrSetAllocIncrement(pMsg->pCSStrucData, 128); } /* if we reach this point, we have the object */ iRet = rsCStrSetSzStr(pMsg->pCSStrucData, (uchar*) pszStrucData); diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index d3ddf33a..f3824831 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -71,7 +71,6 @@ rsRetVal cstrConstruct(cstr_t **ppThis) pThis->pszBuf = NULL; pThis->iBufSize = 0; pThis->iStrLen = 0; - pThis->iAllocIncrement = RS_STRINGBUF_ALLOC_INCREMENT; *ppThis = pThis; finalize_it: @@ -154,7 +153,7 @@ void rsCStrDestruct(cstr_t **ppThis) * rgerhards, 2008-01-07 * changed to utilized realloc() -- rgerhards, 2009-06-16 */ -static rsRetVal +rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) { uchar *pNewBuf; @@ -162,16 +161,16 @@ rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded) DEFiRet; /* first compute the new size needed */ - if(iMinNeeded > pThis->iAllocIncrement) { - /* we allocate "n" iAllocIncrements. Usually, that should + if(iMinNeeded > RS_STRINGBUF_ALLOC_INCREMENT) { + /* we allocate "n" ALLOC_INCREMENTs. Usually, that should * leave some room after the absolutely needed one. It also * reduces memory fragmentation. Note that all of this are * integer operations (very important to understand what is * going on)! Parenthesis are for better readibility. */ - iNewSize = ((iMinNeeded / pThis->iAllocIncrement) + 1) * pThis->iAllocIncrement; + iNewSize = (iMinNeeded / RS_STRINGBUF_ALLOC_INCREMENT + 1) * RS_STRINGBUF_ALLOC_INCREMENT; } else { - iNewSize = pThis->iBufSize + pThis->iAllocIncrement; + iNewSize = pThis->iBufSize + RS_STRINGBUF_ALLOC_INCREMENT; } iNewSize += pThis->iBufSize; /* add current size */ @@ -246,33 +245,10 @@ finalize_it: } -/* Append a character to the current string object. This may only be done until - * cstrFinalize() is called. - * rgerhards, 2009-06-16 - */ -rsRetVal cstrAppendChar(cstr_t *pThis, uchar c) -{ - DEFiRet; - - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if(pThis->iStrLen >= pThis->iBufSize) { - CHKiRet(rsCStrExtendBuf(pThis, 1)); /* need more memory! */ - } - - /* ok, when we reach this, we have sufficient memory */ - *(pThis->pBuf + pThis->iStrLen++) = c; - -finalize_it: - RETiRet; -} - - /* Sets the string object to the classigal sz-string provided. * Any previously stored vlaue is discarded. If a NULL pointer * the the new value (pszNew) is provided, an empty string is - * created (this is NOT an error!). Property iAllocIncrement is - * not modified by this function. + * created (this is NOT an error!). * rgerhards, 2005-10-18 */ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) @@ -290,7 +266,6 @@ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) pThis->iStrLen = strlen((char*)pszNew); pThis->iBufSize = pThis->iStrLen; pThis->pszBuf = NULL; - /* iAllocIncrement is NOT modified! */ /* now save the new value */ if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { @@ -472,14 +447,6 @@ finalize_it: } -void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement) -{ - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - assert(iNewIncrement > 0); - pThis->iAllocIncrement = iNewIncrement; -} - - /* return the length of the current string * 2005-09-09 rgerhards * Please note: this is only a function in a debug build. diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 9d2e7865..4fbd9a9b 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -35,6 +35,7 @@ #ifndef _STRINGBUF_H_INCLUDED__ #define _STRINGBUF_H_INCLUDED__ 1 + /** * The dynamic string buffer object. */ @@ -47,7 +48,6 @@ typedef struct cstr_s uchar *pszBuf; /**< pointer to the sz version of the string (after it has been created )*/ size_t iBufSize; /**< current maximum size of the string buffer */ size_t iStrLen; /**< length of the string in characters. */ - unsigned short iAllocIncrement; /**< the amount of bytes the string should be expanded if it needs to */ bool bIsForeignBuf; /**< is pBuf a buffer provided by someone else? */ } cstr_t; @@ -66,13 +66,28 @@ rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom); void rsCStrDestruct(cstr_t **ppThis); #define cstrDestruct(x) rsCStrDestruct((x)) -/** - * Append a character to an existing string. If necessary, the - * method expands the string buffer. - * - * \param c Character to append to string. + +/* Append a character to the current string object. This may only be done until + * cstrFinalize() is called. + * rgerhards, 2009-06-16 */ -rsRetVal cstrAppendChar(cstr_t *pThis, uchar c); +rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded); /* our helper, NOT a public interface! */ +static inline rsRetVal cstrAppendChar(cstr_t *pThis, uchar c) +{ + rsRetVal iRet; + + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + + if(pThis->iStrLen >= pThis->iBufSize) { + CHKiRet(rsCStrExtendBuf(pThis, 1)); /* need more memory! */ + } + + /* ok, when we reach this, we have sufficient memory */ + *(pThis->pBuf + pThis->iStrLen++) = c; + +finalize_it: + return iRet; +} /** * Truncate "n" number of characters from the end of the @@ -101,22 +116,6 @@ rsRetVal rsCStrAppendStr(cstr_t *pThis, uchar* psz); */ rsRetVal rsCStrAppendStrWithLen(cstr_t *pThis, uchar* psz, size_t iStrLen); -/** - * Set a new allocation incremet. This will influence - * the allocation the next time the string will be expanded. - * It can be set and changed at any time. If done immediately - * after custructing the StrB object, this will also be - * the inital allocation. - * - * \param iNewIncrement The new increment size - * - * \note It is possible to use a very low increment, e.g. 1 byte. - * This can generate a considerable overhead. We highly - * advise not to use an increment below 32 bytes, except - * if you are very well aware why you are doing it ;) - */ -void rsCStrSetAllocIncrement(cstr_t *pThis, int iNewIncrement); -#define rsCStrGetAllocIncrement(pThis) ((pThis)->iAllocIncrement) /** * Append an integer to the string. No special formatting is diff --git a/template.c b/template.c index 43a54d98..704f0b19 100644 --- a/template.c +++ b/template.c @@ -373,7 +373,6 @@ static int do_Constant(unsigned char **pp, struct template *pTpl) if(cstrConstruct(&pStrB) != RS_RET_OK) return 1; - rsCStrSetAllocIncrement(pStrB, 32); /* process the message and expand escapes * (additional escapes can be added here if needed) */ -- cgit v1.2.3 From d88803c8e471b0b822108106137f5383c7efd57e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 26 Jun 2009 13:42:52 +0200 Subject: got rid of the recursive requirement for msg_t mutex (finally!) --- action.c | 4 +- runtime/msg.c | 267 +++++++++++++++++++++++------------------------------- runtime/msg.h | 8 +- runtime/rsyslog.h | 5 +- runtime/rule.c | 7 +- runtime/srUtils.h | 2 - 6 files changed, 123 insertions(+), 170 deletions(-) diff --git a/action.c b/action.c index d1d2b78d..f21feea6 100644 --- a/action.c +++ b/action.c @@ -782,8 +782,8 @@ doActionCallAction(action_t *pAction, msg_t *pMsg) (pMsg->msgFlags & MARK) == 0 && getMSGLen(pMsg) == getMSGLen(pAction->f_pMsg) && !strcmp(getMSG(pMsg), getMSG(pAction->f_pMsg)) && !strcmp(getHOSTNAME(pMsg), getHOSTNAME(pAction->f_pMsg)) && - !strcmp(getPROCID(pMsg), getPROCID(pAction->f_pMsg)) && - !strcmp(getAPPNAME(pMsg), getAPPNAME(pAction->f_pMsg))) { + !strcmp(getPROCID(pMsg, LOCK_MUTEX), getPROCID(pAction->f_pMsg, LOCK_MUTEX)) && + !strcmp(getAPPNAME(pMsg, LOCK_MUTEX), getAPPNAME(pAction->f_pMsg, LOCK_MUTEX))) { pAction->f_prevcount++; dbgprintf("msg repeated %d times, %ld sec of %d.\n", pAction->f_prevcount, (long) getActNow(pAction) - pAction->f_time, diff --git a/runtime/msg.c b/runtime/msg.c index fa820893..3f69dbcb 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -271,8 +271,13 @@ static char *syslog_number_names[24] = { "0", "1", "2", "3", "4", "5", "6", "7", "15", "16", "17", "18", "19", "20", "21", "22", "23" }; /* some forward declarations */ -static int getAPPNAMELen(msg_t *pM); -static int getProtocolVersion(msg_t *pM); +static int getAPPNAMELen(msg_t *pM, bool bLockMutex); + +static inline int getProtocolVersion(msg_t *pM) +{ + return(pM->iProtocolVersion); +} + /* The following functions will support advanced output module * multithreading, once this is implemented. Currently, we @@ -334,34 +339,9 @@ static void MsgLockingDummy(msg_t __attribute__((unused)) *pMsg) */ static void MsgPrepareEnqueueLockingCase(msg_t *pThis) { - int iErr; - pthread_mutexattr_t mutAttr; BEGINfunc assert(pThis != NULL); - iErr = pthread_mutexattr_init(&mutAttr); - if(iErr != 0) { - dbgprintf("error initializing mutex attribute in %s:%d, trying to continue\n", - __FILE__, __LINE__); - } - iErr = pthread_mutexattr_settype(&mutAttr, PTHREAD_MUTEX_RECURSIVE); - if(iErr != 0) { - dbgprintf("ERROR setting mutex attribute to recursive in %s:%d, trying to continue " - "but we will probably either abort or hang soon\n", - __FILE__, __LINE__); - /* TODO: it makes very little sense to continue here, - * but it requires an iRet interface to gracefully shut - * down. We should do that over time. -- rgerhards, 2008-07-14 - */ - } -// pthread_mutex_init(&pThis->mut, &mutAttr); -pthread_mutex_init(&pThis->mut, NULL); - - /* we do no longer need the attribute. According to the - * POSIX spec, we can destroy it without affecting the - * initialized mutex (that used the attribute). - * rgerhards, 2008-07-14 - */ - pthread_mutexattr_destroy(&mutAttr); + pthread_mutex_init(&pThis->mut, NULL); pThis->bDoLock = 1; ENDfunc } @@ -810,6 +790,7 @@ msg_t *MsgAddRef(msg_t *pM) * can obtain a PROCID. Take in mind that not every legacy syslog message * actually has a PROCID. * rgerhards, 2005-11-24 + * THIS MUST be called with the message lock locked. */ static rsRetVal aquirePROCIDFromTAG(msg_t *pM) { @@ -874,7 +855,8 @@ finalize_it: * The program name is not parsed by default, because it is infrequently-used. * If it is needed, this function should be called first. It checks if it is * already set and extracts it, if not. - * A message object must be provided, else a crash will occur. + * + * IMPORTANT: A locked message object must be provided, else a crash will occur. * rgerhards, 2005-10-19 */ static rsRetVal aquireProgramName(msg_t *pM) @@ -885,9 +867,7 @@ static rsRetVal aquireProgramName(msg_t *pM) assert(pM != NULL); if(pM->pCSProgName == NULL) { - /* ok, we do not yet have it. So let's parse the TAG - * to obtain it. - */ + /* ok, we do not yet have it. So let's parse the TAG to obtain it. */ pszTag = (uchar*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); CHKiRet(cstrConstruct(&pM->pCSProgName)); for( i = 0 @@ -916,12 +896,6 @@ void setProtocolVersion(msg_t *pM, int iNewVersion) pM->iProtocolVersion = iNewVersion; } -static int getProtocolVersion(msg_t *pM) -{ - assert(pM != NULL); - return(pM->iProtocolVersion); -} - /* note: string is taken from constant pool, do NOT free */ char *getProtocolVersionString(msg_t *pM) { @@ -1232,7 +1206,10 @@ MsgSetAfterPRIOffs(msg_t *pMsg, short offs) /* rgerhards 2004-11-24: set APP-NAME in msg object - * TODO: revisit msg locking code! + * This is not locked, because it either is called during message + * construction (where we need no locking) or later as part of a function + * which already obtained the lock. So in general, this function here must + * only be called when it it safe to do so without it aquiring a lock. */ rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME) { @@ -1250,20 +1227,6 @@ finalize_it: } -static void tryEmulateAPPNAME(msg_t *pM); /* forward reference */ -/* rgerhards, 2005-11-24 - */ -char *getAPPNAME(msg_t *pM) -{ - assert(pM != NULL); - MsgLock(pM); - if(pM->pCSAPPNAME == NULL) - tryEmulateAPPNAME(pM); - MsgUnlock(pM); - return (pM->pCSAPPNAME == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSAPPNAME); -} - - /* rgerhards 2004-11-24: set PROCID in msg object */ rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID) @@ -1282,33 +1245,44 @@ finalize_it: RETiRet; } + +/* check if we have a procid, and, if not, try to aquire/emulate it. + * This must be called WITHOUT the message lock being held. + * rgerhards, 2009-06-26 + */ +static inline void preparePROCID(msg_t *pM, bool bLockMutex) +{ + if(pM->pCSPROCID == NULL) { + if(bLockMutex == LOCK_MUTEX) + MsgLock(pM); + /* re-query, things may have changed in the mean time... */ + if(pM->pCSPROCID == NULL) + aquirePROCIDFromTAG(pM); + if(bLockMutex == LOCK_MUTEX) + MsgUnlock(pM); + } +} + + +#if 0 /* rgerhards, 2005-11-24 */ -static inline int getPROCIDLen(msg_t *pM) +static inline int getPROCIDLen(msg_t *pM, bool bLockMutex) { assert(pM != NULL); - MsgLock(pM); - if(pM->pCSPROCID == NULL) - aquirePROCIDFromTAG(pM); - MsgUnlock(pM); + preparePROCID(pM, bLockMutex); return (pM->pCSPROCID == NULL) ? 1 : rsCStrLen(pM->pCSPROCID); } +#endif /* rgerhards, 2005-11-24 */ -char *getPROCID(msg_t *pM) +char *getPROCID(msg_t *pM, bool bLockMutex) { - char* pszRet; - ISOBJ_TYPE_assert(pM, msg); - MsgLock(pM); - if(pM->pCSPROCID == NULL) - aquirePROCIDFromTAG(pM); - pszRet = (pM->pCSPROCID == NULL) ? "-" : (char*) cstrGetSzStrNoNULL(pM->pCSPROCID); - //pszRet = (pM->pCSPROCID == NULL) ? "-" : (char*) rsCStrGetSzStrNoNULL(pM->pCSPROCID); - MsgUnlock(pM); - return pszRet; + preparePROCID(pM, bLockMutex); + return (pM->pCSPROCID == NULL) ? "-" : (char*) cstrGetSzStrNoNULL(pM->pCSPROCID); } @@ -1383,26 +1357,31 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf) * if there is a TAG and, if not, if it can emulate it. * rgerhards, 2005-11-24 */ -static inline void tryEmulateTAG(msg_t *pM) +static inline void tryEmulateTAG(msg_t *pM, bool bLockMutex) { size_t lenTAG; uchar bufTAG[CONF_TAG_MAXSIZE]; assert(pM != NULL); + if(bLockMutex == LOCK_MUTEX) + MsgLock(pM); if(pM->iLenTAG > 0) return; /* done, no need to emulate */ if(getProtocolVersion(pM) == 1) { - if(!strcmp(getPROCID(pM), "-")) { + if(!strcmp(getPROCID(pM, MUTEX_ALREADY_LOCKED), "-")) { /* no process ID, use APP-NAME only */ - MsgSetTAG(pM, (uchar*) getAPPNAME(pM), getAPPNAMELen(pM)); + MsgSetTAG(pM, (uchar*) getAPPNAME(pM, MUTEX_ALREADY_LOCKED), getAPPNAMELen(pM, MUTEX_ALREADY_LOCKED)); } else { /* now we can try to emulate */ - lenTAG = snprintf((char*)bufTAG, CONF_TAG_MAXSIZE, "%s[%s]", getAPPNAME(pM), getPROCID(pM)); + lenTAG = snprintf((char*)bufTAG, CONF_TAG_MAXSIZE, "%s[%s]", + getAPPNAME(pM, MUTEX_ALREADY_LOCKED), getPROCID(pM, MUTEX_ALREADY_LOCKED)); bufTAG[32] = '\0'; /* just to make sure... */ MsgSetTAG(pM, bufTAG, lenTAG); } } + if(bLockMutex == LOCK_MUTEX) + MsgUnlock(pM); } @@ -1413,13 +1392,12 @@ static inline char *getTAG(msg_t *pM) if(pM == NULL) ret = ""; else { - MsgLock(pM); - tryEmulateTAG(pM); + if(pM->iLenTAG == 0) + tryEmulateTAG(pM, LOCK_MUTEX); if(pM->iLenTAG == 0) ret = ""; else ret = (char*) ((pM->iLenTAG < CONF_TAG_BUFSIZE) ? pM->TAG.szBuf : pM->TAG.pszTAG); - MsgUnlock(pM); } return(ret); } @@ -1521,101 +1499,50 @@ static inline char *getStructuredData(msg_t *pM) } - -/* get the length of the "programname" sz string - * rgerhards, 2005-10-19 +/* check if we have a ProgramName, and, if not, try to aquire/emulate it. + * rgerhards, 2009-06-26 */ -int getProgramNameLen(msg_t *pM) +static inline void prepareProgramName(msg_t *pM, bool bLockMutex) { - int iRet; + if(pM->pCSProgName == NULL) { + if(bLockMutex == LOCK_MUTEX) + MsgLock(pM); - assert(pM != NULL); - MsgLock(pM); - if((iRet = aquireProgramName(pM)) != RS_RET_OK) { - dbgprintf("error %d returned by aquireProgramName() in getProgramNameLen()\n", iRet); - MsgUnlock(pM); - return 0; /* best we can do (consistent wiht what getProgramName() returns) */ - } - MsgUnlock(pM); + /* re-query as things might have changed during locking */ + if(pM->pCSProgName == NULL) + aquireProgramName(pM); - return (pM->pCSProgName == NULL) ? 0 : rsCStrLen(pM->pCSProgName); + if(bLockMutex == LOCK_MUTEX) + MsgUnlock(pM); + } } -/* get the "programname" as sz string +/* get the length of the "programname" sz string * rgerhards, 2005-10-19 */ -char *getProgramName(msg_t *pM) /* this is the non-locking version for internal use */ +int getProgramNameLen(msg_t *pM, bool bLockMutex) { - int iRet; - char *pszRet; - assert(pM != NULL); - MsgLock(pM); - if((iRet = aquireProgramName(pM)) != RS_RET_OK) { - dbgprintf("error %d returned by aquireProgramName() in getProgramName()\n", iRet); - pszRet = ""; /* best we can do */ - } else { - pszRet = (pM->pCSProgName == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSProgName); - } - - MsgUnlock(pM); - return pszRet; + prepareProgramName(pM, bLockMutex); + return (pM->pCSProgName == NULL) ? 0 : rsCStrLen(pM->pCSProgName); } -/* The code below was an approach without PTHREAD_MUTEX_RECURSIVE - * However, it turned out to be quite complex. So far, we use recursive - * locking, which is OK from a performance point of view, especially as - * we do not anticipate that multithreading msg objects is used often. - * However, we may re-think about using non-recursive locking and I leave this - * code in here to conserve the idea. -- rgerhards, 2008-01-05 - */ -#if 0 -static char *getProgramNameNoLock(msg_t *pM) /* this is the non-locking version for internal use */ -{ - int iRet; - assert(pM != NULL); - if((iRet = aquireProgramName(pM)) != RS_RET_OK) { - dbgprintf("error %d returned by aquireProgramName() in getProgramName()\n", iRet); - return ""; /* best we can do */ - } - return (pM->pCSProgName == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSProgName); -} -char *getProgramName(msg_t *pM) /* this is the external callable version */ +/* get the "programname" as sz string + * rgerhards, 2005-10-19 + */ +char *getProgramName(msg_t *pM, bool bLockMutex) { - char *pszRet; - - MsgLock(pM); - pszRet = getProgramNameNoLock(pM); - MsgUnlock(pM); - return pszRet; + prepareProgramName(pM, bLockMutex); + return (pM->pCSProgName == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSProgName); } -/* an alternative approach has been: */ -/* The macro below is used to generate external function definitions - * for such functions that may also be called internally (and thus have - * both a locking and non-locking implementation. Over time, we could - * reconsider how we handle that. -- rgerhards, 2008-01-05 - */ -#define EXT_LOCKED_FUNC(fName, ret) \ -ret fName(msg_t *pM) \ -{ \ - ret valRet; \ - MsgLock(pM); \ - valRet = fName##NoLock(pM); \ - MsgUnlock(pM); \ - return(valRet); \ -} -EXT_LOCKED_FUNC(getProgramName, char*) -/* in this approach, the external function is provided by the macro and - * needs not to be writen. - */ -#endif /* #if 0 -- saved code */ /* This function tries to emulate APPNAME if it is not present. Its * main use is when we have received a log record via legacy syslog and * now would like to send out the same one via syslog-protocol. + * MUST be called with the Msg Lock locked! */ static void tryEmulateAPPNAME(msg_t *pM) { @@ -1625,18 +1552,46 @@ static void tryEmulateAPPNAME(msg_t *pM) if(getProtocolVersion(pM) == 0) { /* only then it makes sense to emulate */ - MsgSetAPPNAME(pM, getProgramName(pM)); + MsgSetAPPNAME(pM, getProgramName(pM, MUTEX_ALREADY_LOCKED)); + } +} + + + +/* check if we have a APPNAME, and, if not, try to aquire/emulate it. + * This must be called WITHOUT the message lock being held. + * rgerhards, 2009-06-26 + */ +static inline void prepareAPPNAME(msg_t *pM, bool bLockMutex) +{ + if(pM->pCSAPPNAME == NULL) { + if(bLockMutex == LOCK_MUTEX) + MsgLock(pM); + + /* re-query as things might have changed during locking */ + if(pM->pCSAPPNAME == NULL) + tryEmulateAPPNAME(pM); + + if(bLockMutex == LOCK_MUTEX) + MsgUnlock(pM); } } +/* rgerhards, 2005-11-24 + */ +char *getAPPNAME(msg_t *pM, bool bLockMutex) +{ + assert(pM != NULL); + prepareAPPNAME(pM, bLockMutex); + return (pM->pCSAPPNAME == NULL) ? "" : (char*) rsCStrGetSzStrNoNULL(pM->pCSAPPNAME); +} /* rgerhards, 2005-11-24 */ -static int getAPPNAMELen(msg_t *pM) +static int getAPPNAMELen(msg_t *pM, bool bLockMutex) { assert(pM != NULL); - if(pM->pCSAPPNAME == NULL) - tryEmulateAPPNAME(pM); + prepareAPPNAME(pM, bLockMutex); return (pM->pCSAPPNAME == NULL) ? 0 : rsCStrLen(pM->pCSAPPNAME); } @@ -1987,15 +1942,15 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } else if(!strcmp((char*) pName, "timegenerated")) { pRes = getTimeGenerated(pMsg, pTpe->data.field.eDateFormat); } else if(!strcmp((char*) pName, "programname")) { - pRes = getProgramName(pMsg); + pRes = getProgramName(pMsg, LOCK_MUTEX); } else if(!strcmp((char*) pName, "protocol-version")) { pRes = getProtocolVersionString(pMsg); } else if(!strcmp((char*) pName, "structured-data")) { pRes = getStructuredData(pMsg); } else if(!strcmp((char*) pName, "app-name")) { - pRes = getAPPNAME(pMsg); + pRes = getAPPNAME(pMsg, LOCK_MUTEX); } else if(!strcmp((char*) pName, "procid")) { - pRes = getPROCID(pMsg); + pRes = getPROCID(pMsg, LOCK_MUTEX); } else if(!strcmp((char*) pName, "msgid")) { pRes = getMSGID(pMsg); /* here start system properties (those, that do not relate to the message itself */ diff --git a/runtime/msg.h b/runtime/msg.h index 4bfc1e3f..56ba5b80 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -168,14 +168,14 @@ rsRetVal MsgEnableThreadSafety(void); /* TODO: remove these five (so far used in action.c) */ char *getMSG(msg_t *pM); char *getHOSTNAME(msg_t *pM); -char *getPROCID(msg_t *pM); -char *getAPPNAME(msg_t *pM); +char *getPROCID(msg_t *pM, bool bLockMutex); +char *getAPPNAME(msg_t *pM, bool bLockMutex); int getMSGLen(msg_t *pM); char *getHOSTNAME(msg_t *pM); int getHOSTNAMELen(msg_t *pM); -char *getProgramName(msg_t *pM); -int getProgramNameLen(msg_t *pM); +char *getProgramName(msg_t *pM, bool bLockMutex); +int getProgramNameLen(msg_t *pM, bool bLockMutex); uchar *getRcvFrom(msg_t *pM); /* The MsgPrepareEnqueue() function is a macro for performance reasons. diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 913ab300..c2fb9383 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -414,9 +414,8 @@ typedef enum rsObjectID rsObjID; #endif /* some constants */ -// TODO: do we really need them - if not, delete -- rgerhards, 2009-06-10 -#define IGNORE_ERROR_CODES 1 -#define ABORT_ON_ERROR 0 +#define MUTEX_ALREADY_LOCKED 0 +#define LOCK_MUTEX 1 /* The following prototype is convenient, even though it may not be the 100% correct place.. -- rgerhards 2008-01-07 */ void dbgprintf(char *, ...) __attribute__((format(printf, 1, 2))); diff --git a/runtime/rule.c b/runtime/rule.c index 12494543..221368e6 100644 --- a/runtime/rule.c +++ b/runtime/rule.c @@ -147,13 +147,14 @@ shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) offset = 1; } } - if(!rsCStrOffsetSzStrCmp(pRule->pCSProgNameComp, offset, (uchar*) getProgramName(pMsg), getProgramNameLen(pMsg))) + if(!rsCStrOffsetSzStrCmp(pRule->pCSProgNameComp, offset, + (uchar*) getProgramName(pMsg, LOCK_MUTEX), getProgramNameLen(pMsg, LOCK_MUTEX))) bEqv = 1; if((!bEqv && !bInv) || (bEqv && bInv)) { /* not equal or inverted selection, so we are already done... */ - dbgprintf("programname filter '%s' does not match '%s'\n", - rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp), getProgramName(pMsg)); + DBGPRINTF("programname filter '%s' does not match '%s'\n", + rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp), getProgramName(pMsg, LOCK_MUTEX)); FINALIZE; } } diff --git a/runtime/srUtils.h b/runtime/srUtils.h index b37559cf..16766312 100644 --- a/runtime/srUtils.h +++ b/runtime/srUtils.h @@ -109,8 +109,6 @@ rsRetVal getFileSize(uchar *pszName, off_t *pSize); #define mutex_cancelsafe_unlock(mut) pthread_cleanup_pop(1) /* some useful constants */ -#define MUTEX_ALREADY_LOCKED 0 -#define LOCK_MUTEX 1 #define DEFVARS_mutexProtection\ int iCancelStateSave; \ int bLockedOpIsLocked=0 -- cgit v1.2.3 From 6be06d50e328c4b8c79b263e176e6b4e384a03f9 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 26 Jun 2009 14:08:30 +0200 Subject: added some basic multithreading tests to testbench --- tests/Makefile.am | 6 ++++++ tests/testsuites/threadingmq.conf | 20 ++++++++++++++++++++ tests/testsuites/threadingmqaq.conf | 24 ++++++++++++++++++++++++ tests/threadingmq.sh | 15 +++++++++++++++ tests/threadingmqaq.sh | 15 +++++++++++++++ 5 files changed, 80 insertions(+) create mode 100644 tests/testsuites/threadingmq.conf create mode 100644 tests/testsuites/threadingmqaq.conf create mode 100755 tests/threadingmq.sh create mode 100755 tests/threadingmqaq.sh diff --git a/tests/Makefile.am b/tests/Makefile.am index 4d38f875..b34daa06 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -14,6 +14,8 @@ TESTS += omod-if-array.sh \ parsertest.sh \ timestamp.sh \ inputname.sh \ + threadingmq.sh \ + threadingmqaq.sh \ fieldtest.sh endif @@ -106,6 +108,10 @@ EXTRA_DIST= 1.rstest 2.rstest 3.rstest err1.rstest \ queue-persist.sh \ queue-persist-drvr.sh \ testsuites/queue-persist.conf \ + threadingmq.sh \ + testsuites/threadingmq.conf \ + threadingmqaq.sh \ + testsuites/threadingmqaq.conf \ DiagTalker.java \ cfg.sh diff --git a/tests/testsuites/threadingmq.conf b/tests/testsuites/threadingmq.conf new file mode 100644 index 00000000..aa5197bb --- /dev/null +++ b/tests/testsuites/threadingmq.conf @@ -0,0 +1,20 @@ +# Threading test, we run a tcp flood to via an +# engine instructed to use multiple threads +# rgerhards, 2009-06-26 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$MaxOpenFiles 2000 +$InputTCPMaxSessions 1100 +$InputTCPServerRun 13514 + +$MainMsgQueueWorkerThreadMinimumMessages 10 +$MainMsgQueueWorkerThreads 5 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +# write quickly to the output file: +$OMFileFlushOnTXEnd off +$OMFileIOBufferSize 256k +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/testsuites/threadingmqaq.conf b/tests/testsuites/threadingmqaq.conf new file mode 100644 index 00000000..f1bb72df --- /dev/null +++ b/tests/testsuites/threadingmqaq.conf @@ -0,0 +1,24 @@ +# Threading test, we run a tcp flood to via an +# engine instructed to use multiple threads +# rgerhards, 2009-06-26 +$IncludeConfig diag-common.conf + +$ModLoad ../plugins/imtcp/.libs/imtcp +$MainMsgQueueTimeoutShutdown 10000 +$MaxOpenFiles 2000 +$InputTCPMaxSessions 1100 +$InputTCPServerRun 13514 + +$MainMsgQueueWorkerThreadMinimumMessages 10 +$MainMsgQueueWorkerThreads 5 + +$template outfmt,"%msg:F,58:2%\n" +$template dynfile,"rsyslog.out.log" # trick to use relative path names! +# write quickly to the output file: +$OMFileFlushOnTXEnd off +$OMFileIOBufferSize 256k +# This time, also run the action queue detached +$ActionQueueWorkerThreadMinimumMessages 10 +$ActionQueueWorkerThreads 5 +$ActionQueueType LinkedList +:msg, contains, "msgnum:" ?dynfile;outfmt diff --git a/tests/threadingmq.sh b/tests/threadingmq.sh new file mode 100755 index 00000000..5c29ec60 --- /dev/null +++ b/tests/threadingmq.sh @@ -0,0 +1,15 @@ +# test many concurrent tcp connections +# we send 100,000 messages in the hopes that his puts at least a little bit +# of pressure on the threading subsystem. To really prove it, we would need to +# push messages for several minutes, but that takes too long during the +# automatted tests (hint: do this manually after suspect changes). Thankfully, +# in practice many threading bugs result in an abort rather quickly and these +# should be covered by this test here. +# rgerhards, 2009-06-26 +echo TEST: threadingmq.sh - main queue concurrency +source $srcdir/diag.sh init +source $srcdir/diag.sh startup threadingmq.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 2 100000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 99999 +source $srcdir/diag.sh exit diff --git a/tests/threadingmqaq.sh b/tests/threadingmqaq.sh new file mode 100755 index 00000000..009551fd --- /dev/null +++ b/tests/threadingmqaq.sh @@ -0,0 +1,15 @@ +# test many concurrent tcp connections +# we send 100,000 messages in the hopes that his puts at least a little bit +# of pressure on the threading subsystem. To really prove it, we would need to +# push messages for several minutes, but that takes too long during the +# automatted tests (hint: do this manually after suspect changes). Thankfully, +# in practice many threading bugs result in an abort rather quickly and these +# should be covered by this test here. +# rgerhards, 2009-06-26 +echo TEST: threadingmqaq.sh - main/action queue concurrency +source $srcdir/diag.sh init +source $srcdir/diag.sh startup threadingmqaq.conf +source $srcdir/diag.sh tcpflood 127.0.0.1 13514 2 100000 +source $srcdir/diag.sh shutdown-when-empty # shut down rsyslogd when done processing messages +source $srcdir/diag.sh seq-check 0 99999 +source $srcdir/diag.sh exit -- cgit v1.2.3 From 1f361c5eb9028e7750a1c84811c9c3ac5cdd0a31 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 26 Jun 2009 14:22:00 +0200 Subject: some optimization, declared some frequently called small stringbuf functions inline --- runtime/stringbuf.c | 57 ---------------------------------------------- runtime/stringbuf.h | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 60 deletions(-) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index f3824831..88f5a3a2 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -343,42 +343,6 @@ uchar* rsCStrGetSzStr(cstr_t *pThis) } -/* Converts the CStr object to a classical sz string and returns that. - * Same restrictions as in cstrGetSzStr() applies (see there!). This - * function here guarantees that a valid string is returned, even if - * the CStr object currently holds a NULL pointer string buffer. If so, - * "" is returned. - * rgerhards 2005-10-19 - * WARNING: The returned pointer MUST NOT be freed, as it may be - * obtained from that constant memory pool (in case of NULL!) - */ -uchar* cstrGetSzStrNoNULL(cstr_t *pThis) -{ - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - if(pThis->pBuf == NULL) - return (uchar*) ""; - else - return cstrGetSzStr(pThis); -} - -/* Returns the cstr data as a classical C sz string. We use that the - * Finalizer did properly terminate our string (but we may stil be NULL). - * So it is vital that the finalizer is called BEFORe this function here! - * The caller must not free or otherwise manipulate the returned string and must not - * destroy the CStr object as long as the ascii string is used. - * This function may return NULL, if the string is currently NULL. This - * is a feature, not a bug. If you need non-NULL in any case, use - * cstrGetSzStrNoNULL() instead. - * Note that due to the new single-buffer interface this function almost does nothing! - * rgerhards, 2006-09-16 - */ -uchar* cstrGetSzStr(cstr_t *pThis) -{ - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - return(pThis->pBuf); -} - - /* Converts the CStr object to a classical zero-terminated C string, * returns that string and destroys the CStr object. The returned string * MUST be freed by the caller. The function might return NULL if @@ -426,27 +390,6 @@ finalize_it: } -/* Finalize the string object. This must be called after all data is added to it - * but before that data is used. - * rgerhards, 2009-06-16 - */ -rsRetVal -cstrFinalize(cstr_t *pThis) -{ - DEFiRet; - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - - if(pThis->iStrLen > 0) { - /* terminate string only if one exists */ - CHKiRet(cstrAppendChar(pThis, '\0')); - --pThis->iStrLen; /* do NOT count the \0 byte */ - } - -finalize_it: - RETiRet; -} - - /* return the length of the current string * 2005-09-09 rgerhards * Please note: this is only a function in a debug build. diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 4fbd9a9b..86c7e943 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -74,7 +74,7 @@ void rsCStrDestruct(cstr_t **ppThis); rsRetVal rsCStrExtendBuf(cstr_t *pThis, size_t iMinNeeded); /* our helper, NOT a public interface! */ static inline rsRetVal cstrAppendChar(cstr_t *pThis, uchar c) { - rsRetVal iRet; + rsRetVal iRet = RS_RET_OK; rsCHECKVALIDOBJECT(pThis, OIDrsCStr); @@ -89,6 +89,67 @@ finalize_it: return iRet; } + +/* some inline functions for things that are really frequently called... */ + +/* Finalize the string object. This must be called after all data is added to it + * but before that data is used. + * rgerhards, 2009-06-16 + */ +static inline rsRetVal +cstrFinalize(cstr_t *pThis) +{ + rsRetVal iRet = RS_RET_OK; + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + + if(pThis->iStrLen > 0) { + /* terminate string only if one exists */ + CHKiRet(cstrAppendChar(pThis, '\0')); + --pThis->iStrLen; /* do NOT count the \0 byte */ + } + +finalize_it: + return iRet; +} + + +/* Returns the cstr data as a classical C sz string. We use that the + * Finalizer did properly terminate our string (but we may stil be NULL). + * So it is vital that the finalizer is called BEFORe this function here! + * The caller must not free or otherwise manipulate the returned string and must not + * destroy the CStr object as long as the ascii string is used. + * This function may return NULL, if the string is currently NULL. This + * is a feature, not a bug. If you need non-NULL in any case, use + * cstrGetSzStrNoNULL() instead. + * Note that due to the new single-buffer interface this function almost does nothing! + * rgerhards, 2006-09-16 + */ +static inline uchar* cstrGetSzStr(cstr_t *pThis) +{ + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + return(pThis->pBuf); +} + + +/* Converts the CStr object to a classical sz string and returns that. + * Same restrictions as in cstrGetSzStr() applies (see there!). This + * function here guarantees that a valid string is returned, even if + * the CStr object currently holds a NULL pointer string buffer. If so, + * "" is returned. + * rgerhards 2005-10-19 + * WARNING: The returned pointer MUST NOT be freed, as it may be + * obtained from that constant memory pool (in case of NULL!) + */ +static inline uchar* cstrGetSzStrNoNULL(cstr_t *pThis) +{ + rsCHECKVALIDOBJECT(pThis, OIDrsCStr); + if(pThis->pBuf == NULL) + return (uchar*) ""; + else + return cstrGetSzStr(pThis); +} + + /** * Truncate "n" number of characters from the end of the * string. The buffer remains unchanged, just the @@ -148,8 +209,6 @@ rsRetVal rsCStrConvertToBool(cstr_t *pStr, number_t *pBool); /* new calling interface */ rsRetVal cstrFinalize(cstr_t *pThis); rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL); -uchar* cstrGetSzStr(cstr_t *pThis); -uchar* cstrGetSzStrNoNULL(cstr_t *pThis); rsRetVal cstrAppendCStr(cstr_t *pThis, cstr_t *pstrAppend); /* now come inline-like functions */ -- cgit v1.2.3 From 7ea63db2f9d6b375dc95696e47357d73927f2d3d Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Fri, 26 Jun 2009 15:31:40 +0200 Subject: optimization: propert names are now internally identified by integers --- runtime/conf.c | 10 +- runtime/msg.c | 356 ++++++++++++++++++++++++++++++++++------------------ runtime/msg.h | 5 +- runtime/rsyslog.h | 41 ++++++ runtime/rule.c | 14 +-- runtime/rule.h | 4 +- runtime/stringbuf.h | 3 - template.c | 16 +-- template.h | 2 +- 9 files changed, 303 insertions(+), 148 deletions(-) diff --git a/runtime/conf.c b/runtime/conf.c index d70bc281..83ed2e9b 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -837,6 +837,7 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) { rsParsObj *pPars; cstr_t *pCSCompOp; + cstr_t *pCSPropName; rsRetVal iRet; int iOffset; /* for compare operations */ @@ -856,12 +857,19 @@ static rsRetVal cflineProcessPropFilter(uchar **pline, register rule_t *f) } /* read property */ - iRet = parsDelimCStr(pPars, &f->f_filterData.prop.pCSPropName, ',', 1, 1, 1); + iRet = parsDelimCStr(pPars, &pCSPropName, ',', 1, 1, 1); if(iRet != RS_RET_OK) { errmsg.LogError(0, iRet, "error %d parsing filter property - ignoring selector", iRet); rsParsDestruct(pPars); return(iRet); } + iRet = propNameToID(pCSPropName, &f->f_filterData.prop.propID); + if(iRet != RS_RET_OK) { + errmsg.LogError(0, iRet, "error %d parsing filter property - ignoring selector", iRet); + rsParsDestruct(pPars); + return(iRet); + } + cstrDestruct(&pCSPropName); /* read operation */ iRet = parsDelimCStr(pPars, &pCSCompOp, ',', 1, 1, 1); diff --git a/runtime/msg.c b/runtime/msg.c index 3f69dbcb..c4ba18df 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1864,10 +1864,9 @@ static uchar *getNOW(eNOWType eNow) * rgerhards 2005-09-15 */ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - cstr_t *pCSPropName, size_t *pPropLen, + propid_t propID, size_t *pPropLen, unsigned short *pbMustBeFreed) { - uchar *pName; char *pRes; /* result pointer */ int bufLen = -1; /* length of string or -1, if not known */ char *pBufStart; @@ -1876,132 +1875,155 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, short iOffs; BEGINfunc + assert(pMsg != NULL); + assert(pbMustBeFreed != NULL); + #ifdef FEATURE_REGEXP /* Variables necessary for regular expression matching */ size_t nmatch = 10; regmatch_t pmatch[10]; #endif - assert(pMsg != NULL); - assert(pbMustBeFreed != NULL); - - if(pCSPropName == NULL) { - assert(pTpe != NULL); - pName = pTpe->data.field.pPropRepl; - } else { - pName = rsCStrGetSzStrNoNULL(pCSPropName); - } *pbMustBeFreed = 0; - /* sometimes there are aliases to the original MonitoWare - * property names. These come after || in the ifs below. */ - if(!strcmp((char*) pName, "msg")) { - pRes = getMSG(pMsg); - bufLen = getMSGLen(pMsg); - } else if(!strcmp((char*) pName, "timestamp") - || !strcmp((char*) pName, "timereported")) { - pRes = getTimeReported(pMsg, pTpe->data.field.eDateFormat); - } else if(!strcmp((char*) pName, "hostname") || !strcmp((char*) pName, "source")) { - pRes = getHOSTNAME(pMsg); - } else if(!strcmp((char*) pName, "syslogtag")) { - pRes = getTAG(pMsg); - } else if(!strcmp((char*) pName, "rawmsg")) { - pRes = getRawMsg(pMsg); - /* enable this, if someone actually uses UxTradMsg, delete after some time has - * passed and nobody complained -- rgerhards, 2009-06-16 - } else if(!strcmp((char*) pName, "uxtradmsg")) { - pRes = getUxTradMsg(pMsg); - */ - } else if(!strcmp((char*) pName, "inputname")) { - pRes = (char*) getInputName(pMsg); - } else if(!strcmp((char*) pName, "fromhost")) { - pRes = (char*) getRcvFrom(pMsg); - } else if(!strcmp((char*) pName, "fromhost-ip")) { - pRes = (char*) getRcvFromIP(pMsg); - } else if(!strcmp((char*) pName, "pri")) { - pRes = getPRI(pMsg); - } else if(!strcmp((char*) pName, "pri-text")) { - pBuf = malloc(20 * sizeof(char)); - if(pBuf == NULL) { - *pbMustBeFreed = 0; - return "**OUT OF MEMORY**"; - } else { - *pbMustBeFreed = 1; - pRes = textpri(pBuf, 20, getPRIi(pMsg)); - } - } else if(!strcmp((char*) pName, "iut")) { - pRes = "1"; /* always 1 for syslog messages (a MonitorWare thing;)) */ - } else if(!strcmp((char*) pName, "syslogfacility")) { - pRes = getFacility(pMsg); - } else if(!strcmp((char*) pName, "syslogfacility-text")) { - pRes = getFacilityStr(pMsg); - } else if(!strcmp((char*) pName, "syslogseverity") || !strcmp((char*) pName, "syslogpriority")) { - pRes = getSeverity(pMsg); - } else if(!strcmp((char*) pName, "syslogseverity-text") || !strcmp((char*) pName, "syslogpriority-text")) { - pRes = getSeverityStr(pMsg); - } else if(!strcmp((char*) pName, "timegenerated")) { - pRes = getTimeGenerated(pMsg, pTpe->data.field.eDateFormat); - } else if(!strcmp((char*) pName, "programname")) { - pRes = getProgramName(pMsg, LOCK_MUTEX); - } else if(!strcmp((char*) pName, "protocol-version")) { - pRes = getProtocolVersionString(pMsg); - } else if(!strcmp((char*) pName, "structured-data")) { - pRes = getStructuredData(pMsg); - } else if(!strcmp((char*) pName, "app-name")) { - pRes = getAPPNAME(pMsg, LOCK_MUTEX); - } else if(!strcmp((char*) pName, "procid")) { - pRes = getPROCID(pMsg, LOCK_MUTEX); - } else if(!strcmp((char*) pName, "msgid")) { - pRes = getMSGID(pMsg); - /* here start system properties (those, that do not relate to the message itself */ - } else if(!strcmp((char*) pName, "$now")) { - if((pRes = (char*) getNOW(NOW_NOW)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$year")) { - if((pRes = (char*) getNOW(NOW_YEAR)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$month")) { - if((pRes = (char*) getNOW(NOW_MONTH)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$day")) { - if((pRes = (char*) getNOW(NOW_DAY)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$hour")) { - if((pRes = (char*) getNOW(NOW_HOUR)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$hhour")) { - if((pRes = (char*) getNOW(NOW_HHOUR)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$qhour")) { - if((pRes = (char*) getNOW(NOW_QHOUR)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$minute")) { - if((pRes = (char*) getNOW(NOW_MINUTE)) == NULL) { - return "***OUT OF MEMORY***"; - } else - *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ - } else if(!strcmp((char*) pName, "$myhostname")) { - pRes = (char*) glbl.GetLocalHostName(); - } else { - /* there is no point in continuing, we may even otherwise render the - * error message unreadable. rgerhards, 2007-07-10 - */ - dbgprintf("invalid property name: '%s'\n", pName); - return "**INVALID PROPERTY NAME**"; + switch(propID) { + case PROP_MSG: + pRes = getMSG(pMsg); + bufLen = getMSGLen(pMsg); + break; + case PROP_TIMESTAMP: + pRes = getTimeReported(pMsg, pTpe->data.field.eDateFormat); + break; + case PROP_HOSTNAME: + pRes = getHOSTNAME(pMsg); + break; + case PROP_SYSLOGTAG: + pRes = getTAG(pMsg); + break; + case PROP_RAWMSG: + pRes = getRawMsg(pMsg); + break; + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + case PROP_UXTRADMSG: + pRes = getUxTradMsg(pMsg); + break; + */ + case PROP_INPUTNAME: + pRes = (char*) getInputName(pMsg); + break; + case PROP_FROMHOST: + pRes = (char*) getRcvFrom(pMsg); + break; + case PROP_FROMHOST_IP: + pRes = (char*) getRcvFromIP(pMsg); + break; + case PROP_PRI: + pRes = getPRI(pMsg); + break; + case PROP_PRI_TEXT: + pBuf = malloc(20 * sizeof(char)); + if(pBuf == NULL) { + *pbMustBeFreed = 0; + return "**OUT OF MEMORY**"; + } else { + *pbMustBeFreed = 1; + pRes = textpri(pBuf, 20, getPRIi(pMsg)); + } + break; + case PROP_IUT: + pRes = "1"; /* always 1 for syslog messages (a MonitorWare thing;)) */ + break; + case PROP_SYSLOGFACILITY: + pRes = getFacility(pMsg); + break; + case PROP_SYSLOGFACILITY_TEXT: + pRes = getFacilityStr(pMsg); + break; + case PROP_SYSLOGSEVERITY: + pRes = getSeverity(pMsg); + break; + case PROP_SYSLOGSEVERITY_TEXT: + pRes = getSeverityStr(pMsg); + break; + case PROP_TIMEGENERATED: + pRes = getTimeGenerated(pMsg, pTpe->data.field.eDateFormat); + break; + case PROP_PROGRAMNAME: + pRes = getProgramName(pMsg, LOCK_MUTEX); + break; + case PROP_PROTOCOL_VERSION: + pRes = getProtocolVersionString(pMsg); + break; + case PROP_STRUCTURED_DATA: + pRes = getStructuredData(pMsg); + break; + case PROP_APP_NAME: + pRes = getAPPNAME(pMsg, LOCK_MUTEX); + break; + case PROP_PROCID: + pRes = getPROCID(pMsg, LOCK_MUTEX); + break; + case PROP_MSGID: + pRes = getMSGID(pMsg); + break; + case PROP_SYS_NOW: + if((pRes = (char*) getNOW(NOW_NOW)) == NULL) { + return "***OUT OF MEMORY***"; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_YEAR: + if((pRes = (char*) getNOW(NOW_YEAR)) == NULL) { + return "***OUT OF MEMORY***"; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_MONTH: + if((pRes = (char*) getNOW(NOW_MONTH)) == NULL) { + return "***OUT OF MEMORY***"; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_DAY: + if((pRes = (char*) getNOW(NOW_DAY)) == NULL) { + return "***OUT OF MEMORY***"; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_HOUR: + if((pRes = (char*) getNOW(NOW_HOUR)) == NULL) { + return "***OUT OF MEMORY***"; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_HHOUR: + if((pRes = (char*) getNOW(NOW_HHOUR)) == NULL) { + return "***OUT OF MEMORY***"; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_QHOUR: + if((pRes = (char*) getNOW(NOW_QHOUR)) == NULL) { + return "***OUT OF MEMORY***"; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_MINUTE: + if((pRes = (char*) getNOW(NOW_MINUTE)) == NULL) { + return "***OUT OF MEMORY***"; + } else + *pbMustBeFreed = 1; /* all of these functions allocate dyn. memory */ + break; + case PROP_SYS_MYHOSTNAME: + pRes = (char*) glbl.GetLocalHostName(); + break; + default: + /* there is no point in continuing, we may even otherwise render the + * error message unreadable. rgerhards, 2007-07-10 + */ + dbgprintf("invalid property id: '%d'\n", propID); + return "**INVALID PROPERTY NAME**"; } /* If we did not receive a template pointer, we are already done... */ @@ -2587,6 +2609,7 @@ msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar) size_t propLen; uchar *pszProp = NULL; cstr_t *pstrProp; + propid_t propid; unsigned short bMustBeFreed = 0; ISOBJ_TYPE_assert(pThis, msg); @@ -2598,7 +2621,9 @@ msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar) CHKiRet(var.ConstructFinalize(pVar)); /* always call MsgGetProp() without a template specifier */ - pszProp = (uchar*) MsgGetProp(pThis, NULL, pstrPropName, &propLen, &bMustBeFreed); + /* TODO: optimize propNameToID() call -- rgerhards, 2009-06-26 */ + propNameToID(pstrPropName, &propid); + pszProp = (uchar*) MsgGetProp(pThis, NULL, propid, &propLen, &bMustBeFreed); /* now create a string object out of it and hand that over to the var */ CHKiRet(rsCStrConstructFromszStr(&pstrProp, pszProp)); @@ -2614,6 +2639,95 @@ finalize_it: RETiRet; } +/* map a property name (string) to a property ID */ +rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID) +{ + uchar *pName; + DEFiRet; + + assert(pCSPropName != NULL); + assert(pPropID != NULL); + pName = rsCStrGetSzStrNoNULL(pCSPropName); + + /* sometimes there are aliases to the original MonitoWare + * property names. These come after || in the ifs below. */ + if(!strcmp((char*) pName, "msg")) { + *pPropID = PROP_MSG; + } else if(!strcmp((char*) pName, "timestamp") + || !strcmp((char*) pName, "timereported")) { + *pPropID = PROP_TIMESTAMP; + } else if(!strcmp((char*) pName, "hostname") || !strcmp((char*) pName, "source")) { + *pPropID = PROP_HOSTNAME; + } else if(!strcmp((char*) pName, "syslogtag")) { + *pPropID = PROP_SYSLOGTAG; + } else if(!strcmp((char*) pName, "rawmsg")) { + *pPropID = PROP_RAWMSG; + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + } else if(!strcmp((char*) pName, "uxtradmsg")) { + pRes = getUxTradMsg(pMsg); + */ + } else if(!strcmp((char*) pName, "inputname")) { + *pPropID = PROP_INPUTNAME; + } else if(!strcmp((char*) pName, "fromhost")) { + *pPropID = PROP_FROMHOST; + } else if(!strcmp((char*) pName, "fromhost-ip")) { + *pPropID = PROP_FROMHOST_IP; + } else if(!strcmp((char*) pName, "pri")) { + *pPropID = PROP_PRI; + } else if(!strcmp((char*) pName, "pri-text")) { + *pPropID = PROP_PRI_TEXT; + } else if(!strcmp((char*) pName, "iut")) { + *pPropID = PROP_IUT; + } else if(!strcmp((char*) pName, "syslogfacility")) { + *pPropID = PROP_SYSLOGFACILITY; + } else if(!strcmp((char*) pName, "syslogfacility-text")) { + *pPropID = PROP_SYSLOGFACILITY_TEXT; + } else if(!strcmp((char*) pName, "syslogseverity") || !strcmp((char*) pName, "syslogpriority")) { + *pPropID = PROP_SYSLOGSEVERITY; + } else if(!strcmp((char*) pName, "syslogseverity-text") || !strcmp((char*) pName, "syslogpriority-text")) { + *pPropID = PROP_SYSLOGSEVERITY_TEXT; + } else if(!strcmp((char*) pName, "timegenerated")) { + *pPropID = PROP_TIMEGENERATED; + } else if(!strcmp((char*) pName, "programname")) { + *pPropID = PROP_PROGRAMNAME; + } else if(!strcmp((char*) pName, "protocol-version")) { + *pPropID = PROP_PROTOCOL_VERSION; + } else if(!strcmp((char*) pName, "structured-data")) { + *pPropID = PROP_STRUCTURED_DATA; + } else if(!strcmp((char*) pName, "app-name")) { + *pPropID = PROP_APP_NAME; + } else if(!strcmp((char*) pName, "procid")) { + *pPropID = PROP_PROCID; + } else if(!strcmp((char*) pName, "msgid")) { + *pPropID = PROP_MSGID; + /* here start system properties (those, that do not relate to the message itself */ + } else if(!strcmp((char*) pName, "$now")) { + *pPropID = PROP_SYS_NOW; + } else if(!strcmp((char*) pName, "$year")) { + *pPropID = PROP_SYS_YEAR; + } else if(!strcmp((char*) pName, "$month")) { + *pPropID = PROP_SYS_MONTH; + } else if(!strcmp((char*) pName, "$day")) { + *pPropID = PROP_SYS_DAY; + } else if(!strcmp((char*) pName, "$hour")) { + *pPropID = PROP_SYS_HOUR; + } else if(!strcmp((char*) pName, "$hhour")) { + *pPropID = PROP_SYS_HHOUR; + } else if(!strcmp((char*) pName, "$qhour")) { + *pPropID = PROP_SYS_QHOUR; + } else if(!strcmp((char*) pName, "$minute")) { + *pPropID = PROP_SYS_MINUTE; + } else if(!strcmp((char*) pName, "$myhostname")) { + *pPropID = PROP_SYS_MYHOSTNAME; + } else { + *pPropID = PROP_INVALID; + iRet = RS_RET_VAR_NOT_FOUND; + } + + RETiRet; +} + /* This function can be used as a generic way to set properties. * We have to handle a lot of legacy, so our return value is not always diff --git a/runtime/msg.h b/runtime/msg.h index 56ba5b80..e6b25e6c 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -33,6 +33,7 @@ #include "syslogd-types.h" #include "template.h" + /* rgerhards 2004-11-08: The following structure represents a * syslog message. * @@ -160,7 +161,7 @@ void MsgSetRawMsgWOSize(msg_t *pMsg, char* pszRawMsg); void MsgSetRawMsg(msg_t *pMsg, char* pszRawMsg, size_t lenMsg); rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG); char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, - cstr_t *pCSPropName, size_t *pPropLen, unsigned short *pbMustBeFreed); + propid_t propID, size_t *pPropLen, unsigned short *pbMustBeFreed); char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); @@ -177,6 +178,8 @@ int getHOSTNAMELen(msg_t *pM); char *getProgramName(msg_t *pM, bool bLockMutex); int getProgramNameLen(msg_t *pM, bool bLockMutex); uchar *getRcvFrom(msg_t *pM); +rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID); + /* The MsgPrepareEnqueue() function is a macro for performance reasons. * It needs one global variable to work. This is acceptable, as it gains diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index c2fb9383..4e0f6e6c 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -111,6 +111,7 @@ typedef long long int64; typedef long long unsigned uint64; typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ typedef char intTiny; /* 0..127! */ +typedef uchar uintTiny; /* 0..255! */ #ifdef __hpux typedef unsigned int u_int32_t; /* TODO: is this correct? */ @@ -156,6 +157,45 @@ struct multi_submit_s { #define _PATH_CONSOLE "/dev/console" #endif +/* properties are now encoded as (tiny) integers. I do not use an enum as I would like + * to keep the memory footprint small (and thus cache hits high). + * rgerhards, 2009-06-26 + */ +typedef uintTiny propid_t; +#define PROP_INVALID 0 +#define PROP_MSG 1 +#define PROP_TIMESTAMP 2 +#define PROP_HOSTNAME 3 +#define PROP_SYSLOGTAG 4 +#define PROP_RAWMSG 5 +#define PROP_INPUTNAME 6 +#define PROP_FROMHOST 7 +#define PROP_FROMHOST_IP 8 +#define PROP_PRI 9 +#define PROP_PRI_TEXT 10 +#define PROP_IUT 11 +#define PROP_SYSLOGFACILITY 12 +#define PROP_SYSLOGFACILITY_TEXT 13 +#define PROP_SYSLOGSEVERITY 14 +#define PROP_SYSLOGSEVERITY_TEXT 15 +#define PROP_TIMEGENERATED 16 +#define PROP_PROGRAMNAME 17 +#define PROP_PROTOCOL_VERSION 18 +#define PROP_STRUCTURED_DATA 19 +#define PROP_APP_NAME 20 +#define PROP_PROCID 21 +#define PROP_MSGID 22 +#define PROP_SYS_NOW 150 +#define PROP_SYS_YEAR 151 +#define PROP_SYS_MONTH 152 +#define PROP_SYS_DAY 153 +#define PROP_SYS_HOUR 154 +#define PROP_SYS_HHOUR 155 +#define PROP_SYS_QHOUR 156 +#define PROP_SYS_MINUTE 157 +#define PROP_SYS_MYHOSTNAME 158 + + /* The error codes below are orginally "borrowed" from * liblogging. As such, we reserve values up to -2999 * just in case we need to borrow something more ;) @@ -325,6 +365,7 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_STREAM_DISABLED = -2127, /**< a file has been disabled (e.g. by size limit restriction) */ RS_RET_FILENAME_INVALID = -2140, /**< filename invalid, not found, no access, ... */ RS_RET_ZLIB_ERR = -2141, /**< error during zlib call */ + RS_RET_VAR_NOT_FOUND = -2142, /**< variable not found */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/rule.c b/runtime/rule.c index 221368e6..ce19146b 100644 --- a/runtime/rule.c +++ b/runtime/rule.c @@ -179,8 +179,7 @@ shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) bRet = (pResult->val.num) ? 1 : 0; } else { assert(pRule->f_filter_type == FILTER_PROP); /* assert() just in case... */ - pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.pCSPropName, &propLen, &pbMustBeFreed); - // TODO: optimize, we now have the length of the property! + pszPropVal = MsgGetProp(pMsg, NULL, pRule->f_filterData.prop.propID, &propLen, &pbMustBeFreed); /* Now do the compares (short list currently ;)) */ switch(pRule->f_filterData.prop.operation ) { @@ -220,9 +219,8 @@ shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) bRet = (bRet == 1) ? 0 : 1; if(Debug) { - dbgprintf("Filter: check for property '%s' (value '%s') ", - rsCStrGetSzStrNoNULL(pRule->f_filterData.prop.pCSPropName), - pszPropVal); + dbgprintf("Filter: check for property '%d' (value '%s') ", + pRule->f_filterData.prop.propID, pszPropVal); if(pRule->f_filterData.prop.isNegated) dbgprintf("NOT "); dbgprintf("%s '%s': %s\n", @@ -308,8 +306,6 @@ CODESTARTobjDestruct(rule) rsCStrDestruct(&pThis->pCSProgNameComp); if(pThis->f_filter_type == FILTER_PROP) { - if(pThis->f_filterData.prop.pCSPropName != NULL) - rsCStrDestruct(&pThis->f_filterData.prop.pCSPropName); if(pThis->f_filterData.prop.pCSCompValue != NULL) rsCStrDestruct(&pThis->f_filterData.prop.pCSCompValue); if(pThis->f_filterData.prop.regex_cache != NULL) @@ -377,8 +373,8 @@ CODESTARTobjDebugPrint(rule) dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); } else { dbgprintf("PROPERTY-BASED Filter:\n"); - dbgprintf("\tProperty.: '%s'\n", - rsCStrGetSzStrNoNULL(pThis->f_filterData.prop.pCSPropName)); + dbgprintf("\tProperty.: '%d'\n", pThis->f_filterData.prop.propID); +// TODO: XXXX ADD idtostring()! dbgprintf("\tOperation: "); if(pThis->f_filterData.prop.isNegated) dbgprintf("NOT "); diff --git a/runtime/rule.h b/runtime/rule.h index 38b11c63..99ac44e7 100644 --- a/runtime/rule.h +++ b/runtime/rule.h @@ -44,11 +44,11 @@ struct rule_s { union { u_char f_pmask[LOG_NFACILITIES+1]; /* priority mask */ struct { - cstr_t *pCSPropName; fiop_t operation; regex_t *regex_cache; /* cache for compiled REs, if such are used */ cstr_t *pCSCompValue; /* value to "compare" against */ - char isNegated; /* actually a boolean ;) */ + bool isNegated; + propid_t propID; /* ID of the requested property */ } prop; expr_t *f_expr; /* expression object */ } f_filterData; diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 86c7e943..400c74eb 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -76,8 +76,6 @@ static inline rsRetVal cstrAppendChar(cstr_t *pThis, uchar c) { rsRetVal iRet = RS_RET_OK; - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); - if(pThis->iStrLen >= pThis->iBufSize) { CHKiRet(rsCStrExtendBuf(pThis, 1)); /* need more memory! */ } @@ -100,7 +98,6 @@ static inline rsRetVal cstrFinalize(cstr_t *pThis) { rsRetVal iRet = RS_RET_OK; - rsCHECKVALIDOBJECT(pThis, OIDrsCStr); if(pThis->iStrLen > 0) { /* terminate string only if one exists */ diff --git a/template.c b/template.c index 704f0b19..832183b0 100644 --- a/template.c +++ b/template.c @@ -105,7 +105,7 @@ rsRetVal tplToString(struct template *pTpl, msg_t *pMsg, uchar **ppBuf, size_t * iLenVal = pTpe->data.constant.iLenConstant; bMustBeFreed = 0; } else if(pTpe->eEntryType == FIELD) { - pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &iLenVal, &bMustBeFreed); + pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, &iLenVal, &bMustBeFreed); /* we now need to check if we should use SQL option. In this case, * we must go over the generated string and escape '\'' characters. * rgerhards, 2005-09-22: the option values below look somewhat misplaced, @@ -173,7 +173,7 @@ rsRetVal tplToArray(struct template *pTpl, msg_t *pMsg, uchar*** ppArr) if(pTpe->eEntryType == CONSTANT) { CHKmalloc(pArr[iArr] = (uchar*)strdup((char*) pTpe->data.constant.pConstant)); } else if(pTpe->eEntryType == FIELD) { - pVal = (uchar*) MsgGetProp(pMsg, pTpe, NULL, &propLen, &bMustBeFreed); + pVal = (uchar*) MsgGetProp(pMsg, pTpe, pTpe->data.field.propid, &propLen, &bMustBeFreed); if(bMustBeFreed) { /* if it must be freed, it is our own private copy... */ pArr[iArr] = pVal; /* ... so we can use it! */ } else { @@ -561,11 +561,11 @@ static int do_Parameter(unsigned char **pp, struct template *pTpl) ++p; /* do NOT do this in tolower()! */ } - /* got the name*/ + /* got the name */ cstrFinalize(pStrB); - if(cstrConvSzStrAndDestruct(pStrB, &pTpe->data.field.pPropRepl, 0) != RS_RET_OK) + + if(propNameToID(pStrB, &pTpe->data.field.propid) != RS_RET_OK) return 1; -// TODO: another optimization: map name to integer id OPT /* Check frompos, if it has an R, then topos should be a regex */ if(*p == ':') { @@ -1004,8 +1004,6 @@ void tplDeleteAll(void) regexp.regfree(&(pTpeDel->data.field.re)); } } - /*dbgprintf("(FIELD), value: '%s'", pTpeDel->data.field.pPropRepl);*/ - free(pTpeDel->data.field.pPropRepl); break; } /*dbgprintf("\n");*/ @@ -1061,8 +1059,6 @@ void tplDeleteNew(void) regexp.regfree(&(pTpeDel->data.field.re)); } } - /*dbgprintf("(FIELD), value: '%s'", pTpeDel->data.field.pPropRepl);*/ - free(pTpeDel->data.field.pPropRepl); break; } /*dbgprintf("\n");*/ @@ -1111,7 +1107,7 @@ void tplPrintList(void) pTpe->data.constant.pConstant); break; case FIELD: - dbgprintf("(FIELD), value: '%s' ", pTpe->data.field.pPropRepl); + dbgprintf("(FIELD), value: '%d' ", pTpe->data.field.propid); switch(pTpe->data.field.eDateFormat) { case tplFmtDefault: break; diff --git a/template.h b/template.h index 73ed0f84..271e8271 100644 --- a/template.h +++ b/template.h @@ -63,7 +63,7 @@ struct templateEntry { int iLenConstant; /* its length */ } constant; struct { - uchar *pPropRepl; /* pointer to property replacer string */ + propid_t propid; /* property to be used */ unsigned iFromPos; /* for partial strings only chars from this position ... */ unsigned iToPos; /* up to that one... */ #ifdef FEATURE_REGEXP -- cgit v1.2.3 From e6aeb1b995d6629ff04fd0f6068be2e9ffe9ca42 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Sat, 27 Jun 2009 12:31:38 +0200 Subject: fixed small compile issue --- runtime/rsyslog.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 3c8db169..76f320e6 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -81,7 +81,7 @@ typedef long long int64; typedef long long unsigned uint64; typedef int64 number_t; /* type to use for numbers - TODO: maybe an autoconf option? */ typedef char intTiny; /* 0..127! */ -typedef uchar uintTiny; /* 0..255! */ +typedef unsigned char uintTiny; /* 0..255! */ /* define some base data types */ typedef unsigned char uchar;/* get rid of the unhandy "unsigned char" */ -- cgit v1.2.3 From 279896ebda98804c89899c53f857ca189ab24f45 Mon Sep 17 00:00:00 2001 From: Ronny Egner Date: Sat, 27 Jun 2009 12:36:34 +0200 Subject: improved omoracle documentation Signed-off-by: Rainer Gerhards --- doc/omoracle.html | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/doc/omoracle.html b/doc/omoracle.html index 40f6360f..cfcf277f 100644 --- a/doc/omoracle.html +++ b/doc/omoracle.html @@ -67,6 +67,128 @@ it is suggested to post questions to the need to define the properties on the template in the correct order you want them passed to the statement! +

      Some additional documentation contributed by Ronny Egner: +

      +REQUIREMENTS:
+--------------
+
+- Oracle Instantclient 10g (NOT 11g) Base + Devel
+  (if you´re on 64-bit linux you should choose the 64-bit libs!) 
+- JDK 1.6 (not neccessary for oracle plugin but "make" didd not finsished successfully without it)
+
+- "oracle-instantclient-config" script 
+  (seems to shipped with instantclient 10g Release 1 but i was unable to find it for 10g Release 2 so here it is)
+
+  
+======================  /usr/local/bin/oracle-instantclient-config =====================
+#!/bin/sh
+#
+# Oracle InstantClient SDK config file
+# Jean-Christophe Duberga - Bordeaux 2 University
+#
+
+# just adapt it to your environment
+incdirs="-I/usr/include/oracle/10.2.0.4/client64"
+libdirs="-L/usr/lib/oracle/10.2.0.4/client64/lib"
+
+usage="\
+Usage: oracle-instantclient-config [--prefix[=DIR]] [--exec-prefix[=DIR]] [--version] [--cflags] [--libs] [--static-libs]"
+
+if test $# -eq 0; then
+      echo "${usage}" 1>&2
+      exit 1
+fi
+
+while test $# -gt 0; do
+  case "$1" in
+  -*=*) optarg=`echo "$1" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
+  *) optarg= ;;
+  esac
+
+  case $1 in
+    --prefix=*)
+      prefix=$optarg
+      if test $exec_prefix_set = no ; then
+        exec_prefix=$optarg
+      fi
+      ;;
+    --prefix)
+      echo $prefix
+      ;;
+    --exec-prefix=*)
+      exec_prefix=$optarg
+      exec_prefix_set=yes
+      ;;
+    --exec-prefix)
+      echo ${exec_prefix}
+      ;;
+    --version)
+      echo ${version}
+      ;;
+    --cflags)
+      echo ${incdirs}
+      ;;
+    --libs)
+      echo $libdirs -lclntsh -lnnz10 -locci -lociei -locijdbc10
+      ;;
+    --static-libs)
+      echo "No static libs" 1>&2
+      exit 1
+      ;;
+    *)
+      echo "${usage}" 1>&2
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+===============   END ==============
+
+
+
+
+COMPILING RSYSLOGD
+-------------------
+
+
+./configure --enable-oracle
+
+
+
+
+RUNNING
+-------
+
+- make sure rsyslogd is able to locate the oracle libs (either via LD_LIBRARY_PATH or /etc/ld.so.conf)
+- set TNS_ADMIN to point to your tnsnames.ora
+- create a tnsnames.ora and test you are able to connect to the database
+
+- create user in oracle as shown in the following example:
+		create user syslog identified by syslog default tablespace users quota unlimited on users;
+		grant create session to syslog;
+		create role syslog_role;
+		grant syslog_role to syslog;
+		grant create table to syslog_role;
+		grant create sequence to syslog_role;
+		
+- create tables as needed
+
+- configure rsyslog as shown in the following example
+		$ModLoad omoracle
+
+		$OmoracleDBUser syslog
+		$OmoracleDBPassword syslog
+		$OmoracleDB syslog
+		$OmoracleBatchSize 1
+		$OmoracleBatchItemSize 4096
+
+		$OmoracleStatementTemplate OmoracleStatement
+		$template OmoracleStatement,"insert into foo(hostname,message) values (:host,:message)"
+		$template TestStmt,"%hostname%%msg%"
+		*.*                     :omoracle:;TestStmt
+	(you guess it: username = password = database = "syslog".... see $rsyslogd_source/plugins/omoracle/omoracle.c for me info)
+

      [rsyslog.conf overview] [manual index] [rsyslog site]

      This documentation is part of the -- cgit v1.2.3 From ba4806a70439dd24dc98bd707893b1319dd5e3ef Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Thu, 18 Jun 2009 13:51:17 -0400 Subject: add support for KLogPath --- plugins/imklog/bsd.c | 9 +++++++-- plugins/imklog/imklog.c | 9 +++++++++ plugins/imklog/imklog.h | 1 + plugins/imklog/linux.c | 11 ++++++++--- tools/syslogd.c | 4 ++-- 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/plugins/imklog/bsd.c b/plugins/imklog/bsd.c index 090c4e9b..6d7b6c98 100644 --- a/plugins/imklog/bsd.c +++ b/plugins/imklog/bsd.c @@ -83,6 +83,11 @@ static int fklog = -1; /* /dev/klog */ # define _PATH_KLOG "/dev/klog" #endif +static uchar *GetPath(void) +{ + return pszPath ? pszPath : _PATH_KLOG; +} + /* open the kernel log - will be called inside the willRun() imklog * entry point. -- rgerhards, 2008-04-09 */ @@ -91,9 +96,9 @@ klogWillRun(void) { DEFiRet; - fklog = open(_PATH_KLOG, O_RDONLY, 0); + fklog = open(GetPath(), O_RDONLY, 0); if (fklog < 0) { - dbgprintf("can't open %s (%d)\n", _PATH_KLOG, errno); + dbgprintf("can't open %s (%d)\n", GetPath(), errno); iRet = RS_RET_ERR; // TODO: better error code } diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index f63d60ac..ab806183 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -44,6 +44,7 @@ #include #include #include +#include #include "dirty.h" #include "cfsysline.h" @@ -69,6 +70,7 @@ int use_syscall = 0; int symbol_lookup = 0; /* on recent kernels > 2.6, the kernel does this */ int bPermitNonKernel = 0; /* permit logging of messages not having LOG_KERN facility */ int iFacilIntMsg; /* the facility to use for internal messages (set by driver) */ +uchar *pszPath = NULL; /* TODO: configuration for the following directives must be implemented. It * was not done yet because we either do not yet have a config handler for * that type or I thought it was acceptable to push it to a later stage when @@ -243,6 +245,8 @@ CODESTARTmodExit /* release objects we used */ objRelease(glbl, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); + if(pszPath != NULL) + free(pszPath); ENDmodExit @@ -259,6 +263,10 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a symfile = NULL; symbol_lookup = 0; bPermitNonKernel = 0; + if(pszPath != NULL) { + free(pszPath); + pszPath = NULL; + } iFacilIntMsg = klogFacilIntMsg(); return RS_RET_OK; } @@ -273,6 +281,7 @@ CODEmodInit_QueryRegCFSLineHdlr iFacilIntMsg = klogFacilIntMsg(); CHKiRet(omsdRegCFSLineHdlr((uchar *)"debugprintkernelsymbols", 0, eCmdHdlrBinary, NULL, &dbgPrintSymbols, STD_LOADABLE_MODULE_ID)); + CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogpath", 0, eCmdHdlrGetWord, NULL, &pszPath, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogsymbollookup", 0, eCmdHdlrBinary, NULL, &symbol_lookup, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogsymbolstwice", 0, eCmdHdlrBinary, NULL, &symbols_twice, STD_LOADABLE_MODULE_ID)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"klogusesyscallinterface", 0, eCmdHdlrBinary, NULL, &use_syscall, STD_LOADABLE_MODULE_ID)); diff --git a/plugins/imklog/imklog.h b/plugins/imklog/imklog.h index 37bd58b0..c183026d 100644 --- a/plugins/imklog/imklog.h +++ b/plugins/imklog/imklog.h @@ -52,6 +52,7 @@ extern int symbol_lookup; extern char *symfile; extern int console_log_level; extern int dbgPrintSymbols; +extern uchar *pszPath; /* the functions below may be called by the drivers */ rsRetVal imklogLogIntMsg(int priority, char *fmt, ...) __attribute__((format(printf,2, 3))); diff --git a/plugins/imklog/linux.c b/plugins/imklog/linux.c index 0dd4320d..fcd9d6cd 100644 --- a/plugins/imklog/linux.c +++ b/plugins/imklog/linux.c @@ -84,6 +84,11 @@ static enum LOGSRC {none, proc, kernel} logsrc; extern int ksyslog(int type, char *buf, int len); +static uchar *GetPath(void) +{ + return pszPath ? pszPath : _PATH_KLOG; +} + static void CloseLogSrc(void) { /* Turn on logging of messages to console, but only if we had the -c @@ -135,7 +140,7 @@ static enum LOGSRC GetKernelLogSrc(void) * file system is available to get kernel messages from. */ if ( use_syscall || - ((stat(_PATH_KLOG, &sb) < 0) && (errno == ENOENT)) ) + ((stat(GetPath(), &sb) < 0) && (errno == ENOENT)) ) { /* Initialize kernel logging. */ ksyslog(1, NULL, 0); @@ -144,14 +149,14 @@ static enum LOGSRC GetKernelLogSrc(void) return(kernel); } - if ( (kmsg = open(_PATH_KLOG, O_RDONLY|O_CLOEXEC)) < 0 ) + if ( (kmsg = open(GetPath(), O_RDONLY|O_CLOEXEC)) < 0 ) { imklogLogIntMsg(LOG_ERR, "imklog: Cannot open proc file system, %d.\n", errno); ksyslog(7, NULL, 0); /* TODO: check this, implement more */ return(none); } - imklogLogIntMsg(LOG_INFO, "imklog %s, log source = %s started.", VERSION, _PATH_KLOG); + imklogLogIntMsg(LOG_INFO, "imklog %s, log source = %s started.", VERSION, GetPath()); return(proc); } diff --git a/tools/syslogd.c b/tools/syslogd.c index 8bc4939f..f6f4b469 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2750,8 +2750,6 @@ static rsRetVal mainThread() pTmp = template_StdPgSQLFmt; tplLastStaticInit(tplAddLine(" StdPgSQLFmt", &pTmp)); - CHKiRet(init()); - if(Debug && debugging_on) { DBGPRINTF("Debugging enabled, SIGUSR1 to turn off debugging.\n"); } @@ -2779,6 +2777,8 @@ static rsRetVal mainThread() } + CHKiRet(init()); + /* END OF INTIALIZATION * ... but keep in mind that we might do a restart and thus init() might * be called again. If that happens, we must shut down the worker thread, -- cgit v1.2.3 From e747478051f4f81a872c791710933881c9564d64 Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Mon, 29 Jun 2009 11:18:55 +0200 Subject: separate willRun and runInput calls for input modules Signed-off-by: Rainer Gerhards --- runtime/modules.c | 1 + runtime/modules.h | 1 + tools/syslogd.c | 46 +++++++++++++++++++++++++++++++++++++--------- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/runtime/modules.c b/runtime/modules.c index 32ae659f..bd201252 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -399,6 +399,7 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ CHKiRet((*pNew->modQueryEtryPt)((uchar*)"runInput", &pNew->mod.im.runInput)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"willRun", &pNew->mod.im.willRun)); CHKiRet((*pNew->modQueryEtryPt)((uchar*)"afterRun", &pNew->mod.im.afterRun)); + pNew->mod.im.bCanRun = 0; break; case eMOD_OUT: CHKiRet((*pNew->modQueryEtryPt)((uchar*)"freeInstance", &pNew->freeInstance)); diff --git a/runtime/modules.h b/runtime/modules.h index 372529ee..4d874019 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -106,6 +106,7 @@ typedef struct modInfo_s { rsRetVal (*runInput)(thrdInfo_t*); /* function to gather input and submit to queue */ rsRetVal (*willRun)(void); /* function to gather input and submit to queue */ rsRetVal (*afterRun)(thrdInfo_t*); /* function to gather input and submit to queue */ + int bCanRun; /* cached value of whether willRun() succeeded */ } im; struct {/* data for output modules */ /* below: perform the configured action diff --git a/tools/syslogd.c b/tools/syslogd.c index f6f4b469..6ccacf5f 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2081,6 +2081,29 @@ static void dbgPrintInitInfo(void) } +/* Actually run the input modules. This happens after privileges are dropped, + * if that is requested. + */ +static rsRetVal +runInputModules(void) +{ + modInfo_t *pMod; + + /* loop through all modules and activate them (brr...) */ + pMod = module.GetNxtType(NULL, eMOD_IN); + while(pMod != NULL) { + if(pMod->mod.im.bCanRun) { + /* activate here */ + thrdCreate(pMod->mod.im.runInput, pMod->mod.im.afterRun); + } + pMod = module.GetNxtType(pMod, eMOD_IN); + } + + ENDfunc + return RS_RET_OK; /* intentional: we do not care about module errors */ +} + + /* Start the input modules. This function will probably undergo big changes * while we implement the input module interface. For now, it does the most * important thing to get at least my poor initial input modules up and @@ -2088,7 +2111,7 @@ static void dbgPrintInitInfo(void) * rgerhards, 2007-12-14 */ static rsRetVal -startInputModules(void) +startInputModules(int bRunInputModules) { DEFiRet; modInfo_t *pMod; @@ -2096,15 +2119,18 @@ startInputModules(void) /* loop through all modules and activate them (brr...) */ pMod = module.GetNxtType(NULL, eMOD_IN); while(pMod != NULL) { - if((iRet = pMod->mod.im.willRun()) == RS_RET_OK) { - /* activate here */ - thrdCreate(pMod->mod.im.runInput, pMod->mod.im.afterRun); - } else { + iRet = pMod->mod.im.willRun(); + pMod->mod.im.bCanRun = (iRet == RS_RET_OK); + if(!pMod->mod.im.bCanRun) { DBGPRINTF("module %lx will not run, iRet %d\n", (unsigned long) pMod, iRet); } pMod = module.GetNxtType(pMod, eMOD_IN); } + if (bRunInputModules) { + runInputModules(); + } + ENDfunc return RS_RET_OK; /* intentional: we do not care about module errors */ } @@ -2116,7 +2142,7 @@ startInputModules(void) * else happens. -- rgerhards, 2008-07-28 */ static rsRetVal -init(void) +init(int bRunInputModules) { rsRetVal localRet; int iNbrActions; @@ -2321,7 +2347,7 @@ init(void) * shuffled to down here once we have everything in input modules. * rgerhards, 2007-12-14 */ - startInputModules(); + startInputModules(bRunInputModules); if(Debug) { dbgPrintInitInfo(); @@ -2492,7 +2518,7 @@ doHUP(void) if(glbl.GetHUPisRestart()) { DBGPRINTF("Received SIGHUP, configured to be restart, reloading rsyslogd.\n"); - init(); /* main queue is stopped as part of init() */ + init(1); /* main queue is stopped as part of init() */ } else { DBGPRINTF("Received SIGHUP, configured to be a non-restart type of HUP - notifying actions.\n"); ruleset.IterateAllActions(doHUPActions, NULL); @@ -2750,6 +2776,8 @@ static rsRetVal mainThread() pTmp = template_StdPgSQLFmt; tplLastStaticInit(tplAddLine(" StdPgSQLFmt", &pTmp)); + CHKiRet(init(0)); + if(Debug && debugging_on) { DBGPRINTF("Debugging enabled, SIGUSR1 to turn off debugging.\n"); } @@ -2777,7 +2805,7 @@ static rsRetVal mainThread() } - CHKiRet(init()); + runInputModules(); /* END OF INTIALIZATION * ... but keep in mind that we might do a restart and thus init() might -- cgit v1.2.3 From 6a29afc83f16c57d243b39b8e8d47f2ed909bae0 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 29 Jun 2009 11:26:18 +0200 Subject: some cleanup --- tools/syslogd.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/syslogd.c b/tools/syslogd.c index 6ccacf5f..10d07c95 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -2111,7 +2111,7 @@ runInputModules(void) * rgerhards, 2007-12-14 */ static rsRetVal -startInputModules(int bRunInputModules) +startInputModules(void) { DEFiRet; modInfo_t *pMod; @@ -2127,10 +2127,6 @@ startInputModules(int bRunInputModules) pMod = module.GetNxtType(pMod, eMOD_IN); } - if (bRunInputModules) { - runInputModules(); - } - ENDfunc return RS_RET_OK; /* intentional: we do not care about module errors */ } @@ -2142,7 +2138,7 @@ startInputModules(int bRunInputModules) * else happens. -- rgerhards, 2008-07-28 */ static rsRetVal -init(int bRunInputModules) +init() { rsRetVal localRet; int iNbrActions; @@ -2343,11 +2339,14 @@ init(int bRunInputModules) DBGPRINTF("Main processing queue is initialized and running\n"); /* the output part and the queue is now ready to run. So it is a good time - * to start the inputs. Please note that the net code above should be + * to initialize the inputs. Please note that the net code above should be * shuffled to down here once we have everything in input modules. * rgerhards, 2007-12-14 + * NOTE: as of 2009-06-29, the input modules are initialized, but not yet run. + * Keep in mind. though, that the outputs already run if the queue was + * persisted to disk. -- rgerhards */ - startInputModules(bRunInputModules); + startInputModules(); if(Debug) { dbgPrintInitInfo(); @@ -2518,7 +2517,8 @@ doHUP(void) if(glbl.GetHUPisRestart()) { DBGPRINTF("Received SIGHUP, configured to be restart, reloading rsyslogd.\n"); - init(1); /* main queue is stopped as part of init() */ + init(); /* main queue is stopped as part of init() */ + runInputModules(); } else { DBGPRINTF("Received SIGHUP, configured to be a non-restart type of HUP - notifying actions.\n"); ruleset.IterateAllActions(doHUPActions, NULL); @@ -2776,11 +2776,12 @@ static rsRetVal mainThread() pTmp = template_StdPgSQLFmt; tplLastStaticInit(tplAddLine(" StdPgSQLFmt", &pTmp)); - CHKiRet(init(0)); + CHKiRet(init()); if(Debug && debugging_on) { DBGPRINTF("Debugging enabled, SIGUSR1 to turn off debugging.\n"); } + /* Send a signal to the parent so it can terminate. */ if(myPid != ppid) @@ -2788,7 +2789,7 @@ static rsRetVal mainThread() /* If instructed to do so, we now drop privileges. Note that this is not 100% secure, - * because inputs and outputs are already running at this time. However, we can implement + * because outputs are already running at this time. However, we can implement * dropping of privileges rather quickly and it will work in many cases. While it is not * the ultimate solution, the current one is still much better than not being able to * drop privileges at all. Doing it correctly, requires a change in architecture, which @@ -2805,13 +2806,12 @@ static rsRetVal mainThread() } + /* finally let the inputs run... */ runInputModules(); /* END OF INTIALIZATION * ... but keep in mind that we might do a restart and thus init() might - * be called again. If that happens, we must shut down the worker thread, - * do the init() and then restart things. - * rgerhards, 2005-10-24 + * be called again. -- rgerhards, 2005-10-24 */ DBGPRINTF("initialization completed, transitioning to regular run mode\n"); -- cgit v1.2.3 From 4822715a6e1ecc48d50a5450f93f02f785b8745a Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 29 Jun 2009 11:37:29 +0200 Subject: updated testbench to new init() behavior the deferred activation of input modules broke part of the testbench - but this was a testbench issue, not one of the patch --- ChangeLog | 2 ++ tests/nettester.c | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/ChangeLog b/ChangeLog index d7c6e408..2c4d1145 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ --------------------------------------------------------------------------- Version 4.5.0 [DEVEL] (rgerhards), 2009-??-?? +- activation order of inputs changed, they are now activated only after + privileges are dropped. Thanks to Michael Terry for the patch. - greatly improved performance - greatly reduced memory requirements of msg object to around half of the previous demand. This means that more messages can diff --git a/tests/nettester.c b/tests/nettester.c index b8816b7c..c9a978c5 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -107,6 +107,7 @@ tcpSend(char *buf, int lenBuf) { static int sock = INVALID_SOCKET; struct sockaddr_in addr; + int retries; if(sock == INVALID_SOCKET) { /* first time, need to connect to target */ @@ -122,10 +123,19 @@ tcpSend(char *buf, int lenBuf) fprintf(stderr, "inet_aton() failed\n"); return(1); } - if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) != 0) { - fprintf(stderr, "connect() failed\n"); - return(1); - } + retries = 0; + while(1) { /* loop broken inside */ + if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) == 0) { + break; + } else { + if(retries++ == 3) { + fprintf(stderr, "connect() failed\n"); + return(1); + } else { + sleep(1); + } + } + } } /* send test data */ -- cgit v1.2.3 From 6511278082a7e1e9602385cd24cdb5e363cb702f Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 29 Jun 2009 12:40:41 +0200 Subject: bugfix: ssh session hangs after rsyslgod is started from it stderr/stdout were not closed to be able to emit error messages, but this caused ssh sessions to hang. Now we close them after the initial initialization. See forum thread: http://kb.monitorware.com/controlling-terminal-issues-t9875.html --- ChangeLog | 6 ++++++ tools/syslogd.c | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 6ee2e6ba..66836317 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,10 @@ --------------------------------------------------------------------------- +Version 4.2.1 [v4-stable] (rgerhards), 2009-0?-?? +- bugfix: stderr/stdout were not closed to be able to emit error messages, + but this caused ssh sessions to hang. Now we close them after the + initial initialization. See forum thread: + http://kb.monitorware.com/controlling-terminal-issues-t9875.html +--------------------------------------------------------------------------- Version 4.2.0 [v4-stable] (rgerhards), 2009-06-23 - bugfix: light and full delay watermarks had invalid values, badly affecting performance for delayable inputs diff --git a/tools/syslogd.c b/tools/syslogd.c index 77273bec..45c55664 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -3015,7 +3015,6 @@ static rsRetVal mainThread() glbl.SetHUPisRestart(0); /* we can not do restart-type HUPs with dropped privs */ } - /* END OF INTIALIZATION * ... but keep in mind that we might do a restart and thus init() might * be called again. If that happens, we must shut down the worker thread, @@ -3024,6 +3023,19 @@ static rsRetVal mainThread() */ dbgprintf("initialization completed, transitioning to regular run mode\n"); + /* close stderr and stdout if they are kept open during a fork. Note that this + * may introduce subtle security issues: if we are in a jail, one may break out of + * it via these descriptors. But if I close them earlier, error messages will (once + * again) not be emitted to the user that starts the daemon. As root jail support + * is still in its infancy (and not really done), we currently accept this issue. + * rgerhards, 2009-06-29 + */ + if(!(Debug || NoFork)) { + close(1); + close(2); + bErrMsgToStderr = 0; + } + mainloop(); finalize_it: -- cgit v1.2.3 From de84a12f8a5f140c0f7b8e00f4cac92ef13cd866 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Mon, 29 Jun 2009 16:53:26 +0200 Subject: introduced the idea of detached properties some things inside the message can be used over a large number of messages and need to to be allocated and re-written every time. I now begin to implement this as a "prop_t" object, first use for the inputName. Some input modules are already converted, some others to go. Will do a little performance check on the new method before I go further. Also, this commit has some cleanup and a few bug fixes that prevented compiliation in debug mode (I overlooked this as I did not compile for debug, what I normally do, and the automatted test also does not do that) --- plugins/imdiag/imdiag.c | 2 +- plugins/imfile/imfile.c | 2 +- plugins/imklog/imklog.c | 2 +- plugins/imklog/linux.c | 7 ++-- plugins/imudp/imudp.c | 16 ++++++++- runtime/datetime.c | 5 --- runtime/datetime.h | 2 -- runtime/msg.c | 91 +++++++++++++++++++++++++++++++++++-------------- runtime/msg.h | 11 ++++-- runtime/prop.c | 63 ++++++++++++++++++++++++++++++++++ runtime/prop.h | 11 +++++- runtime/rsyslog.c | 3 ++ runtime/rsyslog.h | 1 + runtime/stringbuf.h | 2 +- tcps_sess.c | 8 +++-- tcpsrv.c | 15 ++++++-- tcpsrv.h | 4 +-- tests/nettester.c | 5 +-- tools/syslogd.c | 5 +-- 19 files changed, 200 insertions(+), 55 deletions(-) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index bfb4a2e5..4b004bc4 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -206,7 +206,7 @@ doInjectMsg(int iNum) /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*) szMsg, ustrlen(szMsg)); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imdiag"), sizeof("imdiag")-1); + MsgSetInputNameStr(pMsg, UCHAR_CONSTANT("imdiag"), sizeof("imdiag")-1); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index 631d02ff..fdd0c5f0 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -97,7 +97,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); + MsgSetInputNameStr(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine), cstrLen(cstrLine)); MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index ab806183..21744c4b 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -97,7 +97,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); + MsgSetInputNameStr(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); diff --git a/plugins/imklog/linux.c b/plugins/imklog/linux.c index fcd9d6cd..977395ea 100644 --- a/plugins/imklog/linux.c +++ b/plugins/imklog/linux.c @@ -37,6 +37,7 @@ #include "msg.h" #include "module-template.h" #include "imklog.h" +#include "unicode-helper.h" /* Includes. */ @@ -86,7 +87,7 @@ extern int ksyslog(int type, char *buf, int len); static uchar *GetPath(void) { - return pszPath ? pszPath : _PATH_KLOG; + return pszPath ? pszPath : UCHAR_CONSTANT(_PATH_KLOG); } static void CloseLogSrc(void) @@ -140,7 +141,7 @@ static enum LOGSRC GetKernelLogSrc(void) * file system is available to get kernel messages from. */ if ( use_syscall || - ((stat(GetPath(), &sb) < 0) && (errno == ENOENT)) ) + ((stat((char*)GetPath(), &sb) < 0) && (errno == ENOENT)) ) { /* Initialize kernel logging. */ ksyslog(1, NULL, 0); @@ -149,7 +150,7 @@ static enum LOGSRC GetKernelLogSrc(void) return(kernel); } - if ( (kmsg = open(GetPath(), O_RDONLY|O_CLOEXEC)) < 0 ) + if ( (kmsg = open((char*)GetPath(), O_RDONLY|O_CLOEXEC)) < 0 ) { imklogLogIntMsg(LOG_ERR, "imklog: Cannot open proc file system, %d.\n", errno); ksyslog(7, NULL, 0); /* TODO: check this, implement more */ diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 2340aac4..828b9636 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -43,6 +43,7 @@ #include "msg.h" #include "parser.h" #include "datetime.h" +#include "prop.h" #include "unicode-helper.h" MODULE_TYPE_INPUT @@ -55,6 +56,7 @@ DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(net) DEFobjCurrIf(datetime) +DEFobjCurrIf(prop) static int iMaxLine; /* maximum UDP message size supported */ static time_t ttLastDiscard = 0; /* timestamp when a message from a non-permitted sender was last discarded @@ -68,6 +70,7 @@ static uchar *pRcvBuf = NULL; /* receive buffer (for a single packet). We use a * it so that we can check available memory in willRun() and request * termination if we can not get it. -- rgerhards, 2007-12-27 */ +static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ // TODO: static ruleset_t *pBindRuleset = NULL; /* ruleset to bind listener to (use system default if unspecified) */ #define TIME_REQUERY_DFLT 2 static int iTimeRequery = TIME_REQUERY_DFLT;/* how often is time to be queried inside tight recv loop? 0=always */ @@ -242,7 +245,8 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*)pRcvBuf, lenRcvBuf); - MsgSetInputName(pMsg, UCHAR_CONSTANT("imudp"), sizeof("imudp")-1); + prop.AddRef(pInputName); + MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; @@ -338,6 +342,12 @@ ENDrunInput /* initialize and return if will run or not */ BEGINwillRun CODESTARTwillRun + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imudp"), sizeof("imudp") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); + prop.AddRef(pInputName); + net.PrintAllowedSenders(1); /* UDP */ /* if we could not set up any listners, there is no point in running... */ @@ -365,6 +375,8 @@ CODESTARTafterRun free(pRcvBuf); pRcvBuf = NULL; } + if(pInputName != NULL) + prop.Destruct(&pInputName); ENDafterRun @@ -374,6 +386,7 @@ CODESTARTmodExit objRelease(errmsg, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); ENDmodExit @@ -405,6 +418,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* register config file handlers */ diff --git a/runtime/datetime.c b/runtime/datetime.c index e0f3f5fa..ea67eec5 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -556,7 +556,6 @@ int formatTimestampToMySQL(struct syslogTime *ts, char* pBuf) */ assert(ts != NULL); assert(pBuf != NULL); - assert(iLenDst >= 15); pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; @@ -582,7 +581,6 @@ int formatTimestampToPgSQL(struct syslogTime *ts, char *pBuf) /* see note in formatTimestampToMySQL, applies here as well */ assert(ts != NULL); assert(pBuf != NULL); - assert(iLenDst >= 20); pBuf[0] = (ts->year / 1000) % 10 + '0'; pBuf[1] = (ts->year / 100) % 10 + '0'; @@ -665,7 +663,6 @@ int formatTimestamp3339(struct syslogTime *ts, char* pBuf) BEGINfunc assert(ts != NULL); assert(pBuf != NULL); - assert(iLenBuf >= 33); /* start with fixed parts */ /* year yyyy */ @@ -740,8 +737,6 @@ int formatTimestamp3164(struct syslogTime *ts, char* pBuf) assert(ts != NULL); assert(pBuf != NULL); - assert(iLenBuf >= 16); - pBuf[0] = monthNames[(ts->month - 1)% 12][0]; pBuf[1] = monthNames[(ts->month - 1) % 12][1]; pBuf[2] = monthNames[(ts->month - 1) % 12][2]; diff --git a/runtime/datetime.h b/runtime/datetime.h index 79a86d05..58f368e7 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -23,8 +23,6 @@ #ifndef INCLUDED_DATETIME_H #define INCLUDED_DATETIME_H -#include "datetime.h" - /* TODO: define error codes */ #define NO_ERRCODE -1 diff --git a/runtime/msg.c b/runtime/msg.c index c4ba18df..47b4ed85 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -48,6 +48,7 @@ #include "atomic.h" #include "unicode-helper.h" #include "ruleset.h" +#include "prop.h" /* static data */ DEFobjStaticHelpers @@ -55,6 +56,7 @@ DEFobjCurrIf(var) DEFobjCurrIf(datetime) DEFobjCurrIf(glbl) DEFobjCurrIf(regexp) +DEFobjCurrIf(prop) static struct { uchar *pszName; @@ -273,12 +275,26 @@ static char *syslog_number_names[24] = { "0", "1", "2", "3", "4", "5", "6", "7", /* some forward declarations */ static int getAPPNAMELen(msg_t *pM, bool bLockMutex); + static inline int getProtocolVersion(msg_t *pM) { return(pM->iProtocolVersion); } +static inline void +getInputName(msg_t *pM, uchar **ppsz, int *plen) +{ + BEGINfunc + if(pM == NULL) { + *ppsz = UCHAR_CONSTANT(""); + *plen = 0; + } else { + prop.GetString(pM->pInputName, ppsz, plen); + } + ENDfunc +} + /* The following functions will support advanced output module * multithreading, once this is implemented. Currently, we * include them as hooks only. The idea is that we need to guard @@ -431,7 +447,6 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->iFacility = -1; pM->offAfterPRI = 0; pM->offMSG = -1; - pM->iLenInputName = 0; pM->iProtocolVersion = 0; pM->msgFlags = 0; pM->iLenRawMsg = 0; @@ -444,7 +459,6 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->pszHOSTNAME = NULL; pM->pszRcvFrom = NULL; pM->pszRcvFromIP = NULL; - pM->pszInputName = NULL; pM->pszRcvdAt3164 = NULL; pM->pszRcvdAt3339 = NULL; pM->pszRcvdAt_MySQL = NULL; @@ -458,6 +472,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->pCSAPPNAME = NULL; pM->pCSPROCID = NULL; pM->pCSMSGID = NULL; + pM->pInputName = NULL; pM->pRuleset = NULL; memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt)); memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP)); @@ -556,7 +571,8 @@ CODESTARTobjDestruct(msg) free(pThis->pszRawMsg); freeTAG(pThis); freeHOSTNAME(pThis); - free(pThis->pszInputName); + if(pThis->pInputName != NULL) + prop.Destruct(&pThis->pInputName); free(pThis->pszRcvFrom); free(pThis->pszRcvFromIP); free(pThis->pszRcvdAt3164); @@ -719,11 +735,16 @@ msg_t* MsgDup(msg_t* pOld) */ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) { + uchar *psz; + int len; DEFiRet; assert(pThis != NULL); assert(pStrm != NULL); + /* "pump" some property values into strings */ + + /* then serialize elements */ CHKiRet(obj.BeginSerialize(pStrm, (obj_t*) pThis)); objSerializeSCALAR(pStrm, iProtocolVersion, SHORT); objSerializeSCALAR(pStrm, iSeverity, SHORT); @@ -743,7 +764,9 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializePTR(pStrm, pszRawMsg, PSZ); objSerializePTR(pStrm, pszHOSTNAME, PSZ); - objSerializePTR(pStrm, pszInputName, PSZ); + getInputName(pThis, &psz, &len); + objSerializeSCALAR_VAR(pStrm, "pszInputName", PSZ, psz); + //objSerializePTR(pStrm, pszInputName, PSZ); objSerializePTR(pStrm, pszRcvFrom, PSZ); objSerializePTR(pStrm, pszRcvFromIP, PSZ); @@ -1427,18 +1450,6 @@ char *getHOSTNAME(msg_t *pM) } -static uchar *getInputName(msg_t *pM) -{ - if(pM == NULL) - return (uchar*) ""; - else - if(pM->pszInputName == NULL) - return (uchar*) ""; - else - return pM->pszInputName; -} - - uchar *getRcvFrom(msg_t *pM) { if(pM == NULL) @@ -1598,14 +1609,31 @@ static int getAPPNAMELen(msg_t *pM, bool bLockMutex) /* rgerhards 2008-09-10: set pszInputName in msg object * rgerhards, 2009-06-16 */ -void MsgSetInputName(msg_t *pMsg, uchar* pszInputName, size_t lenInputName) +void MsgSetInputName(msg_t *pThis, prop_t *inputName) { - assert(pMsg != NULL); - free(pMsg->pszInputName); - pMsg->iLenInputName = lenInputName; - if((pMsg->pszInputName = malloc(pMsg->iLenInputName + 1)) != NULL) { - memcpy(pMsg->pszInputName, pszInputName, pMsg->iLenInputName + 1); - } + assert(pThis != NULL); + + if(pThis->pInputName != NULL) + prop.Destruct(&pThis->pInputName); + pThis->pInputName = inputName; +} + +/* to be removed soon: work-around for those tht can not natively generate an + * input name. + * rgerhards, 2009-06-29 + */ +void MsgSetInputNameStr(msg_t *pThis, uchar *psz, int len) +{ + prop_t *pProp; + assert(pThis != NULL); + + /* we need to create a property */ + prop.Construct(&pProp); + prop.SetString(pProp, psz, len); + prop.ConstructFinalize(pProp); + prop.AddRef(pProp); + MsgSetInputName(pThis, pProp); + prop.Destruct(&pProp); } /* rgerhards 2004-11-16: set pszRcvFrom in msg object @@ -1875,6 +1903,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, short iOffs; BEGINfunc +dbgprintf("XXXX: msgGetProp for %d\n", propID); assert(pMsg != NULL); assert(pbMustBeFreed != NULL); @@ -1910,7 +1939,10 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, break; */ case PROP_INPUTNAME: - pRes = (char*) getInputName(pMsg); +RUNLOG; + getInputName(pMsg, ((uchar**) &pRes), &bufLen); +RUNLOG; +RUNLOG_VAR("%p", pRes); break; case PROP_FROMHOST: pRes = (char*) getRcvFrom(pMsg); @@ -2738,6 +2770,7 @@ rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID) #define isProp(name) !rsCStrSzStrCmp(pProp->pcsName, (uchar*) name, sizeof(name) - 1) rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) { + prop_t *myProp; DEFiRet; ISOBJ_TYPE_assert(pThis, msg); @@ -2765,7 +2798,13 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("pszTAG")) { MsgSetTAG(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), cstrLen(pProp->val.pStr)); } else if(isProp("pszInputName")) { - MsgSetInputName(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); + /* we need to create a property */ + CHKiRet(prop.Construct(&myProp)); + CHKiRet(prop.SetString(myProp, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr))); + CHKiRet(prop.ConstructFinalize(myProp)); + prop.AddRef(myProp); + MsgSetInputName(pThis, myProp); + prop.Destruct(&myProp); } else if(isProp("pszRcvFromIP")) { MsgSetRcvFromIP(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszRcvFrom")) { @@ -2790,6 +2829,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) dbgprintf("no longer supported property pszMSG silently ignored\n"); } +finalize_it: RETiRet; } #undef isProp @@ -2833,6 +2873,7 @@ BEGINObjClassInit(msg, 1, OBJ_IS_CORE_MODULE) CHKiRet(objUse(var, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_SERIALIZE, MsgSerialize); diff --git a/runtime/msg.h b/runtime/msg.h index e6b25e6c..a1fc535b 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -72,7 +72,6 @@ struct msg { short iFacility; /* Facility code 0 .. 23*/ short offAfterPRI; /* offset, at which raw message WITHOUT PRI part starts in pszRawMsg */ short offMSG; /* offset at which the MSG part starts in pszRawMsg */ - short iLenInputName; /* Length of pszInputName */ short iProtocolVersion;/* protocol version of message received 0 - legacy, 1 syslog-protocol) */ int msgFlags; /* flags associated with this message */ int iLenRawMsg; /* length of raw message */ @@ -86,7 +85,6 @@ struct msg { uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ uchar *pszRcvFrom; /* System message was received from */ uchar *pszRcvFromIP; /* IP of system message was received from */ - uchar *pszInputName; /* name of the input module that submitted this message */ char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ @@ -100,6 +98,7 @@ struct msg { cstr_t *pCSAPPNAME; /* APP-NAME */ cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ + prop_t *pInputName; /* input name property */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp @@ -144,7 +143,7 @@ rsRetVal msgDestruct(msg_t **ppM); msg_t* MsgDup(msg_t* pOld); msg_t *MsgAddRef(msg_t *pM); void setProtocolVersion(msg_t *pM, int iNewVersion); -void MsgSetInputName(msg_t *pMsg, uchar*, size_t); +void MsgSetInputName(msg_t *pMsg, prop_t*); rsRetVal MsgSetAPPNAME(msg_t *pMsg, char* pszAPPNAME); rsRetVal MsgSetPROCID(msg_t *pMsg, char* pszPROCID); rsRetVal MsgSetMSGID(msg_t *pMsg, char* pszMSGID); @@ -166,6 +165,12 @@ char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); + +// REMOVE: +void MsgSetInputNameStr(msg_t *pThis, uchar *psz, int len); + + + /* TODO: remove these five (so far used in action.c) */ char *getMSG(msg_t *pM); char *getHOSTNAME(msg_t *pM); diff --git a/runtime/prop.c b/runtime/prop.c index 02be315f..989657dd 100644 --- a/runtime/prop.c +++ b/runtime/prop.c @@ -36,9 +36,12 @@ #include "config.h" #include #include +#include #include "rsyslog.h" #include "obj.h" +#include "obj-types.h" +#include "atomic.h" #include "prop.h" /* static data */ @@ -48,8 +51,45 @@ DEFobjStaticHelpers /* Standard-Constructor */ BEGINobjConstruct(prop) /* be sure to specify the object type also in END macro! */ + pThis->iRefCount = 1; ENDobjConstruct(prop) +/* set string, we make our own private copy! This MUST only be called BEFORE + * ConstructFinalize()! + */ +static rsRetVal SetString(prop_t *pThis, uchar *psz, int len) +{ + DEFiRet; + ISOBJ_TYPE_assert(pThis, prop); + if(pThis->len >= CONF_PROP_BUFSIZE) + free(pThis->szVal.psz); + pThis->len = len; + if(len < CONF_PROP_BUFSIZE) { + memcpy(pThis->szVal.sz, psz, len + 1); + } else { + CHKmalloc(pThis->szVal.psz = malloc(len + 1)); + memcpy(pThis->szVal.sz, psz, len + 1); + } + +finalize_it: + RETiRet; +} + + +/* get string */ +static rsRetVal GetString(prop_t *pThis, uchar **ppsz, int *plen) +{ + BEGINfunc + ISOBJ_TYPE_assert(pThis, prop); + if(pThis->len < CONF_PROP_BUFSIZE) + *ppsz = pThis->szVal.sz; + else + *ppsz = pThis->szVal.psz; + *plen = pThis->len; + ENDfunc + return RS_RET_OK; +} + /* ConstructionFinalizer * rgerhards, 2008-01-09 @@ -63,9 +103,29 @@ propConstructFinalize(prop_t __attribute__((unused)) *pThis) } +/* add a new reference. It is VERY IMPORTANT to call this function whenever + * the property is handed over to some entitiy that later call Destruct() on it. + */ +static rsRetVal AddRef(prop_t *pThis) +{ + ATOMIC_INC(pThis->iRefCount); + return RS_RET_OK; +} + + /* destructor for the prop object */ BEGINobjDestruct(prop) /* be sure to specify the object type also in END and CODESTART macros! */ + int currRefCount; CODESTARTobjDestruct(prop) + currRefCount = ATOMIC_DEC_AND_FETCH(pThis->iRefCount); + if(currRefCount == 0) { + /* (only) in this case we need to actually destruct the object */ +dbgprintf("XXXXX: propDestruct: ptr %p, pThis %p, len %d\n", pThis->szVal.psz, pThis, pThis->len); + if(pThis->len >= CONF_PROP_BUFSIZE) + free(pThis->szVal.psz); + } else { + pThis = NULL; /* tell framework NOT to destructing the object! */ + } ENDobjDestruct(prop) @@ -94,6 +154,9 @@ CODESTARTobjQueryInterface(prop) pIf->ConstructFinalize = propConstructFinalize; pIf->Destruct = propDestruct; pIf->DebugPrint = propDebugPrint; + pIf->SetString = SetString; + pIf->GetString = GetString; + pIf->AddRef = AddRef; finalize_it: ENDobjQueryInterface(prop) diff --git a/runtime/prop.h b/runtime/prop.h index 7fc466b5..1d18c650 100644 --- a/runtime/prop.h +++ b/runtime/prop.h @@ -28,14 +28,23 @@ /* the prop object */ struct prop_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + int iRefCount; /* reference counter */ + union { + uchar *psz; /* stored string */ + uchar sz[CONF_PROP_BUFSIZE]; + } szVal; + int len; /* we use int intentionally, otherwise we may get some troubles... */ }; /* interfaces */ BEGINinterface(prop) /* name must also be changed in ENDinterface macro! */ INTERFACEObjDebugPrint(prop); rsRetVal (*Construct)(prop_t **ppThis); - rsRetVal (*ConstructFinalize)(prop_t __attribute__((unused)) *pThis); + rsRetVal (*ConstructFinalize)(prop_t *pThis); rsRetVal (*Destruct)(prop_t **ppThis); + rsRetVal (*SetString)(prop_t *pThis, uchar* psz, int len); + rsRetVal (*GetString)(prop_t *pThis, uchar** ppsz, int *plen); + rsRetVal (*AddRef)(prop_t *pThis); ENDinterface(prop) #define propCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 6f732f0e..faa74427 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -77,6 +77,7 @@ #include "conf.h" #include "glbl.h" #include "errmsg.h" +#include "prop.h" #include "rule.h" #include "ruleset.h" @@ -150,6 +151,8 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(glblClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "datetime"; CHKiRet(datetimeClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "prop"; + CHKiRet(propClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "ctok_token"; diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 4e0f6e6c..8a043dde 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -39,6 +39,7 @@ #define CONF_RAWMSG_BUFSIZE 101 #define CONF_TAG_BUFSIZE 32 #define CONF_HOSTNAME_BUFSIZE 32 +#define CONF_PROP_BUFSIZE 16 /* should be close to sizeof(ptr) or lighly above it */ /* ############################################################# * diff --git a/runtime/stringbuf.h b/runtime/stringbuf.h index 400c74eb..c5130238 100644 --- a/runtime/stringbuf.h +++ b/runtime/stringbuf.h @@ -35,6 +35,7 @@ #ifndef _STRINGBUF_H_INCLUDED__ #define _STRINGBUF_H_INCLUDED__ 1 +#include /** * The dynamic string buffer object. @@ -48,7 +49,6 @@ typedef struct cstr_s uchar *pszBuf; /**< pointer to the sz version of the string (after it has been created )*/ size_t iBufSize; /**< current maximum size of the string buffer */ size_t iStrLen; /**< length of the string in characters. */ - bool bIsForeignBuf; /**< is pBuf a buffer provided by someone else? */ } cstr_t; diff --git a/tcps_sess.c b/tcps_sess.c index e0bec949..4786e154 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -45,6 +45,7 @@ #include "netstrm.h" #include "msg.h" #include "datetime.h" +#include "prop.h" /* static data */ @@ -52,6 +53,7 @@ DEFobjStaticHelpers DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) DEFobjCurrIf(netstrm) +DEFobjCurrIf(prop) DEFobjCurrIf(datetime) static int iMaxLine; /* maximum size of a single message */ @@ -234,9 +236,9 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttG /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); -dbgprintf("defaultDoSubmit, iMsg %d\n", pThis->iMsg); MsgSetRawMsg(pMsg, (char*)pThis->pMsg, pThis->iMsg); - MsgSetInputName(pMsg, pThis->pLstnInfo->pszInputName, pThis->pLstnInfo->lenInputName); + prop.AddRef(pThis->pLstnInfo->pInputName); + MsgSetInputName(pMsg, pThis->pLstnInfo->pInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; @@ -520,6 +522,7 @@ CODESTARTObjClassExit(tcps_sess) objRelease(errmsg, CORE_COMPONENT); objRelease(netstrm, LM_NETSTRMS_FILENAME); objRelease(datetime, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); ENDObjClassExit(tcps_sess) @@ -532,6 +535,7 @@ BEGINObjClassInit(tcps_sess, 1, OBJ_IS_CORE_MODULE) /* class, version - CHANGE c CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(netstrm, LM_NETSTRMS_FILENAME)); CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); iMaxLine = glbl.GetMaxLine(); /* get maximum size we currently support */ diff --git a/tcpsrv.c b/tcpsrv.c index 119aea91..e8ea2b98 100644 --- a/tcpsrv.c +++ b/tcpsrv.c @@ -89,6 +89,7 @@ DEFobjCurrIf(net) DEFobjCurrIf(netstrms) DEFobjCurrIf(netstrm) DEFobjCurrIf(nssel) +DEFobjCurrIf(prop) /* add new listener port to listener port list @@ -107,8 +108,11 @@ addNewLstnPort(tcpsrv_t *pThis, uchar *pszPort) pEntry->pszPort = pszPort; pEntry->pSrv = pThis; pEntry->pRuleset = pThis->pRuleset; - CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); - pEntry->lenInputName = ustrlen(pEntry->pszInputName); + + /* we need to create a property */ + CHKiRet(prop.Construct(&pEntry->pInputName)); + CHKiRet(prop.SetString(pEntry->pInputName, pThis->pszInputName, ustrlen(pThis->pszInputName))); + CHKiRet(prop.ConstructFinalize(pEntry->pInputName)); /* and add to list */ pEntry->pNext = pThis->pLstnPorts; @@ -250,7 +254,7 @@ static void deinit_tcp_listener(tcpsrv_t *pThis) pEntry = pThis->pLstnPorts; while(pEntry != NULL) { free(pEntry->pszPort); - free(pEntry->pszInputName); + prop.Destruct(&pEntry->pInputName); pDel = pEntry; pEntry = pEntry->pNext; free(pDel); @@ -477,6 +481,7 @@ Run(tcpsrv_t *pThis) * this thread. Thus, we also need to instantiate a cancel cleanup handler * to prevent us from leaking anything. -- rgerharsd, 20080-04-24 */ +RUNLOG_STR("XXXX: tcp server runs\n"); pthread_cleanup_push(RunCancelCleanup, (void*) &pSel); while(1) { CHKiRet(nssel.Construct(&pSel)); @@ -497,6 +502,7 @@ Run(tcpsrv_t *pThis) iTCPSess = TCPSessGetNxtSess(pThis, iTCPSess); } +RUNLOG_STR("XXXX: tcp server select\n"); /* wait for io to become ready */ CHKiRet(nssel.Wait(pSel, &nfds)); @@ -508,6 +514,7 @@ Run(tcpsrv_t *pThis) --nfds; /* indicate we have processed one */ } } +RUNLOG_STR("XXXX: tcp server post select\n"); /* now check the sessions */ iTCPSess = TCPSessGetNxtSess(pThis, -1); @@ -883,6 +890,7 @@ CODESTARTObjClassExit(tcpsrv) /* release objects we no longer need */ objRelease(tcps_sess, DONT_LOAD_LIB); objRelease(conf, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); objRelease(ruleset, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); @@ -908,6 +916,7 @@ BEGINObjClassInit(tcpsrv, 1, OBJ_IS_LOADABLE_MODULE) /* class, version - CHANGE CHKiRet(objUse(conf, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(ruleset, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, tcpsrvDebugPrint); diff --git a/tcpsrv.h b/tcpsrv.h index e2170bef..70682398 100644 --- a/tcpsrv.h +++ b/tcpsrv.h @@ -23,6 +23,7 @@ #define INCLUDED_TCPSRV_H #include "obj.h" +#include "prop.h" #include "tcps_sess.h" /* support for framing anomalies */ @@ -36,8 +37,7 @@ typedef enum ETCPsyslogFramingAnomaly { /* list of tcp listen ports */ struct tcpLstnPortList_s { uchar *pszPort; /**< the ports the listener shall listen on */ - uchar *pszInputName; /**< value to be used as input name */ - size_t lenInputName; /**< length of inputName */ + prop_t *pInputName; tcpsrv_t *pSrv; /**< pointer to higher-level server instance */ ruleset_t *pRuleset; /**< associated ruleset */ tcpLstnPortList_t *pNext; /**< next port or NULL */ diff --git a/tests/nettester.c b/tests/nettester.c index c9a978c5..dbfb4db3 100644 --- a/tests/nettester.c +++ b/tests/nettester.c @@ -128,11 +128,12 @@ tcpSend(char *buf, int lenBuf) if(connect(sock, (struct sockaddr*)&addr, sizeof(addr)) == 0) { break; } else { - if(retries++ == 3) { + if(retries++ == 30) { + ++iFailed; fprintf(stderr, "connect() failed\n"); return(1); } else { - sleep(1); + usleep(100); } } } diff --git a/tools/syslogd.c b/tools/syslogd.c index 3204d94e..2fe949ac 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -580,7 +580,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } if(pszInputName != NULL) - MsgSetInputName(pMsg, pszInputName, ustrlen(pszInputName)); + MsgSetInputNameStr(pMsg, pszInputName, ustrlen(pszInputName)); MsgSetFlowControlType(pMsg, flowCtlType); MsgSetRawMsgWOSize(pMsg, (char*)msg); @@ -881,7 +881,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) DEFiRet; CHKiRet(msgConstruct(&pMsg)); - MsgSetInputName(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1); + MsgSetInputNameStr(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1); MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); @@ -2089,6 +2089,7 @@ runInputModules(void) { modInfo_t *pMod; + BEGINfunc /* loop through all modules and activate them (brr...) */ pMod = module.GetNxtType(NULL, eMOD_IN); while(pMod != NULL) { -- cgit v1.2.3 From e397c34d2a6c7c1e4c116fd2363cb173e32eb2a2 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 30 Jun 2009 15:21:15 +0200 Subject: finished transition to using inputName property ... plus some celanup and adding minor missing functionality (the rule debug info again tell the property name, not just number). --- dirty.h | 2 +- plugins/imdiag/imdiag.c | 14 ++- plugins/imfile/imfile.c | 15 ++- plugins/imklog/imklog.c | 19 ++- plugins/imrelp/imrelp.c | 18 ++- plugins/imudp/imudp.c | 2 - plugins/imuxsock/imuxsock.c | 18 ++- runtime/msg.c | 273 ++++++++++++++++++++++++++++---------------- runtime/msg.h | 6 +- runtime/rule.c | 7 +- tcps_sess.c | 1 - tools/syslogd.c | 46 ++++---- 12 files changed, 278 insertions(+), 143 deletions(-) diff --git a/dirty.h b/dirty.h index 8a0cbc80..0153cb69 100644 --- a/dirty.h +++ b/dirty.h @@ -30,7 +30,7 @@ rsRetVal multiSubmitMsg(multi_submit_t *pMultiSub); rsRetVal submitMsg(msg_t *pMsg); rsRetVal logmsgInternal(int iErr, int pri, uchar *msg, int flags); -rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime); +rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlTypeu, prop_t *pInputName, struct syslogTime *stTime, time_t ttGenTime); int parseRFCSyslogMsg(msg_t *pMsg, int flags); int parseLegacySyslogMsg(msg_t *pMsg, int flags); rsRetVal diagGetMainMsgQSize(int *piSize); /* for imdiag */ diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 4b004bc4..8d874a87 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -66,10 +66,12 @@ DEFobjCurrIf(net) DEFobjCurrIf(netstrm) DEFobjCurrIf(errmsg) DEFobjCurrIf(datetime) +DEFobjCurrIf(prop) /* Module static data */ static tcpsrv_t *pOurTcpsrv = NULL; /* our TCP server(listener) TODO: change for multiple instances */ static permittedPeers_t *pPermPeersRoot = NULL; +static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this input */ /* config settings */ @@ -206,7 +208,7 @@ doInjectMsg(int iNum) /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*) szMsg, ustrlen(szMsg)); - MsgSetInputNameStr(pMsg, UCHAR_CONSTANT("imdiag"), sizeof("imdiag")-1); + MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; @@ -376,13 +378,19 @@ CODESTARTwillRun /* first apply some config settings */ if(pOurTcpsrv == NULL) ABORT_FINALIZE(RS_RET_NO_RUN); + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imdiag"), sizeof("imdiag") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); + finalize_it: ENDwillRun BEGINafterRun CODESTARTafterRun - /* do cleanup here */ + if(pInputName != NULL) + prop.Destruct(&pInputName); ENDafterRun @@ -402,6 +410,7 @@ CODESTARTmodExit objRelease(tcpsrv, LM_TCPSRV_FILENAME); objRelease(errmsg, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); ENDmodExit @@ -439,6 +448,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(tcpsrv, LM_TCPSRV_FILENAME)); CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); /* register config file handlers */ CHKiRet(omsdRegCFSLineHdlr(UCHAR_CONSTANT("imdiagserverrun"), 0, eCmdHdlrGetWord, diff --git a/plugins/imfile/imfile.c b/plugins/imfile/imfile.c index fdd0c5f0..7c588f90 100644 --- a/plugins/imfile/imfile.c +++ b/plugins/imfile/imfile.c @@ -46,6 +46,7 @@ #include "glbl.h" #include "datetime.h" #include "unicode-helper.h" +#include "prop.h" MODULE_TYPE_INPUT /* must be present for input modules, do not remove */ @@ -57,6 +58,7 @@ DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(datetime) DEFobjCurrIf(strm) +DEFobjCurrIf(prop) typedef struct fileInfo_s { uchar *pszFileName; @@ -81,6 +83,7 @@ static int iFilPtr = 0; /* number of files to be monitored; pointer to next fre #define MAX_INPUT_FILES 100 static fileInfo_t files[MAX_INPUT_FILES]; +static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this input */ /* enqueue the read file line as a message. The provided string is * not freed - thuis must be done by the caller. @@ -97,7 +100,7 @@ static rsRetVal enqLine(fileInfo_t *pInfo, cstr_t *cstrLine) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_FULL_DELAY); - MsgSetInputNameStr(pMsg, UCHAR_CONSTANT("imfile"), sizeof("imfile")-1); + MsgSetInputName(pMsg, pInputName); MsgSetRawMsg(pMsg, (char*)rsCStrGetSzStr(cstrLine), cstrLen(cstrLine)); MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); @@ -336,6 +339,11 @@ CODESTARTwillRun ABORT_FINALIZE(RS_RET_NO_RUN); } + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imfile"), sizeof("imfile") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); + finalize_it: ENDwillRun @@ -391,6 +399,9 @@ CODESTARTafterRun strm.Destruct(&(files[i].pStrm)); } } + + if(pInputName != NULL) + prop.Destruct(&pInputName); ENDafterRun @@ -405,6 +416,7 @@ CODESTARTmodExit objRelease(datetime, CORE_COMPONENT); objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); ENDmodExit @@ -515,6 +527,7 @@ CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(strm, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(omsdRegCFSLineHdlr((uchar *)"inputfilename", 0, eCmdHdlrGetWord, NULL, &pszFileName, STD_LOADABLE_MODULE_ID)); diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 21744c4b..10cc8b14 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -18,7 +18,7 @@ * Please note that this file replaces the klogd daemon that was * also present in pre-v3 versions of rsyslog. * - * Copyright (C) 2008 by Rainer Gerhards and Adiscon GmbH + * Copyright (C) 2008, 2009 by Rainer Gerhards and Adiscon GmbH * * This file is part of rsyslog. * @@ -54,6 +54,7 @@ #include "datetime.h" #include "imklog.h" #include "glbl.h" +#include "prop.h" #include "unicode-helper.h" MODULE_TYPE_INPUT @@ -62,6 +63,7 @@ MODULE_TYPE_INPUT DEF_IMOD_STATIC_DATA DEFobjCurrIf(datetime) DEFobjCurrIf(glbl) +DEFobjCurrIf(prop) /* configuration settings */ int dbgPrintSymbols = 0; /* this one is extern so the helpers can access it! */ @@ -81,6 +83,8 @@ char *symfile = NULL; int console_log_level = -1; +static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this module */ + /* enqueue the the kernel message into the message queue. * The provided msg string is not freed - thus must be done * by the caller. @@ -97,7 +101,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) CHKiRet(msgConstruct(&pMsg)); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); - MsgSetInputNameStr(pMsg, UCHAR_CONSTANT("imklog"), sizeof("imklog")-1); + MsgSetInputName(pMsg, pInputName); MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); @@ -230,13 +234,22 @@ ENDrunInput BEGINwillRun CODESTARTwillRun + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imklog"), sizeof("imklog") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); + iRet = klogWillRun(); +finalize_it: ENDwillRun BEGINafterRun CODESTARTafterRun iRet = klogAfterRun(); + + if(pInputName != NULL) + prop.Destruct(&pInputName); ENDafterRun @@ -245,6 +258,7 @@ CODESTARTmodExit /* release objects we used */ objRelease(glbl, CORE_COMPONENT); objRelease(datetime, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); if(pszPath != NULL) free(pszPath); ENDmodExit @@ -277,6 +291,7 @@ CODESTARTmodInit CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); iFacilIntMsg = klogFacilIntMsg(); diff --git a/plugins/imrelp/imrelp.c b/plugins/imrelp/imrelp.c index 2255e643..b9e7b2f8 100644 --- a/plugins/imrelp/imrelp.c +++ b/plugins/imrelp/imrelp.c @@ -4,7 +4,7 @@ * * File begun on 2008-03-13 by RGerhards * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of rsyslog. * @@ -43,15 +43,19 @@ #include "module-template.h" #include "net.h" #include "msg.h" +#include "unicode-helper.h" +#include "prop.h" MODULE_TYPE_INPUT /* static data */ DEF_IMOD_STATIC_DATA DEFobjCurrIf(net) +DEFobjCurrIf(prop) /* Module static data */ static relpEngine_t *pRelpEngine; /* our relp engine */ +static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this module */ /* config settings */ @@ -85,7 +89,7 @@ onSyslogRcv(uchar *pHostname, uchar __attribute__((unused)) *pIP, uchar *pMsg, s { DEFiRet; parseAndSubmitMessage(pHostname, (uchar*) "[unset]", pMsg, lenMsg, PARSE_HOSTNAME, - eFLOWCTL_LIGHT_DELAY, (uchar*)"imrelp", NULL, 0); + eFLOWCTL_LIGHT_DELAY, pInputName, NULL, 0); RETiRet; } @@ -130,6 +134,11 @@ CODESTARTwillRun //net.PrintAllowedSenders(2); /* TCP */ if(pRelpEngine == NULL) ABORT_FINALIZE(RS_RET_NO_RUN); + + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imrelp"), sizeof("imrelp") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); finalize_it: ENDwillRun @@ -143,6 +152,9 @@ CODESTARTafterRun net.pAllowedSenders_TCP = NULL; } #endif + + if(pInputName != NULL) + prop.Destruct(&pInputName); ENDafterRun @@ -152,6 +164,7 @@ CODESTARTmodExit iRet = relpEngineDestruct(&pRelpEngine); /* release objects we used */ + objRelease(prop, CORE_COMPONENT); objRelease(net, LM_NET_FILENAME); ENDmodExit @@ -177,6 +190,7 @@ CODESTARTmodInit CODEmodInit_QueryRegCFSLineHdlr pRelpEngine = NULL; /* request objects we use */ + CHKiRet(objUse(prop, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); /* register config file handlers */ diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 828b9636..dcdec1fc 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -245,7 +245,6 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, &stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*)pRcvBuf, lenRcvBuf); - prop.AddRef(pInputName); MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; @@ -346,7 +345,6 @@ CODESTARTwillRun CHKiRet(prop.Construct(&pInputName)); CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imudp"), sizeof("imudp") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); - prop.AddRef(pInputName); net.PrintAllowedSenders(1); /* UDP */ diff --git a/plugins/imuxsock/imuxsock.c b/plugins/imuxsock/imuxsock.c index 1d88a2b5..424d0904 100644 --- a/plugins/imuxsock/imuxsock.c +++ b/plugins/imuxsock/imuxsock.c @@ -37,12 +37,14 @@ #include #include "dirty.h" #include "cfsysline.h" +#include "unicode-helper.h" #include "module-template.h" #include "srUtils.h" #include "errmsg.h" #include "net.h" #include "glbl.h" #include "msg.h" +#include "prop.h" MODULE_TYPE_INPUT @@ -66,7 +68,9 @@ MODULE_TYPE_INPUT DEF_IMOD_STATIC_DATA DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) +DEFobjCurrIf(prop) +static prop_t *pInputName = NULL; /* our inputName currently is always "imudp", and this will hold it */ static int startIndexUxLocalSockets; /* process funix from that index on (used to * suppress local logging. rgerhards 2005-08-01 * read-only after startup @@ -223,7 +227,7 @@ static rsRetVal readSocket(int fd, int iSock) parseAndSubmitMessage(funixHName[iSock] == NULL ? glbl.GetLocalHostName() : funixHName[iSock], (uchar*)"127.0.0.1", pRcv, iRcvd, funixParseHost[iSock] ? (funixFlags[iSock] | PARSE_HOSTNAME) : funixFlags[iSock], - funixFlowCtl[iSock], (uchar*)"imuxsock", NULL, 0); + funixFlowCtl[iSock], pInputName, NULL, 0); } else if (iRcvd < 0 && errno != EINTR) { char errStr[1024]; rs_strerror_r(errno, errStr, sizeof(errStr)); @@ -306,7 +310,12 @@ CODESTARTwillRun dbgprintf("Opened UNIX socket '%s' (fd %d).\n", funixn[i], funix[i]); } - RETiRet; + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInputName)); + CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imudp"), sizeof("imudp") - 1)); + CHKiRet(prop.ConstructFinalize(pInputName)); + +finalize_it: ENDwillRun @@ -332,6 +341,9 @@ CODESTARTafterRun discardFunixn(); nfunix = 1; + + if(pInputName != NULL) + prop.Destruct(&pInputName); ENDafterRun @@ -339,6 +351,7 @@ BEGINmodExit CODESTARTmodExit objRelease(glbl, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); + objRelease(prop, CORE_COMPONENT); ENDmodExit @@ -375,6 +388,7 @@ CODESTARTmodInit CODEmodInit_QueryRegCFSLineHdlr CHKiRet(objUse(errmsg, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(prop, CORE_COMPONENT)); dbgprintf("imuxsock version %s initializing\n", PACKAGE_VERSION); diff --git a/runtime/msg.c b/runtime/msg.c index 47b4ed85..fd0cbcdc 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -295,6 +295,175 @@ getInputName(msg_t *pM, uchar **ppsz, int *plen) ENDfunc } + +/* map a property name (string) to a property ID */ +rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID) +{ + uchar *pName; + DEFiRet; + + assert(pCSPropName != NULL); + assert(pPropID != NULL); + pName = rsCStrGetSzStrNoNULL(pCSPropName); + + /* sometimes there are aliases to the original MonitoWare + * property names. These come after || in the ifs below. */ + if(!strcmp((char*) pName, "msg")) { + *pPropID = PROP_MSG; + } else if(!strcmp((char*) pName, "timestamp") + || !strcmp((char*) pName, "timereported")) { + *pPropID = PROP_TIMESTAMP; + } else if(!strcmp((char*) pName, "hostname") || !strcmp((char*) pName, "source")) { + *pPropID = PROP_HOSTNAME; + } else if(!strcmp((char*) pName, "syslogtag")) { + *pPropID = PROP_SYSLOGTAG; + } else if(!strcmp((char*) pName, "rawmsg")) { + *pPropID = PROP_RAWMSG; + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + } else if(!strcmp((char*) pName, "uxtradmsg")) { + pRes = getUxTradMsg(pMsg); + */ + } else if(!strcmp((char*) pName, "inputname")) { + *pPropID = PROP_INPUTNAME; + } else if(!strcmp((char*) pName, "fromhost")) { + *pPropID = PROP_FROMHOST; + } else if(!strcmp((char*) pName, "fromhost-ip")) { + *pPropID = PROP_FROMHOST_IP; + } else if(!strcmp((char*) pName, "pri")) { + *pPropID = PROP_PRI; + } else if(!strcmp((char*) pName, "pri-text")) { + *pPropID = PROP_PRI_TEXT; + } else if(!strcmp((char*) pName, "iut")) { + *pPropID = PROP_IUT; + } else if(!strcmp((char*) pName, "syslogfacility")) { + *pPropID = PROP_SYSLOGFACILITY; + } else if(!strcmp((char*) pName, "syslogfacility-text")) { + *pPropID = PROP_SYSLOGFACILITY_TEXT; + } else if(!strcmp((char*) pName, "syslogseverity") || !strcmp((char*) pName, "syslogpriority")) { + *pPropID = PROP_SYSLOGSEVERITY; + } else if(!strcmp((char*) pName, "syslogseverity-text") || !strcmp((char*) pName, "syslogpriority-text")) { + *pPropID = PROP_SYSLOGSEVERITY_TEXT; + } else if(!strcmp((char*) pName, "timegenerated")) { + *pPropID = PROP_TIMEGENERATED; + } else if(!strcmp((char*) pName, "programname")) { + *pPropID = PROP_PROGRAMNAME; + } else if(!strcmp((char*) pName, "protocol-version")) { + *pPropID = PROP_PROTOCOL_VERSION; + } else if(!strcmp((char*) pName, "structured-data")) { + *pPropID = PROP_STRUCTURED_DATA; + } else if(!strcmp((char*) pName, "app-name")) { + *pPropID = PROP_APP_NAME; + } else if(!strcmp((char*) pName, "procid")) { + *pPropID = PROP_PROCID; + } else if(!strcmp((char*) pName, "msgid")) { + *pPropID = PROP_MSGID; + /* here start system properties (those, that do not relate to the message itself */ + } else if(!strcmp((char*) pName, "$now")) { + *pPropID = PROP_SYS_NOW; + } else if(!strcmp((char*) pName, "$year")) { + *pPropID = PROP_SYS_YEAR; + } else if(!strcmp((char*) pName, "$month")) { + *pPropID = PROP_SYS_MONTH; + } else if(!strcmp((char*) pName, "$day")) { + *pPropID = PROP_SYS_DAY; + } else if(!strcmp((char*) pName, "$hour")) { + *pPropID = PROP_SYS_HOUR; + } else if(!strcmp((char*) pName, "$hhour")) { + *pPropID = PROP_SYS_HHOUR; + } else if(!strcmp((char*) pName, "$qhour")) { + *pPropID = PROP_SYS_QHOUR; + } else if(!strcmp((char*) pName, "$minute")) { + *pPropID = PROP_SYS_MINUTE; + } else if(!strcmp((char*) pName, "$myhostname")) { + *pPropID = PROP_SYS_MYHOSTNAME; + } else { + *pPropID = PROP_INVALID; + iRet = RS_RET_VAR_NOT_FOUND; + } + + RETiRet; +} + + +/* map a property ID to a name string (useful for displaying) */ +uchar *propIDToName(propid_t propID) +{ + switch(propID) { + case PROP_MSG: + return UCHAR_CONSTANT("msg"); + case PROP_TIMESTAMP: + return UCHAR_CONSTANT("timestamp"); + case PROP_HOSTNAME: + return UCHAR_CONSTANT("hostname"); + case PROP_SYSLOGTAG: + return UCHAR_CONSTANT("syslogtag"); + case PROP_RAWMSG: + return UCHAR_CONSTANT("rawmsg"); + /* enable this, if someone actually uses UxTradMsg, delete after some time has + * passed and nobody complained -- rgerhards, 2009-06-16 + case PROP_UXTRADMSG: + pRes = getUxTradMsg(pMsg); + break; + */ + case PROP_INPUTNAME: + return UCHAR_CONSTANT("inputname"); + case PROP_FROMHOST: + return UCHAR_CONSTANT("fromhost"); + case PROP_FROMHOST_IP: + return UCHAR_CONSTANT("fromhost-ip"); + case PROP_PRI: + return UCHAR_CONSTANT("pri"); + case PROP_PRI_TEXT: + return UCHAR_CONSTANT("pri-text"); + case PROP_IUT: + return UCHAR_CONSTANT("iut"); + case PROP_SYSLOGFACILITY: + return UCHAR_CONSTANT("syslogfacility"); + case PROP_SYSLOGFACILITY_TEXT: + return UCHAR_CONSTANT("syslogfacility-text"); + case PROP_SYSLOGSEVERITY: + return UCHAR_CONSTANT("syslogseverity"); + case PROP_SYSLOGSEVERITY_TEXT: + return UCHAR_CONSTANT("syslogseverity-text"); + case PROP_TIMEGENERATED: + return UCHAR_CONSTANT("timegenerated"); + case PROP_PROGRAMNAME: + return UCHAR_CONSTANT("programname"); + case PROP_PROTOCOL_VERSION: + return UCHAR_CONSTANT("protocol-version"); + case PROP_STRUCTURED_DATA: + return UCHAR_CONSTANT("structured-data"); + case PROP_APP_NAME: + return UCHAR_CONSTANT("app-name"); + case PROP_PROCID: + return UCHAR_CONSTANT("procid"); + case PROP_MSGID: + return UCHAR_CONSTANT("msgid"); + case PROP_SYS_NOW: + return UCHAR_CONSTANT("$NOW"); + case PROP_SYS_YEAR: + return UCHAR_CONSTANT("$YEAR"); + case PROP_SYS_MONTH: + return UCHAR_CONSTANT("$MONTH"); + case PROP_SYS_DAY: + return UCHAR_CONSTANT("$DAY"); + case PROP_SYS_HOUR: + return UCHAR_CONSTANT("$HOUR"); + case PROP_SYS_HHOUR: + return UCHAR_CONSTANT("$HHOUR"); + case PROP_SYS_QHOUR: + return UCHAR_CONSTANT("$QHOUR"); + case PROP_SYS_MINUTE: + return UCHAR_CONSTANT("$MINUTE"); + case PROP_SYS_MYHOSTNAME: + return UCHAR_CONSTANT("$MYHOSTNAME"); + default: + return UCHAR_CONSTANT("*invalid property id*"); + } +} + + /* The following functions will support advanced output module * multithreading, once this is implemented. Currently, we * include them as hooks only. The idea is that we need to guard @@ -1606,18 +1775,22 @@ static int getAPPNAMELen(msg_t *pM, bool bLockMutex) return (pM->pCSAPPNAME == NULL) ? 0 : rsCStrLen(pM->pCSAPPNAME); } -/* rgerhards 2008-09-10: set pszInputName in msg object +/* rgerhards 2008-09-10: set pszInputName in msg object. This calls AddRef() + * on the property, because this must be done in all current cases and there + * is no case expected where this may not be necessary. * rgerhards, 2009-06-16 */ void MsgSetInputName(msg_t *pThis, prop_t *inputName) { assert(pThis != NULL); + prop.AddRef(inputName); if(pThis->pInputName != NULL) prop.Destruct(&pThis->pInputName); pThis->pInputName = inputName; } +#if 0 /* to be removed soon: work-around for those tht can not natively generate an * input name. * rgerhards, 2009-06-29 @@ -1635,6 +1808,7 @@ void MsgSetInputNameStr(msg_t *pThis, uchar *psz, int len) MsgSetInputName(pThis, pProp); prop.Destruct(&pProp); } +#endif /* rgerhards 2004-11-16: set pszRcvFrom in msg object */ @@ -1903,7 +2077,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, short iOffs; BEGINfunc -dbgprintf("XXXX: msgGetProp for %d\n", propID); +dbgprintf("XXXX: msgGetProp for %s\n", propIDToName(propID)); assert(pMsg != NULL); assert(pbMustBeFreed != NULL); @@ -1939,10 +2113,7 @@ dbgprintf("XXXX: msgGetProp for %d\n", propID); break; */ case PROP_INPUTNAME: -RUNLOG; getInputName(pMsg, ((uchar**) &pRes), &bufLen); -RUNLOG; -RUNLOG_VAR("%p", pRes); break; case PROP_FROMHOST: pRes = (char*) getRcvFrom(pMsg); @@ -2670,97 +2841,6 @@ finalize_it: RETiRet; } - -/* map a property name (string) to a property ID */ -rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID) -{ - uchar *pName; - DEFiRet; - - assert(pCSPropName != NULL); - assert(pPropID != NULL); - pName = rsCStrGetSzStrNoNULL(pCSPropName); - - /* sometimes there are aliases to the original MonitoWare - * property names. These come after || in the ifs below. */ - if(!strcmp((char*) pName, "msg")) { - *pPropID = PROP_MSG; - } else if(!strcmp((char*) pName, "timestamp") - || !strcmp((char*) pName, "timereported")) { - *pPropID = PROP_TIMESTAMP; - } else if(!strcmp((char*) pName, "hostname") || !strcmp((char*) pName, "source")) { - *pPropID = PROP_HOSTNAME; - } else if(!strcmp((char*) pName, "syslogtag")) { - *pPropID = PROP_SYSLOGTAG; - } else if(!strcmp((char*) pName, "rawmsg")) { - *pPropID = PROP_RAWMSG; - /* enable this, if someone actually uses UxTradMsg, delete after some time has - * passed and nobody complained -- rgerhards, 2009-06-16 - } else if(!strcmp((char*) pName, "uxtradmsg")) { - pRes = getUxTradMsg(pMsg); - */ - } else if(!strcmp((char*) pName, "inputname")) { - *pPropID = PROP_INPUTNAME; - } else if(!strcmp((char*) pName, "fromhost")) { - *pPropID = PROP_FROMHOST; - } else if(!strcmp((char*) pName, "fromhost-ip")) { - *pPropID = PROP_FROMHOST_IP; - } else if(!strcmp((char*) pName, "pri")) { - *pPropID = PROP_PRI; - } else if(!strcmp((char*) pName, "pri-text")) { - *pPropID = PROP_PRI_TEXT; - } else if(!strcmp((char*) pName, "iut")) { - *pPropID = PROP_IUT; - } else if(!strcmp((char*) pName, "syslogfacility")) { - *pPropID = PROP_SYSLOGFACILITY; - } else if(!strcmp((char*) pName, "syslogfacility-text")) { - *pPropID = PROP_SYSLOGFACILITY_TEXT; - } else if(!strcmp((char*) pName, "syslogseverity") || !strcmp((char*) pName, "syslogpriority")) { - *pPropID = PROP_SYSLOGSEVERITY; - } else if(!strcmp((char*) pName, "syslogseverity-text") || !strcmp((char*) pName, "syslogpriority-text")) { - *pPropID = PROP_SYSLOGSEVERITY_TEXT; - } else if(!strcmp((char*) pName, "timegenerated")) { - *pPropID = PROP_TIMEGENERATED; - } else if(!strcmp((char*) pName, "programname")) { - *pPropID = PROP_PROGRAMNAME; - } else if(!strcmp((char*) pName, "protocol-version")) { - *pPropID = PROP_PROTOCOL_VERSION; - } else if(!strcmp((char*) pName, "structured-data")) { - *pPropID = PROP_STRUCTURED_DATA; - } else if(!strcmp((char*) pName, "app-name")) { - *pPropID = PROP_APP_NAME; - } else if(!strcmp((char*) pName, "procid")) { - *pPropID = PROP_PROCID; - } else if(!strcmp((char*) pName, "msgid")) { - *pPropID = PROP_MSGID; - /* here start system properties (those, that do not relate to the message itself */ - } else if(!strcmp((char*) pName, "$now")) { - *pPropID = PROP_SYS_NOW; - } else if(!strcmp((char*) pName, "$year")) { - *pPropID = PROP_SYS_YEAR; - } else if(!strcmp((char*) pName, "$month")) { - *pPropID = PROP_SYS_MONTH; - } else if(!strcmp((char*) pName, "$day")) { - *pPropID = PROP_SYS_DAY; - } else if(!strcmp((char*) pName, "$hour")) { - *pPropID = PROP_SYS_HOUR; - } else if(!strcmp((char*) pName, "$hhour")) { - *pPropID = PROP_SYS_HHOUR; - } else if(!strcmp((char*) pName, "$qhour")) { - *pPropID = PROP_SYS_QHOUR; - } else if(!strcmp((char*) pName, "$minute")) { - *pPropID = PROP_SYS_MINUTE; - } else if(!strcmp((char*) pName, "$myhostname")) { - *pPropID = PROP_SYS_MYHOSTNAME; - } else { - *pPropID = PROP_INVALID; - iRet = RS_RET_VAR_NOT_FOUND; - } - - RETiRet; -} - - /* This function can be used as a generic way to set properties. * We have to handle a lot of legacy, so our return value is not always * 100% correct (called functions do not always provide one, should @@ -2802,7 +2882,6 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) CHKiRet(prop.Construct(&myProp)); CHKiRet(prop.SetString(myProp, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr))); CHKiRet(prop.ConstructFinalize(myProp)); - prop.AddRef(myProp); MsgSetInputName(pThis, myProp); prop.Destruct(&myProp); } else if(isProp("pszRcvFromIP")) { diff --git a/runtime/msg.h b/runtime/msg.h index a1fc535b..7f84da35 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -166,11 +166,6 @@ rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); -// REMOVE: -void MsgSetInputNameStr(msg_t *pThis, uchar *psz, int len); - - - /* TODO: remove these five (so far used in action.c) */ char *getMSG(msg_t *pM); char *getHOSTNAME(msg_t *pM); @@ -184,6 +179,7 @@ char *getProgramName(msg_t *pM, bool bLockMutex); int getProgramNameLen(msg_t *pM, bool bLockMutex); uchar *getRcvFrom(msg_t *pM); rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID); +uchar *propIDToName(propid_t propID); /* The MsgPrepareEnqueue() function is a macro for performance reasons. diff --git a/runtime/rule.c b/runtime/rule.c index ce19146b..3a257a90 100644 --- a/runtime/rule.c +++ b/runtime/rule.c @@ -219,8 +219,8 @@ shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) bRet = (bRet == 1) ? 0 : 1; if(Debug) { - dbgprintf("Filter: check for property '%d' (value '%s') ", - pRule->f_filterData.prop.propID, pszPropVal); + dbgprintf("Filter: check for property '%s' (value '%s') ", + propIDToName(pRule->f_filterData.prop.propID), pszPropVal); if(pRule->f_filterData.prop.isNegated) dbgprintf("NOT "); dbgprintf("%s '%s': %s\n", @@ -373,8 +373,7 @@ CODESTARTobjDebugPrint(rule) dbgprintf("EXPRESSION-BASED Filter: can currently not be displayed"); } else { dbgprintf("PROPERTY-BASED Filter:\n"); - dbgprintf("\tProperty.: '%d'\n", pThis->f_filterData.prop.propID); -// TODO: XXXX ADD idtostring()! + dbgprintf("\tProperty.: '%s'\n", propIDToName(pThis->f_filterData.prop.propID)); dbgprintf("\tOperation: "); if(pThis->f_filterData.prop.isNegated) dbgprintf("NOT "); diff --git a/tcps_sess.c b/tcps_sess.c index 4786e154..23241f4f 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -237,7 +237,6 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttG /* we now create our own message object and submit it to the queue */ CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); MsgSetRawMsg(pMsg, (char*)pThis->pMsg, pThis->iMsg); - prop.AddRef(pThis->pLstnInfo->pInputName); MsgSetInputName(pMsg, pThis->pLstnInfo->pInputName); MsgSetFlowControlType(pMsg, eFLOWCTL_LIGHT_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; diff --git a/tools/syslogd.c b/tools/syslogd.c index 2fe949ac..f1c43602 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -140,6 +140,7 @@ #include "rule.h" #include "net.h" #include "vm.h" +#include "prop.h" /* definitions for objects we access */ DEFobjCurrIf(obj) @@ -151,25 +152,13 @@ DEFobjCurrIf(module) DEFobjCurrIf(errmsg) DEFobjCurrIf(rule) DEFobjCurrIf(ruleset) +DEFobjCurrIf(prop) DEFobjCurrIf(net) /* TODO: make go away! */ /* forward definitions */ static rsRetVal GlobalClassExit(void); -/* We define our own set of syslog defintions so that we - * do not need to rely on (possibly different) implementations. - * 2007-07-19 rgerhards - */ -/* missing definitions for solaris - * 2006-02-16 Rger - */ -#ifdef __sun -# define LOG_AUTHPRIV LOG_AUTH -#endif -#define INTERNAL_NOPRI 0x10 /* the "no priority" priority */ -#define LOG_FTP (11<<3) /* ftp daemon */ - #ifndef UTMP_FILE #ifdef UTMP_FILENAME @@ -217,14 +206,11 @@ static rsRetVal GlobalClassExit(void); # endif #endif -#ifndef _PATH_DEV -# define _PATH_DEV "/dev/" -#endif - #ifndef _PATH_TTY # define _PATH_TTY "/dev/tty" #endif +static prop_t *pInternalInputName = NULL; /* there is only one global inputName for all internally-generated messages */ static uchar *ConfFile = (uchar*) _PATH_LOGCONF; /* read-only after startup */ static char *PidFile = _PATH_LOGPID; /* read-only after startup */ @@ -566,7 +552,7 @@ void untty(void) * interface change: bParseHostname removed, now in flags */ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int flags, flowControl_t flowCtlType, - uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime) + prop_t *pInputName, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; register uchar *p; @@ -579,8 +565,8 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f } else { CHKiRet(msgConstructWithTime(&pMsg, stTime, ttGenTime)); } - if(pszInputName != NULL) - MsgSetInputNameStr(pMsg, pszInputName, ustrlen(pszInputName)); + if(pInputName != NULL) + MsgSetInputName(pMsg, pInputName); MsgSetFlowControlType(pMsg, flowCtlType); MsgSetRawMsgWOSize(pMsg, (char*)msg); @@ -675,7 +661,7 @@ finalize_it: */ rsRetVal parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int flags, flowControl_t flowCtlType, - uchar *pszInputName, struct syslogTime *stTime, time_t ttGenTime) + prop_t *pInputName, struct syslogTime *stTime, time_t ttGenTime) { DEFiRet; register int iMsg; @@ -786,7 +772,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla */ if(iMsg == iMaxLine) { *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ - printline(hname, hnameIP, tmpline, flags, flowCtlType, pszInputName, stTime, ttGenTime); + printline(hname, hnameIP, tmpline, flags, flowCtlType, pInputName, stTime, ttGenTime); } else { /* This case in theory never can happen. If it happens, we have * a logic error. I am checking for it, because if I would not, @@ -838,7 +824,7 @@ parseAndSubmitMessage(uchar *hname, uchar *hnameIP, uchar *msg, int len, int fla *(pMsg + iMsg) = '\0'; /* space *is* reserved for this! */ /* typically, we should end up here! */ - printline(hname, hnameIP, tmpline, flags, flowCtlType, pszInputName, stTime, ttGenTime); + printline(hname, hnameIP, tmpline, flags, flowCtlType, pInputName, stTime, ttGenTime); finalize_it: if(tmpline != NULL) @@ -881,7 +867,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) DEFiRet; CHKiRet(msgConstruct(&pMsg)); - MsgSetInputNameStr(pMsg, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslogd")-1); + MsgSetInputName(pMsg, pInternalInputName); MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); @@ -1711,6 +1697,10 @@ die(int sig) legacyOptsFree(); + /* destruct our input name */ + if(pInternalInputName != NULL) + prop.Destruct(&pInternalInputName); + /* terminate the remaining classes */ GlobalClassExit(); @@ -2867,6 +2857,8 @@ InitGlobalClasses(void) CHKiRet(objUse(ruleset, CORE_COMPONENT)); pErrObj = "conf"; CHKiRet(objUse(conf, CORE_COMPONENT)); + pErrObj = "prop"; + CHKiRet(objUse(prop, CORE_COMPONENT)); /* intialize some dummy classes that are not part of the runtime */ pErrObj = "action"; @@ -2907,6 +2899,7 @@ GlobalClassExit(void) /* first, release everything we used ourself */ objRelease(net, LM_NET_FILENAME);/* TODO: the dependency on net shall go away! -- rgerhards, 2008-03-07 */ + objRelease(prop, CORE_COMPONENT); objRelease(conf, CORE_COMPONENT); objRelease(ruleset, CORE_COMPONENT); objRelease(rule, CORE_COMPONENT); @@ -3213,6 +3206,11 @@ int realMain(int argc, char **argv) /* doing some core initializations */ + /* we need to create the inputName property (only once during our lifetime) */ + CHKiRet(prop.Construct(&pInternalInputName)); + CHKiRet(prop.SetString(pInternalInputName, UCHAR_CONSTANT("rsyslgod"), sizeof("rsyslgod") - 1)); + CHKiRet(prop.ConstructFinalize(pInternalInputName)); + /* get our host and domain names - we need to do this early as we may emit * error log messages, which need the correct hostname. -- rgerhards, 2008-04-04 */ -- cgit v1.2.3 From aaffc4281e0b26f419a3fc341461f2fc479080b8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 30 Jun 2009 18:45:41 +0200 Subject: introduced a new way of handling the RcvFrom property ... plus a fix for a long-time bug in obj-types.h. That lead to the object pointer only then to become NULL when the object was actually destructed, I discovered this issue during introduction of the pRcvFrom property in msg_t, but it potentially had other effects, too. I am not sure if some experienced instability resulted from this bug OR if its fix will cause harm to so-far "correctly" running code. The later may very well be. Thus I will change it only for the current branch and also the beta, but not in all old builds. Let's see how things evolve. --- plugins/imdiag/imdiag.c | 9 ++++- plugins/imklog/imklog.c | 2 +- plugins/imudp/imudp.c | 2 +- runtime/glbl.c | 46 +++++++++++++++++++++++++ runtime/glbl.h | 7 +++- runtime/msg.c | 91 ++++++++++++++++++++++++++++--------------------- runtime/msg.h | 7 ++-- runtime/obj-types.h | 11 +++++- runtime/parser.c | 2 +- runtime/prop.c | 18 +++++++--- runtime/prop.h | 1 + runtime/rsyslog.c | 4 +-- tcps_sess.c | 18 +++++++--- tcps_sess.h | 3 +- tools/syslogd.c | 8 +++-- 15 files changed, 168 insertions(+), 61 deletions(-) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 8d874a87..140222e1 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -72,6 +72,7 @@ DEFobjCurrIf(prop) static tcpsrv_t *pOurTcpsrv = NULL; /* our TCP server(listener) TODO: change for multiple instances */ static permittedPeers_t *pPermPeersRoot = NULL; static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this input */ +static prop_t *pRcvDummy = NULL; /* config settings */ @@ -212,7 +213,7 @@ doInjectMsg(int iNum) MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; - MsgSetRcvFrom(pMsg, UCHAR_CONSTANT("127.0.0.1")); /* TODO: way may use the real sender here... */ + MsgSetRcvFrom(pMsg, pRcvDummy); CHKiRet(MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1"))); CHKiRet(submitMsg(pMsg)); @@ -383,6 +384,10 @@ CODESTARTwillRun CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imdiag"), sizeof("imdiag") - 1)); CHKiRet(prop.ConstructFinalize(pInputName)); + CHKiRet(prop.Construct(&pRcvDummy)); + CHKiRet(prop.SetString(pRcvDummy, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + CHKiRet(prop.ConstructFinalize(pRcvDummy)); + finalize_it: ENDwillRun @@ -391,6 +396,8 @@ BEGINafterRun CODESTARTafterRun if(pInputName != NULL) prop.Destruct(&pInputName); + if(pRcvDummy != NULL) + prop.Destruct(&pRcvDummy); ENDafterRun diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 10cc8b14..e4db03b3 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -104,7 +104,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) MsgSetInputName(pMsg, pInputName); MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ - MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); + MsgSetRcvFrom(pMsg, glbl.GetLocalHostNameProp()); MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); MsgSetTAG(pMsg, pszTag, ustrlen(pszTag)); diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index dcdec1fc..e091c7d6 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -249,7 +249,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; - MsgSetRcvFrom(pMsg, fromHost); + MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost)); CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP)); CHKiRet(submitMsg(pMsg)); } diff --git a/runtime/glbl.c b/runtime/glbl.c index 28f14320..32b85622 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -35,8 +35,10 @@ #include "rsyslog.h" #include "obj.h" +#include "unicode-helper.h" #include "cfsysline.h" #include "glbl.h" +#include "prop.h" /* some defaults */ #ifndef DFLT_NETSTRM_DRVR @@ -45,6 +47,7 @@ /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(prop) /* static data * For this object, these variables are obviously what makes the "meat" of the @@ -59,6 +62,7 @@ static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) static int bDropMalPTRMsgs = 0;/* Drop messages which have malicious PTR records during DNS lookup */ static int option_DisallowWarning = 1; /* complain if message from disallowed sender is received */ static int bDisableDNS = 0; /* don't look up IP addresses of remote messages */ +static prop_t *propLocalHostName = NULL;/* our hostname as FQDN - read-only after startup */ static uchar *LocalHostName = NULL;/* our hostname - read-only after startup */ static uchar *LocalFQDNName = NULL;/* our hostname as FQDN - read-only after startup */ static uchar *LocalDomain; /* our local domain name - read-only after startup */ @@ -132,6 +136,44 @@ GetLocalHostName(void) } +/* generate the local hostname property. This must be done after the hostname info + * has been set as well as PreserveFQDN. + * rgerhards, 2009-06-30 + */ +static rsRetVal +GenerateLocalHostNameProperty(void) +{ + DEFiRet; + uchar *pszName; + + if(propLocalHostName != NULL) + prop.Destruct(&propLocalHostName); + + CHKiRet(prop.Construct(&propLocalHostName)); + if(LocalHostName == NULL) + pszName = (uchar*) "[localhost]"; + else { + if(GetPreserveFQDN() == 1) + pszName = LocalFQDNName; + else + pszName = LocalHostName; + } + CHKiRet(prop.SetString(propLocalHostName, pszName, ustrlen(pszName))); + CHKiRet(prop.ConstructFinalize(propLocalHostName)); + +finalize_it: + RETiRet; +} + +/* return our local hostname as a string property + */ +static prop_t* +GetLocalHostNameProp(void) +{ + return(propLocalHostName); +} + + /* return the current localhost name as FQDN (requires FQDN to be set) * TODO: we should set the FQDN ourselfs in here! */ @@ -197,6 +239,8 @@ CODESTARTobjQueryInterface(glbl) * of course, also affects the "if" above). */ pIf->GetWorkDir = GetWorkDir; + pIf->GenerateLocalHostNameProperty = GenerateLocalHostNameProperty; + pIf->GetLocalHostNameProp = GetLocalHostNameProp; #define SIMP_PROP(name) \ pIf->Get##name = Get##name; \ pIf->Set##name = Set##name; @@ -262,6 +306,7 @@ static rsRetVal resetConfigVariables(uchar __attribute__((unused)) *pp, void __a */ BEGINAbstractObjClassInit(glbl, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ + CHKiRet(objUse(prop, CORE_COMPONENT)); /* register config handlers (TODO: we need to implement a way to unregister them) */ CHKiRet(regCfSysLineHdlr((uchar *)"workdirectory", 0, eCmdHdlrGetWord, NULL, &pszWorkDir, NULL)); @@ -295,6 +340,7 @@ BEGINObjClassExit(glbl, OBJ_IS_CORE_MODULE) /* class, version */ free(LocalHostName); if(LocalFQDNName != NULL) free(LocalFQDNName); + objRelease(prop, CORE_COMPONENT); ENDObjClassExit(glbl) /* vi:set ai: diff --git a/runtime/glbl.h b/runtime/glbl.h index 5bdf4f57..dcfb6d5f 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -32,6 +32,8 @@ #ifndef GLBL_H_INCLUDED #define GLBL_H_INCLUDED +#include "prop.h" + #define glblGetIOBufSize() 4096 /* size of the IO buffer, e.g. for strm class */ /* interfaces */ @@ -57,9 +59,12 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ SIMP_PROP(DfltNetstrmDrvrCAF, uchar*) SIMP_PROP(DfltNetstrmDrvrKeyFile, uchar*) SIMP_PROP(DfltNetstrmDrvrCertFile, uchar*) + /* added v3, 2009-06-30 */ + rsRetVal (*GenerateLocalHostNameProperty)(void); + prop_t* (*GetLocalHostNameProp)(void); #undef SIMP_PROP ENDinterface(glbl) -#define glblCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +#define glblCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ /* version 2 had PreserveFQDN added - rgerhards, 2008-12-08 */ /* the remaining prototypes */ diff --git a/runtime/msg.c b/runtime/msg.c index fd0cbcdc..3eb41a87 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -622,11 +622,9 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->iLenMSG = 0; pM->iLenTAG = 0; pM->iLenHOSTNAME = 0; - pM->iLenRcvFrom = 0; pM->iLenRcvFromIP = 0; pM->pszRawMsg = NULL; pM->pszHOSTNAME = NULL; - pM->pszRcvFrom = NULL; pM->pszRcvFromIP = NULL; pM->pszRcvdAt3164 = NULL; pM->pszRcvdAt3339 = NULL; @@ -642,6 +640,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->pCSPROCID = NULL; pM->pCSMSGID = NULL; pM->pInputName = NULL; + pM->pRcvFrom = NULL; pM->pRuleset = NULL; memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt)); memset(&pM->tTIMESTAMP, 0, sizeof(pM->tTIMESTAMP)); @@ -742,8 +741,8 @@ CODESTARTobjDestruct(msg) freeHOSTNAME(pThis); if(pThis->pInputName != NULL) prop.Destruct(&pThis->pInputName); - free(pThis->pszRcvFrom); - free(pThis->pszRcvFromIP); + if(pThis->pRcvFrom != NULL) + prop.Destruct(&pThis->pRcvFrom); free(pThis->pszRcvdAt3164); free(pThis->pszRcvdAt3339); free(pThis->pszRcvdAt_MySQL); @@ -844,6 +843,10 @@ msg_t* MsgDup(msg_t* pOld) pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; pNew->offMSG = pOld->offMSG; + pNew->pRcvFrom = pOld->pRcvFrom; + prop.AddRef(pNew->pRcvFrom); + pNew->pInputName = pOld->pInputName; + prop.AddRef(pNew->pInputName); /* enable this, if someone actually uses UxTradMsg, delete after some time has * passed and nobody complained -- rgerhards, 2009-06-16 pNew->offAfterPRI = pOld->offAfterPRI; @@ -871,7 +874,6 @@ msg_t* MsgDup(msg_t* pOld) } else { tmpCOPYSZ(HOSTNAME); } - tmpCOPYSZ(RcvFrom); tmpCOPYCSTR(ProgName); tmpCOPYCSTR(StrucData); @@ -935,8 +937,8 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializePTR(pStrm, pszHOSTNAME, PSZ); getInputName(pThis, &psz, &len); objSerializeSCALAR_VAR(pStrm, "pszInputName", PSZ, psz); - //objSerializePTR(pStrm, pszInputName, PSZ); - objSerializePTR(pStrm, pszRcvFrom, PSZ); + psz = getRcvFrom(pThis); + objSerializeSCALAR_VAR(pStrm, "pszRcvFrom", PSZ, psz); objSerializePTR(pStrm, pszRcvFromIP, PSZ); objSerializePTR(pStrm, pCSStrucData, CSTR); @@ -1601,7 +1603,10 @@ int getHOSTNAMELen(msg_t *pM) return 0; else if(pM->pszHOSTNAME == NULL) - return pM->iLenRcvFrom; + if(pM->pRcvFrom == NULL) + return 0; + else + return prop.GetStringLen(pM->pRcvFrom); else return pM->iLenHOSTNAME; } @@ -1612,22 +1617,33 @@ char *getHOSTNAME(msg_t *pM) if(pM == NULL) return ""; else - if(pM->pszHOSTNAME == NULL) - return (char*) pM->pszRcvFrom; - else + if(pM->pszHOSTNAME == NULL) { + if(pM->pRcvFrom == NULL) { + return ""; + } else { + uchar *psz; + int len; + prop.GetString(pM->pRcvFrom, &psz, &len); + return (char*) psz; + } + } else { return (char*) pM->pszHOSTNAME; + } } uchar *getRcvFrom(msg_t *pM) { - if(pM == NULL) - return UCHAR_CONSTANT(""); - else - if(pM->pszRcvFrom == NULL) - return UCHAR_CONSTANT(""); - else - return pM->pszRcvFrom; + uchar *psz; + int len; + BEGINfunc + if(pM == NULL) { + psz = UCHAR_CONSTANT(""); + } else { + prop.GetString(pM->pInputName, &psz, &len); + } + ENDfunc + return psz; } @@ -1790,12 +1806,27 @@ void MsgSetInputName(msg_t *pThis, prop_t *inputName) pThis->pInputName = inputName; } -#if 0 + +/* rgerhards 2008-09-10: set RcvFrom name in msg object. This calls AddRef() + * on the property, because this must be done in all current cases and there + * is no case expected where this may not be necessary. + * rgerhards, 2009-06-30 + */ +void MsgSetRcvFrom(msg_t *pThis, prop_t *new) +{ + assert(pThis != NULL); + + prop.AddRef(new); + if(pThis->pRcvFrom != NULL) + prop.Destruct(&pThis->pRcvFrom); + pThis->pRcvFrom = new; +} + /* to be removed soon: work-around for those tht can not natively generate an * input name. * rgerhards, 2009-06-29 */ -void MsgSetInputNameStr(msg_t *pThis, uchar *psz, int len) +void MsgSetRcvFromStr(msg_t *pThis, uchar *psz, int len) { prop_t *pProp; assert(pThis != NULL); @@ -1804,24 +1835,9 @@ void MsgSetInputNameStr(msg_t *pThis, uchar *psz, int len) prop.Construct(&pProp); prop.SetString(pProp, psz, len); prop.ConstructFinalize(pProp); - prop.AddRef(pProp); - MsgSetInputName(pThis, pProp); + MsgSetRcvFrom(pThis, pProp); prop.Destruct(&pProp); } -#endif - -/* rgerhards 2004-11-16: set pszRcvFrom in msg object - */ -void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom) -{ - assert(pMsg != NULL); - free(pMsg->pszRcvFrom); - - pMsg->iLenRcvFrom = ustrlen(pszRcvFrom); - if((pMsg->pszRcvFrom = malloc(pMsg->iLenRcvFrom + 1)) != NULL) { - memcpy(pMsg->pszRcvFrom, pszRcvFrom, pMsg->iLenRcvFrom + 1); - } -} /* rgerhards 2005-05-16: set pszRcvFromIP in msg object */ @@ -2077,7 +2093,6 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, short iOffs; BEGINfunc -dbgprintf("XXXX: msgGetProp for %s\n", propIDToName(propID)); assert(pMsg != NULL); assert(pbMustBeFreed != NULL); @@ -2887,7 +2902,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) } else if(isProp("pszRcvFromIP")) { MsgSetRcvFromIP(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); } else if(isProp("pszRcvFrom")) { - MsgSetRcvFrom(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetRcvFromStr(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pszHOSTNAME")) { MsgSetHOSTNAME(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pCSStrucData")) { diff --git a/runtime/msg.h b/runtime/msg.h index 7f84da35..59046fc1 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -78,12 +78,10 @@ struct msg { int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ int iLenHOSTNAME; /* Length of HOSTNAME */ - int iLenRcvFrom; /* Length of pszRcvFrom */ int iLenRcvFromIP; /* Length of pszRcvFromIP */ uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we * need to preserve cryptographic verifiers. */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ - uchar *pszRcvFrom; /* System message was received from */ uchar *pszRcvFromIP; /* IP of system message was received from */ char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ @@ -99,6 +97,7 @@ struct msg { cstr_t *pCSPROCID; /* PROCID */ cstr_t *pCSMSGID; /* MSGID */ prop_t *pInputName; /* input name property */ + prop_t *pRcvFrom; /* name of system message was received from */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp @@ -151,7 +150,8 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf); void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); -void MsgSetRcvFrom(msg_t *pMsg, uchar* pszRcvFrom); +void MsgSetRcvFrom(msg_t *pMsg, prop_t*); +void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME, int lenHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); @@ -164,6 +164,7 @@ char *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, char *textpri(char *pRes, size_t pResLen, int pri); rsRetVal msgGetMsgVar(msg_t *pThis, cstr_t *pstrPropName, var_t **ppVar); rsRetVal MsgEnableThreadSafety(void); +uchar *getRcvFrom(msg_t *pM); /* TODO: remove these five (so far used in action.c) */ diff --git a/runtime/obj-types.h b/runtime/obj-types.h index 6c1381ac..e1b54d4f 100644 --- a/runtime/obj-types.h +++ b/runtime/obj-types.h @@ -293,6 +293,15 @@ rsRetVal objName##ClassExit(void) \ ISOBJ_TYPE_assert(pThis, OBJ); \ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); +/* note: there was a long-time bug in the macro below that lead to *ppThis = NULL + * only when the object was actually destructed. I discovered this issue during + * introduction of the pRcvFrom property in msg_t, but it potentially had other + * effects, too. I am not sure if some experienced instability resulted from this + * bug OR if its fix will cause harm to so-far "correctly" running code. The later + * may very well be. Thus I will change it only for the current branch and also + * the beta, but not in all old builds. Let's see how things evolve. + * rgerhards, 2009-06-30 + */ #define ENDobjDestruct(OBJ) \ goto finalize_it; /* prevent compiler warning ;) */ \ /* no more code here! */ \ @@ -300,8 +309,8 @@ rsRetVal objName##ClassExit(void) \ if(pThis != NULL) { \ obj.DestructObjSelf((obj_t*) pThis); \ free(pThis); \ - *ppThis = NULL; \ } \ + *ppThis = NULL; \ pthread_setcancelstate(iCancelStateSave, NULL); \ RETiRet; \ } diff --git a/runtime/parser.c b/runtime/parser.c index d4ca7673..a5183105 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -261,7 +261,7 @@ rsRetVal parseMsg(msg_t *pMsg) CHKiRet(sanitizeMessage(pMsg)); /* we needed to sanitize first, because we otherwise do not have a C-string we can print... */ - DBGPRINTF("msg parser: flags %x, from '%s', msg '%s'\n", pMsg->msgFlags, pMsg->pszRcvFrom, pMsg->pszRawMsg); + DBGPRINTF("msg parser: flags %x, from '%s', msg '%s'\n", pMsg->msgFlags, getRcvFrom(pMsg), pMsg->pszRawMsg); /* pull PRI */ pri = DEFUPRI; diff --git a/runtime/prop.c b/runtime/prop.c index 989657dd..96ebe212 100644 --- a/runtime/prop.c +++ b/runtime/prop.c @@ -68,7 +68,7 @@ static rsRetVal SetString(prop_t *pThis, uchar *psz, int len) memcpy(pThis->szVal.sz, psz, len + 1); } else { CHKmalloc(pThis->szVal.psz = malloc(len + 1)); - memcpy(pThis->szVal.sz, psz, len + 1); + memcpy(pThis->szVal.psz, psz, len + 1); } finalize_it: @@ -76,15 +76,25 @@ finalize_it: } +/* get string length */ +static int GetStringLen(prop_t *pThis) +{ + return pThis->len; +} + + /* get string */ static rsRetVal GetString(prop_t *pThis, uchar **ppsz, int *plen) { BEGINfunc ISOBJ_TYPE_assert(pThis, prop); - if(pThis->len < CONF_PROP_BUFSIZE) + if(pThis->len < CONF_PROP_BUFSIZE) { *ppsz = pThis->szVal.sz; - else +RUNLOG; + } else { *ppsz = pThis->szVal.psz; +RUNLOG; + } *plen = pThis->len; ENDfunc return RS_RET_OK; @@ -120,7 +130,6 @@ CODESTARTobjDestruct(prop) currRefCount = ATOMIC_DEC_AND_FETCH(pThis->iRefCount); if(currRefCount == 0) { /* (only) in this case we need to actually destruct the object */ -dbgprintf("XXXXX: propDestruct: ptr %p, pThis %p, len %d\n", pThis->szVal.psz, pThis, pThis->len); if(pThis->len >= CONF_PROP_BUFSIZE) free(pThis->szVal.psz); } else { @@ -156,6 +165,7 @@ CODESTARTobjQueryInterface(prop) pIf->DebugPrint = propDebugPrint; pIf->SetString = SetString; pIf->GetString = GetString; + pIf->GetStringLen = GetStringLen; pIf->AddRef = AddRef; finalize_it: diff --git a/runtime/prop.h b/runtime/prop.h index 1d18c650..62c0d799 100644 --- a/runtime/prop.h +++ b/runtime/prop.h @@ -44,6 +44,7 @@ BEGINinterface(prop) /* name must also be changed in ENDinterface macro! */ rsRetVal (*Destruct)(prop_t **ppThis); rsRetVal (*SetString)(prop_t *pThis, uchar* psz, int len); rsRetVal (*GetString)(prop_t *pThis, uchar** ppsz, int *plen); + int (*GetStringLen)(prop_t *pThis); rsRetVal (*AddRef)(prop_t *pThis); ENDinterface(prop) #define propCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index faa74427..443d0f41 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -147,12 +147,12 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) * class immediately after it is initialized. And, of course, we load those classes * first that we use ourselfs... -- rgerhards, 2008-03-07 */ + if(ppErrObj != NULL) *ppErrObj = "prop"; + CHKiRet(propClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "glbl"; CHKiRet(glblClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "datetime"; CHKiRet(datetimeClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "prop"; - CHKiRet(propClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "ctok_token"; diff --git a/tcps_sess.c b/tcps_sess.c index 23241f4f..c4dc4bd4 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -36,6 +36,7 @@ #include "rsyslog.h" #include "dirty.h" +#include "unicode-helper.h" #include "module-template.h" #include "net.h" #include "tcpsrv.h" @@ -102,7 +103,8 @@ CODESTARTobjDestruct(tcps_sess) pThis->pSrv->pOnSessDestruct(&pThis->pUsr); } /* now destruct our own properties */ - free(pThis->fromHost); + if(pThis->fromHost != NULL) + CHKiRet(prop.Destruct(&pThis->fromHost)); free(pThis->fromHostIP); free(pThis->pMsg); ENDobjDestruct(tcps_sess) @@ -126,9 +128,14 @@ SetHost(tcps_sess_t *pThis, uchar *pszHost) ISOBJ_TYPE_assert(pThis, tcps_sess); - free(pThis->fromHost); - pThis->fromHost = pszHost; + if(pThis->fromHost == NULL) { + CHKiRet(prop.Construct(&pThis->fromHost)); + } + + CHKiRet(prop.SetString(pThis->fromHost, pszHost, ustrlen(pszHost))); +finalize_it: + free(pszHost); /* we must free according to our (old) calling conventions */ RETiRet; } @@ -325,8 +332,9 @@ Close(tcps_sess_t *pThis) ISOBJ_TYPE_assert(pThis, tcps_sess); netstrm.Destruct(&pThis->pStrm); - free(pThis->fromHost); - pThis->fromHost = NULL; /* not really needed, but... */ + if(pThis->fromHost != NULL) { + prop.Destruct(&pThis->fromHost); + } free(pThis->fromHostIP); pThis->fromHostIP = NULL; /* not really needed, but... */ diff --git a/tcps_sess.h b/tcps_sess.h index 5e59aaab..2051bd1f 100644 --- a/tcps_sess.h +++ b/tcps_sess.h @@ -24,6 +24,7 @@ #define INCLUDED_TCPS_SESS_H #include "obj.h" +#include "prop.h" /* a forward-definition, we are somewhat cyclic */ struct tcpsrv_s; @@ -44,7 +45,7 @@ struct tcps_sess_s { int iOctetsRemain; /* Number of Octets remaining in message */ TCPFRAMINGMODE eFraming; uchar *pMsg; /* message (fragment) received */ - uchar *fromHost; + prop_t *fromHost; /* host name we received messages from */ uchar *fromHostIP; void *pUsr; /* a user-pointer */ rsRetVal (*DoSubmitMessage)(tcps_sess_t*, uchar*, int); /* submit message callback */ diff --git a/tools/syslogd.c b/tools/syslogd.c index f1c43602..3694c70a 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -596,7 +596,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f */ if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) MsgSetHOSTNAME(pMsg, hname, ustrlen(hname)); - MsgSetRcvFrom(pMsg, hname); + MsgSetRcvFromStr(pMsg, hname, ustrlen(hname)); MsgSetAfterPRIOffs(pMsg, p - msg); CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP)); @@ -870,7 +870,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) MsgSetInputName(pMsg, pInternalInputName); MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); - MsgSetRcvFrom(pMsg, glbl.GetLocalHostName()); + MsgSetRcvFrom(pMsg, glbl.GetLocalHostNameProp()); MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1")); /* check if we have an error code associated and, if so, * adjust the tag. -- rgerhards, 2008-06-27 @@ -3259,6 +3259,7 @@ int realMain(int argc, char **argv) */ glbl.SetLocalHostName(LocalHostName); glbl.SetLocalDomain(LocalDomain); + glbl.GenerateLocalHostNameProperty(); /* must be redone after conf processing, FQDN setting may have changed */ /* initialize the objects */ if((iRet = modInitIminternal()) != RS_RET_OK) { @@ -3455,6 +3456,9 @@ int realMain(int argc, char **argv) if(!iConfigVerify) CHKiRet(doGlblProcessInit()); + /* re-generate local host name property, as the config may have changed our FQDN settings */ + glbl.GenerateLocalHostNameProperty(); + CHKiRet(mainThread()); /* do any de-init's that need to be done AFTER this comment */ -- cgit v1.2.3 From a0496e7bd4cba22d2559adb50cd286f75cc6a40e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Tue, 30 Jun 2009 18:54:49 +0200 Subject: internal bugfix: object pointer was only reset to NULL when an object was actually destructed. This most likely had no effect to existing code, but it may also have caused trouble in remote cases. Similarly, the fix may also cause trouble... Due to this trouble scenario, the fix deserves its own commit. --- ChangeLog | 6 ++++++ runtime/obj-types.h | 11 ++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index b9352957..294616df 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,10 @@ --------------------------------------------------------------------------- +Version 4.3.3 [beta] (rgerhards), 2009-0?-?? +- internal bugfix: object pointer was only reset to NULL when an object + was actually destructed. This most likely had no effect to existing code, + but it may also have caused trouble in remote cases. Similarly, the fix + may also cause trouble... +--------------------------------------------------------------------------- Version 4.3.2 [beta] (rgerhards), 2009-06-24 - removed long-obsoleted property UxTradMsg - added a generic network stream server (in addition to rather specific diff --git a/runtime/obj-types.h b/runtime/obj-types.h index 78829f94..ff1819a9 100644 --- a/runtime/obj-types.h +++ b/runtime/obj-types.h @@ -292,6 +292,15 @@ rsRetVal objName##ClassExit(void) \ ISOBJ_TYPE_assert(pThis, OBJ); \ pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); +/* note: there was a long-time bug in the macro below that lead to *ppThis = NULL + * only when the object was actually destructed. I discovered this issue during + * introduction of the pRcvFrom property in msg_t, but it potentially had other + * effects, too. I am not sure if some experienced instability resulted from this + * bug OR if its fix will cause harm to so-far "correctly" running code. The later + * may very well be. Thus I will change it only for the current branch and also + * the beta, but not in all old builds. Let's see how things evolve. + * rgerhards, 2009-06-30 + */ #define ENDobjDestruct(OBJ) \ goto finalize_it; /* prevent compiler warning ;) */ \ /* no more code here! */ \ @@ -299,8 +308,8 @@ rsRetVal objName##ClassExit(void) \ if(pThis != NULL) { \ obj.DestructObjSelf((obj_t*) pThis); \ free(pThis); \ - *ppThis = NULL; \ } \ + *ppThis = NULL; \ pthread_setcancelstate(iCancelStateSave, NULL); \ RETiRet; \ } -- cgit v1.2.3 From f76881fff39b46630d95a8d8308f383bec1b8be8 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Jul 2009 10:34:17 +0200 Subject: removed a debugging-left-over, caused compilation failure on some platforms ... because LARGFILE macros were not defined consistenly --- runtime/stringbuf.c | 1 - 1 file changed, 1 deletion(-) diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 88f5a3a2..93995b38 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -1,4 +1,3 @@ -#include /* This is the byte-counted string class for rsyslog. It is a replacement * for classical \0 terminated string functions. We introduce it in * the hope it will make the program more secure, obtain some performance -- cgit v1.2.3 From d6faee67b413d1f257c96a14e46f15ec1868a365 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Jul 2009 13:28:17 +0200 Subject: RcvFromIP now also a property This sets stage to enable use of the property-interface to speed up things (mildly), the next step to be done. I have also fixed one regression of yesterday's changes. --- plugins/imdiag/imdiag.c | 9 ++++- plugins/imklog/imklog.c | 2 +- plugins/imudp/imudp.c | 2 +- runtime/msg.c | 97 +++++++++++++++++++++++++++++++++---------------- runtime/msg.h | 6 +-- tcps_sess.c | 19 ++++++---- tcps_sess.h | 2 +- tools/syslogd.c | 15 ++++++-- 8 files changed, 102 insertions(+), 50 deletions(-) diff --git a/plugins/imdiag/imdiag.c b/plugins/imdiag/imdiag.c index 140222e1..bf972191 100644 --- a/plugins/imdiag/imdiag.c +++ b/plugins/imdiag/imdiag.c @@ -73,6 +73,7 @@ static tcpsrv_t *pOurTcpsrv = NULL; /* our TCP server(listener) TODO: change fo static permittedPeers_t *pPermPeersRoot = NULL; static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this input */ static prop_t *pRcvDummy = NULL; +static prop_t *pRcvIPDummy = NULL; /* config settings */ @@ -214,7 +215,7 @@ doInjectMsg(int iNum) pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; MsgSetRcvFrom(pMsg, pRcvDummy); - CHKiRet(MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1"))); + CHKiRet(MsgSetRcvFromIP(pMsg, pRcvIPDummy)); CHKiRet(submitMsg(pMsg)); finalize_it: @@ -388,6 +389,10 @@ CODESTARTwillRun CHKiRet(prop.SetString(pRcvDummy, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); CHKiRet(prop.ConstructFinalize(pRcvDummy)); + CHKiRet(prop.Construct(&pRcvIPDummy)); + CHKiRet(prop.SetString(pRcvIPDummy, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + CHKiRet(prop.ConstructFinalize(pRcvIPDummy)); + finalize_it: ENDwillRun @@ -398,6 +403,8 @@ CODESTARTafterRun prop.Destruct(&pInputName); if(pRcvDummy != NULL) prop.Destruct(&pRcvDummy); + if(pRcvIPDummy != NULL) + prop.Destruct(&pRcvIPDummy); ENDafterRun diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index e4db03b3..269cfee0 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -105,7 +105,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ MsgSetRcvFrom(pMsg, glbl.GetLocalHostNameProp()); - MsgSetRcvFromIP(pMsg, (uchar*)"127.0.0.1"); + MsgSetRcvFromIPStr(pMsg, (uchar*)"127.0.0.1", sizeof("127.0.0.1") - 1); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); MsgSetTAG(pMsg, pszTag, ustrlen(pszTag)); pMsg->iFacility = LOG_FAC(iFacility); diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index e091c7d6..31a4869f 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -250,7 +250,7 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost)); - CHKiRet(MsgSetRcvFromIP(pMsg, fromHostIP)); + CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP))); CHKiRet(submitMsg(pMsg)); } } diff --git a/runtime/msg.c b/runtime/msg.c index 3eb41a87..d3645664 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -296,6 +296,26 @@ getInputName(msg_t *pM, uchar **ppsz, int *plen) } +static inline uchar* +getRcvFromIP(msg_t *pM) +{ + uchar *psz; + int len; + BEGINfunc + if(pM == NULL) { + psz = UCHAR_CONSTANT(""); + } else { + if(pM->pRcvFromIP == NULL) + psz = UCHAR_CONSTANT(""); + else + prop.GetString(pM->pRcvFromIP, &psz, &len); + } + ENDfunc + return psz; +} + + + /* map a property name (string) to a property ID */ rsRetVal propNameToID(cstr_t *pCSPropName, propid_t *pPropID) { @@ -622,10 +642,8 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->iLenMSG = 0; pM->iLenTAG = 0; pM->iLenHOSTNAME = 0; - pM->iLenRcvFromIP = 0; pM->pszRawMsg = NULL; pM->pszHOSTNAME = NULL; - pM->pszRcvFromIP = NULL; pM->pszRcvdAt3164 = NULL; pM->pszRcvdAt3339 = NULL; pM->pszRcvdAt_MySQL = NULL; @@ -640,6 +658,7 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) pM->pCSPROCID = NULL; pM->pCSMSGID = NULL; pM->pInputName = NULL; + pM->pRcvFromIP = NULL; pM->pRcvFrom = NULL; pM->pRuleset = NULL; memset(&pM->tRcvdAt, 0, sizeof(pM->tRcvdAt)); @@ -743,6 +762,8 @@ CODESTARTobjDestruct(msg) prop.Destruct(&pThis->pInputName); if(pThis->pRcvFrom != NULL) prop.Destruct(&pThis->pRcvFrom); + if(pThis->pRcvFromIP != NULL) + prop.Destruct(&pThis->pRcvFromIP); free(pThis->pszRcvdAt3164); free(pThis->pszRcvdAt3339); free(pThis->pszRcvdAt_MySQL); @@ -845,6 +866,8 @@ msg_t* MsgDup(msg_t* pOld) pNew->offMSG = pOld->offMSG; pNew->pRcvFrom = pOld->pRcvFrom; prop.AddRef(pNew->pRcvFrom); + pNew->pRcvFromIP = pOld->pRcvFromIP; + prop.AddRef(pNew->pRcvFromIP); pNew->pInputName = pOld->pInputName; prop.AddRef(pNew->pInputName); /* enable this, if someone actually uses UxTradMsg, delete after some time has @@ -939,7 +962,8 @@ static rsRetVal MsgSerialize(msg_t *pThis, strm_t *pStrm) objSerializeSCALAR_VAR(pStrm, "pszInputName", PSZ, psz); psz = getRcvFrom(pThis); objSerializeSCALAR_VAR(pStrm, "pszRcvFrom", PSZ, psz); - objSerializePTR(pStrm, pszRcvFromIP, PSZ); + psz = getRcvFromIP(pThis); + objSerializeSCALAR_VAR(pStrm, "pszRcvFromIP", PSZ, psz); objSerializePTR(pStrm, pCSStrucData, CSTR); objSerializePTR(pStrm, pCSAPPNAME, CSTR); @@ -1640,24 +1664,16 @@ uchar *getRcvFrom(msg_t *pM) if(pM == NULL) { psz = UCHAR_CONSTANT(""); } else { - prop.GetString(pM->pInputName, &psz, &len); + if(pM->pRcvFrom == NULL) + psz = UCHAR_CONSTANT(""); + else + prop.GetString(pM->pRcvFrom, &psz, &len); } ENDfunc return psz; } -uchar *getRcvFromIP(msg_t *pM) -{ - if(pM == NULL) - return (uchar*) ""; - else - if(pM->pszRcvFromIP == NULL) - return (uchar*) ""; - else - return pM->pszRcvFromIP; -} - /* rgerhards 2004-11-24: set STRUCTURED DATA in msg object */ rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData) @@ -1822,8 +1838,8 @@ void MsgSetRcvFrom(msg_t *pThis, prop_t *new) pThis->pRcvFrom = new; } -/* to be removed soon: work-around for those tht can not natively generate an - * input name. +/* to be removed soon: work-around for those tht can not natively generate a + * property. * rgerhards, 2009-06-29 */ void MsgSetRcvFromStr(msg_t *pThis, uchar *psz, int len) @@ -1840,24 +1856,43 @@ void MsgSetRcvFromStr(msg_t *pThis, uchar *psz, int len) } -/* rgerhards 2005-05-16: set pszRcvFromIP in msg object */ -rsRetVal -MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP) +/* set RcvFromIP name in msg object. This calls AddRef() + * on the property, because this must be done in all current cases and there + * is no case expected where this may not be necessary. + * rgerhards, 2009-06-30 + */ +rsRetVal MsgSetRcvFromIP(msg_t *pThis, prop_t *new) { - DEFiRet; - assert(pMsg != NULL); - if(pMsg->pszRcvFromIP != NULL) { - free(pMsg->pszRcvFromIP); - pMsg->iLenRcvFromIP = 0; - } + assert(pThis != NULL); - CHKmalloc(pMsg->pszRcvFromIP = (uchar*)strdup((char*)pszRcvFromIP)); - pMsg->iLenRcvFromIP = strlen((char*)pszRcvFromIP); -finalize_it: - RETiRet; + BEGINfunc + prop.AddRef(new); + if(pThis->pRcvFromIP != NULL) + prop.Destruct(&pThis->pRcvFromIP); + pThis->pRcvFromIP = new; + ENDfunc + return RS_RET_OK; } +/* to be removed soon: work-around for those tht can not natively generate a + * property. + * rgerhards, 2009-06-29 + */ +rsRetVal MsgSetRcvFromIPStr(msg_t *pThis, uchar *psz, int len) +{ + prop_t *pProp; + assert(pThis != NULL); + + /* we need to create a property */ + prop.Construct(&pProp); + prop.SetString(pProp, psz, len); + prop.ConstructFinalize(pProp); + MsgSetRcvFromIP(pThis, pProp); + prop.Destruct(&pProp); + return RS_RET_OK; +} + /* rgerhards 2004-11-09: set HOSTNAME in msg object * rgerhards, 2007-06-21: * Does not return anything. If an error occurs, the hostname is @@ -2900,7 +2935,7 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) MsgSetInputName(pThis, myProp); prop.Destruct(&myProp); } else if(isProp("pszRcvFromIP")) { - MsgSetRcvFromIP(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr)); + MsgSetRcvFromIPStr(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pszRcvFrom")) { MsgSetRcvFromStr(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pszHOSTNAME")) { diff --git a/runtime/msg.h b/runtime/msg.h index 59046fc1..43f24435 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -78,11 +78,9 @@ struct msg { int iLenMSG; /* Length of the MSG part */ int iLenTAG; /* Length of the TAG part */ int iLenHOSTNAME; /* Length of HOSTNAME */ - int iLenRcvFromIP; /* Length of pszRcvFromIP */ uchar *pszRawMsg; /* message as it was received on the wire. This is important in case we * need to preserve cryptographic verifiers. */ uchar *pszHOSTNAME; /* HOSTNAME from syslog message */ - uchar *pszRcvFromIP; /* IP of system message was received from */ char *pszRcvdAt3164; /* time as RFC3164 formatted string (always 15 charcters) */ char *pszRcvdAt3339; /* time as RFC3164 formatted string (32 charcters at most) */ char *pszRcvdAt_MySQL; /* rcvdAt as MySQL formatted string (always 14 charcters) */ @@ -98,6 +96,7 @@ struct msg { cstr_t *pCSMSGID; /* MSGID */ prop_t *pInputName; /* input name property */ prop_t *pRcvFrom; /* name of system message was received from */ + prop_t *pRcvFromIP; /* IP of system message was received from */ ruleset_t *pRuleset; /* ruleset to be used for processing this message */ time_t ttGenTime; /* time msg object was generated, same as tRcvdAt, but a Unix timestamp. While this field looks redundant, it is required because a Unix timestamp @@ -152,7 +151,8 @@ rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); void MsgSetRcvFrom(msg_t *pMsg, prop_t*); void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int); -rsRetVal MsgSetRcvFromIP(msg_t *pMsg, uchar* pszRcvFromIP); +rsRetVal MsgSetRcvFromIP(msg_t *pMsg, prop_t*); +rsRetVal MsgSetRcvFromIPStr(msg_t *pMsg, uchar*, int); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME, int lenHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSGoffs(msg_t *pMsg, short offs); diff --git a/tcps_sess.c b/tcps_sess.c index c4dc4bd4..f701fe57 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -105,7 +105,8 @@ CODESTARTobjDestruct(tcps_sess) /* now destruct our own properties */ if(pThis->fromHost != NULL) CHKiRet(prop.Destruct(&pThis->fromHost)); - free(pThis->fromHostIP); + if(pThis->fromHostIP != NULL) + CHKiRet(prop.Destruct(&pThis->fromHostIP)); free(pThis->pMsg); ENDobjDestruct(tcps_sess) @@ -128,9 +129,8 @@ SetHost(tcps_sess_t *pThis, uchar *pszHost) ISOBJ_TYPE_assert(pThis, tcps_sess); - if(pThis->fromHost == NULL) { + if(pThis->fromHost == NULL) CHKiRet(prop.Construct(&pThis->fromHost)); - } CHKiRet(prop.SetString(pThis->fromHost, pszHost, ustrlen(pszHost))); @@ -150,9 +150,12 @@ SetHostIP(tcps_sess_t *pThis, uchar *pszHostIP) ISOBJ_TYPE_assert(pThis, tcps_sess); - free(pThis->fromHostIP); - pThis->fromHostIP = pszHostIP; + if(pThis->fromHostIP == NULL) + CHKiRet(prop.Construct(&pThis->fromHostIP)); + + CHKiRet(prop.SetString(pThis->fromHostIP, pszHostIP, ustrlen(pszHostIP))); +finalize_it: RETiRet; } @@ -249,8 +252,8 @@ defaultDoSubmitMessage(tcps_sess_t *pThis, struct syslogTime *stTime, time_t ttG pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; MsgSetRcvFrom(pMsg, pThis->fromHost); - MsgSetRuleset(pMsg, pThis->pLstnInfo->pRuleset); CHKiRet(MsgSetRcvFromIP(pMsg, pThis->fromHostIP)); + MsgSetRuleset(pMsg, pThis->pLstnInfo->pRuleset); if(pMultiSub == NULL) { CHKiRet(submitMsg(pMsg)); @@ -335,8 +338,8 @@ Close(tcps_sess_t *pThis) if(pThis->fromHost != NULL) { prop.Destruct(&pThis->fromHost); } - free(pThis->fromHostIP); - pThis->fromHostIP = NULL; /* not really needed, but... */ + if(pThis->fromHostIP != NULL) + prop.Destruct(&pThis->fromHostIP); RETiRet; } diff --git a/tcps_sess.h b/tcps_sess.h index 2051bd1f..ec3a6af4 100644 --- a/tcps_sess.h +++ b/tcps_sess.h @@ -46,7 +46,7 @@ struct tcps_sess_s { TCPFRAMINGMODE eFraming; uchar *pMsg; /* message (fragment) received */ prop_t *fromHost; /* host name we received messages from */ - uchar *fromHostIP; + prop_t *fromHostIP; void *pUsr; /* a user-pointer */ rsRetVal (*DoSubmitMessage)(tcps_sess_t*, uchar*, int); /* submit message callback */ }; diff --git a/tools/syslogd.c b/tools/syslogd.c index 3694c70a..be5b5aad 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -211,6 +211,7 @@ static rsRetVal GlobalClassExit(void); #endif static prop_t *pInternalInputName = NULL; /* there is only one global inputName for all internally-generated messages */ +static prop_t *pLocalHostIP = NULL; /* there is only one global IP for all internally-generated messages */ static uchar *ConfFile = (uchar*) _PATH_LOGCONF; /* read-only after startup */ static char *PidFile = _PATH_LOGPID; /* read-only after startup */ @@ -598,7 +599,7 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f MsgSetHOSTNAME(pMsg, hname, ustrlen(hname)); MsgSetRcvFromStr(pMsg, hname, ustrlen(hname)); MsgSetAfterPRIOffs(pMsg, p - msg); - CHKiRet(MsgSetRcvFromIP(pMsg, hnameIP)); + CHKiRet(MsgSetRcvFromIPStr(pMsg, hnameIP, ustrlen(hname))); logmsg(pMsg, flags); @@ -871,7 +872,7 @@ logmsgInternal(int iErr, int pri, uchar *msg, int flags) MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); MsgSetRcvFrom(pMsg, glbl.GetLocalHostNameProp()); - MsgSetRcvFromIP(pMsg, UCHAR_CONSTANT("127.0.0.1")); + MsgSetRcvFromIP(pMsg, pLocalHostIP); /* check if we have an error code associated and, if so, * adjust the tag. -- rgerhards, 2008-06-27 */ @@ -1697,9 +1698,11 @@ die(int sig) legacyOptsFree(); - /* destruct our input name */ + /* destruct our global properties */ if(pInternalInputName != NULL) prop.Destruct(&pInternalInputName); + if(pLocalHostIP != NULL) + prop.Destruct(&pLocalHostIP); /* terminate the remaining classes */ GlobalClassExit(); @@ -3208,9 +3211,13 @@ int realMain(int argc, char **argv) /* we need to create the inputName property (only once during our lifetime) */ CHKiRet(prop.Construct(&pInternalInputName)); - CHKiRet(prop.SetString(pInternalInputName, UCHAR_CONSTANT("rsyslgod"), sizeof("rsyslgod") - 1)); + CHKiRet(prop.SetString(pInternalInputName, UCHAR_CONSTANT("rsyslogd"), sizeof("rsyslgod") - 1)); CHKiRet(prop.ConstructFinalize(pInternalInputName)); + CHKiRet(prop.Construct(&pLocalHostIP)); + CHKiRet(prop.SetString(pLocalHostIP, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); + CHKiRet(prop.ConstructFinalize(pLocalHostIP)); + /* get our host and domain names - we need to do this early as we may emit * error log messages, which need the correct hostname. -- rgerhards, 2008-04-04 */ -- cgit v1.2.3 From 7bfa03bdc0b73647ffdbe4b73e5c1649af665fbf Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Jul 2009 14:33:19 +0200 Subject: now put the new property-based methods to good use ... hopefully reducing the number of allocs/frees as well as overall memory usage in a busy system (plus that these shared properties hopefully remain in cache longer than its single-instance counterparts...) --- plugins/imklog/imklog.c | 10 ++++--- plugins/imudp/imudp.c | 12 ++++++-- runtime/msg.c | 58 +++++++++++++++++++++---------------- runtime/msg.h | 4 +-- runtime/prop.c | 76 ++++++++++++++++++++++++++++++++++++++++--------- runtime/prop.h | 2 ++ tcps_sess.c | 1 + tools/syslogd.c | 6 ++-- 8 files changed, 120 insertions(+), 49 deletions(-) diff --git a/plugins/imklog/imklog.c b/plugins/imklog/imklog.c index 269cfee0..2a832bee 100644 --- a/plugins/imklog/imklog.c +++ b/plugins/imklog/imklog.c @@ -84,6 +84,7 @@ int console_log_level = -1; static prop_t *pInputName = NULL; /* there is only one global inputName for all messages generated by this module */ +static prop_t *pLocalHostIP = NULL; /* a pseudo-constant propterty for 127.0.0.1 */ /* enqueue the the kernel message into the message queue. * The provided msg string is not freed - thus must be done @@ -105,7 +106,7 @@ enqMsg(uchar *msg, uchar* pszTag, int iFacility, int iSeverity) MsgSetRawMsgWOSize(pMsg, (char*)msg); MsgSetMSGoffs(pMsg, 0); /* we do not have a header... */ MsgSetRcvFrom(pMsg, glbl.GetLocalHostNameProp()); - MsgSetRcvFromIPStr(pMsg, (uchar*)"127.0.0.1", sizeof("127.0.0.1") - 1); + MsgSetRcvFromIP(pMsg, pLocalHostIP); MsgSetHOSTNAME(pMsg, glbl.GetLocalHostName(), ustrlen(glbl.GetLocalHostName())); MsgSetTAG(pMsg, pszTag, ustrlen(pszTag)); pMsg->iFacility = LOG_FAC(iFacility); @@ -235,9 +236,8 @@ ENDrunInput BEGINwillRun CODESTARTwillRun /* we need to create the inputName property (only once during our lifetime) */ - CHKiRet(prop.Construct(&pInputName)); - CHKiRet(prop.SetString(pInputName, UCHAR_CONSTANT("imklog"), sizeof("imklog") - 1)); - CHKiRet(prop.ConstructFinalize(pInputName)); + CHKiRet(prop.CreateStringProp(&pInputName, UCHAR_CONSTANT("imklog"), sizeof("imklog") - 1)); + CHKiRet(prop.CreateStringProp(&pLocalHostIP, UCHAR_CONSTANT("127.0.0.1"), sizeof("127.0.0.1") - 1)); iRet = klogWillRun(); finalize_it: @@ -250,6 +250,8 @@ CODESTARTafterRun if(pInputName != NULL) prop.Destruct(&pInputName); + if(pLocalHostIP != NULL) + prop.Destruct(&pLocalHostIP); ENDafterRun diff --git a/plugins/imudp/imudp.c b/plugins/imudp/imudp.c index 31a4869f..718c3090 100644 --- a/plugins/imudp/imudp.c +++ b/plugins/imudp/imudp.c @@ -191,6 +191,8 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, ssize_t lenRcvBuf; struct sockaddr_storage frominet; msg_t *pMsg; + prop_t *propFromHost = NULL; + prop_t *propFromHostIP = NULL; char errStr[1024]; iNbrTimeUsed = 0; @@ -249,14 +251,18 @@ processSocket(int fd, struct sockaddr_storage *frominetPrev, int *pbIsPermitted, MsgSetFlowControlType(pMsg, eFLOWCTL_NO_DELAY); pMsg->msgFlags = NEEDS_PARSING | PARSE_HOSTNAME; pMsg->bParseHOSTNAME = 1; - MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost)); - CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP))); + MsgSetRcvFromStr(pMsg, fromHost, ustrlen(fromHost), &propFromHost); + CHKiRet(MsgSetRcvFromIPStr(pMsg, fromHostIP, ustrlen(fromHostIP), &propFromHostIP)); CHKiRet(submitMsg(pMsg)); } } - finalize_it: + if(propFromHost != NULL) + prop.Destruct(&propFromHost); + if(propFromHostIP != NULL) + prop.Destruct(&propFromHostIP); + RETiRet; } diff --git a/runtime/msg.c b/runtime/msg.c index d3645664..6c272d1f 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -1838,21 +1838,22 @@ void MsgSetRcvFrom(msg_t *pThis, prop_t *new) pThis->pRcvFrom = new; } -/* to be removed soon: work-around for those tht can not natively generate a - * property. - * rgerhards, 2009-06-29 + +/* This is used to set the property via a string. This function should not be + * called if there is a reliable way for a caller to make sure that the + * same name can be used across multiple messages. However, if it can not + * ensure that, calling this function is the second best thing, because it + * will re-use the previously created property if it contained the same + * name (but it works only for the immediate previous). + * rgerhards, 2009-06-31 */ -void MsgSetRcvFromStr(msg_t *pThis, uchar *psz, int len) +void MsgSetRcvFromStr(msg_t *pThis, uchar *psz, int len, prop_t **ppProp) { - prop_t *pProp; assert(pThis != NULL); + assert(ppProp != NULL); - /* we need to create a property */ - prop.Construct(&pProp); - prop.SetString(pProp, psz, len); - prop.ConstructFinalize(pProp); - MsgSetRcvFrom(pThis, pProp); - prop.Destruct(&pProp); + prop.CreateOrReuseStringProp(ppProp, psz, len); + MsgSetRcvFrom(pThis, *ppProp); } @@ -1875,24 +1876,27 @@ rsRetVal MsgSetRcvFromIP(msg_t *pThis, prop_t *new) } -/* to be removed soon: work-around for those tht can not natively generate a - * property. - * rgerhards, 2009-06-29 +/* This is used to set the property via a string. This function should not be + * called if there is a reliable way for a caller to make sure that the + * same name can be used across multiple messages. However, if it can not + * ensure that, calling this function is the second best thing, because it + * will re-use the previously created property if it contained the same + * name (but it works only for the immediate previous). + * rgerhards, 2009-06-31 */ -rsRetVal MsgSetRcvFromIPStr(msg_t *pThis, uchar *psz, int len) +rsRetVal MsgSetRcvFromIPStr(msg_t *pThis, uchar *psz, int len, prop_t **ppProp) { - prop_t *pProp; + DEFiRet; assert(pThis != NULL); - /* we need to create a property */ - prop.Construct(&pProp); - prop.SetString(pProp, psz, len); - prop.ConstructFinalize(pProp); - MsgSetRcvFromIP(pThis, pProp); - prop.Destruct(&pProp); - return RS_RET_OK; + CHKiRet(prop.CreateOrReuseStringProp(ppProp, psz, len)); + MsgSetRcvFrom(pThis, *ppProp); + +finalize_it: + RETiRet; } + /* rgerhards 2004-11-09: set HOSTNAME in msg object * rgerhards, 2007-06-21: * Does not return anything. If an error occurs, the hostname is @@ -2901,6 +2905,8 @@ finalize_it: rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) { prop_t *myProp; + prop_t *propRcvFrom = NULL; + prop_t *propRcvFromIP = NULL; DEFiRet; ISOBJ_TYPE_assert(pThis, msg); @@ -2935,9 +2941,11 @@ rsRetVal MsgSetProperty(msg_t *pThis, var_t *pProp) MsgSetInputName(pThis, myProp); prop.Destruct(&myProp); } else if(isProp("pszRcvFromIP")) { - MsgSetRcvFromIPStr(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); + MsgSetRcvFromIPStr(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr), &propRcvFromIP); + prop.Destruct(&propRcvFromIP); } else if(isProp("pszRcvFrom")) { - MsgSetRcvFromStr(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); + MsgSetRcvFromStr(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr), &propRcvFrom); + prop.Destruct(&propRcvFrom); } else if(isProp("pszHOSTNAME")) { MsgSetHOSTNAME(pThis, rsCStrGetSzStrNoNULL(pProp->val.pStr), rsCStrLen(pProp->val.pStr)); } else if(isProp("pCSStrucData")) { diff --git a/runtime/msg.h b/runtime/msg.h index 43f24435..c20fb005 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -150,9 +150,9 @@ void MsgSetRuleset(msg_t *pMsg, ruleset_t*); rsRetVal MsgSetFlowControlType(msg_t *pMsg, flowControl_t eFlowCtl); rsRetVal MsgSetStructuredData(msg_t *pMsg, char* pszStrucData); void MsgSetRcvFrom(msg_t *pMsg, prop_t*); -void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int); +void MsgSetRcvFromStr(msg_t *pMsg, uchar* pszRcvFrom, int, prop_t **); rsRetVal MsgSetRcvFromIP(msg_t *pMsg, prop_t*); -rsRetVal MsgSetRcvFromIPStr(msg_t *pMsg, uchar*, int); +rsRetVal MsgSetRcvFromIPStr(msg_t *pThis, uchar *psz, int len, prop_t **ppProp); void MsgSetHOSTNAME(msg_t *pMsg, uchar* pszHOSTNAME, int lenHOSTNAME); rsRetVal MsgSetAfterPRIOffs(msg_t *pMsg, short offs); void MsgSetMSGoffs(msg_t *pMsg, short offs); diff --git a/runtime/prop.c b/runtime/prop.c index 96ebe212..804f3491 100644 --- a/runtime/prop.c +++ b/runtime/prop.c @@ -41,6 +41,7 @@ #include "rsyslog.h" #include "obj.h" #include "obj-types.h" +#include "unicode-helper.h" #include "atomic.h" #include "prop.h" @@ -54,6 +55,21 @@ BEGINobjConstruct(prop) /* be sure to specify the object type also in END macro! pThis->iRefCount = 1; ENDobjConstruct(prop) + +/* destructor for the prop object */ +BEGINobjDestruct(prop) /* be sure to specify the object type also in END and CODESTART macros! */ + int currRefCount; +CODESTARTobjDestruct(prop) + currRefCount = ATOMIC_DEC_AND_FETCH(pThis->iRefCount); + if(currRefCount == 0) { + /* (only) in this case we need to actually destruct the object */ + if(pThis->len >= CONF_PROP_BUFSIZE) + free(pThis->szVal.psz); + } else { + pThis = NULL; /* tell framework NOT to destructing the object! */ + } +ENDobjDestruct(prop) + /* set string, we make our own private copy! This MUST only be called BEFORE * ConstructFinalize()! */ @@ -90,10 +106,8 @@ static rsRetVal GetString(prop_t *pThis, uchar **ppsz, int *plen) ISOBJ_TYPE_assert(pThis, prop); if(pThis->len < CONF_PROP_BUFSIZE) { *ppsz = pThis->szVal.sz; -RUNLOG; } else { *ppsz = pThis->szVal.psz; -RUNLOG; } *plen = pThis->len; ENDfunc @@ -123,19 +137,53 @@ static rsRetVal AddRef(prop_t *pThis) } -/* destructor for the prop object */ -BEGINobjDestruct(prop) /* be sure to specify the object type also in END and CODESTART macros! */ - int currRefCount; -CODESTARTobjDestruct(prop) - currRefCount = ATOMIC_DEC_AND_FETCH(pThis->iRefCount); - if(currRefCount == 0) { - /* (only) in this case we need to actually destruct the object */ - if(pThis->len >= CONF_PROP_BUFSIZE) - free(pThis->szVal.psz); +/* this is a "do it all in one shot" function that creates a new property, + * assigns the provided string to it and finalizes the property. Among the + * convenience, it is alos (very, very) slightly faster. + * rgerhards, 2009-07-01 + */ +static rsRetVal CreateStringProp(prop_t **ppThis, uchar* psz, int len) +{ + DEFiRet; + propConstruct(ppThis); + SetString(*ppThis, psz, len); + propConstructFinalize(*ppThis); + RETiRet; +} + +/* another one-stop function, quite useful: it takes a property pointer and + * a string. If the string is already contained in the property, nothing happens. + * If the string is different (or the pointer NULL), the current property + * is destructed and a new one created. This can be used to get a specific + * name in those cases where there is a good chance that the property + * immediatly previously processed already contained the value we need - in + * which case we save us all the creation overhead by just reusing the already + * existing property). + * rgerhards, 2009-07-01 + */ +rsRetVal CreateOrReuseStringProp(prop_t **ppThis, uchar *psz, int len) +{ + uchar *pszPrev; + int lenPrev; + DEFiRet; + assert(ppThis != NULL); + + if(*ppThis == NULL) { + /* we need to create a property */ + CHKiRet(CreateStringProp(ppThis, psz, len)); } else { - pThis = NULL; /* tell framework NOT to destructing the object! */ + /* already exists, check if we can re-use it */ + GetString(*ppThis, &pszPrev, &lenPrev); + if(len != lenPrev && ustrcmp(psz, pszPrev)) { + /* different, need to discard old & create new one */ + propDestruct(ppThis); + CHKiRet(CreateStringProp(ppThis, psz, len)); + } /* else we can re-use the existing one! */ } -ENDobjDestruct(prop) + +finalize_it: + RETiRet; +} /* debugprint for the prop object */ @@ -167,6 +215,8 @@ CODESTARTobjQueryInterface(prop) pIf->GetString = GetString; pIf->GetStringLen = GetStringLen; pIf->AddRef = AddRef; + pIf->CreateStringProp = CreateStringProp; + pIf->CreateOrReuseStringProp = CreateOrReuseStringProp; finalize_it: ENDobjQueryInterface(prop) diff --git a/runtime/prop.h b/runtime/prop.h index 62c0d799..e3519664 100644 --- a/runtime/prop.h +++ b/runtime/prop.h @@ -46,6 +46,8 @@ BEGINinterface(prop) /* name must also be changed in ENDinterface macro! */ rsRetVal (*GetString)(prop_t *pThis, uchar** ppsz, int *plen); int (*GetStringLen)(prop_t *pThis); rsRetVal (*AddRef)(prop_t *pThis); + rsRetVal (*CreateStringProp)(prop_t **ppThis, uchar* psz, int len); + rsRetVal (*CreateOrReuseStringProp)(prop_t **ppThis, uchar *psz, int len); ENDinterface(prop) #define propCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ diff --git a/tcps_sess.c b/tcps_sess.c index f701fe57..8d307380 100644 --- a/tcps_sess.c +++ b/tcps_sess.c @@ -156,6 +156,7 @@ SetHostIP(tcps_sess_t *pThis, uchar *pszHostIP) CHKiRet(prop.SetString(pThis->fromHostIP, pszHostIP, ustrlen(pszHostIP))); finalize_it: + free(pszHostIP); RETiRet; } diff --git a/tools/syslogd.c b/tools/syslogd.c index be5b5aad..f7253a8e 100644 --- a/tools/syslogd.c +++ b/tools/syslogd.c @@ -559,6 +559,8 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f register uchar *p; int pri; msg_t *pMsg; + prop_t *propFromHost = NULL; + prop_t *propFromHostIP = NULL; /* Now it is time to create the message object (rgerhards) */ if(stTime == NULL) { @@ -597,9 +599,9 @@ static inline rsRetVal printline(uchar *hname, uchar *hnameIP, uchar *msg, int f */ if((pMsg->msgFlags & PARSE_HOSTNAME) == 0) MsgSetHOSTNAME(pMsg, hname, ustrlen(hname)); - MsgSetRcvFromStr(pMsg, hname, ustrlen(hname)); + MsgSetRcvFromStr(pMsg, hname, ustrlen(hname), &propFromHost); + CHKiRet(MsgSetRcvFromIPStr(pMsg, hnameIP, ustrlen(hname), &propFromHostIP)); MsgSetAfterPRIOffs(pMsg, p - msg); - CHKiRet(MsgSetRcvFromIPStr(pMsg, hnameIP, ustrlen(hname))); logmsg(pMsg, flags); -- cgit v1.2.3 From 51882ce4dece319008118f2b7d2fc7d4de4ce244 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 1 Jul 2009 18:57:58 +0200 Subject: first shot at multi-ruleset doc --- doc/Makefile.am | 1 + doc/multi_ruleset.html | 198 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 doc/multi_ruleset.html diff --git a/doc/Makefile.am b/doc/Makefile.am index 0703b8fc..62ec7500 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -111,6 +111,7 @@ html_files = \ rsyslog_conf_templates.html \ rsyslog_conf_nomatch.html \ queues_analogy.html \ + multi_ruleset.html \ src/classes.dia grfx_files = \ diff --git a/doc/multi_ruleset.html b/doc/multi_ruleset.html new file mode 100644 index 00000000..532edbcf --- /dev/null +++ b/doc/multi_ruleset.html @@ -0,0 +1,198 @@ + + +Multiple Rulesets in rsyslog + +

      Multiple Rulesets in rsyslog

      +

      Starting with version 4.5.0 and 5.1.1, rsyslog supports +multiple rulesets within a single configuration. +This is especially useful for routing the recpetion of remote messages to a set of specific rules. +Note that the input module must support binding to non-standard rulesets, so the functionality +may not be available with all inputs. +

      In this document, I am using the imtcp in this text, an input module +that supports binding to non-standard rulesets as long as rsyslog supports multiple rulesets. +

      What is a Ruleset?

      +If you have worked with (r)syslog.conf, you know that it is made up of what I call rules (others +tend to call them selectors, an sysklogd term). Each rule consist of a filter and one or more +actions to be carried out when the filter evaluates to true. A filter may be a simple traditional +syslog priority based filter (like "*.*" or "mail.info" or a complex +script-like expression. Details on that are covered in the config file documentation. After the +filter come action specifiers, and an action is something that does something to a message, e.g. +write it to a file or forward it to a remote logging server. + +

      A traditional configuration file is made up of one or more of these rules. When a new +message arrives, its processing starts with the first rule (in order of appearance in +rsyslog.conf) and continues for each rule until either all rules have been processed or +a so-called "e;discard" action happens, in which case processing stops and the +message is thrown away (what also happens after the last rule has been processed). + +

      The multi-ruleset support now permits to specify more than one such rule sequence. +You can think of a traditional config file just as a single default rule set, which is +automatically bound to each of the inputs. This is even what actually happens. When +rsyslog.conf is processed, the config file parser looks for the directive + +

      $RuleSet <name>
+
      + +

      Where name is any name the user likes. If it finds this directive, it begins a new +rule set (if the name was not yet know) or switches to an already-existing one (if the name +was known). All rules defined between this $RuleSet directive and the next one are appended +to the named ruleset. Note that the reserved name "RSYSLOG_DefaultRuleset" is used to +specify rsyslogd's default ruleset. You can use that name whereever you can use a ruleset name, +including when binding an input to it. + +

      Inside a ruleset, messages are processed as described above: they start with the first rule +and rules are processed in the order of appearance of the configuration file until either +there are no more rules or the discard action is executed. Note that with multiple rulesets +no longer all rsyslog.conf rules are executed but only those that are +contained within the specific ruleset. + +

      Inputs must explicitely bind to rulesets. If they don't do, the default ruleset is used. + +This brings up the next question: + +

      What does "To bind to a Ruleset" mean?

      +

      This term is used in the same sense as "to bind an IP address to an interface": +it means that a specific input, or part of an input (like a tcp listener) will use a specific +ruleset to "pass its messages to". So when a new message arrives, it will be processed +via the bound ruleset. Rule from all other rulesets are irrelevant and will never be processed. +

      This makes multiple rulesets very handy to process local and remote message via +seperate means: bind the respective receivers to different rule sets, and you do not need +to seperate the messages by any other method. + +

      Binding to rulesets is input-specifc. For imtcp, this is done via the + +

      $InputTCPServerBindRuleset <name>
+
      + +directive. Note that "name"e; must be the name of a ruleset that is already defined +at the time the bind directive is given. There are many ways to make sure this happens, but +I personally think that it is best to define all rule sets at the top of rsyslog.conf and +define the input at the bottom. This kind of reverses its traditional recommended ordering, but +seems to be a really useful and straightforward ways of doing things. +

      Examples

      +

      Split local and remote logging

      +

      Let's say you have a pretty standard system that logs its local messages to the usual +bunch of files that are specified in the default rsyslog.conf. As an example, your rsyslog.conf +might look like this: + +

      +# ... module loading ...
+# The authpriv file has restricted access.
+authpriv.*                                              /var/log/secure
+# Log all the mail messages in one place.
+mail.*                                                  /var/log/maillog
+# Log cron stuff
+cron.*                                                  /var/log/cron
+# Everybody gets emergency messages
+*.emerg                                                 *
+... more ...
+
      + +

      Now, you want to add receive messages from a remote system and log these to +a special file, but you do not want to have these messages written to the files +specified above. The traditional approach is to add a rule in front of all others that +filters on the message, processes it and then discards it: + +

      +# ... module loading ...
+# process remote messages
+:fromhost-ip, isequal, "192.0.2.1"                      /var/log/remotefile
+& ~
+# only messages not from 192.0.21 make it past this point
+
+# The authpriv file has restricted access.
+authpriv.*                                              /var/log/secure
+# Log all the mail messages in one place.
+mail.*                                                  /var/log/maillog
+# Log cron stuff
+cron.*                                                  /var/log/cron
+# Everybody gets emergency messages
+*.emerg                                                 *
+... more ...
+
      + +

      Note the tilde character, which is the discard action!. Also note that we assume that +192.0.2.1 is the sole remote sender (to keep it simple). + +

      With multiple rulesets, we can simply define a dedicated ruleset for the remote reception +case and bind it to the receiver. This may be written as follows: + +

      +# ... module loading ...
+# process remote messages
+# define new ruleset and add rules to it:
+$RuleSet remote
+*.*                                                     /var/log/remotefile
+# only messages not from 192.0.21 make it past this point
+
+# bind ruleset to tcp listener
+$InputTCPServerBindRuleset remote
+# and activate it:
+$InputTCPServerRun 10514
+
+# switch back to the default ruleset:
+$RuleSet RSYSLOG_DefaultRuleset
+# The authpriv file has restricted access.
+authpriv.*                                              /var/log/secure
+# Log all the mail messages in one place.
+mail.*                                                  /var/log/maillog
+# Log cron stuff
+cron.*                                                  /var/log/cron
+# Everybody gets emergency messages
+*.emerg                                                 *
+... more ...
+
      + +

      Here, we need to switch back to the default ruleset after we have defined our custom +one. This is why I recommend a different ordering, which I find more intuitive. The sample +below has it, and it leads to the same results: + +

      +# ... module loading ...
+# at first, this is a copy of the unmodified rsyslog.conf
+# The authpriv file has restricted access.
+authpriv.*                                              /var/log/secure
+# Log all the mail messages in one place.
+mail.*                                                  /var/log/maillog
+# Log cron stuff
+cron.*                                                  /var/log/cron
+# Everybody gets emergency messages
+*.emerg                                                 *
+... more ...
+# end of the "regular" rsyslog.conf. Now come the new definitions:
+# process remote messages
+# define new ruleset and add rules to it:
+$RuleSet remote
+*.*                                                     /var/log/remotefile
+
+# bind ruleset to tcp listener
+$InputTCPServerBindRuleset remote
+# and activate it:
+$InputTCPServerRun 10514
+
      + +

      Here, we do not switch back to the default ruleset, because this is not needed as it is +completely defined. + +

      Now look at the examples and compare them to the single-ruleset solution. You will notice +that we do not need a real filter in the multi-ruleset case: we can simply use +"*.*" as all messages now means all messages that are being processed by this +rule set and all of them come in via the TCP receiver! + +

      Performance

      +

      No rule processing can be faster than not processing a rule at all. As such, it is useful +for a high performance system to identify disjunct actions and try to split these off to +different rule sets. In the example section, we had a case where three different tcp listeners +need to write to three different files. This is a perfect example of where multiple rule sets +are easier to use and offer more performance. The performance is better simply because there +is no need to check the reception service - instead messages are automatically pushed to the +right rule set and can be processed by very simple rules (maybe even with +"*.*"-filters, the fastest ones available). + +

      [manual index] [rsyslog site]

      +

      This documentation is part of the rsyslog +project.
      +Copyright © 2009 by Rainer Gerhards and +Adiscon. +Released under the GNU GPL version 3 or higher.

      + -- cgit v1.2.3 From 1dee20014346a2f20b0db190cfdd8d9c7f57232e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Thu, 2 Jul 2009 15:29:37 +0200 Subject: completed ruleset documentation --- doc/manual.html | 1 + doc/multi_ruleset.html | 139 +++++++++++++++++++++++++++++++++---------- doc/rsyslog_conf_global.html | 6 +- 3 files changed, 113 insertions(+), 33 deletions(-) diff --git a/doc/manual.html b/doc/manual.html index 307f9a82..0d371146 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -52,6 +52,7 @@ generic syslog application design
    diff --git a/runtime/queue.c b/runtime/queue.c index a4d16132..e2641c5b 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -2460,7 +2460,7 @@ finalize_it: } -/* enqueue a single data object. This currently is a helper to qqueueMultiEnqObj. +/* enqueue a single data object. * Note that the queue mutex MUST already be locked when this function is called. * rgerhards, 2009-06-16 */ -- cgit v1.2.3 From e13e417f4c47b8fc8d3163f38ede254a42e00a6e Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Jul 2009 16:28:17 +0200 Subject: updating project status --- doc/status.html | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/doc/status.html b/doc/status.html index 272b49bc..56ad78d9 100644 --- a/doc/status.html +++ b/doc/status.html @@ -2,14 +2,16 @@ rsyslog status page

    rsyslog status page

    -

    This page reflects the status as of 2009-06-24.

    +

    This page reflects the status as of 2009-07-08.

    Current Releases

    -

    v5 development: 5.1.0 [2009-05-29] - -change log - -download +

    v5 development: 5.1.2 [2009-07-08] - +change log - +download -
    v4 development: currently no version other than beta +
    v4 development: 4.5.0 [2009-07-03] - +change log - +download


    beta: 4.3.2 [2009-06-24] - change log - @@ -19,9 +21,9 @@ change log - download -
    v3 stable: 3.22.0 [2009-04-21] - -change log - -download +
    v3 stable: 3.22.1 [2009-07-02] - +change log - +download
    v2 stable: 2.0.7 [2009-04-14] - change log - download -- cgit v1.2.3 From 9f286c0c4c21128c66305166ae379d3f7b07f673 Mon Sep 17 00:00:00 2001 From: Rainer Gerhards Date: Wed, 8 Jul 2009 16:58:53 +0200 Subject: optimizing queue thread handling ... first commit in a series of more. Makes worker threads detached. Needs more testing (will be done soon) and if it works as expected, we can further reduce code. --- runtime/wti.c | 4 ++-- runtime/wtp.c | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/runtime/wti.c b/runtime/wti.c index 917b456b..93c66028 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -263,7 +263,7 @@ wtiJoinThrd(wti_t *pThis) if (pThis->thrdID == 0) { dbgprintf("worker %s was already stopped\n", wtiGetDbgHdr(pThis)); } else { - pthread_join(pThis->thrdID, NULL); + //pthread_join(pThis->thrdID, NULL); wtiSetState(pThis, eWRKTHRD_STOPPED, 0, MUTEX_ALREADY_LOCKED); /* back to virgin... */ pThis->thrdID = 0; /* invalidate the thread ID so that we do not accidently find reused ones */ dbgprintf("worker %s has stopped\n", wtiGetDbgHdr(pThis)); @@ -399,7 +399,7 @@ wtiWorker(wti_t *pThis) /* now we have our identity, on to real processing */ while(1) { /* loop will be broken below - need to do mutex locks */ /* process any pending thread requests */ - wtpProcessThrdChanges(pWtp); + // wtpProcessThrdChanges(pWtp); if(pWtp->pfRateLimiter != NULL) { /* call rate-limiter, if defined */ pWtp->pfRateLimiter(pWtp->pUsr); diff --git a/runtime/wtp.c b/runtime/wtp.c index 81e4446d..59553984 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -493,11 +493,12 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in static rsRetVal wtpStartWrkr(wtp_t *pThis, int bLockMutex) { - DEFiRet; DEFVARS_mutexProtection; wti_t *pWti; int i; int iState; + pthread_attr_t attr; + DEFiRet; ISOBJ_TYPE_assert(pThis, wtp); @@ -521,7 +522,10 @@ wtpStartWrkr(wtp_t *pThis, int bLockMutex) pWti = pThis->pWrkr[i]; wtiSetState(pWti, eWRKTHRD_RUN_CREATED, 0, LOCK_MUTEX); - iState = pthread_create(&(pWti->thrdID), NULL, wtpWorker, (void*) pWti); + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + iState = pthread_create(&(pWti->thrdID), &attr, wtpWorker, (void*) pWti); + pthread_attr_destroy(&attr); /* TODO: we could globally reuse such an attribute 2009-07-08 */ dbgprintf("%s: started with state %d, num workers now %d\n", wtpGetDbgHdr(pThis), iState, pThis->iCurNumWrkThrd); -- cgit v1.2.3