diff options
Diffstat (limited to 'runtime')
42 files changed, 1216 insertions, 797 deletions
diff --git a/runtime/apc.c b/runtime/apc.c index c2f61266..3c6b7ec4 100644 --- a/runtime/apc.c +++ b/runtime/apc.c @@ -40,9 +40,11 @@ #include "obj.h" #include "apc.h" #include "srUtils.h" +#include "datetime.h" /* static data */ DEFobjStaticHelpers +DEFobjCurrIf(datetime) /* following is a used to implement a monotonically increasing id for the apcs. That * ID can be used to cancel an apc request. Note that the ID is generated with modulo @@ -200,7 +202,7 @@ unlistCurrent(apc_list_t **ppList) DEFiRet; assert(ppList != NULL); - time(&tCurr); + datetime.GetTime(&tCurr); if(apcListRoot == NULL || apcListRoot->pApc->ttExec > tCurr) { *ppList = NULL; @@ -375,7 +377,7 @@ ENDobjQueryInterface(apc) * rgerhards, 2009-04-06 */ BEGINObjClassExit(apc, OBJ_IS_CORE_MODULE) /* class, version */ - //objRelease(apcstk, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); pthread_mutex_destroy(&listMutex); ENDObjClassExit(apc) @@ -386,7 +388,7 @@ ENDObjClassExit(apc) */ BEGINObjClassInit(apc, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* request objects we use */ - //CHKiRet(objUse(apcstk, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); /* set our own handlers */ OBJSetMethodHandler(objMethod_DEBUGPRINT, apcDebugPrint); diff --git a/runtime/batch.h b/runtime/batch.h index 031718a7..2b3aa83e 100644 --- a/runtime/batch.h +++ b/runtime/batch.h @@ -34,7 +34,7 @@ typedef enum { BATCH_STATE_RDY = 0, /* object ready for processing */ BATCH_STATE_BAD = 1, /* unrecoverable failure while processing, do NOT resubmit to same action */ - BATCH_STATE_SUB = 2, /* message submitted for processing, outcome yet unkonwn */ + BATCH_STATE_SUB = 2, /* message submitted for processing, outcome yet unknown */ BATCH_STATE_COMM = 3, /* message successfully commited */ BATCH_STATE_DISC = 4, /* discarded - processed OK, but do not submit to any other action */ } batch_state_t; diff --git a/runtime/conf.c b/runtime/conf.c index 2e37edf2..d6e6ccf6 100644 --- a/runtime/conf.c +++ b/runtime/conf.c @@ -95,17 +95,16 @@ DEFobjCurrIf(ruleset) static int iNbrActions = 0; /* number of currently defined actions */ -/* The following global variables are used for building +/* The following module-global variables are used for building * tag and host selector lines during startup and config reload. * This is stored as a global variable pool because of its ease. It is * also fairly compatible with multi-threading as the stratup code must - * be run in a single thread anyways. So there can be no race conditions. These - * variables are no longer used once the configuration has been loaded (except, - * of course, during a reload). rgerhards 2005-10-18 + * be run in a single thread anyways. So there can be no race conditions. + * rgerhards 2005-10-18 */ -EHostnameCmpMode eDfltHostnameCmpMode; -cstr_t *pDfltHostnameCmp; -cstr_t *pDfltProgNameCmp; +static EHostnameCmpMode eDfltHostnameCmpMode = HN_NO_COMP; +static cstr_t *pDfltHostnameCmp = NULL; +static cstr_t *pDfltProgNameCmp = NULL; /* process a directory and include all of its files into @@ -1038,7 +1037,7 @@ static rsRetVal cflineDoFilter(uchar **pp, rule_t *f) DEFiRet; ASSERT(pp != NULL); - ASSERT(f != NULL); + ISOBJ_TYPE_assert(f, rule); /* check which filter we need to pull... */ switch(**pp) { @@ -1060,6 +1059,7 @@ static rsRetVal cflineDoFilter(uchar **pp, rule_t *f) * and, if so, we copy them over. rgerhards, 2005-10-18 */ if(pDfltProgNameCmp != NULL) { +RUNLOG_STR("dflt ProgNameCmp != NULL, setting opCSProgNameComp"); CHKiRet(rsCStrConstructFromCStr(&(f->pCSProgNameComp), pDfltProgNameCmp)); } @@ -1248,6 +1248,15 @@ ENDobjQueryInterface(conf) */ BEGINObjClassExit(conf, OBJ_IS_CORE_MODULE) /* CHANGE class also in END MACRO! */ CODESTARTObjClassExit(conf) + /* free no-longer needed module-global variables */ + if(pDfltHostnameCmp != NULL) { + rsCStrDestruct(&pDfltHostnameCmp); + } + + if(pDfltProgNameCmp != NULL) { + rsCStrDestruct(&pDfltProgNameCmp); + } + /* release objects we no longer need */ objRelease(expr, CORE_COMPONENT); objRelease(ctok, CORE_COMPONENT); diff --git a/runtime/conf.h b/runtime/conf.h index 6db1623e..d85d1f82 100644 --- a/runtime/conf.h +++ b/runtime/conf.h @@ -49,10 +49,6 @@ ENDinterface(conf) PROTOTYPEObj(conf); -/* TODO: remove them below (means move the config init code) -- rgerhards, 2008-02-19 */ -extern EHostnameCmpMode eDfltHostnameCmpMode; -extern cstr_t *pDfltHostnameCmp; -extern cstr_t *pDfltProgNameCmp; /* TODO: the following 2 need to go in conf obj interface... */ rsRetVal cflineParseTemplateName(uchar** pp, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *dfltTplName); rsRetVal cflineParseFileName(uchar* p, uchar *pFileName, omodStringRequest_t *pOMSR, int iEntry, int iTplOpts, uchar *pszTpl); diff --git a/runtime/datetime.c b/runtime/datetime.c index 6160bd7c..4ab4318d 100644 --- a/runtime/datetime.c +++ b/runtime/datetime.c @@ -127,6 +127,24 @@ static void getCurrTime(struct syslogTime *t, time_t *ttSeconds) } +/* A fast alternative to getCurrTime() and time() that only obtains + * a timestamp like time() does. I was told that gettimeofday(), at + * least under Linux, is much faster than time() and I could confirm + * this testing. So I created that function as a replacement. + * rgerhards, 2009-11-12 + */ +static time_t +getTime(time_t *ttSeconds) +{ + struct timeval tp; + + if(gettimeofday(&tp, NULL) == -1) + return -1; + + if(ttSeconds != NULL) + *ttSeconds = tp.tv_sec; + return tp.tv_sec; +} /******************************************************************* @@ -831,6 +849,7 @@ CODESTARTobjQueryInterface(datetime) * of course, also affects the "if" above). */ pIf->getCurrTime = getCurrTime; + pIf->GetTime = getTime; pIf->ParseTIMESTAMP3339 = ParseTIMESTAMP3339; pIf->ParseTIMESTAMP3164 = ParseTIMESTAMP3164; pIf->formatTimestampToMySQL = formatTimestampToMySQL; diff --git a/runtime/datetime.h b/runtime/datetime.h index 8140eb71..1de3db95 100644 --- a/runtime/datetime.h +++ b/runtime/datetime.h @@ -41,8 +41,10 @@ BEGINinterface(datetime) /* name must also be changed in ENDinterface macro! */ int (*formatTimestamp3339)(struct syslogTime *ts, char* pBuf); int (*formatTimestamp3164)(struct syslogTime *ts, char* pBuf); int (*formatTimestampSecFrac)(struct syslogTime *ts, char* pBuf); + /* v3, 2009-11-12 */ + time_t (*GetTime)(time_t *ttSeconds); ENDinterface(datetime) -#define datetimeCURR_IF_VERSION 2 /* increment whenever you change the interface structure! */ +#define datetimeCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ /* interface changes: * 1 - initial version * 2 - not compatible to 1 - bugfix required ParseTIMESTAMP3164 to accept char ** as diff --git a/runtime/debug.c b/runtime/debug.c index 959d56a3..545ac876 100644 --- a/runtime/debug.c +++ b/runtime/debug.c @@ -51,6 +51,7 @@ #include "rsyslog.h" #include "debug.h" #include "atomic.h" +#include "cfsysline.h" #include "obj.h" @@ -960,7 +961,7 @@ void dbgprintf(char *fmt, ...) { va_list ap; - char pszWriteBuf[1024]; + char pszWriteBuf[32*1024]; size_t lenWriteBuf; if(!(Debug && debugging_on)) @@ -969,6 +970,16 @@ dbgprintf(char *fmt, ...) va_start(ap, fmt); lenWriteBuf = vsnprintf(pszWriteBuf, sizeof(pszWriteBuf), fmt, ap); va_end(ap); + + if(lenWriteBuf >= sizeof(pszWriteBuf)) { + /* if we need to truncate, do it in a somewhat useful way... */ + pszWriteBuf[sizeof(pszWriteBuf) - 5] = '!'; + pszWriteBuf[sizeof(pszWriteBuf) - 4] = '.'; + pszWriteBuf[sizeof(pszWriteBuf) - 3] = '.'; + pszWriteBuf[sizeof(pszWriteBuf) - 2] = '.'; + pszWriteBuf[sizeof(pszWriteBuf) - 1] = '\n'; + lenWriteBuf = sizeof(pszWriteBuf); + } dbgprint(NULL, pszWriteBuf, lenWriteBuf); } @@ -1244,6 +1255,20 @@ dbgPrintNameIsInList(const uchar *pName, dbgPrintName_t *pRoot) } +/* this is a special version of malloc that fills the alloced memory with + * HIGHVALUE, as this helps to identify bugs. -- rgerhards, 2009-10-22 + */ +void * +dbgmalloc(size_t size) +{ + void *pRet; + pRet = malloc(size); + if(pRet != NULL) + memset(pRet, 0xff, size); + return pRet; +} + + /* read in the runtime options * rgerhards, 2008-02-28 */ @@ -1276,6 +1301,7 @@ dbgGetRuntimeOptions(void) "NoLogTimestamp\n" "Nostdoout\n" "filetrace=file (may be provided multiple times)\n" + "DebugOnDemand - enables debugging on USR1, but does not turn on output\n" "\nSee debug.html in your doc set or http://www.rsyslog.com for details\n"); exit(1); } else if(!strcasecmp((char*)optname, "debug")) { @@ -1284,6 +1310,13 @@ dbgGetRuntimeOptions(void) */ Debug = 1; debugging_on = 1; + } else if(!strcasecmp((char*)optname, "debugondemand")) { + /* Enables debugging, but turns off debug output */ + Debug = 1; + debugging_on = 1; + dbgprintf("Note: debug on demand turned on via configuraton file, " + "use USR1 signal to activate.\n"); + debugging_on = 0; } else if(!strcasecmp((char*)optname, "logfuncflow")) { bLogFuncFlow = 1; } else if(!strcasecmp((char*)optname, "logallocfree")) { diff --git a/runtime/debug.h b/runtime/debug.h index dcbfb930..8d9c1ceb 100644 --- a/runtime/debug.h +++ b/runtime/debug.h @@ -3,7 +3,7 @@ * Definitions for the debug and run-time analysis support module. * Contains a lot of macros. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -99,6 +99,7 @@ void dbgExitFunc(dbgFuncDB_t *pFuncDB, int iStackPtrRestore, int iRet); void dbgSetExecLocation(int iStackPtr, int line); void dbgSetThrdName(uchar *pszName); void dbgPrintAllDebugInfo(void); +void *dbgmalloc(size_t size); /* macros */ #define DBGPRINTF(...) if(Debug) { dbgprintf(__VA_ARGS__); } @@ -126,6 +127,12 @@ void dbgPrintAllDebugInfo(void); # define RUNLOG_STR(str) #endif +#ifdef MEMCHECK +# define MALLOC(x) dbgmalloc(x) +#else +# define MALLOC(x) malloc(x) +#endif + /* mutex operations */ #define MUTOP_LOCKWAIT 1 #define MUTOP_LOCK 2 diff --git a/runtime/glbl.c b/runtime/glbl.c index f27b8e73..71c2ed0d 100644 --- a/runtime/glbl.c +++ b/runtime/glbl.c @@ -56,6 +56,7 @@ DEFobjCurrIf(prop) */ static uchar *pszWorkDir = NULL; static int bOptimizeUniProc = 1; /* enable uniprocessor optimizations */ +static int bParseHOSTNAMEandTAG = 1; /* parser modification (based on startup params!) */ static int bPreserveFQDN = 0; /* should FQDNs always be preserved? */ static int iMaxLine = 2048; /* maximum length of a syslog message */ static int iDefPFFamily = PF_UNSPEC; /* protocol family (IPv4, IPv6 or both) */ @@ -94,6 +95,7 @@ static dataType Get##nameFunc(void) \ return(nameVar); \ } +SIMP_PROP(ParseHOSTNAMEandTAG, bParseHOSTNAMEandTAG, int) SIMP_PROP(OptimizeUniProc, bOptimizeUniProc, int) SIMP_PROP(PreserveFQDN, bPreserveFQDN, int) SIMP_PROP(MaxLine, iMaxLine, int) @@ -183,6 +185,7 @@ finalize_it: RETiRet; } + /* return our local hostname as a string property */ static prop_t* @@ -266,6 +269,7 @@ CODESTARTobjQueryInterface(glbl) pIf->Set##name = Set##name; SIMP_PROP(MaxLine); SIMP_PROP(OptimizeUniProc); + SIMP_PROP(ParseHOSTNAMEandTAG); SIMP_PROP(PreserveFQDN); SIMP_PROP(DefPFFamily); SIMP_PROP(DropMalPTRMsgs); diff --git a/runtime/glbl.h b/runtime/glbl.h index 0d0c8210..7506f16b 100644 --- a/runtime/glbl.h +++ b/runtime/glbl.h @@ -64,9 +64,11 @@ BEGINinterface(glbl) /* name must also be changed in ENDinterface macro! */ /* added v4, 2009-07-20 */ int (*GetGlobalInputTermState)(void); void (*SetGlobalInputTermination)(void); + /* added v5, 2009-11-03 */ + SIMP_PROP(ParseHOSTNAMEandTAG, int) #undef SIMP_PROP ENDinterface(glbl) -#define glblCURR_IF_VERSION 4 /* increment whenever you change the interface structure! */ +#define glblCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ /* version 2 had PreserveFQDN added - rgerhards, 2008-12-08 */ /* the remaining prototypes */ diff --git a/runtime/module-template.h b/runtime/module-template.h index d49da2c9..18aad650 100644 --- a/runtime/module-template.h +++ b/runtime/module-template.h @@ -46,6 +46,9 @@ DEFobjCurrIf(obj) #define DEF_LMOD_STATIC_DATA \ DEF_MOD_STATIC_DATA +#define DEF_PMOD_STATIC_DATA \ + DEFobjCurrIf(obj) \ + DEF_MOD_STATIC_DATA /* Macro to define the module type. Each module can only have a single type. If @@ -65,6 +68,7 @@ static rsRetVal modGetType(eModType_t *modType) \ #define MODULE_TYPE_INPUT MODULE_TYPE(eMOD_IN) #define MODULE_TYPE_OUTPUT MODULE_TYPE(eMOD_OUT) +#define MODULE_TYPE_PARSER MODULE_TYPE(eMOD_PARSER) #define MODULE_TYPE_LIB \ DEF_LMOD_STATIC_DATA \ MODULE_TYPE(eMOD_LIB) @@ -400,6 +404,18 @@ static rsRetVal queryEtryPt(uchar *name, rsRetVal (**pEtryPoint)())\ #define CODEqueryEtryPt_STD_LIB_QUERIES \ CODEqueryEtryPt_STD_MOD_QUERIES +/* the following definition is the standard block for queryEtryPt for PARSER + * modules. This can be used if no specific handling (e.g. to cover version + * differences) is needed. + */ +#define CODEqueryEtryPt_STD_PMOD_QUERIES \ + CODEqueryEtryPt_STD_MOD_QUERIES \ + else if(!strcmp((char*) name, "parse")) {\ + *pEtryPoint = parse;\ + } else if(!strcmp((char*) name, "GetParserName")) {\ + *pEtryPoint = GetParserName;\ + } + /* modInit() * This has an extra parameter, which is the specific name of the modInit * function. That is needed for built-in modules, which must have unique @@ -590,5 +606,31 @@ static rsRetVal doHUP(instanceData __attribute__((unused)) *pData)\ } +/* parse() - main entry point of parser modules + */ +#define BEGINparse \ +static rsRetVal parse(msg_t *pMsg)\ +{\ + DEFiRet; + +#define CODESTARTparse \ + assert(pMsg != NULL); + +#define ENDparse \ + RETiRet;\ +} + + +/* function to specify the parser name. This is done via a single command which + * receives a ANSI string as parameter. + */ +#define PARSER_NAME(x) \ +static rsRetVal GetParserName(uchar **ppSz)\ +{\ + *ppSz = UCHAR_CONSTANT(x);\ + return RS_RET_OK;\ +} + + /* vim:set ai: */ diff --git a/runtime/modules.c b/runtime/modules.c index bdb15e7f..fd3468d8 100644 --- a/runtime/modules.c +++ b/runtime/modules.c @@ -11,7 +11,7 @@ * * File begun on 2007-07-22 by RGerhards * - * Copyright 2007 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -57,10 +57,12 @@ #include "cfsysline.h" #include "modules.h" #include "errmsg.h" +#include "parser.h" /* static data */ DEFobjStaticHelpers DEFobjCurrIf(errmsg) +DEFobjCurrIf(parser) /* we must ensure that only one thread at one time tries to load or unload * modules, otherwise we may see race conditions. This first came up with @@ -94,7 +96,6 @@ static rsRetVal dummyEndTransaction() } static rsRetVal dummyIsCompatibleWithFeature() { -dbgprintf("XXX: dummy isCompatibleWithFeature called!\n"); return RS_RET_INCOMPATIBLE; } @@ -403,10 +404,13 @@ finalize_it: static rsRetVal doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_t*), uchar *name, void *pModHdlr) { - DEFiRet; rsRetVal localRet; modInfo_t *pNew = NULL; + uchar *pParserName; + parser_t *pParser; /* used for parser modules */ + rsRetVal (*GetParserName)(uchar**); rsRetVal (*modGetType)(eModType_t *pType); + DEFiRet; assert(modInit != NULL); @@ -475,6 +479,33 @@ doModInit(rsRetVal (*modInit)(int, int*, rsRetVal(**)(), rsRetVal(*)(), modInfo_ break; case eMOD_LIB: break; + case eMOD_PARSER: + /* first, we need to obtain the parser object. We could not do that during + * init as that would have caused class bootstrap issues which are not + * absolutely necessary. Note that we can call objUse() multiple times, it + * handles that. + */ + CHKiRet(objUse(parser, CORE_COMPONENT)); + /* here, we create a new parser object */ + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"parse", &pNew->mod.pm.parse)); + CHKiRet((*pNew->modQueryEtryPt)((uchar*)"GetParserName", &GetParserName)); + CHKiRet(GetParserName(&pParserName)); + CHKiRet(parser.Construct(&pParser)); + + /* check some features */ + localRet = pNew->isCompatibleWithFeature(sFEATUREAutomaticSanitazion); + if(localRet == RS_RET_OK){ + CHKiRet(parser.SetDoSanitazion(pParser, TRUE)); + } + localRet = pNew->isCompatibleWithFeature(sFEATUREAutomaticPRIParsing); + if(localRet == RS_RET_OK){ + CHKiRet(parser.SetDoPRIParsing(pParser, TRUE)); + } + + CHKiRet(parser.SetName(pParser, pParserName)); + CHKiRet(parser.SetModPtr(pParser, pNew)); + CHKiRet(parser.ConstructFinalize(pParser)); + break; } pNew->pszName = (uchar*) strdup((char*)name); /* we do not care if strdup() fails, we can accept that */ @@ -521,6 +552,9 @@ static void modPrintList(void) case eMOD_LIB: dbgprintf("library"); break; + case eMOD_PARSER: + dbgprintf("parser"); + break; } dbgprintf(" module.\n"); dbgprintf("Entry points:\n"); @@ -867,6 +901,7 @@ BEGINObjClassExit(module, OBJ_IS_LOADABLE_MODULE) /* CHANGE class also in END MA CODESTARTObjClassExit(module) /* release objects we no longer need */ objRelease(errmsg, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); /* We have a problem in our reference counting, which leads to this function * being called too early. This usually is no problem, but if we destroy * the mutex object, we get into trouble. So rather than finding the root cause, diff --git a/runtime/modules.h b/runtime/modules.h index 71e3199c..62f86ded 100644 --- a/runtime/modules.h +++ b/runtime/modules.h @@ -12,7 +12,7 @@ * * File begun on 2007-07-22 by RGerhards * - * Copyright 2007, 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2007-2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -51,9 +51,10 @@ #define CURR_MOD_IF_VERSION 5 typedef enum eModType_ { - eMOD_IN, /* input module */ - eMOD_OUT, /* output module */ - eMOD_LIB /* library module - this module provides one or many interfaces */ + eMOD_IN = 0, /* input module */ + eMOD_OUT = 1, /* output module */ + eMOD_LIB = 2, /* library module */ + eMOD_PARSER = 3 /* parser module */ } eModType_t; @@ -73,7 +74,7 @@ typedef enum eModLinkType_ { eMOD_LINK_ALL /* special: all linkage types, e.g. for unload */ } eModLinkType_t; -typedef struct modInfo_s { +struct modInfo_s { struct modInfo_s *pPrev; /* support for creating a double linked module list */ struct modInfo_s *pNext; /* support for creating a linked module list */ int iIFVers; /* Interface version of module */ @@ -117,7 +118,10 @@ typedef struct modInfo_s { rsRetVal (*parseSelectorAct)(uchar**, void**,omodStringRequest_t**); } om; struct { /* data for library modules */ - } fm; + } lm; + struct { /* data for parser modules */ + rsRetVal (*parse)(msg_t*); + } pm; } mod; void *pModHdlr; /* handler to the dynamic library holding the module */ # ifdef DEBUG @@ -126,7 +130,7 @@ typedef struct modInfo_s { */ modUsr_t *pModUsrRoot; # endif -} modInfo_t; +}; /* interfaces */ BEGINinterface(module) /* name must also be changed in ENDinterface macro! */ @@ -152,6 +156,5 @@ extern uchar *pModDir; /* read-only after startup */ #endif /* #ifndef MODULES_H_INCLUDED */ -/* - * vi:set ai: +/* vi:set ai: */ diff --git a/runtime/msg.c b/runtime/msg.c index d28ee350..623c5b4a 100644 --- a/runtime/msg.c +++ b/runtime/msg.c @@ -288,7 +288,7 @@ static inline void getInputName(msg_t *pM, uchar **ppsz, int *plen) { BEGINfunc - if(pM == NULL) { + if(pM == NULL || pM->pInputName == NULL) { *ppsz = UCHAR_CONSTANT(""); *plen = 0; } else { @@ -626,13 +626,12 @@ static inline rsRetVal msgBaseConstruct(msg_t **ppThis) msg_t *pM; assert(ppThis != NULL); - CHKmalloc(pM = malloc(sizeof(msg_t))); + CHKmalloc(pM = MALLOC(sizeof(msg_t))); objConstructSetObjInfo(pM); /* intialize object helper entities */ /* initialize members in ORDER they appear in structure (think "cache line"!) */ pM->flowCtlType = 0; pM->bDoLock = 0; - pM->bParseHOSTNAME = 0; pM->iRefCount = 1; pM->iSeverity = -1; pM->iFacility = -1; @@ -861,7 +860,6 @@ msg_t* MsgDup(msg_t* pOld) pNew->iRefCount = 1; pNew->iSeverity = pOld->iSeverity; pNew->iFacility = pOld->iFacility; - pNew->bParseHOSTNAME = pOld->bParseHOSTNAME; pNew->msgFlags = pOld->msgFlags; pNew->iProtocolVersion = pOld->iProtocolVersion; pNew->ttGenTime = pOld->ttGenTime; @@ -935,7 +933,7 @@ msg_t* MsgDup(msg_t* pOld) * We do not serialize the cache properties. We re-create them when needed. * This saves us a lot of memory. Performance is no concern, as serializing * is a so slow operation that recration of the caches does not count. Also, - * we do not serialize bParseHOSTNAME, as this is only a helper variable + * we do not serialize --currently none--, as this is only a helper variable * during msg construction - and never again used later. * rgerhards, 2008-01-03 */ @@ -1229,7 +1227,7 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtMySQLDate: MsgLock(pM); if(pM->pszTIMESTAMP_MySQL == NULL) { - if((pM->pszTIMESTAMP_MySQL = malloc(15)) == NULL) { + if((pM->pszTIMESTAMP_MySQL = MALLOC(15)) == NULL) { MsgUnlock(pM); return ""; } @@ -1240,7 +1238,7 @@ static inline char *getTimeReported(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtPgSQLDate: MsgLock(pM); if(pM->pszTIMESTAMP_PgSQL == NULL) { - if((pM->pszTIMESTAMP_PgSQL = malloc(21)) == NULL) { + if((pM->pszTIMESTAMP_PgSQL = MALLOC(21)) == NULL) { MsgUnlock(pM); return ""; } @@ -1281,7 +1279,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtDefault: MsgLock(pM); if(pM->pszRcvdAt3164 == NULL) { - if((pM->pszRcvdAt3164 = malloc(16)) == NULL) { + if((pM->pszRcvdAt3164 = MALLOC(16)) == NULL) { MsgUnlock(pM); return ""; } @@ -1292,7 +1290,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtMySQLDate: MsgLock(pM); if(pM->pszRcvdAt_MySQL == NULL) { - if((pM->pszRcvdAt_MySQL = malloc(15)) == NULL) { + if((pM->pszRcvdAt_MySQL = MALLOC(15)) == NULL) { MsgUnlock(pM); return ""; } @@ -1303,7 +1301,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtPgSQLDate: MsgLock(pM); if(pM->pszRcvdAt_PgSQL == NULL) { - if((pM->pszRcvdAt_PgSQL = malloc(21)) == NULL) { + if((pM->pszRcvdAt_PgSQL = MALLOC(21)) == NULL) { MsgUnlock(pM); return ""; } @@ -1314,7 +1312,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtRFC3164Date: MsgLock(pM); if(pM->pszRcvdAt3164 == NULL) { - if((pM->pszRcvdAt3164 = malloc(16)) == NULL) { + if((pM->pszRcvdAt3164 = MALLOC(16)) == NULL) { MsgUnlock(pM); return ""; } @@ -1325,7 +1323,7 @@ static inline char *getTimeGenerated(msg_t *pM, enum tplFormatTypes eFmt) case tplFmtRFC3339Date: MsgLock(pM); if(pM->pszRcvdAt3339 == NULL) { - if((pM->pszRcvdAt3339 = malloc(33)) == NULL) { + if((pM->pszRcvdAt3339 = MALLOC(33)) == NULL) { MsgUnlock(pM); return ""; } @@ -1576,7 +1574,7 @@ void MsgSetTAG(msg_t *pMsg, uchar* pszBuf, size_t lenBuf) /* small enough: use fixed buffer (faster!) */ pBuf = pMsg->TAG.szBuf; } else { - if((pBuf = (uchar*) malloc(pMsg->iLenTAG + 1)) == NULL) { + if((pBuf = (uchar*) MALLOC(pMsg->iLenTAG + 1)) == NULL) { /* truncate message, better than completely loosing it... */ pBuf = pMsg->TAG.szBuf; pMsg->iLenTAG = CONF_TAG_BUFSIZE - 1; @@ -1941,7 +1939,7 @@ void MsgSetHOSTNAME(msg_t *pThis, uchar* pszHOSTNAME, int lenHOSTNAME) if(pThis->iLenHOSTNAME < CONF_HOSTNAME_BUFSIZE) { /* small enough: use fixed buffer (faster!) */ pThis->pszHOSTNAME = pThis->szHOSTNAME; - } else if((pThis->pszHOSTNAME = (uchar*) malloc(pThis->iLenHOSTNAME + 1)) == NULL) { + } else if((pThis->pszHOSTNAME = (uchar*) MALLOC(pThis->iLenHOSTNAME + 1)) == NULL) { /* truncate message, better than completely loosing it... */ pThis->pszHOSTNAME = pThis->szHOSTNAME; pThis->iLenHOSTNAME = CONF_HOSTNAME_BUFSIZE - 1; @@ -1993,7 +1991,7 @@ rsRetVal MsgReplaceMSG(msg_t *pThis, uchar* pszMSG, int lenMSG) lenNew = pThis->iLenRawMsg + lenMSG - pThis->iLenMSG; if(lenMSG > pThis->iLenMSG && lenNew >= CONF_RAWMSG_BUFSIZE) { /* we have lost our "bet" and need to alloc a new buffer ;) */ - CHKmalloc(bufNew = malloc(lenNew + 1)); + CHKmalloc(bufNew = MALLOC(lenNew + 1)); memcpy(bufNew, pThis->pszRawMsg, pThis->offMSG); if(pThis->pszRawMsg != pThis->szRawMsg) free(pThis->pszRawMsg); @@ -2024,7 +2022,7 @@ void MsgSetRawMsg(msg_t *pThis, char* pszRawMsg, size_t lenMsg) if(pThis->iLenRawMsg < CONF_RAWMSG_BUFSIZE) { /* small enough: use fixed buffer (faster!) */ pThis->pszRawMsg = pThis->szRawMsg; - } else if((pThis->pszRawMsg = (uchar*) malloc(pThis->iLenRawMsg + 1)) == NULL) { + } else if((pThis->pszRawMsg = (uchar*) MALLOC(pThis->iLenRawMsg + 1)) == NULL) { /* truncate message, better than completely loosing it... */ pThis->pszRawMsg = pThis->szRawMsg; pThis->iLenRawMsg = CONF_RAWMSG_BUFSIZE - 1; @@ -2080,7 +2078,7 @@ static uchar *getNOW(eNOWType eNow) uchar *pBuf; struct syslogTime t; - if((pBuf = (uchar*) malloc(sizeof(uchar) * tmpBUFSIZE)) == NULL) { + if((pBuf = (uchar*) MALLOC(sizeof(uchar) * tmpBUFSIZE)) == NULL) { return NULL; } @@ -2214,7 +2212,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, pRes = (uchar*)getPRI(pMsg); break; case PROP_PRI_TEXT: - pBuf = malloc(20 * sizeof(uchar)); + pBuf = MALLOC(20 * sizeof(uchar)); if(pBuf == NULL) { *pbMustBeFreed = 0; return UCHAR_CONSTANT("**OUT OF MEMORY**"); @@ -2370,7 +2368,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* we got our end pointer, now do the copy */ /* TODO: code copied from below, this is a candidate for a separate function */ iLen = pFldEnd - pFld + 1; /* the +1 is for an actual char, NOT \0! */ - pBufStart = pBuf = malloc((iLen + 1) * sizeof(char)); + pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(char)); if(pBuf == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2416,7 +2414,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, ; /*DO NOTHING*/ } else { iLen = iTo - iFrom + 1; /* the +1 is for an actual char, NOT \0! */ - pBufStart = pBuf = malloc((iLen + 1) * sizeof(char)); + pBufStart = pBuf = MALLOC((iLen + 1) * sizeof(char)); if(pBuf == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2521,7 +2519,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, iLenBuf = pmatch[pTpe->data.field.iSubMatchToUse].rm_eo - pmatch[pTpe->data.field.iSubMatchToUse].rm_so; - pB = malloc((iLenBuf + 1) * sizeof(uchar)); + pB = MALLOC((iLenBuf + 1) * sizeof(uchar)); if (pB == NULL) { if (*pbMustBeFreed == 1) @@ -2578,7 +2576,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, uchar *pBStart; uchar *pB; uchar *pSrc; - pBStart = pB = malloc((bufLen + 1) * sizeof(char)); + pBStart = pB = MALLOC((bufLen + 1) * sizeof(char)); if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2625,7 +2623,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } if(bDropped) { - pDst = pDstStart = malloc(iLenBuf + 1); + pDst = pDstStart = MALLOC(iLenBuf + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2661,7 +2659,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } else { if(bufLen == -1) bufLen = ustrlen(pRes); - pDst = pDstStart = malloc(bufLen + 1); + pDst = pDstStart = MALLOC(bufLen + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2701,7 +2699,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, int i; iLenBuf += iNumCC * 4; - pBStart = pB = malloc((iLenBuf + 1) * sizeof(uchar)); + pBStart = pB = MALLOC((iLenBuf + 1) * sizeof(uchar)); if(pB == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2747,7 +2745,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } if(bDropped) { - pDst = pDstStart = malloc(iLenBuf + 1); + pDst = pDstStart = MALLOC(iLenBuf + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2783,7 +2781,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, } else { if(bufLen == -1) bufLen = ustrlen(pRes); - pDst = pDstStart = malloc(bufLen + 1); + pDst = pDstStart = MALLOC(bufLen + 1); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); @@ -2840,7 +2838,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, /* check if we need to obtain a private copy */ if(*pbMustBeFreed == 0) { /* ok, original copy, need a private one */ - pB = malloc((iLn + 1) * sizeof(uchar)); + pB = MALLOC((iLn + 1) * sizeof(uchar)); if(pB == NULL) { *pbMustBeFreed = 0; return UCHAR_CONSTANT("**OUT OF MEMORY**"); @@ -2869,7 +2867,7 @@ uchar *MsgGetProp(msg_t *pMsg, struct templateEntry *pTpe, bufLen = ustrlen(pRes); iBufLen = bufLen; /* the malloc may be optimized, we currently use the worst case... */ - pBStart = pDst = malloc((2 * iBufLen + 3) * sizeof(uchar)); + pBStart = pDst = MALLOC((2 * iBufLen + 3) * sizeof(uchar)); if(pDst == NULL) { if(*pbMustBeFreed == 1) free(pRes); diff --git a/runtime/msg.h b/runtime/msg.h index b006cbec..9101cef7 100644 --- a/runtime/msg.h +++ b/runtime/msg.h @@ -60,7 +60,6 @@ struct msg { once data has entered the queue, this property is no longer needed. */ pthread_mutex_t mut; bool bDoLock; /* use the mutex? */ - bool bParseHOSTNAME; /* should the hostname be parsed from the message? */ short iRefCount; /* reference counter (0 = unused) */ /* background: the hostname is not present on "regular" messages * received via UNIX domain sockets from the same machine. However, @@ -209,6 +208,16 @@ MsgSetRawMsgSize(msg_t *pMsg, size_t newLen) } +/* get the ruleset that is associated with the ruleset. + * May be NULL. -- rgerhards, 2009-10-27 + */ +static inline ruleset_t* +MsgGetRuleset(msg_t *pMsg) +{ + return pMsg->pRuleset; +} + + #endif /* #ifndef MSG_H_INCLUDED */ /* vim:set ai: */ diff --git a/runtime/net.c b/runtime/net.c index 5cafe522..dfae53e2 100644 --- a/runtime/net.c +++ b/runtime/net.c @@ -173,7 +173,7 @@ AddPermittedPeerWildcard(permittedPeers_t *pPeer, uchar* pszStr, size_t lenStr) /* alloc memory for the domain component. We may waste a byte or * two, but that's ok. */ - CHKmalloc(pNew->pszDomainPart = malloc(lenStr +1 )); + CHKmalloc(pNew->pszDomainPart = MALLOC(lenStr +1 )); } if(pszStr[0] == '*') { @@ -695,7 +695,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS case AF_INET: /* add IPv4 */ iSignificantBits = 32; allowIP.flags = 0; - if((allowIP.addr.NetAddr = malloc(res->ai_addrlen)) == NULL) { + if((allowIP.addr.NetAddr = MALLOC(res->ai_addrlen)) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } memcpy(allowIP.addr.NetAddr, res->ai_addr, res->ai_addrlen); @@ -710,7 +710,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS iSignificantBits = 32; allowIP.flags = 0; - if((allowIP.addr.NetAddr = malloc(sizeof(struct sockaddr_in))) + if((allowIP.addr.NetAddr = MALLOC(sizeof(struct sockaddr_in))) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -732,7 +732,7 @@ static rsRetVal AddAllowedSender(struct AllowedSenders **ppRoot, struct AllowedS iSignificantBits = 128; allowIP.flags = 0; - if((allowIP.addr.NetAddr = malloc(res->ai_addrlen)) == NULL) { + if((allowIP.addr.NetAddr = MALLOC(res->ai_addrlen)) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } memcpy(allowIP.addr.NetAddr, res->ai_addr, res->ai_addrlen); @@ -1306,7 +1306,7 @@ getLocalHostname(uchar **ppName) do { if(buf == NULL) { buf_len = 128; /* Initial guess */ - CHKmalloc(buf = malloc(buf_len)); + CHKmalloc(buf = MALLOC(buf_len)); } else { buf_len += buf_len; CHKmalloc(buf = realloc (buf, buf_len)); @@ -1370,7 +1370,7 @@ int *create_udp_socket(uchar *hostname, uchar *pszPort, int bIsServer) /* Count max number of sockets we may open */ for (maxs = 0, r = res; r != NULL ; r = r->ai_next, maxs++) /* EMPTY */; - socks = malloc((maxs+1) * sizeof(int)); + socks = MALLOC((maxs+1) * sizeof(int)); if (socks == NULL) { errmsg.LogError(0, NO_ERRCODE, "couldn't allocate memory for UDP sockets, suspending UDP message reception"); freeaddrinfo(res); diff --git a/runtime/net.h b/runtime/net.h index ec364b1c..a50b6fcb 100644 --- a/runtime/net.h +++ b/runtime/net.h @@ -149,8 +149,8 @@ BEGINinterface(net) /* name must also be changed in ENDinterface macro! */ /* v5 interface additions */ int (*CmpHost)(struct sockaddr_storage *, struct sockaddr_storage*, size_t); /* data members - these should go away over time... TODO */ - int *pACLAddHostnameOnFail; /* add hostname to acl when DNS resolving has failed */ - int *pACLDontResolve; /* add hostname to acl instead of resolving it to IP(s) */ + int pACLAddHostnameOnFail; /* add hostname to acl when DNS resolving has failed */ + int pACLDontResolve; /* add hostname to acl instead of resolving it to IP(s) */ ENDinterface(net) #define netCURR_IF_VERSION 5 /* increment whenever you change the interface structure! */ diff --git a/runtime/nsd_gtls.c b/runtime/nsd_gtls.c index 79ceffb3..74c142f2 100644 --- a/runtime/nsd_gtls.c +++ b/runtime/nsd_gtls.c @@ -44,6 +44,7 @@ #include "stringbuf.h" #include "errmsg.h" #include "net.h" +#include "datetime.h" #include "nsd_ptcp.h" #include "nsdsel_gtls.h" #include "nsd_gtls.h" @@ -61,6 +62,7 @@ DEFobjStaticHelpers DEFobjCurrIf(errmsg) DEFobjCurrIf(glbl) DEFobjCurrIf(net) +DEFobjCurrIf(datetime) DEFobjCurrIf(nsd_ptcp) static int bGlblSrvrInitDone = 0; /**< 0 - server global init not yet done, 1 - already done */ @@ -129,7 +131,7 @@ readFile(uchar *pszFile, gnutls_datum_t *pBuf) ABORT_FINALIZE(RS_RET_FILE_TOO_LARGE); } - CHKmalloc(pBuf->data = malloc(stat_st.st_size)); + CHKmalloc(pBuf->data = MALLOC(stat_st.st_size)); pBuf->size = stat_st.st_size; if(read(fd, pBuf->data, stat_st.st_size) != stat_st.st_size) { errmsg.LogError(0, RS_RET_IO_ERROR, "error or incomplete read of file '%s'", pszFile); @@ -1016,7 +1018,7 @@ gtlsChkPeerCertValidity(nsd_gtls_t *pThis) } /* get current time for certificate validation */ - if(time(&ttNow) == -1) + if(datetime.GetTime(&ttNow) == -1) ABORT_FINALIZE(RS_RET_SYS_ERR); /* as it looks, we need to validate the expiration dates ourselves... @@ -1479,7 +1481,7 @@ Rcv(nsd_t *pNsd, uchar *pBuf, ssize_t *pLenBuf) if(pThis->pszRcvBuf == NULL) { /* we have no buffer, so we need to malloc one */ - CHKmalloc(pThis->pszRcvBuf = malloc(NSD_GTLS_MAX_RCVBUF)); + CHKmalloc(pThis->pszRcvBuf = MALLOC(NSD_GTLS_MAX_RCVBUF)); pThis->lenRcvBuf = -1; } @@ -1694,6 +1696,7 @@ CODESTARTObjClassExit(nsd_gtls) objRelease(nsd_ptcp, LM_NSD_PTCP_FILENAME); objRelease(net, LM_NET_FILENAME); objRelease(glbl, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); objRelease(errmsg, CORE_COMPONENT); ENDObjClassExit(nsd_gtls) @@ -1705,6 +1708,7 @@ ENDObjClassExit(nsd_gtls) BEGINObjClassInit(nsd_gtls, 1, OBJ_IS_LOADABLE_MODULE) /* class, version */ /* request objects we use */ CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(net, LM_NET_FILENAME)); CHKiRet(objUse(nsd_ptcp, LM_NSD_PTCP_FILENAME)); diff --git a/runtime/nsd_ptcp.c b/runtime/nsd_ptcp.c index 54ee0666..fe31ab40 100644 --- a/runtime/nsd_ptcp.c +++ b/runtime/nsd_ptcp.c @@ -296,12 +296,12 @@ FillRemHost(nsd_ptcp_t *pThis, struct sockaddr *pAddr) * memory consumption) */ len = strlen((char*)szIP) + 1; /* +1 for \0 byte */ - if((pThis->pRemHostIP = malloc(len)) == NULL) + if((pThis->pRemHostIP = MALLOC(len)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pRemHostIP, szIP, len); len = strlen((char*)szHname) + 1; /* +1 for \0 byte */ - if((pThis->pRemHostName = malloc(len)) == NULL) { + if((pThis->pRemHostName = MALLOC(len)) == NULL) { free(pThis->pRemHostIP); /* prevent leak */ pThis->pRemHostIP = NULL; ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); diff --git a/runtime/obj.c b/runtime/obj.c index aebea332..45dac776 100644 --- a/runtime/obj.c +++ b/runtime/obj.c @@ -48,7 +48,7 @@ * * File begun on 2008-01-04 by RGerhards * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008, 2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -90,6 +90,7 @@ #include "cfsysline.h" #include "unicode-helper.h" #include "apc.h" +#include "datetime.h" /* static data */ DEFobjCurrIf(obj) /* we define our own interface, as this is expected by some macros! */ @@ -1129,7 +1130,7 @@ UseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) /* DEV debug only: dbgprintf("source file %s requests object '%s', ifIsLoaded %d\n", srcFile, pObjName, pIf->ifIsLoaded); */ - d_pthread_mutex_lock(&mutObjGlobalOp); + pthread_mutex_lock(&mutObjGlobalOp); if(pIf->ifIsLoaded == 1) { ABORT_FINALIZE(RS_RET_OK); /* we are already set */ @@ -1170,7 +1171,7 @@ UseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) pIf->ifIsLoaded = 1; /* we are happy */ finalize_it: - d_pthread_mutex_unlock(&mutObjGlobalOp); + pthread_mutex_unlock(&mutObjGlobalOp); if(pStr != NULL) rsCStrDestruct(&pStr); @@ -1193,7 +1194,7 @@ ReleaseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) /* dev debug only dbgprintf("source file %s releasing object '%s', ifIsLoaded %d\n", srcFile, pObjName, pIf->ifIsLoaded); */ - d_pthread_mutex_lock(&mutObjGlobalOp); + pthread_mutex_lock(&mutObjGlobalOp); if(pObjFile == NULL) FINALIZE; /* if it is not a lodable module, we do not need to do anything... */ @@ -1214,7 +1215,7 @@ ReleaseObj(char *srcFile, uchar *pObjName, uchar *pObjFile, interface_t *pIf) pIf->ifIsLoaded = 0; /* indicated "no longer valid" */ finalize_it: - d_pthread_mutex_unlock(&mutObjGlobalOp); + pthread_mutex_unlock(&mutObjGlobalOp); if(pStr != NULL) rsCStrDestruct(&pStr); @@ -1330,8 +1331,9 @@ objClassInit(modInfo_t *pModInfo) CHKiRet(objGetObjInterface(&obj)); /* get ourselves ;) */ /* init classes we use (limit to as few as possible!) */ - CHKiRet(apcClassInit(pModInfo)); CHKiRet(errmsgClassInit(pModInfo)); + CHKiRet(datetimeClassInit(pModInfo)); + CHKiRet(apcClassInit(pModInfo)); CHKiRet(cfsyslineInit()); CHKiRet(varClassInit(pModInfo)); CHKiRet(moduleClassInit(pModInfo)); diff --git a/runtime/objomsr.c b/runtime/objomsr.c index 8dddc4b8..1d442c61 100644 --- a/runtime/objomsr.c +++ b/runtime/objomsr.c @@ -67,31 +67,25 @@ rsRetVal OMSRconstruct(omodStringRequest_t **ppThis, int iNumEntries) assert(ppThis != NULL); assert(iNumEntries >= 0); - if((pThis = calloc(1, sizeof(omodStringRequest_t))) == NULL) { - iRet = RS_RET_OUT_OF_MEMORY; - goto abort_it; - } + CHKmalloc(pThis = calloc(1, sizeof(omodStringRequest_t))); /* got the structure, so fill it */ pThis->iNumEntries = iNumEntries; /* allocate string for template name array. The individual strings will be * allocated as the code progresses (we do not yet know the string sizes) */ - if((pThis->ppTplName = calloc(iNumEntries, sizeof(uchar*))) == NULL) { - OMSRdestruct(pThis); - pThis = NULL; - iRet = RS_RET_OUT_OF_MEMORY; - goto abort_it; - } + CHKmalloc(pThis->ppTplName = calloc(iNumEntries, sizeof(uchar*))); + /* allocate the template options array. */ - if((pThis->piTplOpts = calloc(iNumEntries, sizeof(int))) == NULL) { - OMSRdestruct(pThis); - pThis = NULL; - iRet = RS_RET_OUT_OF_MEMORY; - goto abort_it; - } + CHKmalloc(pThis->piTplOpts = calloc(iNumEntries, sizeof(int))); -abort_it: +finalize_it: + if(iRet != RS_RET_OK) { + if(pThis != NULL) { + OMSRdestruct(pThis); + pThis = NULL; + } + } *ppThis = pThis; RETiRet; } @@ -155,7 +149,7 @@ OMSRgetSupportedTplOpts(unsigned long *pOpts) { DEFiRet; assert(pOpts != NULL); - *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY; + *pOpts = OMSR_RQD_TPL_OPT_SQL | OMSR_TPL_AS_ARRAY | OMSR_TPL_AS_MSG; RETiRet; } diff --git a/runtime/objomsr.h b/runtime/objomsr.h index 75ad0fb8..e59b774f 100644 --- a/runtime/objomsr.h +++ b/runtime/objomsr.h @@ -27,8 +27,12 @@ /* define flags for required template options */ #define OMSR_NO_RQD_TPL_OPTS 0 #define OMSR_RQD_TPL_OPT_SQL 1 +/* only one of OMSR_TPL_AS_ARRAY or _AS_MSG must be specified, if both are given + * results are unpredictable. + */ #define OMSR_TPL_AS_ARRAY 2 /* introduced in 4.1.6, 2009-04-03 */ -/* next option is 4, 8, 16, ... */ +#define OMSR_TPL_AS_MSG 4 /* introduced in 5.3.4, 2009-11-02 */ +/* next option is 8, 16, 32, ... */ struct omodStringRequest_s { /* strings requested by output module for doAction() */ int iNumEntries; /* number of array entries for data elements below */ diff --git a/runtime/parser.c b/runtime/parser.c index 466066e7..38f72986 100644 --- a/runtime/parser.c +++ b/runtime/parser.c @@ -37,7 +37,13 @@ #include "dirty.h" #include "msg.h" #include "obj.h" +#include "datetime.h" #include "errmsg.h" +#include "parser.h" +#include "ruleset.h" +#include "unicode-helper.h" +#include "dirty.h" +#include "cfsysline.h" /* some defines */ #define DEFUPRI (LOG_USER|LOG_NOTICE) @@ -46,26 +52,165 @@ DEFobjStaticHelpers DEFobjCurrIf(glbl) DEFobjCurrIf(errmsg) +DEFobjCurrIf(datetime) +DEFobjCurrIf(ruleset) /* static data */ +/* config data */ +static uchar cCCEscapeChar = '#';/* character to be used to start an escape sequence for control chars */ +static int bEscapeCCOnRcv = 1; /* escape control characters on reception: 0 - no, 1 - yes */ +static int bDropTrailingLF = 1; /* drop trailing LF's on reception? */ + +/* This is the list of all parsers known to us. + * This is also used to unload all modules on shutdown. + */ +parserList_t *pParsLstRoot = NULL; + +/* this is the list of the default parsers, to be used if no others + * are specified. + */ +parserList_t *pDfltParsLst = NULL; + + +/* intialize (but NOT allocate) a parser list. Primarily meant as a hook + * which can be used to extend the list in the future. So far, just sets + * it to NULL. + */ +static rsRetVal +InitParserList(parserList_t **pListRoot) +{ + *pListRoot = NULL; + return RS_RET_OK; +} + + +/* destruct a parser list. The list elements are destroyed, but the parser objects + * themselves are not modified. (That is done at a late stage during rsyslogd + * shutdown and need not be considered here.) + */ +static rsRetVal +DestructParserList(parserList_t **ppListRoot) +{ + parserList_t *pParsLst; + parserList_t *pParsLstDel; + + pParsLst = *ppListRoot; + while(pParsLst != NULL) { + pParsLstDel = pParsLst; + pParsLst = pParsLst->pNext; + free(pParsLstDel); + } + *ppListRoot = NULL; + return RS_RET_OK; +} + -/* this is a dummy class init +/* Add a parser to the list. We use a VERY simple and ineffcient algorithm, + * but it is employed only for a few milliseconds during config processing. So + * I prefer to keep it very simple and with simple data structures. Unfortunately, + * we need to preserve the order, but I don't like to add a tail pointer as that + * would require a container object. So I do the extra work to skip to the tail + * when adding elements... + * rgerhards, 2009-11-03 */ -rsRetVal parserClassInit(void) +static rsRetVal +AddParserToList(parserList_t **ppListRoot, parser_t *pParser) { + parserList_t *pThis; + parserList_t *pTail; DEFiRet; - /* request objects we use */ - CHKiRet(objGetObjInterface(&obj)); /* this provides the root pointer for all other queries */ - CHKiRet(objUse(glbl, CORE_COMPONENT)); - CHKiRet(objUse(errmsg, CORE_COMPONENT)); -// TODO: free components! see action.c + CHKmalloc(pThis = MALLOC(sizeof(parserList_t))); + pThis->pParser = pParser; + pThis->pNext = NULL; + + if(*ppListRoot == NULL) { + pThis->pNext = *ppListRoot; + *ppListRoot = pThis; + } else { + /* find tail first */ + for(pTail = *ppListRoot ; pTail->pNext != NULL ; pTail = pTail->pNext) + /* just search, do nothing else */; + /* add at tail */ + pTail->pNext = pThis; + } + +finalize_it: + RETiRet; +} + + +/* find a parser based on the provided name */ +static rsRetVal +FindParser(parser_t **ppParser, uchar *pName) +{ + parserList_t *pThis; + DEFiRet; + + for(pThis = pParsLstRoot ; pThis != NULL ; pThis = pThis->pNext) { + if(ustrcmp(pThis->pParser->pName, pName) == 0) { + *ppParser = pThis->pParser; + FINALIZE; /* found it, iRet still eq. OK! */ + } + } + + iRet = RS_RET_PARSER_NOT_FOUND; + finalize_it: RETiRet; } +/* --- END helper functions for parser list handling --- */ + +/* Add a an already existing parser to the default list. As usual, order + * of calls is important (most importantly, that means the legacy parser, + * which can process everything, MUST be added last!). + * rgerhards, 2009-11-04 + */ +static rsRetVal +AddDfltParser(uchar *pName) +{ + parser_t *pParser; + DEFiRet; + + CHKiRet(FindParser(&pParser, pName)); + CHKiRet(AddParserToList(&pDfltParsLst, pParser)); + dbgprintf("Parser '%s' added to default parser set.\n", pName); + +finalize_it: + RETiRet; +} + + + +BEGINobjConstruct(parser) /* be sure to specify the object type also in END macro! */ +ENDobjConstruct(parser) + +/* ConstructionFinalizer. The most important chore is to add the parser object + * to our global list of available parsers. + * rgerhards, 2009-11-03 + */ +rsRetVal parserConstructFinalize(parser_t *pThis) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, parser); + CHKiRet(AddParserToList(&pParsLstRoot, pThis)); + DBGPRINTF("Parser '%s' added to list of available parsers.\n", pThis->pName); + +finalize_it: + RETiRet; +} + +BEGINobjDestruct(parser) /* be sure to specify the object type also in END and CODESTART macros! */ +CODESTARTobjDestruct(parser) + dbgprintf("destructing parser '%s'\n", pThis->pName); + free(pThis->pName); +ENDobjDestruct(parser) + + /* uncompress a received message if it is compressed. * pMsg->pszRawMsg buffer is updated. * rgerhards, 2008-10-09 @@ -96,7 +241,7 @@ static inline rsRetVal uncompressMessage(msg_t *pMsg) */ int ret; iLenDefBuf = glbl.GetMaxLine(); - CHKmalloc(deflateBuf = malloc(sizeof(uchar) * (iLenDefBuf + 1))); + CHKmalloc(deflateBuf = MALLOC(sizeof(uchar) * (iLenDefBuf + 1))); ret = uncompress((uchar *) deflateBuf, &iLenDefBuf, (uchar *) pszMsg+1, lenMsg-1); DBGPRINTF("Compressed message uncompressed with status %d, length: new %ld, old %d.\n", ret, (long) iLenDefBuf, (int) (lenMsg-1)); @@ -153,7 +298,7 @@ finalize_it: * rgerhards, 2007-09-14 */ static inline rsRetVal -sanitizeMessage(msg_t *pMsg) +SanitizeMsg(msg_t *pMsg) { DEFiRet; uchar *pszMsg; @@ -169,10 +314,6 @@ sanitizeMessage(msg_t *pMsg) assert(pMsg != NULL); assert(pMsg->iLenRawMsg > 0); -# ifdef USE_NETZIP - CHKiRet(uncompressMessage(pMsg)); -# endif - pszMsg = pMsg->pszRawMsg; lenMsg = pMsg->iLenRawMsg; @@ -223,7 +364,7 @@ sanitizeMessage(msg_t *pMsg) if(maxDest < sizeof(szSanBuf)) pDst = szSanBuf; else - CHKmalloc(pDst = malloc(sizeof(uchar) * (iMaxLine + 1))); + CHKmalloc(pDst = MALLOC(sizeof(uchar) * (iMaxLine + 1))); iSrc = iDst = 0; while(iSrc < lenMsg && iDst < maxDest - 3) { /* leave some space if last char must be escaped */ if(iscntrl((int) pszMsg[iSrc])) { @@ -231,14 +372,14 @@ sanitizeMessage(msg_t *pMsg) * can not handle it! -- rgerhards, 2009-08-26 */ if(pszMsg[iSrc] == '\0' || bEscapeCCOnRcv) { - /* we are configured to escape control characters. Please note - * that this most probably break non-western character sets like - * Japanese, Korean or Chinese. rgerhards, 2007-07-17 - */ - pDst[iDst++] = cCCEscapeChar; - pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); - pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); - pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); + /* we are configured to escape control characters. Please note + * that this most probably break non-western character sets like + * Japanese, Korean or Chinese. rgerhards, 2007-07-17 + */ + pDst[iDst++] = cCCEscapeChar; + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0300) >> 6); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0070) >> 3); + pDst[iDst++] = '0' + ((pszMsg[iSrc] & 0007)); } } else { pDst[iDst++] = pszMsg[iSrc]; @@ -255,27 +396,18 @@ finalize_it: RETiRet; } - -/* Parse a received message. The object's rawmsg property is taken and - * parsed according to the relevant standards. This can later be - * extended to support configured parsers. - * rgerhards, 2008-10-09 +/* A standard parser to parse out the PRI. This is made available in + * this module as it is expected that allmost all parsers will need + * that functionality and so they do not need to implement it themsleves. */ -rsRetVal parseMsg(msg_t *pMsg) +static inline rsRetVal +ParsePRI(msg_t *pMsg) { - DEFiRet; - uchar *msg; int pri; + uchar *msg; int lenMsg; int iPriText; - - if(pMsg->iLenRawMsg == 0) - ABORT_FINALIZE(RS_RET_EMPTY_MSG); - - CHKiRet(sanitizeMessage(pMsg)); - - /* we needed to sanitize first, because we otherwise do not have a C-string we can print... */ - DBGPRINTF("msg parser: flags %x, from '%s', msg '%s'\n", pMsg->msgFlags, getRcvFrom(pMsg), pMsg->pszRawMsg); + DEFiRet; /* pull PRI */ lenMsg = pMsg->iLenRawMsg; @@ -299,31 +431,247 @@ rsRetVal parseMsg(msg_t *pMsg) pMsg->iFacility = LOG_FAC(pri); pMsg->iSeverity = LOG_PRI(pri); MsgSetAfterPRIOffs(pMsg, msg - pMsg->pszRawMsg); + RETiRet; +} + + +/* Parse a received message. The object's rawmsg property is taken and + * parsed according to the relevant standards. This can later be + * extended to support configured parsers. + * rgerhards, 2008-10-09 + */ +static rsRetVal +ParseMsg(msg_t *pMsg) +{ + rsRetVal localRet; + parserList_t *pParserList; + parser_t *pParser; + bool bIsSanitized; + bool bPRIisParsed; + static int iErrMsgRateLimiter = 0; + DEFiRet; - /* rger 2005-11-24 (happy thanksgiving!): we now need to check if we have - * a traditional syslog message or one formatted according to syslog-protocol. - * We need to apply different parsers depending on that. We use the - * -protocol VERSION field for the detection. + if(pMsg->iLenRawMsg == 0) + ABORT_FINALIZE(RS_RET_EMPTY_MSG); + +# ifdef USE_NETZIP + CHKiRet(uncompressMessage(pMsg)); +# endif + + /* we take the risk to print a non-sanitized string, because this is the best we can get + * (and that functionality is too important for debugging to drop it...). */ - if(msg[0] == '1' && msg[1] == ' ') { - dbgprintf("Message has syslog-protocol format.\n"); - setProtocolVersion(pMsg, 1); - if(parseRFCSyslogMsg(pMsg, pMsg->msgFlags) == 1) { - msgDestruct(&pMsg); - ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! + DBGPRINTF("msg parser: flags %x, from '%s', msg '%.50s'\n", pMsg->msgFlags, + getRcvFrom(pMsg), pMsg->pszRawMsg); + + /* we now need to go through our list of parsers and see which one is capable of + * parsing the message. Note that the first parser that requires message sanitization + * will cause it to happen. After that, access to the unsanitized message is no + * loger possible. + */ + pParserList = ruleset.GetParserList(pMsg); + if(pParserList == NULL) { + pParserList = pDfltParsLst; + } + DBGPRINTF("parse using parser list %p%s.\n", pParserList, + (pParserList == pDfltParsLst) ? " (the default list)" : ""); + + bIsSanitized = FALSE; + bPRIisParsed = FALSE; + while(pParserList != NULL) { + pParser = pParserList->pParser; + if(pParser->bDoSanitazion && bIsSanitized == FALSE) { + CHKiRet(SanitizeMsg(pMsg)); + if(pParser->bDoPRIParsing && bPRIisParsed == FALSE) { + CHKiRet(ParsePRI(pMsg)); + bPRIisParsed = TRUE; + } + bIsSanitized = TRUE; } - } else { /* we have legacy syslog */ - dbgprintf("Message has legacy syslog format.\n"); - setProtocolVersion(pMsg, 0); - if(parseLegacySyslogMsg(pMsg, pMsg->msgFlags) == 1) { - msgDestruct(&pMsg); - ABORT_FINALIZE(RS_RET_ERR); // TODO: we need to handle these cases! + localRet = pParser->pModule->mod.pm.parse(pMsg); + dbgprintf("Parser '%s' returned %d\n", pParser->pName, localRet); + if(localRet != RS_RET_COULD_NOT_PARSE) + break; + pParserList = pParserList->pNext; + } + + /* We need to log a warning message and drop the message if we did not find a parser. + * Note that we log at most the first 1000 message, as this may very well be a problem + * that causes a message generation loop. We do not synchronize that counter, it doesn't + * matter if we log a handful messages more than we should... + */ + if(localRet != RS_RET_OK) { + if(++iErrMsgRateLimiter > 1000) { + errmsg.LogError(0, localRet, "Error: one message could not be processed by " + "any parser, message is being discarded (start of raw msg: '%.50s')", + pMsg->pszRawMsg); } + DBGPRINTF("No parser could process the message (state %d), we need to discard it.\n", localRet); + ABORT_FINALIZE(localRet); } - /* finalize message object */ + /* "finalize" message object */ pMsg->msgFlags &= ~NEEDS_PARSING; /* this message is now parsed */ finalize_it: RETiRet; } + +/* set the parser name - string is copied over, call can continue to use it, + * but must free it if desired. + */ +static rsRetVal +SetName(parser_t *pThis, uchar *name) +{ + DEFiRet; + + ISOBJ_TYPE_assert(pThis, parser); + assert(name != NULL); + + if(pThis->pName != NULL) { + free(pThis->pName); + pThis->pName = NULL; + } + + CHKmalloc(pThis->pName = ustrdup(name)); + +finalize_it: + RETiRet; +} + + +/* set a pointer to "our" module. Note that no module + * pointer must already be set. + */ +static rsRetVal +SetModPtr(parser_t *pThis, modInfo_t *pMod) +{ + ISOBJ_TYPE_assert(pThis, parser); + assert(pMod != NULL); + assert(pThis->pModule == NULL); + pThis->pModule = pMod; + return RS_RET_OK; +} + + +/* Specify if we should do standard message sanitazion before we pass the data + * down to the parser. + */ +static rsRetVal +SetDoSanitazion(parser_t *pThis, int bDoIt) +{ + ISOBJ_TYPE_assert(pThis, parser); + pThis->bDoSanitazion = bDoIt; + return RS_RET_OK; +} + + +/* Specify if we should do standard PRI parsing before we pass the data + * down to the parser module. + */ +static rsRetVal +SetDoPRIParsing(parser_t *pThis, int bDoIt) +{ + ISOBJ_TYPE_assert(pThis, parser); + pThis->bDoPRIParsing = bDoIt; + return RS_RET_OK; +} + + +/* queryInterface function-- rgerhards, 2009-11-03 + */ +BEGINobjQueryInterface(parser) +CODESTARTobjQueryInterface(parser) + if(pIf->ifVersion != parserCURR_IF_VERSION) { /* check for current version, increment on each change */ + ABORT_FINALIZE(RS_RET_INTERFACE_NOT_SUPPORTED); + } + + /* ok, we have the right interface, so let's fill it + * Please note that we may also do some backwards-compatibility + * work here (if we can support an older interface version - that, + * of course, also affects the "if" above). + */ + pIf->Construct = parserConstruct; + pIf->ConstructFinalize = parserConstructFinalize; + pIf->Destruct = parserDestruct; + pIf->SetName = SetName; + pIf->SetModPtr = SetModPtr; + pIf->SetDoSanitazion = SetDoSanitazion; + pIf->SetDoPRIParsing = SetDoPRIParsing; + pIf->ParseMsg = ParseMsg; + pIf->SanitizeMsg = SanitizeMsg; + pIf->InitParserList = InitParserList; + pIf->DestructParserList = DestructParserList; + pIf->AddParserToList = AddParserToList; + pIf->AddDfltParser = AddDfltParser; + pIf->FindParser = FindParser; +finalize_it: +ENDobjQueryInterface(parser) + + + +/* Reset config variables to default values. + * rgerhards, 2007-07-17 + */ +static rsRetVal +resetConfigVariables(uchar __attribute__((unused)) *pp, void __attribute__((unused)) *pVal) +{ + cCCEscapeChar = '#'; + bEscapeCCOnRcv = 1; /* default is to escape control characters */ + bDropTrailingLF = 1; /* default is to drop trailing LF's on reception */ + + return RS_RET_OK; +} + +/* This destroys the master parserlist and all of its parser entries. MUST only be + * done when the module is shut down. Parser modules are NOT unloaded, rsyslog + * does that at a later stage for all dynamically loaded modules. + */ +static void +destroyMasterParserList(void) +{ + parserList_t *pParsLst; + parserList_t *pParsLstDel; + + pParsLst = pParsLstRoot; + while(pParsLst != NULL) { + parserDestruct(&pParsLst->pParser); + pParsLstDel = pParsLst; + pParsLst = pParsLst->pNext; + free(pParsLstDel); + } +} + +/* Exit our class. + * rgerhards, 2009-11-04 + */ +BEGINObjClassExit(parser, OBJ_IS_CORE_MODULE) /* class, version */ + DestructParserList(&pDfltParsLst); + destroyMasterParserList(); + objRelease(glbl, CORE_COMPONENT); + objRelease(errmsg, CORE_COMPONENT); + objRelease(datetime, CORE_COMPONENT); + objRelease(ruleset, CORE_COMPONENT); +ENDObjClassExit(parser) + + +/* Initialize the parser class. Must be called as the very first method + * before anything else is called inside this class. + * rgerhards, 2009-11-02 + */ +BEGINObjClassInit(parser, 1, OBJ_IS_CORE_MODULE) /* class, version */ + /* request objects we use */ + CHKiRet(objUse(glbl, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(ruleset, CORE_COMPONENT)); + + CHKiRet(regCfSysLineHdlr((uchar *)"controlcharacterescapeprefix", 0, eCmdHdlrGetChar, NULL, &cCCEscapeChar, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"droptrailinglfonreception", 0, eCmdHdlrBinary, NULL, &bDropTrailingLF, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"escapecontrolcharactersonreceive", 0, eCmdHdlrBinary, NULL, &bEscapeCCOnRcv, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"resetconfigvariables", 1, eCmdHdlrCustomHandler, resetConfigVariables, NULL, NULL)); + + InitParserList(&pParsLstRoot); + InitParserList(&pDfltParsLst); +ENDObjClassInit(parser) + diff --git a/runtime/parser.h b/runtime/parser.h index cec9c083..c4f63021 100644 --- a/runtime/parser.h +++ b/runtime/parser.h @@ -1,8 +1,6 @@ /* header for parser.c - * This is not yet an object, but contains all those code necessary to - * parse syslog messages. * - * Copyright 2008 Rainer Gerhards and Adiscon GmbH. + * Copyright 2008,2009 Rainer Gerhards and Adiscon GmbH. * * This file is part of the rsyslog runtime library. * @@ -21,10 +19,53 @@ * * A copy of the LGPL can be found in the file "COPYING.LESSER" in this distribution. */ -#ifndef INCLUDED_PARSE_H -#define INCLUDED_PARSE_H +#ifndef INCLUDED_PARSER_H +#define INCLUDED_PARSER_H -extern rsRetVal parserClassInit(void); -extern rsRetVal parseMsg(msg_t*); -#endif /* #ifndef INCLUDED_PARSE_H */ +/* we create a small helper object, a list of parsers, that we can use to + * build a chain of them whereever this is needed (initially thought to be + * used in ruleset.c as well as ourselvs). + */ +struct parserList_s { + parser_t *pParser; + parserList_t *pNext; +}; + + +/* the parser object, a dummy because we have only static methods */ +struct parser_s { + BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ + uchar *pName; /* name of this parser */ + modInfo_t *pModule; /* pointer to parser's module */ + bool bDoSanitazion; /* do standard message sanitazion before calling parser? */ + bool bDoPRIParsing; /* do standard PRI parsing before calling parser? */ +}; + +/* interfaces */ +BEGINinterface(parser) /* name must also be changed in ENDinterface macro! */ + INTERFACEObjDebugPrint(var); + rsRetVal (*Construct)(parser_t **ppThis); + rsRetVal (*ConstructFinalize)(parser_t *pThis); + rsRetVal (*Destruct)(parser_t **ppThis); + rsRetVal (*SetName)(parser_t *pThis, uchar *name); + rsRetVal (*SetModPtr)(parser_t *pThis, modInfo_t *pMod); + rsRetVal (*SetDoSanitazion)(parser_t *pThis, int); + rsRetVal (*SetDoPRIParsing)(parser_t *pThis, int); + rsRetVal (*FindParser)(parser_t **ppThis, uchar*name); + rsRetVal (*InitParserList)(parserList_t **pListRoot); + rsRetVal (*DestructParserList)(parserList_t **pListRoot); + rsRetVal (*AddParserToList)(parserList_t **pListRoot, parser_t *pParser); + /* static functions */ + rsRetVal (*ParseMsg)(msg_t *pMsg); + rsRetVal (*SanitizeMsg)(msg_t *pMsg); + rsRetVal (*AddDfltParser)(uchar *); +ENDinterface(parser) +#define parserCURR_IF_VERSION 1 /* increment whenever you change the interface above! */ + + +/* prototypes */ +PROTOTYPEObj(parser); + + +#endif /* #ifndef INCLUDED_PARSER_H */ diff --git a/runtime/prop.c b/runtime/prop.c index d188b2ed..94d1bd49 100644 --- a/runtime/prop.c +++ b/runtime/prop.c @@ -83,7 +83,7 @@ static rsRetVal SetString(prop_t *pThis, uchar *psz, int len) if(len < CONF_PROP_BUFSIZE) { memcpy(pThis->szVal.sz, psz, len + 1); } else { - CHKmalloc(pThis->szVal.psz = malloc(len + 1)); + CHKmalloc(pThis->szVal.psz = MALLOC(len + 1)); memcpy(pThis->szVal.psz, psz, len + 1); } diff --git a/runtime/queue.c b/runtime/queue.c index 101052a1..b29ec7ac 100644 --- a/runtime/queue.c +++ b/runtime/queue.c @@ -55,6 +55,9 @@ #include "wti.h" #include "msg.h" #include "atomic.h" +#include "errmsg.h" +#include "datetime.h" +#include "unicode-helper.h" #include "msg.h" /* TODO: remove once we remove MsgAddRef() call */ #ifdef OS_SOLARIS @@ -65,14 +68,15 @@ DEFobjStaticHelpers DEFobjCurrIf(glbl) DEFobjCurrIf(strm) +DEFobjCurrIf(errmsg) +DEFobjCurrIf(datetime) /* forward-definitions */ +static inline rsRetVal doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr); static rsRetVal qqueueChkPersist(qqueue_t *pThis, int nUpdates); -static rsRetVal SetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex); static rsRetVal RateLimiter(qqueue_t *pThis); static int qqueueChkStopWrkrDA(qqueue_t *pThis); static rsRetVal GetDeqBatchSize(qqueue_t *pThis, int *pVal); -static int qqueueIsIdleDA(qqueue_t *pThis); static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti); static rsRetVal batchProcessed(qqueue_t *pThis, wti_t *pWti); @@ -143,7 +147,7 @@ static inline rsRetVal tdlAdd(qqueue_t *pQueue, qDeqID deqID, int nElemDeq) ISOBJ_TYPE_assert(pQueue, qqueue); assert(pQueue->toDeleteLst != NULL); - CHKmalloc(pNew = malloc(sizeof(toDeleteLst_t))); + CHKmalloc(pNew = MALLOC(sizeof(toDeleteLst_t))); pNew->deqID = deqID; pNew->nElemDeq = nElemDeq; @@ -228,7 +232,8 @@ static inline void queueDrain(qqueue_t *pThis) * this point in time. The mutex must be locked when * ths function is called. -- rgerhards, 2008-01-25 */ -static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) +static inline rsRetVal +qqueueAdviseMaxWorkers(qqueue_t *pThis) { DEFiRet; int iMaxWorkers; @@ -236,48 +241,18 @@ static inline rsRetVal qqueueAdviseMaxWorkers(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); if(!pThis->bEnqOnly) { - if(pThis->bRunsDA) { - /* if we have not yet reached the high water mark, there is no need to start a - * worker. -- rgerhards, 2008-01-26 - */ - if(getLogicalQueueSize(pThis) >= pThis->iHighWtrMrk || pThis->bQueueStarted == 0) { - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ - } - } - /* regular workers always run */ - if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { - iMaxWorkers = 1; + if(pThis->bIsDA && getLogicalQueueSize(pThis) >= pThis->iHighWtrMrk) { + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* disk queues have always one worker */ } else { - iMaxWorkers = getLogicalQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; + if(getLogicalQueueSize(pThis) == 0) { + iMaxWorkers = 0; + } else if(pThis->qType == QUEUETYPE_DISK || pThis->iMinMsgsPerWrkr == 0) { + iMaxWorkers = 1; + } else { + iMaxWorkers = getLogicalQueueSize(pThis) / pThis->iMinMsgsPerWrkr + 1; + } + wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); } - wtpAdviseMaxWorkers(pThis->pWtpReg, iMaxWorkers); /* disk queues have always one worker */ - } - - RETiRet; -} - - -/* Destruct DA queue. This is the last part of DA-to-normal-mode - * transistion. This is called asynchronously and some time quite a - * while after the actual transistion. The key point is that we need to - * do it at some later time, because we need to destruct the DA queue. That, - * however, can not be done in a thread that has been signalled - * This is to be called when we revert back to our own queue. - * This function must be called with the queue mutex locked (the wti - * class ensures this). - * rgerhards, 2008-01-15 - */ -static rsRetVal -TurnOffDAMode(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - ASSERT(pThis->bRunsDA); - if(getLogicalQueueSize(pThis->pqDA) == 0) { - pThis->bRunsDA = 0; /* tell the world we are back in non-DA mode */ - DBGOPRINT((obj_t*) pThis, "disk-assistance has been turned off, disk queue was empty (iRet %d)\n", - iRet); } RETiRet; @@ -308,15 +283,7 @@ qqueueChkIsDA(qqueue_t *pThis) } -/* Start disk-assisted queue mode. All internal settings are changed. This is supposed - * to be called from the DA worker, which must have been started before. The most important - * chore of this function is to create the DA queue object. If that function fails, - * the DA worker should return with an appropriate state, which in turn should lead to - * a re-set to non-DA mode in the Enq process. The queue mutex must be locked when this - * function is called, else a number of races will happen. - * Please note that this function may be called *while* we in DA mode. This is due to the - * fact that the DA worker calls it and the DA worker may be suspended (and restarted) due - * to inactivity timeouts. +/* Start disk-assisted queue mode. * rgerhards, 2008-01-15 */ static rsRetVal @@ -348,33 +315,22 @@ StartDA(qqueue_t *pThis) CHKiRet(qqueueSetbSyncQueueFiles(pThis->pqDA, pThis->bSyncQueueFiles)); CHKiRet(qqueueSettoActShutdown(pThis->pqDA, pThis->toActShutdown)); CHKiRet(qqueueSettoEnq(pThis->pqDA, pThis->toEnq)); - CHKiRet(SetEnqOnly(pThis->pqDA, pThis->bDAEnqOnly, MUTEX_ALREADY_LOCKED)); CHKiRet(qqueueSetiDeqtWinFromHr(pThis->pqDA, pThis->iDeqtWinFromHr)); CHKiRet(qqueueSetiDeqtWinToHr(pThis->pqDA, pThis->iDeqtWinToHr)); + CHKiRet(qqueueSettoQShutdown(pThis->pqDA, pThis->toQShutdown)); CHKiRet(qqueueSetiHighWtrMrk(pThis->pqDA, 0)); CHKiRet(qqueueSetiDiscardMrk(pThis->pqDA, 0)); - // experimental: XXX - CHKiRet(qqueueSettoWrkShutdown(pThis->pqDA, 0)); - - if(pThis->toQShutdown == 0) { - CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 0)); /* if the user really wants... */ - } else { - /* we use the shortest possible shutdown (0 is endless!) because when we run on disk AND - * have an obviously large backlog, we can't finish it in any case. So there is no point - * in holding shutdown longer than necessary. -- rgerhards, 2008-01-15 - */ - CHKiRet(qqueueSettoQShutdown(pThis->pqDA, 1)); - } - iRet = qqueueStart(pThis->pqDA); /* file not found is expected, that means it is no previous QIF available */ - if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) + if(iRet != RS_RET_OK && iRet != RS_RET_FILE_NOT_FOUND) { + errno = 0; /* else an errno is shown in errmsg! */ + errmsg.LogError(errno, iRet, "error starting up disk queue, using pure in-memory mode"); + pThis->bIsDA = 0; /* disable memory mode */ FINALIZE; /* something is wrong */ + } - //pthread_cond_broadcast(&pThis->condDAReady); /* signal we are now initialized and ready to go ;) */ - - DBGOPRINT((obj_t*) pThis, "is now running in disk assisted mode, disk queue 0x%lx\n", + DBGOPRINT((obj_t*) pThis, "DA queue initialized, disk queue 0x%lx\n", qqueueGetID(pThis->pqDA)); finalize_it: @@ -397,7 +353,7 @@ finalize_it: * rgerhards, 2008-01-16 */ static rsRetVal -InitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) +InitDA(qqueue_t *pThis, int bLockMutex) { DEFiRet; DEFVARS_mutexProtection; @@ -412,91 +368,32 @@ InitDA(qqueue_t *pThis, int bEnqOnly, int bLockMutex) * rgerhards, 2008-01-24 * NOTE: this is the DA worker *pool*, not the DA queue! */ - if(pThis->pWtpDA == NULL) { - lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DAwpool", obj.GetName((obj_t*) pThis)); - CHKiRet(wtpConstruct (&pThis->pWtpDA)); - CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); - CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); - CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, wtp_t*)) qqueueIsIdleDA)); - CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerDA)); - CHKiRet(wtpSetpfObjProcessed (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpDA, (rsRetVal (*)(void *pUsr)) TurnOffDAMode)); - CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); - CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty)); - CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1)); - CHKiRet(wtpSettoWrkShutdown (pThis->pWtpDA, pThis->toWrkShutdown)); - CHKiRet(wtpSetpUsr (pThis->pWtpDA, pThis)); - CHKiRet(wtpConstructFinalize (pThis->pWtpDA)); - } + lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:DAwpool", obj.GetName((obj_t*) pThis)); + CHKiRet(wtpConstruct (&pThis->pWtpDA)); + CHKiRet(wtpSetDbgHdr (pThis->pWtpDA, pszBuf, lenBuf)); + CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int)) qqueueChkStopWrkrDA)); + CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); + CHKiRet(wtpSetpfDoWork (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerDA)); + CHKiRet(wtpSetpfObjProcessed (pThis->pWtpDA, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed)); + CHKiRet(wtpSetpmutUsr (pThis->pWtpDA, pThis->mut)); + CHKiRet(wtpSetpcondBusy (pThis->pWtpDA, &pThis->notEmpty)); + CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpDA, 1)); + CHKiRet(wtpSettoWrkShutdown (pThis->pWtpDA, pThis->toWrkShutdown)); + CHKiRet(wtpSetpUsr (pThis->pWtpDA, pThis)); + CHKiRet(wtpConstructFinalize (pThis->pWtpDA)); /* if we reach this point, we have a "good" DA worker pool */ - /* indicate we now run in DA mode - this is reset by the DA worker if it fails */ - pThis->bDAEnqOnly = bEnqOnly; - /* now construct the actual queue (if it does not already exist) */ if(pThis->pqDA == NULL) { CHKiRet(StartDA(pThis)); } - pThis->bRunsDA = 1; - - /* now we must now adivse the wtp that we need one worker. If none is yet active, - * that will also start one up. If we forgot that step, everything would be stalled - * until the next enqueue request. - */ - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* DA queues always have just one worker max */ - finalize_it: END_MTX_PROTECTED_OPERATIONS(pThis->mut); RETiRet; } -/* check if we need to start disk assisted mode and send some signals to - * keep it running if we are already in it. It also checks if DA mode is - * partially initialized, in which case it waits for initialization to - * complete. - * rgerhards, 2008-01-14 - */ -static rsRetVal -ChkStrtDA(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - - /* if we do not hit the high water mark, we have nothing to do */ - if(getPhysicalQueueSize(pThis) != pThis->iHighWtrMrk) - ABORT_FINALIZE(RS_RET_OK); - - if(pThis->bRunsDA) { - /* then we need to signal that we are at the high water mark again. If that happens - * on our way down the queue, that doesn't matter, because then nobody is waiting - * on the condition variable. - * (Remember that a DA queue stops draining the queue once it has reached the low - * water mark and restarts it when the high water mark is reached again - this is - * what this code here is responsible for. Please note that all workers may have been - * terminated due to the inactivity timeout, thus we need to advise the pool that - * we need at least one). - */ - DBGOPRINT((obj_t*) pThis, "%d entries - passed high water mark in DA mode, send notify\n", - getPhysicalQueueSize(pThis)); - qqueueAdviseMaxWorkers(pThis); - } else { - /* this is the case when we are currently not running in DA mode. So it is time - * to turn it back on. - */ - DBGOPRINT((obj_t*) pThis, "%d entries - passed high water mark for disk-assisted mode, initiating...\n", - getPhysicalQueueSize(pThis)); - InitDA(pThis, QUEUE_MODE_ENQDEQ, MUTEX_ALREADY_LOCKED); /* initiate DA mode */ - } - -finalize_it: - RETiRet; -} - - /* --------------- end code for disk-assisted queue modes -------------------- */ @@ -516,7 +413,7 @@ static rsRetVal qConstructFixedArray(qqueue_t *pThis) if(pThis->iMaxQueueSize == 0) ABORT_FINALIZE(RS_RET_QSIZE_ZERO); - if((pThis->tVars.farray.pBuf = malloc(sizeof(void *) * pThis->iMaxQueueSize)) == NULL) { + if((pThis->tVars.farray.pBuf = MALLOC(sizeof(void *) * pThis->iMaxQueueSize)) == NULL) { ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -587,26 +484,6 @@ static rsRetVal qDelFixedArray(qqueue_t *pThis) } -/* reset the logical dequeue pointer to the physical dequeue position. - * This is only needed after we cancelled workers (during queue shutdown). - */ -static rsRetVal -qUnDeqAllFixedArray(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - - DBGOPRINT((obj_t*) pThis, "resetting FixedArray deq index to %ld (was %ld), logical dequeue count %d\n", - pThis->tVars.farray.head, pThis->tVars.farray.deqhead, pThis->nLogDeq); - - pThis->tVars.farray.deqhead = pThis->tVars.farray.head; - pThis->nLogDeq = 0; - - RETiRet; -} - - /* -------------------- linked list -------------------- */ @@ -644,7 +521,7 @@ static rsRetVal qAddLinkedList(qqueue_t *pThis, void* pUsr) qLinkedList_t *pEntry; DEFiRet; - CHKmalloc((pEntry = (qLinkedList_t*) malloc(sizeof(qLinkedList_t)))); + CHKmalloc((pEntry = (qLinkedList_t*) MALLOC(sizeof(qLinkedList_t)))); pEntry->pNext = NULL; pEntry->pUsr = pUsr; @@ -698,26 +575,6 @@ static rsRetVal qDelLinkedList(qqueue_t *pThis) } -/* reset the logical dequeue pointer to the physical dequeue position. - * This is only needed after we cancelled workers (during queue shutdown). - */ -static rsRetVal -qUnDeqAllLinkedList(qqueue_t *pThis) -{ - DEFiRet; - - ASSERT(pThis != NULL); - - DBGOPRINT((obj_t*) pThis, "resetting LinkedList deq ptr to %p (was %p), logical dequeue count %d\n", - pThis->tVars.linklist.pDelRoot, pThis->tVars.linklist.pDeqRoot, pThis->nLogDeq); - - pThis->tVars.linklist.pDeqRoot = pThis->tVars.linklist.pDelRoot; - pThis->nLogDeq = 0; - - RETiRet; -} - - /* -------------------- disk -------------------- */ @@ -733,44 +590,6 @@ finalize_it: } -/* This method checks if we have a QIF file for the current queue (no matter of - * queue mode). Returns RS_RET_OK if we have a QIF file or an error status otherwise. - * rgerhards, 2008-01-15 - */ -static rsRetVal -qqueueHaveQIF(qqueue_t *pThis) -{ - DEFiRet; - uchar pszQIFNam[MAXFNAME]; - size_t lenQIFNam; - struct stat stat_buf; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pThis->pszFilePrefix == NULL) - ABORT_FINALIZE(RS_RET_NO_FILEPREFIX); - - /* Construct file name */ - lenQIFNam = snprintf((char*)pszQIFNam, sizeof(pszQIFNam) / sizeof(uchar), "%s/%s.qi", - (char*) glbl.GetWorkDir(), (char*)pThis->pszFilePrefix); - - /* check if the file exists */ - if(stat((char*) pszQIFNam, &stat_buf) == -1) { - if(errno == ENOENT) { - DBGOPRINT((obj_t*) pThis, "no .qi file found\n"); - ABORT_FINALIZE(RS_RET_FILE_NOT_FOUND); - } else { - DBGOPRINT((obj_t*) pThis, "error %d trying to access .qi file\n", errno); - ABORT_FINALIZE(RS_RET_IO_ERROR); - } - } - /* If we reach this point, we have a .qi file */ - -finalize_it: - RETiRet; -} - - /* The method loads the persistent queue information. * rgerhards, 2008-01-11 */ @@ -920,9 +739,12 @@ static rsRetVal qDestructDisk(qqueue_t *pThis) ASSERT(pThis != NULL); - strm.Destruct(&pThis->tVars.disk.pWrite); - strm.Destruct(&pThis->tVars.disk.pReadDeq); - strm.Destruct(&pThis->tVars.disk.pReadDel); + if(pThis->tVars.disk.pWrite != NULL) + strm.Destruct(&pThis->tVars.disk.pWrite); + if(pThis->tVars.disk.pReadDeq != NULL) + strm.Destruct(&pThis->tVars.disk.pReadDeq); + if(pThis->tVars.disk.pReadDel != NULL) + strm.Destruct(&pThis->tVars.disk.pReadDel); RETiRet; } @@ -999,16 +821,6 @@ finalize_it: } -/* This is a dummy function for disks - we do not need to reset anything - * because everything is already persisted... - */ -static rsRetVal -qUnDeqAllDisk(__attribute__((unused)) qqueue_t *pThis) -{ - return RS_RET_OK; -} - - /* -------------------- direct (no queueing) -------------------- */ static rsRetVal qConstructDirect(qqueue_t __attribute__((unused)) *pThis) { @@ -1041,7 +853,7 @@ static rsRetVal qAddDirect(qqueue_t *pThis, void* pUsr) batchObj.pUsrp = (obj_t*) pUsr; singleBatch.nElem = 1; /* there always is only one in direct mode */ singleBatch.pElem = &batchObj; - iRet = pThis->pConsumer(pThis->pUsr, &singleBatch); + iRet = pThis->pConsumer(pThis->pUsr, &singleBatch, &pThis->bShutdownImmediate); objDestruct(pUsr); RETiRet; @@ -1053,12 +865,6 @@ static rsRetVal qDelDirect(qqueue_t __attribute__((unused)) *pThis) return RS_RET_OK; } -static rsRetVal -qUnDeqAllDirect(__attribute__((unused)) qqueue_t *pThis) -{ - return RS_RET_OK; -} - /* --------------- end type-specific handlers -------------------- */ @@ -1112,15 +918,17 @@ qqueueDeq(qqueue_t *pThis, void **ppUsr) } -/* Try to terminate queue worker threads within the regular shutdown interval. - * Both the regular and DA queue (if it exists) is waited for, but on the same timeout. - * After this function returns, the workers must either be finished or some force - * to finish them must be applied. - * This function also instructs the DA worker pool (if it exists) to terminate. This is done - * in preparation of final queue shutdown. - * rgerhards, 2009-05-27 +/* Try to shut down regular and DA queue workers, within the queue timeout + * period. That means processing continues as usual. This is the expected + * usual case, where during shutdown those messages remaining are being + * processed. At this point, it is acceptable that the queue can not be + * fully depleted, that case is handled in the next step. During this phase, + * we first shut down the main queue DA worker to prevent new data to arrive + * at the DA queue, and then we ask the regular workers of both the Regular + * and DA queue to try complete processing. + * rgerhards, 2009-10-14 */ -static rsRetVal +static inline rsRetVal tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis) { struct timespec tTimeout; @@ -1130,30 +938,26 @@ tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ - d_pthread_mutex_lock(pThis->mut); /* some workers may be running in parallel! */ - if(getPhysicalQueueSize(pThis) > 0) { - if(pThis->bRunsDA) { - /* We may have waited on the low water mark. As it may have changed, we - * see if we reactivate the worker. - */ - wtpAdviseMaxWorkers(pThis->pWtpDA, 1); - } + if(pThis->bIsDA) { + /* We need to lock the mutex, as otherwise we may have a race that prevents + * us from awaking the DA worker. */ + d_pthread_mutex_lock(pThis->mut); + + /* tell regular queue DA worker to stop shuffling messages to DA queue... */ + pThis->pqDA->bEnqOnly = 1; + wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE); + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); + DBGOPRINT((obj_t*) pThis, "awoke DA worker, told it to shut down.\n"); + + /* also tell the DA queue worker to shut down, so that it already knows... */ + wtpSetState(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN); + wtpAdviseMaxWorkers(pThis->pqDA->pWtpReg, 1); /* awake its lone worker */ + DBGOPRINT((obj_t*) pThis, "awoke DA queue regular worker, told it to shut down when done.\n"); + + d_pthread_mutex_unlock(pThis->mut); } - d_pthread_mutex_unlock(pThis->mut); - /* Now wait for the queue's workers to shut down. Note that we run into the code even if we just found - * out there are no active workers - that doesn't matter: the wtp knows about that and so will - * return immediately. - * We do not yet care about the DA worker - that will be handled down later in the process. - * Note that we must not request shutdown right now - that may introduce a race: if the regular queue - * still runs DA assisted and the DA worker gets scheduled first, it will terminate itself (if the DA - * queue happens to be empty at that instant). Then the regular worker enqueues messages, what will lead - * to a restart of the worker. Of course, everything will continue to run, but in a bit sub-optimal way - * (from a performance point of view). So we don't do anything right now. The DA queue will continue to - * process messages and shutdown itself in any case if there is nothing to do. So we don't loose anything - * by not requesting shutdown now. - * rgerhards, 2008-01-25 - */ + /* first calculate absolute timeout - we need the absolute value here, because we need to coordinate * shutdown of both the regular and DA queue on *the same* timeout. */ @@ -1180,16 +984,6 @@ tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis) } else { DBGOPRINT((obj_t*) pThis, "DA queue worker shut down.\n"); } - /* we also instruct the DA worker pool to shutdown ASAP. If we need it for persisting - * the queue, it is restarted at a later stage. We don't care here if a timeout happens. - */ - DBGOPRINT((obj_t*) pThis, "trying shutdown of main queue DA worker pool\n"); - iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); - if(iRetLocal == RS_RET_TIMED_OUT) { - DBGOPRINT((obj_t*) pThis, "shutdown timed out on main queue DA worker pool (this is OK)\n"); - } else { - DBGOPRINT((obj_t*) pThis, "main queue DA worker pool shut down on first try.\n"); - } } RETiRet; @@ -1197,11 +991,12 @@ tryShutdownWorkersWithinQueueTimeout(qqueue_t *pThis) /* Try to shut down regular and DA queue workers, within the action timeout - * period. Note that the main queue DA worker is still unaffected (and may shuffle - * data to the disk queue while we terminate the other workers). Not finishing - * processing all messages is now OK (but they may be preserved later, depending - * on bSaveOnShutdown setting). - * rgerhards, 2009-05-27 + * period. This aborts processing, but at the end of the current action, in + * a well-defined manner. During this phase, we terminate all three worker + * pools, including the regular queue DA worker if it not yet has terminated. + * Not finishing processing all messages is OK (and expected) at this stage + * (they may be preserved later, depending * on bSaveOnShutdown setting). + * rgerhards, 2009-10-14 */ static rsRetVal tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis) @@ -1210,20 +1005,20 @@ tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis) rsRetVal iRetLocal; DEFiRet; +RUNLOG_STR("trying to shutdown workers within Action Timeout"); ISOBJ_TYPE_assert(pThis, qqueue); ASSERT(pThis->pqParent == NULL); /* detect invalid calling sequence */ /* instruct workers to finish ASAP, even if still work exists */ - /* note that we modify bEnqOnly directly, because going through the method would - * startup some workers again. So this is OK here. -- rgerhards, 2009-05-28 - */ pThis->bEnqOnly = 1; - /* need to set this so that the DA queue begins shutdown in parallel! */ - if(pThis->pqDA != NULL) { + pThis->bShutdownImmediate = 1; + /* now DA queue */ + if(pThis->bIsDA) { pThis->pqDA->bEnqOnly = 1; - wtpSetState(pThis->pqDA->pWtpReg, wtpState_SHUTDOWN_IMMEDIATE); + pThis->pqDA->bShutdownImmediate = 1; } +// TODO: make sure we have at minimum a 10ms timeout - workers deserve a chance... /* now give the queue workers a last chance to gracefully shut down (based on action timeout setting) */ timeoutComp(&tTimeout, pThis->toActShutdown); DBGOPRINT((obj_t*) pThis, "trying immediate shutdown of regular workers (if any)\n"); @@ -1247,17 +1042,14 @@ tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis) DBGOPRINT((obj_t*) pThis, "unexpected iRet state %d after trying immediate shutdown of the DA " "queue in disk save mode. Continuing, but results are unpredictable\n", iRetLocal); } - /* and now we need to check the DA worker itself (the one that shuffles data to the disk). This - * is necessary because we may be in a situation where the DA queue regular worker and the - * main queue worker stopped rather quickly. In this case, there is almost no time (and - * probably no thread switch!) between the point where we instructed the main queue DA - * worker to shutdown and this code location. In consequence, it may not even have - * noticed that it should should down, less acutally done this. So we provide it with a - * fixed 100ms timeout to try complete its work, what usually should be sufficient. - * rgerhards, 2009-10-06 + + /* and now we need to terminate the DA worker itself. We always grant it a 100ms timeout, + * which should be sufficient and usually not be required (it is expected to have finished + * long before while we were processing the queue timeout in shutdown phase 1). + * rgerhards, 2009-10-14 */ timeoutComp(&tTimeout, 100); - DBGOPRINT((obj_t*) pThis, "last try for regular shutdown of main queue DA worker pool\n"); + DBGOPRINT((obj_t*) pThis, "trying regular shutdown of main queue DA worker pool\n"); iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN_IMMEDIATE, &tTimeout); if(iRetLocal == RS_RET_TIMED_OUT) { DBGOPRINT((obj_t*) pThis, "shutdown timed out on main queue DA worker pool " @@ -1272,7 +1064,7 @@ tryShutdownWorkersWithinActionTimeout(qqueue_t *pThis) /* This function cancels all remaining regular workers for both the main and the DA - * queue. The main queue's DA worker pool continues to run (if it exists and is active). + * queue. * rgerhards, 2009-05-29 */ static rsRetVal @@ -1306,7 +1098,7 @@ cancelWorkers(qqueue_t *pThis) * big trouble when resetting the logical dequeue pointer. This operation can only be * done when *no* worker is running. So time for a shutdown... -- rgerhards, 2009-05-28 */ - DBGOPRINT((obj_t*) pThis, "checking to see if we need to cancel the main queue's DA worker pool\n"); + DBGOPRINT((obj_t*) pThis, "checking to see if main queue DA worker pool needs to be cancelled\n"); iRetLocal = wtpCancelAll(pThis->pWtpDA); /* returns immediately if all threads already have terminated */ } @@ -1341,13 +1133,6 @@ ShutdownWorkers(qqueue_t *pThis) DBGOPRINT((obj_t*) pThis, "initiating worker thread shutdown sequence\n"); - /* we reduce the low water mark in any case. This is not absolutely necessary, but - * it is useful because we enable DA mode at several spots below and so we do not need - * to think about the low water mark each time. - */ - pThis->iHighWtrMrk = 1; /* if we do not do this, the DA queue will not stop! */ - pThis->iLowWtrMrk = 0; - CHKiRet(tryShutdownWorkersWithinQueueTimeout(pThis)); if(getPhysicalQueueSize(pThis) > 0) { @@ -1375,7 +1160,7 @@ finalize_it: * to modify some parameters before the queue is actually started. */ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*, batch_t*)) + int iMaxQueueSize, rsRetVal (*pConsumer)(void*, batch_t*,int*)) { DEFiRet; qqueue_t *pThis; @@ -1384,9 +1169,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread ASSERT(pConsumer != NULL); ASSERT(iWorkerThreads >= 0); - if((pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t))) == NULL) { - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); - } + CHKmalloc(pThis = (qqueue_t *)calloc(1, sizeof(qqueue_t))); /* we have an object, so let's fill the properties */ objConstructSetObjInfo(pThis); @@ -1397,7 +1180,7 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->iFullDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 3; /* default 97% */ pThis->iLightDlyMrk = iMaxQueueSize - (iMaxQueueSize / 100) * 30; /* default 70% */ - pThis->lenSpoolDir = strlen((char*)pThis->pszSpoolDir); + pThis->lenSpoolDir = ustrlen(pThis->pszSpoolDir); pThis->iMaxFileSize = 1024 * 1024; /* default is 1 MiB */ pThis->iQueueSize = 0; pThis->nLogDeq = 0; @@ -1418,7 +1201,6 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qAdd = qAddFixedArray; pThis->qDeq = qDeqFixedArray; pThis->qDel = qDelFixedArray; - pThis->qUnDeqAll = qUnDeqAllFixedArray; break; case QUEUETYPE_LINKEDLIST: pThis->qConstruct = qConstructLinkedList; @@ -1426,7 +1208,6 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qAdd = qAddLinkedList; pThis->qDeq = (rsRetVal (*)(qqueue_t*,void**)) qDeqLinkedList; pThis->qDel = (rsRetVal (*)(qqueue_t*)) qDelLinkedList; - pThis->qUnDeqAll = qUnDeqAllLinkedList; break; case QUEUETYPE_DISK: pThis->qConstruct = qConstructDisk; @@ -1434,7 +1215,6 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qAdd = qAddDisk; pThis->qDeq = qDeqDisk; pThis->qDel = qDelDisk; - pThis->qUnDeqAll = qUnDeqAllDisk; /* special handling */ pThis->iNumWorkerThreads = 1; /* we need exactly one worker */ break; @@ -1443,7 +1223,6 @@ rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThread pThis->qDestruct = qDestructDirect; pThis->qAdd = qAddDirect; pThis->qDel = qDelDirect; - pThis->qUnDeqAll = qUnDeqAllDirect; break; } @@ -1456,7 +1235,7 @@ finalize_it: /* This function checks if the provided message shall be discarded and does so, if needed. * In DA mode, we do not discard any messages as we assume the disk subsystem is fast enough to * provide real-time creation of spool files. - * Note: cached copies of iQueueSize and bRunsDA are provided so that no mutex locks are required. + * Note: cached copies of iQueueSize is provided so that no mutex locks are required. * The caller must have obtained them while the mutex was locked. Of course, these values may no * longer be current, but that is OK for the discard check. At worst, the message is either processed * or discarded when it should not have been. As discarding is in itself somewhat racy and erratic, @@ -1466,7 +1245,7 @@ finalize_it: * the return state! * rgerhards, 2008-01-24 */ -static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, int bRunsDA, void *pUsr) +static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, void *pUsr) { DEFiRet; rsRetVal iRetLocal; @@ -1475,7 +1254,7 @@ static int qqueueChkDiscardMsg(qqueue_t *pThis, int iQueueSize, int bRunsDA, voi ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_assert(pUsr); - if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk && bRunsDA == 0) { + if(pThis->iDiscardMrk > 0 && iQueueSize >= pThis->iDiscardMrk) { iRetLocal = objGetSeverity(pUsr, &iSeverity); if(iRetLocal == RS_RET_OK && iSeverity >= pThis->iDiscardSeverity) { DBGOPRINT((obj_t*) pThis, "queue nearly full (%d entries), discarded severity %d message\n", @@ -1519,11 +1298,7 @@ dbgprintf("delete batch from store, new sizes: log %d, phys %d\n", getLogicalQue /* remove messages from the physical queue store that are fully processed. This is - * controlled via the to-delete list. We can only delete those elements, that are - * at the current physical tail of the queue. If the batch is from another position, - * we schedule it for deletion, but actual deletion will happen at a later call - * of this function here. We always delete as much as possible, which includes - * picking up things from the to-delete list. + * controlled via the to-delete list. */ static inline rsRetVal DeleteBatchFromQStore(qqueue_t *pThis, batch_t *pBatch) @@ -1537,7 +1312,7 @@ DeleteBatchFromQStore(qqueue_t *pThis, batch_t *pBatch) pTdl = tdlPeek(pThis); /* get current head element */ if(pTdl == NULL) { /* to-delete list empty */ - DoDeleteBatchFromQStore(pThis, pBatch->nElemDeq); + DoDeleteBatchFromQStore(pThis, pBatch->nElem); } else if(pBatch->deqID == pThis->deqIDDel) { deqIDDel = pThis->deqIDDel; pTdl = tdlPeek(pThis); @@ -1547,10 +1322,12 @@ DeleteBatchFromQStore(qqueue_t *pThis, batch_t *pBatch) ++deqIDDel; pTdl = tdlPeek(pThis); } + /* old entries deleted, now delete current ones... */ + DoDeleteBatchFromQStore(pThis, pBatch->nElem); } else { /* can not delete, insert into to-delete list */ dbgprintf("not at head of to-delete list, enqueue %d\n", (int) pBatch->deqID); - CHKiRet(tdlAdd(pThis, pBatch->deqID, pBatch->nElemDeq)); + CHKiRet(tdlAdd(pThis, pBatch->deqID, pBatch->nElem)); } finalize_it: @@ -1559,7 +1336,8 @@ finalize_it: /* Delete a batch of processed user objects from the queue, which includes - * destructing the objects themself. + * destructing the objects themself. Any entries not marked as finally + * processed are enqueued again. The new enqueue is necessary because we have a * rgerhards, 2009-05-13 */ static inline rsRetVal @@ -1567,6 +1345,8 @@ DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch) { int i; void *pUsr; + int nEnqueued = 0; + rsRetVal localRet; DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); @@ -1574,9 +1354,23 @@ DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch) for(i = 0 ; i < pBatch->nElem ; ++i) { pUsr = pBatch->pElem[i].pUsrp; + if( pBatch->pElem[i].state == BATCH_STATE_RDY + || pBatch->pElem[i].state == BATCH_STATE_SUB) { + localRet = doEnqSingleObj(pThis, eFLOWCTL_NO_DELAY, + (obj_t*)MsgAddRef((msg_t*) pUsr)); + ++nEnqueued; + if(localRet != RS_RET_OK) { + DBGPRINTF("error %d re-enqueuing unprocessed data element - discarded\n", localRet); + } + } objDestruct(pUsr); } + dbgprintf("we deleted %d objects and enqueued %d objects\n", i-nEnqueued, nEnqueued); + + if(nEnqueued > 0) + qqueueChkPersist(pThis, nEnqueued); + iRet = DeleteBatchFromQStore(pThis, pBatch); pBatch->nElem = pBatch->nElemDeq = 0; /* reset batch */ @@ -1586,7 +1380,11 @@ DeleteProcessedBatch(qqueue_t *pThis, batch_t *pBatch) /* dequeue as many user pointers as are available, until we hit the configured - * upper limit of pointers. + * upper limit of pointers. Note that this function also deletes all processed + * objects from the previous batch. However, it is perfectly valid that the + * previous batch contained NO objects at all. For example, this happens + * immediately after system startup or when a queue was exhausted and the queue + * worker needed to wait for new data. * This must only be called when the queue mutex is LOOKED, otherwise serious * malfunction will happen. */ @@ -1606,11 +1404,10 @@ DequeueConsumableElements(qqueue_t *pThis, wti_t *pWti, int *piRemainingQueueSiz nDequeued = nDiscarded = 0; while((iQueueSize = getLogicalQueueSize(pThis)) > 0 && nDequeued < pThis->iDeqBatchSize) { -dbgprintf("DequeueConsumableElements, index %d\n", nDequeued); CHKiRet(qqueueDeq(pThis, &pUsr)); /* check if we should discard this element */ - localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr); + localRet = qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr); if(localRet == RS_RET_QUEUE_FULL) { ++nDiscarded; continue; @@ -1731,7 +1528,7 @@ RateLimiter(qqueue_t *pThis) iDelay = 0; if(pThis->iDeqtWinToHr != 25) { /* 25 means disabled */ /* time calls are expensive, so only do them when needed */ - time(&tCurr); + datetime.GetTime(&tCurr); localtime_r(&tCurr, &m); iHrCurr = m.tm_hour; @@ -1775,7 +1572,8 @@ RateLimiter(qqueue_t *pThis) } -/* This dequeues the next batch. +/* This dequeues the next batch. Note that this function must not be + * cancelled, else it will leave back an inconsistent state. * rgerhards, 2009-05-20 */ static inline rsRetVal @@ -1786,7 +1584,6 @@ DequeueForConsumer(qqueue_t *pThis, wti_t *pWti) ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); -dbgprintf("YYY: deqeueu for consumer"); CHKiRet(DequeueConsumable(pThis, pWti)); if(pWti->batch.nElem == 0) @@ -1809,7 +1606,6 @@ batchProcessed(qqueue_t *pThis, wti_t *pWti) ISOBJ_TYPE_assert(pThis, qqueue); ISOBJ_TYPE_assert(pWti, wti); -dbgprintf("XXX: batchProcessed deletes %d records\n", pWti->batch.nElemDeq); DeleteProcessedBatch(pThis, &pWti->batch); qqueueChkPersist(pThis, pWti->batch.nElemDeq); @@ -1825,6 +1621,7 @@ dbgprintf("XXX: batchProcessed deletes %d records\n", pWti->batch.nElemDeq); static rsRetVal ConsumerReg(qqueue_t *pThis, wti_t *pWti) { + int iCancelStateSave; DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); @@ -1835,7 +1632,10 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti) /* we now have a non-idle batch of work, so we can release the queue mutex and process it */ d_pthread_mutex_unlock(pThis->mut); - CHKiRet(pThis->pConsumer(pThis->pUsr, &pWti->batch)); + /* at this spot, we may be cancelled */ + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave); + + CHKiRet(pThis->pConsumer(pThis->pUsr, &pWti->batch, &pThis->bShutdownImmediate)); /* we now need to check if we should deliberately delay processing a bit * and, if so, do that. -- rgerhards, 2008-01-30 @@ -1847,11 +1647,15 @@ ConsumerReg(qqueue_t *pThis, wti_t *pWti) srSleep(pThis->iDeqSlowdown / 1000000, pThis->iDeqSlowdown % 1000000); } + /* but now cancellation is no longer permitted */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + /* now we are done, but need to re-aquire the mutex */ d_pthread_mutex_lock(pThis->mut); finalize_it: -dbgprintf("XXX: regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet, getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); + dbgprintf("regular consumer finished, iret=%d, szlog %d sz phys %d\n", iRet, + getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); RETiRet; } @@ -1869,6 +1673,7 @@ static rsRetVal ConsumerDA(qqueue_t *pThis, wti_t *pWti) { int i; + int iCancelStateSave; DEFiRet; ISOBJ_TYPE_assert(pThis, qqueue); @@ -1879,15 +1684,24 @@ ConsumerDA(qqueue_t *pThis, wti_t *pWti) /* we now have a non-idle batch of work, so we can release the queue mutex and process it */ d_pthread_mutex_unlock(pThis->mut); + /* at this spot, we may be cancelled */ + pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &iCancelStateSave); + /* iterate over returned results and enqueue them in DA queue */ - for(i = 0 ; i < pWti->batch.nElem ; i++) { + for(i = 0 ; i < pWti->batch.nElem && !pThis->bShutdownImmediate ; i++) { /* TODO: we must add a generic "addRef" mechanism, because the disk queue enqueue destructs * the message. So far, we simply assume we always have msg_t, what currently is always the case. * rgerhards, 2009-05-28 */ - CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, (obj_t*)MsgAddRef((msg_t*)(pWti->batch.pElem[i].pUsrp)))); +dbgprintf("DA consumer pushes msg '%s'\n", ((msg_t*)(pWti->batch.pElem[i].pUsrp))->pszRawMsg); + CHKiRet(qqueueEnqObj(pThis->pqDA, eFLOWCTL_NO_DELAY, + (obj_t*)MsgAddRef((msg_t*)(pWti->batch.pElem[i].pUsrp)))); + pWti->batch.pElem[i].state = BATCH_STATE_COMM; /* commited to other queue! */ } + /* but now cancellation is no longer permitted */ + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); + /* now we are done, but need to re-aquire the mutex */ d_pthread_mutex_lock(pThis->mut); @@ -1899,12 +1713,6 @@ finalize_it: /* must only be called when the queue mutex is locked, else results * are not stable! - * If we are a child, we have done our duty when the queue is empty. In that case, - * we can terminate. - * Version for the DA worker thread. NOTE: the pThis->bRunsDA is different from - * the DA queue. - * If our queue is in destruction, we drain to the DA queue and so we shall not terminate - * until we have done so. */ static rsRetVal qqueueChkStopWrkrDA(qqueue_t *pThis) @@ -1913,25 +1721,6 @@ qqueueChkStopWrkrDA(qqueue_t *pThis) if(pThis->bEnqOnly) { iRet = RS_RET_TERMINATE_WHEN_IDLE; - } else { - if(pThis->bRunsDA) { - ASSERT(pThis->pqDA != NULL); - if( pThis->pqDA->bEnqOnly - && pThis->pqDA->sizeOnDiskMax > 0 - && pThis->pqDA->tVars.disk.sizeOnDisk > pThis->pqDA->sizeOnDiskMax) { - /* this queue can never grow, so we can give up... */ - iRet = RS_RET_TERMINATE_NOW; - } else if(getPhysicalQueueSize(pThis) < pThis->iHighWtrMrk && pThis->bQueueStarted == 1) { -dbgprintf("XXX: terminate_NOW DA worker: queue size %d, high water mark %d\n", getPhysicalQueueSize(pThis), pThis->iHighWtrMrk); - iRet = RS_RET_TERMINATE_NOW; -RUNLOG_STR("XXX: re-start reg worker"); -qqueueAdviseMaxWorkers(pThis); -RUNLOG_STR("XXX: done re-start reg worker"); - } - } else { - // experimental iRet = RS_RET_TERMINATE_NOW; - ; - } } RETiRet; @@ -1941,9 +1730,7 @@ RUNLOG_STR("XXX: done re-start reg worker"); /* must only be called when the queue mutex is locked, else results * are not stable! * If we are a child, we have done our duty when the queue is empty. In that case, - * we can terminate. - * Version for the regular worker thread. NOTE: the pThis->bRunsDA is different from - * the DA queue + * we can terminate. Version for the regular worker thread. */ static rsRetVal ChkStopWrkrReg(qqueue_t *pThis) @@ -1968,60 +1755,12 @@ GetDeqBatchSize(qqueue_t *pThis, int *pVal) DEFiRet; assert(pVal != NULL); *pVal = pThis->iDeqBatchSize; -if(pThis->pqParent != NULL) +if(pThis->pqParent != NULL) // TODO: check why we actually do this! *pVal = 16; RETiRet; } -/* must only be called when the queue mutex is locked, else results - * are not stable! DA worker version (pThis *is* the *main* queue, not DA!) - */ -static int -qqueueIsIdleDA(qqueue_t *pThis) -{ - return(getPhysicalQueueSize(pThis) <= pThis->iLowWtrMrk); -} -/* must only be called when the queue mutex is locked, else results - * are not stable! Regular worker version. - */ -static int -IsIdleReg(qqueue_t *pThis) -{ - return(getPhysicalQueueSize(pThis) == 0); -} - - -/* This function is called when a worker thread for the regular queue is shut down. - * If we are the primary queue, this is not really interesting to us. If, however, - * we are the DA (child) queue, that means the DA queue is empty. In that case, we - * need to signal the parent queue's DA worker, so that it can terminate DA mode. - * rgerhards, 2008-01-26 - * rgerhards, 2008-02-27: HOWEVER, in a shutdown condition, it may be that the parent's worker thread pool - * has already been terminated and destructed. This *is* a legal condition and happens - * from time to time in practice. So we need to signal only if there still is a - * parent DA worker queue. Please keep in mind that the the parent's DA worker - * pool is DIFFERENT from our (DA queue) regular worker pool. So when the parent's - * pWtpDA is destructed, there can still be some of our (DAq/wtp) threads be running. - * I am telling this, because I, too, always get confused by those... - */ -static rsRetVal -RegOnWrkrShutdown(qqueue_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, qqueue); - - if(pThis->pqParent != NULL) { - if(pThis->pqParent->pWtpDA != NULL) { /* see comment in function header from 2008-02-27 */ - wtpAdviseMaxWorkers(pThis->pqParent->pWtpDA, 1); /* reactivate DA worker (always 1) */ - } - } - - RETiRet; -} - - /* start up the queue - it must have been constructed and parameters defined * before. */ @@ -2029,8 +1768,6 @@ rsRetVal qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ { DEFiRet; - rsRetVal iRetLocal; - int bInitialized = 0; /* is queue already initialized? */ uchar pszBuf[64]; size_t lenBuf; @@ -2044,7 +1781,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ * influenced by properties which might have been set after queueConstruct () */ if(pThis->pqParent == NULL) { - pThis->mut = (pthread_mutex_t *) malloc (sizeof (pthread_mutex_t)); + pThis->mut = (pthread_mutex_t *) MALLOC (sizeof (pthread_mutex_t)); pthread_mutex_init(pThis->mut, NULL); } else { /* child queue, we need to use parent's mutex */ @@ -2072,8 +1809,7 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ if(pThis->qType == QUEUETYPE_DIRECT) FINALIZE; /* with direct queues, we are already finished... */ - /* create worker thread pools for regular operation. The DA pool is created on an as-needed - * basis, which potentially means never under most circumstances. + /* create worker thread pools for regular and DA operation. */ lenBuf = snprintf((char*)pszBuf, sizeof(pszBuf), "%s:Reg", obj.GetName((obj_t*) pThis)); CHKiRet(wtpConstruct (&pThis->pWtpReg)); @@ -2081,10 +1817,8 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ CHKiRet(wtpSetpfRateLimiter (pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RateLimiter)); CHKiRet(wtpSetpfChkStopWrkr (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int)) ChkStopWrkrReg)); CHKiRet(wtpSetpfGetDeqBatchSize (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, int*)) GetDeqBatchSize)); - CHKiRet(wtpSetpfIsIdle (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wtp_t*)) IsIdleReg)); CHKiRet(wtpSetpfDoWork (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, void *pWti)) ConsumerReg)); CHKiRet(wtpSetpfObjProcessed (pThis->pWtpReg, (rsRetVal (*)(void *pUsr, wti_t *pWti)) batchProcessed)); - CHKiRet(wtpSetpfOnWorkerShutdown(pThis->pWtpReg, (rsRetVal (*)(void *pUsr)) RegOnWrkrShutdown)); CHKiRet(wtpSetpmutUsr (pThis->pWtpReg, pThis->mut)); CHKiRet(wtpSetpcondBusy (pThis->pWtpReg, &pThis->notEmpty)); CHKiRet(wtpSetiNumWorkerThreads (pThis->pWtpReg, pThis->iNumWorkerThreads)); @@ -2092,27 +1826,11 @@ qqueueStart(qqueue_t *pThis) /* this is the ConstructionFinalizer */ CHKiRet(wtpSetpUsr (pThis->pWtpReg, pThis)); CHKiRet(wtpConstructFinalize (pThis->pWtpReg)); - /* initialize worker thread instances */ - if(pThis->bIsDA) { - /* If we are disk-assisted, we need to check if there is a QIF file - * which we need to load. -- rgerhards, 2008-01-15 - */ - iRetLocal = qqueueHaveQIF(pThis); - if(iRetLocal == RS_RET_OK) { - DBGOPRINT((obj_t*) pThis, "on-disk queue present, needs to be reloaded\n"); - InitDA(pThis, QUEUE_MODE_ENQDEQ, LOCK_MUTEX); /* initiate DA mode */ - bInitialized = 1; /* we are done */ - } else { - /* TODO: use logerror? -- rgerhards, 2008-01-16 */ - DBGOPRINT((obj_t*) pThis, "error %d trying to access on-disk queue files, starting without them. " - "Some data may be lost\n", iRetLocal); - } - } + /* set up DA system if we have a disk-assisted queue */ + if(pThis->bIsDA) + InitDA(pThis, LOCK_MUTEX); /* initiate DA mode */ - if(Debug && !bInitialized) { - DBGOPRINT((obj_t*) pThis, "queue starts up without (loading) any DA disk state (this is normal for the DA " - "queue itself!)\n"); - } + DBGOPRINT((obj_t*) pThis, "queue finished initialization\n"); /* if the queue already contains data, we need to start the correct number of worker threads. This can be * the case when a disk queue has been loaded. If we did not start it here, it would never start. @@ -2165,7 +1883,8 @@ static rsRetVal qqueuePersist(qqueue_t *pThis, int bIsCheckpoint) pThis->bNeedDelQIF = 0; } /* indicate spool file needs to be deleted */ - CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1)); + if(pThis->tVars.disk.pReadDel != NULL) /* may be NULL if we had a startup failure! */ + CHKiRet(strm.SetbDeleteOnClose(pThis->tVars.disk.pReadDel, 1)); FINALIZE; /* nothing left to do, so be happy */ } @@ -2255,10 +1974,16 @@ DoSaveOnShutdown(qqueue_t *pThis) ISOBJ_TYPE_assert(pThis, qqueue); - InitDA(pThis, QUEUE_MODE_ENQONLY, LOCK_MUTEX); /* switch to DA mode */ -dbgprintf("after InitDA, queue log %d, phys %d\n", getLogicalQueueSize(pThis), getPhysicalQueueSize(pThis)); - /* make sure we do not timeout before we are done */ - DBGOPRINT((obj_t*) pThis, "bSaveOnShutdown configured, infinite timeout set\n"); + /* we reduce the low water mark, otherwise the DA worker would terminate when + * it is reached. + */ + DBGOPRINT((obj_t*) pThis, "bSaveOnShutdown set, restarting DA worker...\n"); + pThis->bShutdownImmediate = 0; /* would termiante the DA worker! */ + pThis->iLowWtrMrk = 0; + wtpSetState(pThis->pWtpDA, wtpState_SHUTDOWN); /* shutdown worker (only) when done (was _IMMEDIATE!) */ + wtpAdviseMaxWorkers(pThis->pWtpDA, 1); /* restart DA worker */ + + DBGOPRINT((obj_t*) pThis, "waiting for DA worker to terminate...\n"); timeoutComp(&tTimeout, QUEUE_TIMEOUT_ETERNAL); /* and run the primary queue's DA worker to drain the queue */ iRetLocal = wtpShutdownAll(pThis->pWtpDA, wtpState_SHUTDOWN, &tTimeout); @@ -2276,8 +2001,6 @@ dbgprintf("after InitDA, queue log %d, phys %d\n", getLogicalQueueSize(pThis), g /* destructor for the queue object */ BEGINobjDestruct(qqueue) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(qqueue) - pThis->bQueueInDestruction = 1; /* indicate we are in destruction (modifies some behaviour) */ - /* shut down all workers * We do not need to shutdown workers when we are in enqueue-only mode or we are a * direct queue - because in both cases we have none... ;) @@ -2286,13 +2009,6 @@ CODESTARTobjDestruct(qqueue) if(pThis->qType != QUEUETYPE_DIRECT && !pThis->bEnqOnly && pThis->pqParent == NULL) ShutdownWorkers(pThis); - /* now all workers are terminated. Messages may exist. Also, some logically dequeued - * messages may never have been processed because their worker was terminated. So - * we need to reset the logical dequeue pointer, persist the queue if configured to do - * so and then destruct everything. -- rgerhards, 2009-05-26 - */ - CHKiRet(pThis->qUnDeqAll(pThis)); - if(pThis->bIsDA && getPhysicalQueueSize(pThis) > 0 && pThis->bSaveOnShutdown) { CHKiRet(DoSaveOnShutdown(pThis)); } @@ -2371,7 +2087,7 @@ qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix) if(pszPrefix == NULL) /* just unset the prefix! */ ABORT_FINALIZE(RS_RET_OK); - if((pThis->pszFilePrefix = malloc(sizeof(uchar) * iLenPrefix + 1)) == NULL) + if((pThis->pszFilePrefix = MALLOC(sizeof(uchar) * iLenPrefix + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszFilePrefix, pszPrefix, iLenPrefix + 1); pThis->lenFilePrefix = iLenPrefix; @@ -2413,12 +2129,8 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) /* first check if we need to discard this message (which will cause CHKiRet() to exit) */ - CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pThis->bRunsDA, pUsr)); + CHKiRet(qqueueChkDiscardMsg(pThis, pThis->iQueueSize, pUsr)); - /* then check if we need to add an assistance disk queue */ - if(pThis->bIsDA) - CHKiRet(ChkStrtDA(pThis)); - /* handle flow control * There are two different flow control mechanisms: basic and advanced flow control. * Basic flow control has always been implemented and protects the queue structures @@ -2462,6 +2174,7 @@ doEnqSingleObj(qqueue_t *pThis, flowControl_t flowCtlType, void *pUsr) && pThis->tVars.disk.sizeOnDisk > pThis->sizeOnDiskMax)) { DBGOPRINT((obj_t*) pThis, "enqueueMsg: queue FULL - waiting to drain.\n"); timeoutComp(&t, pThis->toEnq); +// TODO : handle enqOnly => discard! if(pthread_cond_timedwait(&pThis->notFull, pThis->mut, &t) != 0) { DBGOPRINT((obj_t*) pThis, "enqueueMsg: cond timeout, dropping message!\n"); objDestruct(pUsr); @@ -2500,7 +2213,6 @@ qqueueMultiEnqObj(qqueue_t *pThis, multi_submit_t *pMultiSub) } for(i = 0 ; i < pMultiSub->nElem ; ++i) { -dbgprintf("queueMultiEnq: %d\n", i); CHKiRet(doEnqSingleObj(pThis, pMultiSub->ppMsgs[i]->flowCtlType, (void*)pMultiSub->ppMsgs[i])); } @@ -2544,7 +2256,6 @@ finalize_it: if(pThis->qType != QUEUETYPE_DIRECT) { /* make sure at least one worker is running. */ qqueueAdviseMaxWorkers(pThis); -dbgprintf("YYY: call advise with mutex %p locked \n", pThis->mut); /* and release the mutex */ d_pthread_mutex_unlock(pThis->mut); pthread_setcancelstate(iCancelStateSave, NULL); @@ -2555,58 +2266,6 @@ dbgprintf("YYY: call advise with mutex %p locked \n", pThis->mut); } -/* set queue mode to enqueue only or not - * There is one subtle issue: this method may be called during queue - * construction or while it is running. In the former case, the queue - * mutex does not yet exist (it is NULL), while in the later case it - * must be locked. The function detects the state and operates as - * required. - * rgerhards, 2008-01-16 - */ -static rsRetVal -SetEnqOnly(qqueue_t *pThis, int bEnqOnly, int bLockMutex) -{ - DEFiRet; - DEFVARS_mutexProtection; - - ISOBJ_TYPE_assert(pThis, qqueue); - - /* for simplicity, we do one big mutex lock. This method is extremely seldom - * called, so that doesn't matter... -- rgerhards, 2008-01-16 - */ - if(pThis->mut != NULL) { - BEGIN_MTX_PROTECTED_OPERATIONS(pThis->mut, bLockMutex); - } - - if(bEnqOnly == pThis->bEnqOnly) - FINALIZE; /* no change, nothing to do */ - - if(pThis->bQueueStarted) { - /* we need to adjust queue operation only if we are not during initial param setup */ - if(bEnqOnly == 1) { - /* switch to enqueue-only mode */ - /* this means we need to terminate all workers - that's it... */ - DBGOPRINT((obj_t*) pThis, "switching to enqueue-only mode, terminating all worker threads\n"); - if(pThis->pWtpReg != NULL) - wtpWakeupAllWrkr(pThis->pWtpReg); - if(pThis->pWtpDA != NULL) - wtpWakeupAllWrkr(pThis->pWtpDA); - } else { - /* switch back to regular mode */ - ABORT_FINALIZE(RS_RET_NOT_IMPLEMENTED); /* we don't need this so far... */ - } - } - - pThis->bEnqOnly = bEnqOnly; - -finalize_it: - if(pThis->mut != NULL) { - END_MTX_PROTECTED_OPERATIONS(pThis->mut); - } - RETiRet; -} - - /* some simple object access methods */ DEFpropSetMeth(qqueue, bSyncQueueFiles, int) DEFpropSetMeth(qqueue, iPersistUpdCnt, int) @@ -2670,6 +2329,8 @@ BEGINObjClassInit(qqueue, 1, OBJ_IS_CORE_MODULE) /* request objects we use */ CHKiRet(objUse(glbl, CORE_COMPONENT)); CHKiRet(objUse(strm, CORE_COMPONENT)); + CHKiRet(objUse(datetime, CORE_COMPONENT)); + CHKiRet(objUse(errmsg, CORE_COMPONENT)); /* now set our own handlers */ OBJSetMethodHandler(objMethod_SETPROPERTY, qqueueSetProperty); diff --git a/runtime/queue.h b/runtime/queue.h index 73c62b52..93573dae 100644 --- a/runtime/queue.h +++ b/runtime/queue.h @@ -55,14 +55,14 @@ typedef struct qLinkedList_S { /* the queue object */ -typedef struct queue_s { +struct queue_s { BEGINobjInstance; queueType_t qType; int nLogDeq; /* number of elements currently logically dequeued */ + int bShutdownImmediate; /* should all workers cease processing messages? */ bool bEnqOnly; /* does queue run in enqueue-only mode (1) or not (0)? */ bool bSaveOnShutdown;/* persists everthing on shutdown (if DA!)? 1-yes, 0-no */ bool bQueueStarted; /* has queueStart() been called on this queue? 1-yes, 0-no */ - bool bQueueInDestruction;/* 1 if queue is in destruction process, 0 otherwise */ int iQueueSize; /* Current number of elements in the queue */ int iMaxQueueSize; /* how large can the queue grow? */ int iNumWorkerThreads;/* number of worker threads to use */ @@ -101,10 +101,11 @@ typedef struct queue_s { * the user really wanted...). -- rgerhards, 2008-04-02 */ /* end dequeue time window */ - rsRetVal (*pConsumer)(void *,batch_t*); /* user-supplied consumer function for dequeued messages */ + rsRetVal (*pConsumer)(void *,batch_t*,int*); /* user-supplied consumer function for dequeued messages */ /* calling interface for pConsumer: arg1 is the global user pointer from this structure, arg2 is the * user pointer array that was dequeued (actual sample: for actions, arg1 is the pAction and arg2 - * is pointer to an array of message message pointers) + * is pointer to an array of message message pointers), arg3 is a pointer to an interger which is zero + * during normal operations and one if the consumer must urgently shut down. */ /* type-specific handlers (set during construction) */ rsRetVal (*qConstruct)(struct queue_s *pThis); @@ -112,7 +113,6 @@ typedef struct queue_s { rsRetVal (*qAdd)(struct queue_s *pThis, void *pUsr); rsRetVal (*qDeq)(struct queue_s *pThis, void **ppUsr); rsRetVal (*qDel)(struct queue_s *pThis); - rsRetVal (*qUnDeqAll)(struct queue_s *pThis); /* end type-specific handler */ /* synchronization variables */ pthread_mutex_t mutThrdMgmt; /* mutex for the queue's thread management */ @@ -138,7 +138,6 @@ typedef struct queue_s { qDeqID deqIDAdd; /* next dequeue ID to use during add to queue store */ qDeqID deqIDDel; /* queue store delete position */ int bIsDA; /* is this queue disk assisted? */ - int bRunsDA; /* is this queue actually *running* disk assisted? */ struct queue_s *pqDA; /* queue for disk-assisted modes */ struct queue_s *pqParent;/* pointer to the parent (if this is a child queue) */ int bDAEnqOnly; /* EnqOnly setting for DA queue */ @@ -161,14 +160,8 @@ typedef struct queue_s { strm_t *pReadDel; /* current file for deleting */ } disk; } tVars; -} qqueue_t; - -/* some symbolic constants for easier reference */ -#define QUEUE_MODE_ENQDEQ 0 -#define QUEUE_MODE_ENQONLY 1 +}; -#define QUEUE_IDX_DA_WORKER 0 /* index for the DA worker (fixed) */ -#define QUEUE_PTR_DA_WORKER(x) (&((pThis)->pWrkThrds[0])) /* the define below is an "eternal" timeout for the timeout settings which require a value. * It is one day, which is not really eternal, but comes close to it if we think about @@ -185,7 +178,7 @@ rsRetVal qqueueStart(qqueue_t *pThis); rsRetVal qqueueSetMaxFileSize(qqueue_t *pThis, size_t iMaxFileSize); rsRetVal qqueueSetFilePrefix(qqueue_t *pThis, uchar *pszPrefix, size_t iLenPrefix); rsRetVal qqueueConstruct(qqueue_t **ppThis, queueType_t qType, int iWorkerThreads, - int iMaxQueueSize, rsRetVal (*pConsumer)(void*,batch_t*)); + int iMaxQueueSize, rsRetVal (*pConsumer)(void*,batch_t*, int*)); PROTOTYPEObjClassInit(qqueue); PROTOTYPEpropSetMeth(qqueue, iPersistUpdCnt, int); PROTOTYPEpropSetMeth(qqueue, bSyncQueueFiles, int); diff --git a/runtime/rsyslog.c b/runtime/rsyslog.c index 443d0f41..a76ae25a 100644 --- a/runtime/rsyslog.c +++ b/runtime/rsyslog.c @@ -80,6 +80,7 @@ #include "prop.h" #include "rule.h" #include "ruleset.h" +#include "parser.h" /* forward definitions */ static rsRetVal dfltErrLogger(int, uchar *errMsg); @@ -151,8 +152,6 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(propClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "glbl"; CHKiRet(glblClassInit(NULL)); - if(ppErrObj != NULL) *ppErrObj = "datetime"; - CHKiRet(datetimeClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "msg"; CHKiRet(msgClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "ctok_token"; @@ -183,6 +182,8 @@ rsrtInit(char **ppErrObj, obj_if_t *pObjIF) CHKiRet(qqueueClassInit(NULL)); if(ppErrObj != NULL) *ppErrObj = "conf"; CHKiRet(confClassInit(NULL)); + if(ppErrObj != NULL) *ppErrObj = "parser"; + CHKiRet(parserClassInit(NULL)); /* dummy "classes" */ if(ppErrObj != NULL) *ppErrObj = "str"; diff --git a/runtime/rsyslog.h b/runtime/rsyslog.h index 59e8458b..a5e8cf5d 100644 --- a/runtime/rsyslog.h +++ b/runtime/rsyslog.h @@ -40,6 +40,17 @@ #define CONF_TAG_BUFSIZE 32 #define CONF_HOSTNAME_BUFSIZE 32 #define CONF_PROP_BUFSIZE 16 /* should be close to sizeof(ptr) or lighly above it */ +#define CONF_MIN_SIZE_FOR_COMPRESS 60 /* config param: minimum message size to try compression. The smaller + * the message, the less likely is any compression gain. We check for + * gain before we submit the message. But to do so we still need to + * do the (costly) compress() call. The following setting sets a size + * for which no call to compress() is done at all. This may result in + * a few more bytes being transmited but better overall performance. + * Note: I have not yet checked the minimum UDP packet size. It might be + * that we do not save anything by compressing very small messages, because + * UDP might need to pad ;) + * rgerhards, 2006-11-30 + */ /* ############################################################# * @@ -107,6 +118,7 @@ typedef struct wti_s wti_t; typedef obj_t nsd_t; typedef obj_t nsdsel_t; typedef struct msg msg_t; +typedef struct queue_s qqueue_t; typedef struct prop_s prop_t; typedef struct interface_s interface_t; typedef struct objInfo_s objInfo_t; @@ -122,6 +134,9 @@ typedef struct vmstk_s vmstk_t; typedef struct batch_obj_s batch_obj_t; typedef struct batch_s batch_t; typedef struct wtp_s wtp_t; +typedef struct modInfo_s modInfo_t; +typedef struct parser_s parser_t; +typedef struct parserList_s parserList_t; typedef rsRetVal (*prsf_t)(struct vmstk_s*, int); /* pointer to a RainerScript function */ typedef uint64 qDeqID; /* queue Dequeue order ID. 32 bits is considered dangerously few */ @@ -386,6 +401,14 @@ enum rsRetVal_ /** return value. All methods return this if not specified oth RS_RET_NO_SRCNAME_TPL = -2150, /**< sourcename template was not specified where one was needed (omudpspoof spoof addr) */ RS_RET_HOST_NOT_SPECIFIED = -2151, /**< (target) host was not specified where it was needed */ RS_RET_ERR_LIBNET_INIT = -2152, /**< error initializing libnet */ + RS_RET_FORCE_TERM = -2153, /**< thread was forced to terminate by bShallShutdown, a state, not an error */ + RS_RET_RULES_QUEUE_EXISTS = -2154,/**< we were instructed to create a new ruleset queue, but one already exists */ + RS_RET_NO_CURR_RULESET = -2155,/**< no current ruleset exists (but one is required) */ + RS_RET_NO_MSG_PASSING = -2156,/**< output module interface parameter passing mode "MSG" is not available but required */ + RS_RET_RULESET_NOT_FOUND = -2157,/**< a required ruleset could not be found */ + RS_RET_NO_RULESET= -2158,/**< no ruleset name as specified where one was needed */ + RS_RET_PARSER_NOT_FOUND = -2159,/**< parser with the specified name was not found */ + RS_RET_COULD_NOT_PARSE = -2160,/**< (this) parser could not parse the message (no error, means try next one) */ /* RainerScript error messages (range 1000.. 1999) */ RS_RET_SYSVAR_NOT_FOUND = 1001, /**< system variable could not be found (maybe misspelled) */ diff --git a/runtime/rule.c b/runtime/rule.c index 182d616a..ae0da0d6 100644 --- a/runtime/rule.c +++ b/runtime/rule.c @@ -40,7 +40,6 @@ #include "var.h" #include "srUtils.h" #include "unicode-helper.h" -#include "dirty.h" /* for getFIOPName */ /* static data */ DEFobjStaticHelpers @@ -49,6 +48,35 @@ DEFobjCurrIf(expr) DEFobjCurrIf(var) DEFobjCurrIf(vm) + +/* support for simple textual representation of FIOP names + * rgerhards, 2005-09-27 + */ +static char* +getFIOPName(unsigned iFIOP) +{ + char *pRet; + switch(iFIOP) { + case FIOP_CONTAINS: + pRet = "contains"; + break; + case FIOP_ISEQUAL: + pRet = "isequal"; + break; + case FIOP_STARTSWITH: + pRet = "startswith"; + break; + case FIOP_REGEX: + pRet = "regex"; + break; + default: + pRet = "NOP"; + break; + } + return pRet; +} + + /* iterate over all actions, this is often needed, for example when HUP processing * must be done or a shutdown is pending. */ @@ -138,6 +166,7 @@ shouldProcessThisMessage(rule_t *pRule, msg_t *pMsg, int *bProcessMsg) } } +RUNLOG_VAR("%p", pRule->pCSProgNameComp); if(pRule->pCSProgNameComp != NULL) { int bInv = 0, bEqv = 0, offset = 0; if(*(rsCStrGetSzStrNoNULL(pRule->pCSProgNameComp)) == '-') { diff --git a/runtime/ruleset.c b/runtime/ruleset.c index 5ac9a8fd..1a77be2b 100644 --- a/runtime/ruleset.c +++ b/runtime/ruleset.c @@ -40,22 +40,24 @@ #include "rsyslog.h" #include "obj.h" +#include "cfsysline.h" #include "msg.h" #include "ruleset.h" #include "rule.h" #include "errmsg.h" +#include "parser.h" #include "unicode-helper.h" - -static rsRetVal debugPrintAll(void); // TODO: remove! +#include "dirty.h" /* for main ruleset queue creation */ /* static data */ DEFobjStaticHelpers DEFobjCurrIf(errmsg) DEFobjCurrIf(rule) +DEFobjCurrIf(parser) linkedList_t llRulesets; /* this is NOT a pointer - no typo here ;) */ ruleset_t *pCurrRuleset = NULL; /* currently "active" ruleset */ -ruleset_t *pDfltRuleset = NULL; /* currentl default ruleset, e.g. for binding to actions which have no other */ +ruleset_t *pDfltRuleset = NULL; /* current default ruleset, e.g. for binding to actions which have no other */ /* ---------- linked-list key handling functions ---------- */ @@ -141,6 +143,7 @@ DEFFUNC_llExecFunc(processMsgDoRules) rsRetVal iRet; ISOBJ_TYPE_assert(pData, rule); iRet = rule.ProcessMsg((rule_t*) pData, (msg_t*) pParam); +dbgprintf("ruleset: get iRet %d from rule.ProcessMsg()\n", iRet); return iRet; } @@ -161,13 +164,22 @@ processMsg(msg_t *pMsg) CHKiRet(llExecFunc(&pThis->llRules, processMsgDoRules, pMsg)); finalize_it: +dbgprintf("ruleset.ProcessMsg() returns %d\n", iRet); + RETiRet; +} - //if(iRet == RS_RET_DISCARDMSG) - //iRet = RS_RET_OK; - RETiRet; +/* return the ruleset-assigned parser list. NULL means use the default + * parser list. + * rgerhards, 2009-11-04 + */ +static parserList_t* +GetParserList(msg_t *pMsg) +{ + return (pMsg->pRuleset == NULL) ? pDfltRuleset->pParserLst : pMsg->pRuleset->pParserLst; } + /* Add a new rule to the end of the current rule set. We do a number * of checks and ignore the rule if it does not pass them. */ @@ -217,6 +229,19 @@ GetCurrent(void) } +/* get main queue associated with ruleset. If no ruleset-specifc main queue + * is set, the primary main message queue is returned. + * We use a non-standard calling interface, as nothing can go wrong and it + * is really much more natural to return the pointer directly. + */ +static qqueue_t* +GetRulesetQueue(ruleset_t *pThis) +{ + ISOBJ_TYPE_assert(pThis, ruleset); + return (pThis->pQueue == NULL) ? pMsgQueue : pThis->pQueue; +} + + /* Find the ruleset with the given name and return a pointer to its object. */ static rsRetVal @@ -322,6 +347,12 @@ finalize_it: BEGINobjDestruct(ruleset) /* be sure to specify the object type also in END and CODESTART macros! */ CODESTARTobjDestruct(ruleset) dbgprintf("destructing ruleset %p, name %p\n", pThis, pThis->pszName); + if(pThis->pQueue != NULL) { + qqueueDestruct(&pThis->pQueue); + } + if(pThis->pParserLst != NULL) { + parser.DestructParserList(&pThis->pParserLst); + } llDestroy(&pThis->llRules); free(pThis->pszName); ENDobjDestruct(ruleset) @@ -387,6 +418,81 @@ debugPrintAll(void) } +/* Create a ruleset-specific "main" queue for this ruleset. If one is already + * defined, an error message is emitted but nothing else is done. + * Note: we use the main message queue parameters for queue creation and access + * syslogd.c directly to obtain these. This is far from being perfect, but + * considered acceptable for the time being. + * rgerhards, 2009-10-27 + */ +static rsRetVal +rulesetCreateQueue(void __attribute__((unused)) *pVal, int *pNewVal) +{ + DEFiRet; + + if(pCurrRuleset == NULL) { + errmsg.LogError(0, RS_RET_NO_CURR_RULESET, "error: currently no specific ruleset specified, thus a " + "queue can not be added to it"); + ABORT_FINALIZE(RS_RET_NO_CURR_RULESET); + } + + if(pCurrRuleset->pQueue != NULL) { + errmsg.LogError(0, RS_RET_RULES_QUEUE_EXISTS, "error: ruleset already has a main queue, can not " + "add another one"); + ABORT_FINALIZE(RS_RET_RULES_QUEUE_EXISTS); + } + + if(pNewVal == 0) + FINALIZE; /* if it is turned off, we do not need to change anything ;) */ + + dbgprintf("adding a ruleset-specific \"main\" queue"); + CHKiRet(createMainQueue(&pCurrRuleset->pQueue, UCHAR_CONSTANT("ruleset"))); + +finalize_it: + RETiRet; +} + + +/* Add a ruleset specific parser to the ruleset. Note that adding the first + * parser automatically disables the default parsers. If they are needed as well, + * the must be added via explicit config directives. + * Note: this is the only spot in the code that requires the parser object. In order + * to solve some class init bootstrap sequence problems, we get the object handle here + * instead of during module initialization. Note that objUse() is capable of being + * called multiple times. + * rgerhards, 2009-11-04 + */ +static rsRetVal +rulesetAddParser(void __attribute__((unused)) *pVal, uchar *pName) +{ + parser_t *pParser; + DEFiRet; + + assert(pCurrRuleset != NULL); + + CHKiRet(objUse(parser, CORE_COMPONENT)); + iRet = parser.FindParser(&pParser, pName); + if(iRet == RS_RET_PARSER_NOT_FOUND) { + errmsg.LogError(0, RS_RET_PARSER_NOT_FOUND, "error: parser '%s' unknown at this time " + "(maybe defined too late in rsyslog.conf?)", pName); + ABORT_FINALIZE(RS_RET_NO_CURR_RULESET); + } else if(iRet != RS_RET_OK) { + errmsg.LogError(0, iRet, "error trying to find parser '%s'\n", pName); + FINALIZE; + } + + CHKiRet(parser.AddParserToList(&pCurrRuleset->pParserLst, pParser)); + + dbgprintf("added parser '%s' to ruleset '%s'\n", pName, pCurrRuleset->pszName); +RUNLOG_VAR("%p", pCurrRuleset->pParserLst); + +finalize_it: + d_free(pName); /* no longer needed */ + + RETiRet; +} + + /* queryInterface function * rgerhards, 2008-02-21 */ @@ -416,6 +522,8 @@ CODESTARTobjQueryInterface(ruleset) pIf->GetRuleset = GetRuleset; pIf->SetDefaultRuleset = SetDefaultRuleset; pIf->SetCurrRuleset = SetCurrRuleset; + pIf->GetRulesetQueue = GetRulesetQueue; + pIf->GetParserList = GetParserList; finalize_it: ENDobjQueryInterface(ruleset) @@ -427,6 +535,7 @@ BEGINObjClassExit(ruleset, OBJ_IS_CORE_MODULE) /* class, version */ llDestroy(&llRulesets); objRelease(errmsg, CORE_COMPONENT); objRelease(rule, CORE_COMPONENT); + objRelease(parser, CORE_COMPONENT); ENDObjClassExit(ruleset) @@ -445,6 +554,10 @@ BEGINObjClassInit(ruleset, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* prepare global data */ CHKiRet(llInit(&llRulesets, rulesetDestructForLinkedList, keyDestruct, strcasecmp)); + + /* config file handlers */ + CHKiRet(regCfSysLineHdlr((uchar *)"rulesetparser", 0, eCmdHdlrGetWord, rulesetAddParser, NULL, NULL)); + CHKiRet(regCfSysLineHdlr((uchar *)"rulesetcreatemainqueue", 0, eCmdHdlrBinary, rulesetCreateQueue, NULL, NULL)); ENDObjClassInit(ruleset) /* vi:set ai: diff --git a/runtime/ruleset.h b/runtime/ruleset.h index 32571687..222d773e 100644 --- a/runtime/ruleset.h +++ b/runtime/ruleset.h @@ -25,6 +25,7 @@ #ifndef INCLUDED_RULESET_H #define INCLUDED_RULESET_H +#include "queue.h" #include "linkedlist.h" /* the ruleset object */ @@ -32,6 +33,8 @@ struct ruleset_s { BEGINobjInstance; /* Data to implement generic object - MUST be the first data element! */ linkedList_t llRules; /* this is NOT a pointer - no typo here ;) */ uchar *pszName; /* name of our ruleset */ + qqueue_t *pQueue; /* "main" message queue, if the ruleset has its own (else NULL) */ + parserList_t *pParserLst;/* list of parsers to use for this ruleset */ }; /* interfaces */ @@ -50,8 +53,11 @@ BEGINinterface(ruleset) /* name must also be changed in ENDinterface macro! */ rsRetVal (*SetDefaultRuleset)(uchar*); rsRetVal (*SetCurrRuleset)(uchar*); ruleset_t* (*GetCurrent)(void); + qqueue_t* (*GetRulesetQueue)(ruleset_t*); + /* v3, 2009-11-04 */ + parserList_t* (*GetParserList)(msg_t *); ENDinterface(ruleset) -#define rulesetCURR_IF_VERSION 1 /* increment whenever you change the interface structure! */ +#define rulesetCURR_IF_VERSION 3 /* increment whenever you change the interface structure! */ /* prototypes */ diff --git a/runtime/srutils.c b/runtime/srutils.c index c403b312..7ddc3ba2 100644 --- a/runtime/srutils.c +++ b/runtime/srutils.c @@ -158,7 +158,7 @@ uchar *srUtilStrDup(uchar *pOld, size_t len) assert(pOld != NULL); - if((pNew = malloc(len + 1)) != NULL) + if((pNew = MALLOC(len + 1)) != NULL) memcpy(pNew, pOld, len + 1); return pNew; @@ -183,7 +183,7 @@ int makeFileParentDirs(uchar *szFile, size_t lenFile, mode_t mode, assert(lenFile > 0); len = lenFile + 1; /* add one for '\0'-byte */ - if((pszWork = malloc(sizeof(uchar) * len)) == NULL) + if((pszWork = MALLOC(sizeof(uchar) * len)) == NULL) return -1; memcpy(pszWork, szFile, len); for(p = pszWork+1 ; *p ; p++) @@ -326,7 +326,7 @@ rsRetVal genFileName(uchar **ppName, uchar *pDirName, size_t lenDirName, uchar * } lenName = lenDirName + 1 + lenFName + lenBuf + 1; /* last +1 for \0 char! */ - if((pName = malloc(sizeof(uchar) * lenName)) == NULL) + if((pName = MALLOC(sizeof(uchar) * lenName)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); /* got memory, now construct string */ diff --git a/runtime/stream.c b/runtime/stream.c index 58f16cce..3673db91 100644 --- a/runtime/stream.c +++ b/runtime/stream.c @@ -211,6 +211,7 @@ doPhysOpen(strm_t *pThis) } pThis->fd = open((char*)pThis->pszCurrFName, iFlags, pThis->tOpenMode); + DBGPRINTF("file '%s' opened as #%d with mode %d\n", pThis->pszCurrFName, pThis->fd, pThis->tOpenMode); if(pThis->fd == -1) { int ierrnoSave = errno; dbgoprint((obj_t*) pThis, "open error %d, file '%s'\n", errno, pThis->pszCurrFName); @@ -609,7 +610,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) * to make sure we can write out everything with a SINGLE api call! * We add another 128 bytes to take care of the gzip header and "all eventualities". */ - CHKmalloc(pThis->pZipBuf = (Bytef*) malloc(sizeof(uchar) * pThis->sIOBufSize + 128)); + CHKmalloc(pThis->pZipBuf = (Bytef*) MALLOC(sizeof(uchar) * pThis->sIOBufSize + 128)); } } @@ -638,7 +639,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) pthread_cond_init(&pThis->isEmpty, 0); pThis->iCnt = pThis->iEnq = pThis->iDeq = 0; for(i = 0 ; i < STREAM_ASYNC_NUMBUFS ; ++i) { - CHKmalloc(pThis->asyncBuf[i].pBuf = (uchar*) malloc(sizeof(uchar) * pThis->sIOBufSize)); + CHKmalloc(pThis->asyncBuf[i].pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->sIOBufSize)); } pThis->pIOBuf = pThis->asyncBuf[0].pBuf; pThis->bStopWriter = 0; @@ -646,7 +647,7 @@ static rsRetVal strmConstructFinalize(strm_t *pThis) DBGPRINTF("ERROR: stream %p cold not create writer thread\n", pThis); } else { /* we work synchronously, so we need to alloc a fixed pIOBuf */ - CHKmalloc(pThis->pIOBuf = (uchar*) malloc(sizeof(uchar) * pThis->sIOBufSize)); + CHKmalloc(pThis->pIOBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->sIOBufSize)); } finalize_it: @@ -1322,7 +1323,7 @@ strmSetFName(strm_t *pThis, uchar *pszName, size_t iLenName) if(pThis->pszFName != NULL) free(pThis->pszFName); - if((pThis->pszFName = malloc(sizeof(uchar) * (iLenName + 1))) == NULL) + if((pThis->pszFName = MALLOC(sizeof(uchar) * (iLenName + 1))) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszFName, pszName, iLenName + 1); /* always think about the \0! */ @@ -1349,7 +1350,7 @@ strmSetDir(strm_t *pThis, uchar *pszDir, size_t iLenDir) if(iLenDir < 1) ABORT_FINALIZE(RS_RET_FILE_PREFIX_MISSING); - if((pThis->pszDir = malloc(sizeof(uchar) * iLenDir + 1)) == NULL) + if((pThis->pszDir = MALLOC(sizeof(uchar) * iLenDir + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDir, pszDir, iLenDir + 1); /* always think about the \0! */ diff --git a/runtime/stringbuf.c b/runtime/stringbuf.c index 93995b38..ccf115c1 100644 --- a/runtime/stringbuf.c +++ b/runtime/stringbuf.c @@ -90,7 +90,7 @@ rsRetVal rsCStrConstructFromszStr(cstr_t **ppThis, uchar *sz) CHKiRet(rsCStrConstruct(&pThis)); pThis->iBufSize = pThis->iStrLen = strlen((char *) sz); - if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -119,7 +119,7 @@ rsRetVal rsCStrConstructFromCStr(cstr_t **ppThis, cstr_t *pFrom) CHKiRet(rsCStrConstruct(&pThis)); pThis->iBufSize = pThis->iStrLen = pFrom->iStrLen; - if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); } @@ -267,7 +267,7 @@ rsRetVal rsCStrSetSzStr(cstr_t *pThis, uchar *pszNew) pThis->pszBuf = NULL; /* now save the new value */ - if((pThis->pBuf = (uchar*) malloc(sizeof(uchar) * pThis->iStrLen)) == NULL) { + if((pThis->pBuf = (uchar*) MALLOC(sizeof(uchar) * pThis->iStrLen)) == NULL) { RSFREEOBJ(pThis); return RS_RET_OUT_OF_MEMORY; } @@ -315,7 +315,7 @@ uchar* rsCStrGetSzStr(cstr_t *pThis) if(pThis->pBuf != NULL) if(pThis->pszBuf == NULL) { /* we do not yet have a usable sz version - so create it... */ - if((pThis->pszBuf = malloc((pThis->iStrLen + 1) * sizeof(uchar))) == NULL) { + if((pThis->pszBuf = MALLOC((pThis->iStrLen + 1) * sizeof(uchar))) == NULL) { /* TODO: think about what to do - so far, I have no bright * idea... rgerhards 2005-09-07 */ @@ -369,7 +369,7 @@ rsRetVal cstrConvSzStrAndDestruct(cstr_t *pThis, uchar **ppSz, int bRetNULL) if(pThis->pBuf == NULL) { if(bRetNULL == 0) { - CHKmalloc(pRetBuf = malloc(sizeof(uchar))); + CHKmalloc(pRetBuf = MALLOC(sizeof(uchar))); *pRetBuf = '\0'; } else { pRetBuf = NULL; diff --git a/runtime/strmsrv.c b/runtime/strmsrv.c index 3dc53a97..a122ca8a 100644 --- a/runtime/strmsrv.c +++ b/runtime/strmsrv.c @@ -165,7 +165,7 @@ addNewLstnPort(strmsrv_t *pThis, uchar *pszPort) ISOBJ_TYPE_assert(pThis, strmsrv); /* create entry */ - CHKmalloc(pEntry = malloc(sizeof(strmLstnPortList_t))); + CHKmalloc(pEntry = MALLOC(sizeof(strmLstnPortList_t))); pEntry->pszPort = pszPort; pEntry->pSrv = pThis; CHKmalloc(pEntry->pszInputName = ustrdup(pThis->pszInputName)); diff --git a/runtime/sync.c b/runtime/sync.c index a3053e28..15d5c80f 100644 --- a/runtime/sync.c +++ b/runtime/sync.c @@ -28,12 +28,13 @@ #include "rsyslog.h" #include "sync.h" +#include "debug.h" void SyncObjInit(pthread_mutex_t **mut) { - *mut = (pthread_mutex_t *) malloc (sizeof (pthread_mutex_t)); + *mut = (pthread_mutex_t *) MALLOC(sizeof (pthread_mutex_t)); pthread_mutex_init(*mut, NULL); } diff --git a/runtime/syslogd-types.h b/runtime/syslogd-types.h index 161ee06f..fcebd985 100644 --- a/runtime/syslogd-types.h +++ b/runtime/syslogd-types.h @@ -56,8 +56,10 @@ * applications I do not yet envision. -- rgerhards, 2007-07-24 */ typedef enum _syslogFeature { - sFEATURERepeatedMsgReduction = 1, - sFEATURENonCancelInputTermination = 2 + sFEATURERepeatedMsgReduction = 1, /* for output modules */ + sFEATURENonCancelInputTermination = 2, /* for input modules */ + sFEATUREAutomaticSanitazion = 3, /* for parser modules */ + sFEATUREAutomaticPRIParsing = 4 /* for parser modules */ } syslogFeature; /* we define our own facility and severities */ diff --git a/runtime/vm.c b/runtime/vm.c index d7cd52d5..a1d992c3 100644 --- a/runtime/vm.c +++ b/runtime/vm.c @@ -34,6 +34,7 @@ #include "vm.h" #include "sysvar.h" #include "stringbuf.h" +#include "unicode-helper.h" /* static data */ DEFobjStaticHelpers @@ -41,6 +42,8 @@ DEFobjCurrIf(vmstk) DEFobjCurrIf(var) DEFobjCurrIf(sysvar) +static pthread_mutex_t mutGetenv; /* we need to make this global because otherwise we can not guarantee proper init! */ + /* ------------------------------ function registry code and structures ------------------------------ */ /* we maintain a registry of known functions */ @@ -539,6 +542,42 @@ finalize_it: } +/* The getenv function. Note that we guard the OS call by a mutex, as that + * function is not guaranteed to be thread-safe. This implementation here is far from + * being optimal, at least we should cache the result. This is left TODO for + * a later revision. + * rgerhards, 2009-11-03 + */ +static rsRetVal +rsf_getenv(vmstk_t *pStk, int numOperands) +{ + DEFiRet; + var_t *operand1; + char *envResult; + cstr_t *pCstr; + + if(numOperands != 1) + ABORT_FINALIZE(RS_RET_INVLD_NBR_ARGUMENTS); + + /* pop args and do operaton (trivial case here...) */ + vmstk.PopString(pStk, &operand1); + d_pthread_mutex_lock(&mutGetenv); + envResult = getenv((char*) rsCStrGetSzStr(operand1->val.pStr)); + DBGPRINTF("rsf_getenv(): envvar '%s', return '%s'\n", rsCStrGetSzStr(operand1->val.pStr), + envResult == NULL ? "(NULL)" : envResult); + iRet = rsCStrConstructFromszStr(&pCstr, (envResult == NULL) ? UCHAR_CONSTANT("") : (uchar*)envResult); + d_pthread_mutex_unlock(&mutGetenv); + if(iRet != RS_RET_OK) + FINALIZE; /* need to do this after mutex is unlocked! */ + + /* Store result and cleanup */ + var.SetString(operand1, pCstr); + vmstk.Push(pStk, operand1); +finalize_it: + RETiRet; +} + + /* The "tolower" function, which converts its sole argument to lower case. * Quite honestly, currently this is primarily a test driver for me... * rgerhards, 2009-04-06 @@ -756,6 +795,8 @@ BEGINObjClassExit(vm, OBJ_IS_CORE_MODULE) /* class, version */ objRelease(sysvar, CORE_COMPONENT); objRelease(var, CORE_COMPONENT); objRelease(vmstk, CORE_COMPONENT); + + pthread_mutex_destroy(&mutGetenv); ENDObjClassExit(vm) @@ -776,6 +817,9 @@ BEGINObjClassInit(vm, 1, OBJ_IS_CORE_MODULE) /* class, version */ /* register built-in functions // TODO: move to its own module */ CHKiRet(rsfrAddFunction((uchar*)"strlen", rsf_strlen)); CHKiRet(rsfrAddFunction((uchar*)"tolower", rsf_tolower)); + CHKiRet(rsfrAddFunction((uchar*)"getenv", rsf_getenv)); + + pthread_mutex_init(&mutGetenv, NULL); ENDObjClassInit(vm) diff --git a/runtime/wti.c b/runtime/wti.c index 53b695b0..288670b6 100644 --- a/runtime/wti.c +++ b/runtime/wti.c @@ -119,7 +119,7 @@ wtiSetState(wti_t *pThis, bool bNewVal) * Note that when waiting for the thread to terminate, we do a busy wait, checking * progress every 10ms. It is very unlikely that we will ever cancel a thread * and, if so, it will only happen at the end of the rsyslog run. So doing this - * kind of not optimal wait is considered preferable over using condition variables. + * kind of non-optimal wait is considered preferable over using condition variables. * rgerhards, 2008-02-26 */ rsRetVal @@ -134,7 +134,6 @@ wtiCancelThrd(wti_t *pThis) pthread_cancel(pThis->thrdID); /* now wait until the thread terminates... */ while(wtiGetState(pThis)) { -//fprintf(stderr, "sleep loop for getState\n"); srSleep(0, 10000); } } @@ -184,7 +183,7 @@ finalize_it: /* cancellation cleanup handler for queueWorker () - * Updates admin structure and frees ressources. + * Most importantly, it must bring back the batch into a consistent state. * Keep in mind that cancellation is disabled if we run into * the cancel cleanup handler (and have been cancelled). * rgerhards, 2008-01-16 @@ -201,10 +200,9 @@ wtiWorkerCancelCleanup(void *arg) ISOBJ_TYPE_assert(pWtp, wtp); DBGPRINTF("%s: cancelation cleanup handler called.\n", wtiGetDbgHdr(pThis)); + pWtp->pfObjProcessed(pWtp->pUsr, pThis); + DBGPRINTF("%s: done cancelation cleanup handler.\n", wtiGetDbgHdr(pThis)); - /* call user supplied handler */ - pWtp->pfOnWorkerCancel(pThis->pWtp->pUsr, pThis->batch.pElem[0].pUsrp); - ENDfunc } @@ -222,11 +220,8 @@ doIdleProcessing(wti_t *pThis, wtp_t *pWtp, int *pbInactivityTOOccured) BEGINfunc DBGPRINTF("%s: worker IDLE, waiting for work.\n", wtiGetDbgHdr(pThis)); - pWtp->pfOnIdle(pWtp->pUsr, MUTEX_ALREADY_LOCKED); - if(pThis->bAlwaysRunning) { /* never shut down any started worker */ -dbgprintf("YYY/ZZZ: wti Idle wait cond busy, mutex %p\n", pWtp->pmutUsr); d_pthread_cond_wait(pWtp->pcondBusy, pWtp->pmutUsr); } else { timeoutComp(&t, pWtp->toWrkShutdown);/* get absolute timeout */ @@ -235,11 +230,16 @@ dbgprintf("YYY/ZZZ: wti Idle wait cond busy, mutex %p\n", pWtp->pmutUsr); *pbInactivityTOOccured = 1; /* indicate we had a timeout */ } } + dbgoprint((obj_t*) pThis, "worker awoke from idle processing\n"); ENDfunc } -/* generic worker thread framework +/* generic worker thread framework. Note that we prohibit cancellation + * during almost all times, because it can have very undesired side effects. + * However, we may need to cancel a thread if the consumer blocks for too + * long (during shutdown). So what we do is block cancellation, and every + * consumer must enable it during the periods where it is safe. */ #pragma GCC diagnostic ignored "-Wempty-body" rsRetVal @@ -258,8 +258,7 @@ wtiWorker(wti_t *pThis) dbgSetThrdName(pThis->pszDbgHdr); pthread_cleanup_push(wtiWorkerCancelCleanup, pThis); - - pWtp->pfOnWorkerStartup(pWtp->pUsr); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); /* now we have our identity, on to real processing */ while(1) { /* loop will be broken below - need to do mutex locks */ @@ -267,10 +266,8 @@ wtiWorker(wti_t *pThis) pWtp->pfRateLimiter(pWtp->pUsr); } -dbgprintf("YYY/ZZZ: pre lock mutex\n"); d_pthread_mutex_lock(pWtp->pmutUsr); -dbgprintf("YYY/ZZZ: wti locks mutex %p\n", pWtp->pmutUsr); /* first check if we are in shutdown process (but evaluate a bit later) */ terminateRet = wtpChkStopWrkr(pWtp, MUTEX_ALREADY_LOCKED); if(terminateRet == RS_RET_TERMINATE_NOW) { @@ -288,7 +285,6 @@ dbgprintf("YYY/ZZZ: wti locks mutex %p\n", pWtp->pmutUsr); */ localRet = pWtp->pfDoWork(pWtp->pUsr, pThis); -dbgprintf("YYY/ZZZ: wti loop locked mutex %p again\n", pWtp->pmutUsr); if(localRet == RS_RET_IDLE) { if(terminateRet == RS_RET_TERMINATE_WHEN_IDLE || bInactivityTOOccured) { d_pthread_mutex_unlock(pWtp->pmutUsr); @@ -305,12 +301,8 @@ dbgprintf("YYY/ZZZ: wti loop locked mutex %p again\n", pWtp->pmutUsr); } /* indicate termination */ - d_pthread_mutex_lock(pWtp->pmutUsr); - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &iCancelStateSave); pthread_cleanup_pop(0); /* remove cleanup handler */ - pWtp->pfOnWorkerShutdown(pWtp->pUsr); pthread_setcancelstate(iCancelStateSave, NULL); - d_pthread_mutex_unlock(pWtp->pmutUsr); RETiRet; } @@ -340,7 +332,7 @@ wtiSetDbgHdr(wti_t *pThis, uchar *pszMsg, size_t lenMsg) free(pThis->pszDbgHdr); } - if((pThis->pszDbgHdr = malloc(sizeof(uchar) * lenMsg + 1)) == NULL) + if((pThis->pszDbgHdr = MALLOC(sizeof(uchar) * lenMsg + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDbgHdr, pszMsg, lenMsg + 1); /* always think about the \0! */ diff --git a/runtime/wtp.c b/runtime/wtp.c index 47b99fe8..060e6627 100644 --- a/runtime/wtp.c +++ b/runtime/wtp.c @@ -94,13 +94,8 @@ BEGINobjConstruct(wtp) /* be sure to specify the object type also in END macro! /* set all function pointers to "not implemented" dummy so that we can safely call them */ pThis->pfChkStopWrkr = NotImplementedDummy; pThis->pfGetDeqBatchSize = NotImplementedDummy; - pThis->pfIsIdle = NotImplementedDummy; pThis->pfDoWork = NotImplementedDummy; pThis->pfObjProcessed = NotImplementedDummy; - pThis->pfOnIdle = NotImplementedDummy; - pThis->pfOnWorkerCancel = NotImplementedDummy; - pThis->pfOnWorkerStartup = NotImplementedDummy; - pThis->pfOnWorkerShutdown = NotImplementedDummy; ENDobjConstruct(wtp) @@ -122,8 +117,7 @@ wtpConstructFinalize(wtp_t *pThis) /* alloc and construct workers - this can only be done in finalizer as we previously do * not know the max number of workers */ - if((pThis->pWrkr = malloc(sizeof(wti_t*) * pThis->iNumWorkerThreads)) == NULL) - ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); + CHKmalloc(pThis->pWrkr = MALLOC(sizeof(wti_t*) * pThis->iNumWorkerThreads)); for(i = 0 ; i < pThis->iNumWorkerThreads ; ++i) { CHKiRet(wtiConstruct(&pThis->pWrkr[i])); @@ -160,20 +154,6 @@ CODESTARTobjDestruct(wtp) ENDobjDestruct(wtp) -/* wake up all worker threads. - * rgerhards, 2008-01-16 - */ -rsRetVal -wtpWakeupAllWrkr(wtp_t *pThis) -{ - DEFiRet; - - ISOBJ_TYPE_assert(pThis, wtp); - pthread_cond_broadcast(pThis->pcondBusy); - RETiRet; -} - - /* Sent a specific state for the worker thread pool. -- rgerhards, 2008-01-21 * We do not need to do atomic instructions as set operations are only * called when terminating the pool, and then in strict sequence. So we @@ -239,8 +219,11 @@ wtpShutdownAll(wtp_t *pThis, wtpState_t tShutdownCmd, struct timespec *ptTimeout ISOBJ_TYPE_assert(pThis, wtp); + /* lock mutex to prevent races (may otherwise happen during idle processing and such...) */ + d_pthread_mutex_lock(pThis->pmutUsr); wtpSetState(pThis, tShutdownCmd); - wtpWakeupAllWrkr(pThis); + pthread_cond_broadcast(pThis->pcondBusy); /* wake up all workers */ + d_pthread_mutex_unlock(pThis->pmutUsr); /* wait for worker thread termination */ d_pthread_mutex_lock(&pThis->mutWtp); @@ -285,14 +268,14 @@ wtpCancelAll(wtp_t *pThis) } -/* cancellation cleanup handler for executing worker decrements the worker counter. - * This is also called when the the worker is normally shut down. - * rgerhards, 2009-07-20 +/* this function contains shared code for both regular worker shutdown as + * well as shutdown via cancellation. We can not simply use pthread_cleanup_pop(1) + * as this introduces a race in the debug system (RETiRet system). + * rgerhards, 2009-10-26 */ -static void -wtpWrkrExecCancelCleanup(void *arg) +static inline void +wtpWrkrExecCleanup(wti_t *pWti) { - wti_t *pWti = (wti_t*) arg; wtp_t *pThis; BEGINfunc @@ -307,11 +290,37 @@ wtpWrkrExecCancelCleanup(void *arg) DBGPRINTF("%s: Worker thread %lx, terminated, num workers now %d\n", wtpGetDbgHdr(pThis), (unsigned long) pWti, ATOMIC_FETCH_32BIT(pThis->iCurNumWrkThrd)); - pthread_cond_broadcast(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */ ENDfunc } +/* cancellation cleanup handler for executing worker decrements the worker counter. + * rgerhards, 2009-07-20 + */ +static void +wtpWrkrExecCancelCleanup(void *arg) +{ + wti_t *pWti = (wti_t*) arg; + wtp_t *pThis; + + BEGINfunc + ISOBJ_TYPE_assert(pWti, wti); + pThis = pWti->pWtp; + ISOBJ_TYPE_assert(pThis, wtp); + DBGPRINTF("%s: Worker thread %lx requested to be cancelled.\n", + wtpGetDbgHdr(pThis), (unsigned long) pWti); + + wtpWrkrExecCleanup(pWti); + + ENDfunc + /* NOTE: we must call ENDfunc FIRST, because otherwise the schedule may activate the main + * thread after the broadcast, which could destroy the debug class, resulting in a potential + * segfault. So we need to do the broadcast as actually the last action in our processing + */ + pthread_cond_broadcast(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */ +} + + /* wtp worker shell. This is started and calls into the actual * wti worker. * rgerhards, 2008-01-21 @@ -345,9 +354,15 @@ wtpWorker(void *arg) /* the arg is actually a wti object, even though we are in pthread_cleanup_push(wtpWrkrExecCancelCleanup, pWti); wtiWorker(pWti); - pthread_cleanup_pop(1); + pthread_cleanup_pop(0); + wtpWrkrExecCleanup(pWti); ENDfunc + /* NOTE: we must call ENDfunc FIRST, because otherwise the schedule may activate the main + * thread after the broadcast, which could destroy the debug class, resulting in a potential + * segfault. So we need to do the broadcast as actually the last action in our processing + */ + pthread_cond_broadcast(&pThis->condThrdTrm); /* activate anyone waiting on thread shutdown */ pthread_exit(0); } #pragma GCC diagnostic warning "-Wempty-body" @@ -411,7 +426,6 @@ wtpAdviseMaxWorkers(wtp_t *pThis, int nMaxWrkr) ISOBJ_TYPE_assert(pThis, wtp); -int nMaxWrkrTmp = nMaxWrkr; if(nMaxWrkr == 0) FINALIZE; @@ -419,16 +433,16 @@ int nMaxWrkrTmp = nMaxWrkr; nMaxWrkr = pThis->iNumWorkerThreads; nMissing = nMaxWrkr - ATOMIC_FETCH_32BIT(pThis->iCurNumWrkThrd); -dbgprintf("wtpAdviseMaxWorkers, nmax: %d, curr %d, missing %d\n", nMaxWrkrTmp, pThis->iNumWorkerThreads, nMissing); if(nMissing > 0) { - DBGPRINTF("%s: high activity - starting %d additional worker thread(s).\n", wtpGetDbgHdr(pThis), nMissing); + DBGPRINTF("%s: high activity - starting %d additional worker thread(s).\n", + wtpGetDbgHdr(pThis), nMissing); /* start the rqtd nbr of workers */ for(i = 0 ; i < nMissing ; ++i) { CHKiRet(wtpStartWrkr(pThis)); } } else { -dbgprintf("YYY: adivse signal cond busy"); +dbgprintf("YYY: wtpAdviseMaxWorkers, sufficient workers, just doing adivse signal cond busy\n"); pthread_cond_signal(pThis->pcondBusy); } @@ -448,13 +462,8 @@ DEFpropSetMethPTR(wtp, pcondBusy, pthread_cond_t) DEFpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)) DEFpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)) DEFpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)) -DEFpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, wtp_t*)) DEFpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*)) DEFpropSetMethFP(wtp, pfObjProcessed, rsRetVal(*pVal)(void*, wti_t*)) -DEFpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)) -DEFpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*, void*)) -DEFpropSetMethFP(wtp, pfOnWorkerStartup, rsRetVal(*pVal)(void*)) -DEFpropSetMethFP(wtp, pfOnWorkerShutdown, rsRetVal(*pVal)(void*)) /* set the debug header message @@ -478,7 +487,7 @@ wtpSetDbgHdr(wtp_t *pThis, uchar *pszMsg, size_t lenMsg) pThis->pszDbgHdr = NULL; } - if((pThis->pszDbgHdr = malloc(sizeof(uchar) * lenMsg + 1)) == NULL) + if((pThis->pszDbgHdr = MALLOC(sizeof(uchar) * lenMsg + 1)) == NULL) ABORT_FINALIZE(RS_RET_OUT_OF_MEMORY); memcpy(pThis->pszDbgHdr, pszMsg, lenMsg + 1); /* always think about the \0! */ diff --git a/runtime/wtp.h b/runtime/wtp.h index 0505b91c..05c02a8c 100644 --- a/runtime/wtp.h +++ b/runtime/wtp.h @@ -62,12 +62,7 @@ struct wtp_s { rsRetVal (*pfGetDeqBatchSize)(void *pUsr, int*); /* obtains max dequeue count from queue config */ rsRetVal (*pfObjProcessed)(void *pUsr, wti_t *pWti); /* indicate user object is processed */ rsRetVal (*pfRateLimiter)(void *pUsr); - rsRetVal (*pfIsIdle)(void *pUsr, wtp_t *pWtp); rsRetVal (*pfDoWork)(void *pUsr, void *pWti); - rsRetVal (*pfOnIdle)(void *pUsr, int); - rsRetVal (*pfOnWorkerCancel)(void *pUsr, void*pWti); - rsRetVal (*pfOnWorkerStartup)(void *pUsr); - rsRetVal (*pfOnWorkerShutdown)(void *pUsr); /* end user objects */ uchar *pszDbgHdr; /* header string for debug messages */ }; @@ -91,13 +86,8 @@ PROTOTYPEObjClassInit(wtp); PROTOTYPEpropSetMethFP(wtp, pfChkStopWrkr, rsRetVal(*pVal)(void*, int)); PROTOTYPEpropSetMethFP(wtp, pfRateLimiter, rsRetVal(*pVal)(void*)); PROTOTYPEpropSetMethFP(wtp, pfGetDeqBatchSize, rsRetVal(*pVal)(void*, int*)); -PROTOTYPEpropSetMethFP(wtp, pfIsIdle, rsRetVal(*pVal)(void*, wtp_t*)); PROTOTYPEpropSetMethFP(wtp, pfDoWork, rsRetVal(*pVal)(void*, void*)); PROTOTYPEpropSetMethFP(wtp, pfObjProcessed, rsRetVal(*pVal)(void*, wti_t*)); -PROTOTYPEpropSetMethFP(wtp, pfOnIdle, rsRetVal(*pVal)(void*, int)); -PROTOTYPEpropSetMethFP(wtp, pfOnWorkerCancel, rsRetVal(*pVal)(void*,void*)); -PROTOTYPEpropSetMethFP(wtp, pfOnWorkerStartup, rsRetVal(*pVal)(void*)); -PROTOTYPEpropSetMethFP(wtp, pfOnWorkerShutdown, rsRetVal(*pVal)(void*)); PROTOTYPEpropSetMeth(wtp, toWrkShutdown, long); PROTOTYPEpropSetMeth(wtp, wtpState, wtpState_t); PROTOTYPEpropSetMeth(wtp, iMaxWorkerThreads, int); |